diff --git a/.github/workflows/release-bot.yml b/.github/workflows/release-bot.yml new file mode 100644 index 0000000000000000000000000000000000000000..08aa94417c047d4e93645e4b6d0949ee2df29777 --- /dev/null +++ b/.github/workflows/release-bot.yml @@ -0,0 +1,18 @@ +name: Pushes release updates to a pre-defined Matrix room +on: + release: + types: + - edited + - prereleased + - published +jobs: + ping_matrix: + runs-on: ubuntu-latest + steps: + - name: send message + uses: s3krit/matrix-message-action@v0.0.2 + with: + room_id: ${{ secrets.MATRIX_ROOM_ID }} + access_token: ${{ secrets.MATRIX_ACCESS_TOKEN }} + message: "**${{github.event.repository.full_name}}:** A release has been ${{github.event.action}}
Release version [${{github.event.release.tag_name}}](${{github.event.release.html_url}})

***Description:***
${{github.event.release.body}}
" + server: "matrix.parity.io" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 62a6c2de32ddfa86fdbe6736737f7f0f3dc1768d..19f7298ab68eac2ed11646f23aff2e296d9d37bc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,21 @@ # pipelines can be triggered manually in the web # setting DEPLOY_TAG will only deploy the tagged image +# SAMPLE JOB TEMPLATE - This is not a complete example but is enough to build a +# simple CI job. For full documentation, visit https://docs.gitlab.com/ee/ci/yaml/ +# +# my-example-job: +# stage: test # One of the stages listed below this job (required) +# image: parity/tools:latest # Any docker image (required) +# allow_failure: true # Allow the pipeline to continue if this job fails (default: false) +# dependencies: +# - build-rust-doc-release # Any jobs that are required to run before this job (optional) +# variables: +# MY_ENVIRONMENT_VARIABLE: "some useful value" # Environment variables passed to the job (optional) +# script: +# - echo "List of shell commands to run in your job" +# - echo "You can also just specify a script here, like so:" +# - ./.maintain/gitlab/my_amazing_script.sh stages: - test @@ -22,6 +37,9 @@ variables: CI_SERVER_NAME: "GitLab CI" DOCKER_OS: "debian:stretch" ARCH: "x86_64" + # FIXME set to release + CARGO_UNLEASH_INSTALL_PARAMS: "--version 1.0.0-alpha.8" + CARGO_UNLEASH_PKG_DEF: "--skip node node-* pallet-template pallet-example pallet-example-* subkey chain-spec-builder" .collect-artifacts: &collect-artifacts @@ -86,6 +104,16 @@ check-runtime: interruptible: true allow_failure: true +check-signed-tag: + stage: test + image: parity/tools:latest + <<: *kubernetes-build + only: + - tags + - /^v[0-9]+\.[0-9]+\.[0-9]+.*$/ + script: + - ./.maintain/gitlab/check_signed.sh + allow_failure: false check-line-width: stage: test @@ -160,6 +188,16 @@ test-dependency-rules: script: - .maintain/ensure-deps.sh +unleash-check: + stage: test + <<: *docker-env + only: + - master + - tags + script: + - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} + - cargo unleash check ${CARGO_UNLEASH_PKG_DEF} + test-frame-staking: stage: test <<: *docker-env @@ -524,7 +562,28 @@ publish-gh-doc: after_script: - rm -vrf ${HOME}/.gitconfig +publish-draft-release: + stage: publish + image: parity/tools:latest + only: + - tags + - /^v[0-9]+\.[0-9]+\.[0-9]+.*$/ + script: + - ./.maintain/gitlab/publish_draft_release.sh + interruptible: true + allow_failure: true +publish-to-crates-io: + stage: publish + <<: *docker-env + only: + - tags + - /^v[0-9]+\.[0-9]+\.[0-9]+.*$/ + script: + - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} + - cargo unleash em-dragons --no-check ${CARGO_UNLEASH_PKG_DEF} + interruptible: true + allow_failure: true .deploy-template: &deploy stage: kubernetes diff --git a/.maintain/Dockerfile b/.maintain/Dockerfile index 7cba85c544afc2c8cc1ff56401b2172a01d30364..2fc1532aa2837b3314eaf17a907ca765d33c85b5 100644 --- a/.maintain/Dockerfile +++ b/.maintain/Dockerfile @@ -1,7 +1,7 @@ # Note: We don't use Alpine and its packaged Rust/Cargo because they're too often out of date, # preventing them from being used to build Substrate/Polkadot. -FROM phusion/baseimage:0.10.2 as builder +FROM phusion/baseimage:0.11 as builder LABEL maintainer="chevdor@gmail.com" LABEL description="This is the build stage for Substrate. Here we create the binary." @@ -20,13 +20,12 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \ export PATH="$PATH:$HOME/.cargo/bin" && \ rustup toolchain install nightly && \ rustup target add wasm32-unknown-unknown --toolchain nightly && \ - rustup default nightly && \ rustup default stable && \ cargo build "--$PROFILE" # ===== SECOND STAGE ====== -FROM phusion/baseimage:0.10.2 +FROM phusion/baseimage:0.11 LABEL maintainer="chevdor@gmail.com" LABEL description="This is the 2nd stage: a very small image where we copy the Substrate binary." ARG PROFILE=release @@ -34,9 +33,10 @@ ARG PROFILE=release RUN mv /usr/share/ca* /tmp && \ rm -rf /usr/share/* && \ mv /tmp/ca-certificates /usr/share/ && \ - mkdir -p /root/.local/share/Polkadot && \ - ln -s /root/.local/share/Polkadot /data && \ - useradd -m -u 1000 -U -s /bin/sh -d /substrate substrate + useradd -m -u 1000 -U -s /bin/sh -d /substrate substrate && \ + mkdir -p /substrate/.local/share/substrate && \ + chown -R substrate:substrate /substrate/.local && \ + ln -s /substrate/.local/share/substrate /data COPY --from=builder /substrate/target/$PROFILE/substrate /usr/local/bin @@ -49,7 +49,7 @@ RUN rm -rf /usr/lib/python* && \ rm -rf /usr/bin /usr/sbin /usr/share/man USER substrate -EXPOSE 30333 9933 9944 +EXPOSE 30333 9933 9944 9615 VOLUME ["/data"] CMD ["/usr/local/bin/substrate"] diff --git a/.maintain/gitlab/check_signed.sh b/.maintain/gitlab/check_signed.sh new file mode 100755 index 0000000000000000000000000000000000000000..7c4cc47baba38fa41214ed0fefc7e09b75a69e7d --- /dev/null +++ b/.maintain/gitlab/check_signed.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# shellcheck source=lib.sh +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh" + +version="$CI_COMMIT_TAG" + +echo '[+] Checking tag has been signed' +check_tag "paritytech/substrate" "$version" +case $? in + 0) echo '[+] Tag found and has been signed'; exit 0 + ;; + 1) echo '[!] Tag found but has not been signed. Aborting release.'; exit 1 + ;; + 2) echo '[!] Tag not found. Aborting release.'; exit 1 +esac diff --git a/.maintain/gitlab/lib.sh b/.maintain/gitlab/lib.sh new file mode 100755 index 0000000000000000000000000000000000000000..c8b2d73e6097f42fd6590b5e3d78c537dc620028 --- /dev/null +++ b/.maintain/gitlab/lib.sh @@ -0,0 +1,81 @@ +#!/bin/sh + +api_base="https://api.github.com/repos" + +# Function to take 2 git tags/commits and get any lines from commit messages +# that contain something that looks like a PR reference: e.g., (#1234) +sanitised_git_logs(){ + git --no-pager log --pretty=format:"%s" "$1..$2" | + # Only find messages referencing a PR + grep -E '\(#[0-9]+\)' | + # Strip any asterisks + sed 's/^* //g' | + # And add them all back + sed 's/^/* /g' +} + +# Returns the last published release on github +# repo: 'organization/repo' +# Usage: last_github_release "$repo" +last_github_release(){ + curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" \ + -s "$api_base/$1/releases/latest" | jq '.tag_name' +} + +# Checks whether a tag on github has been verified +# repo: 'organization/repo' +# tagver: 'v1.2.3' +# Usage: check_tag $repo $tagver +check_tag () { + repo=$1 + tagver=$2 + tag_out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/git/refs/tags/$tagver") + tag_sha=$(echo "$tag_out" | jq -r .object.sha) + object_url=$(echo "$tag_out" | jq -r .object.url) + if [ "$tag_sha" = "null" ]; then + return 2 + fi + verified_str=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$object_url" | jq -r .verification.verified) + if [ "$verified_str" = "true" ]; then + # Verified, everything is good + return 0 + else + # Not verified. Bad juju. + return 1 + fi +} + +# Checks whether a given PR has a given label. +# repo: 'organization/repo' +# pr_id: 12345 +# label: B1-silent +# Usage: has_label $repo $pr_id $label +has_label(){ + repo="$1" + pr_id="$2" + label="$3" + out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/pulls/$pr_id") + [ -n "$(echo "$out" | jq ".labels | .[] | select(.name==\"$label\")")" ] +} + +# Formats a message into a JSON string for posting to Matrix +# message: 'any plaintext message' +# formatted_message: 'optional message formatted in html' +# Usage: structure_message $content $formatted_content (optional) +structure_message() { + if [ -z "$2" ]; then + body=$(jq -Rs --arg body "$1" '{"msgtype": "m.text", $body}' < /dev/null) + else + body=$(jq -Rs --arg body "$1" --arg formatted_body "$2" '{"msgtype": "m.text", $body, "format": "org.matrix.custom.html", $formatted_body}' < /dev/null) + fi + echo "$body" +} + +# Post a message to a matrix room +# body: '{body: "JSON string produced by structure_message"}' +# room_id: !fsfSRjgjBWEWffws:matrix.parity.io +# access_token: see https://matrix.org/docs/guides/client-server-api/ +# Usage: send_message $body (json formatted) $room_id $access_token +send_message() { +curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3" +} diff --git a/.maintain/gitlab/publish_draft_release.sh b/.maintain/gitlab/publish_draft_release.sh new file mode 100755 index 0000000000000000000000000000000000000000..4f73575f5bbaf836d7e07cb43c7fd1c1173858da --- /dev/null +++ b/.maintain/gitlab/publish_draft_release.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +# shellcheck source=lib.sh +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh" + +# Substrate labels for PRs we want to include in the release notes +labels=( + 'B1-runtimenoteworthy' + 'B1-clientnoteworthy' + 'B1-apinoteworthy' +) + +version="$CI_COMMIT_TAG" + +# Note that this is not the last *tagged* version, but the last *published* version +last_version=$(last_github_release 'paritytech/substrate') +echo "[+] Version: $version; Previous version: $last_version" + +all_changes="$(sanitised_git_logs "$last_version" "$version")" +labelled_changes="" +echo "[+] Iterating through $(wc -l <<< "$all_changes") changes to find labelled PRs" +while IFS= read -r line; do + pr_id=$(echo "$line" | sed -E 's/.*#([0-9]+)\)$/\1/') + + # Skip if the PR has the silent label - this allows us to skip a few requests + if has_label 'paritytech/substrate' "$pr_id" 'B0-silent'; then + continue + fi + for label in "${labels[@]}"; do + if has_label 'paritytech/substrate' "$pr_id" "$label"; then + labelled_changes="$labelled_changes +$line" + fi + done +done <<< "$all_changes" + + +release_text="Substrate $version +----------------- +$labelled_changes" + +echo "[+] Release text generated: " +echo "$release_text" + +echo "[+] Pushing release to github" +# Create release on github +release_name="Substrate $version" +data=$(jq -Rs --arg version "$version" \ + --arg release_name "$release_name" \ + --arg release_text "$release_text" \ +'{ + "tag_name": $version, + "target_commitish": "master", + "name": $release_name, + "body": $release_text, + "draft": true, + "prerelease": false +}' < /dev/null) + +out=$(curl -s -X POST --data "$data" -H "Authorization: token $GITHUB_RELEASE_TOKEN" "$api_base/paritytech/substrate/releases") + +html_url=$(echo "$out" | jq -r .html_url) + +if [ "$html_url" == "null" ] +then + echo "[!] Something went wrong posting:" + echo "$out" +else + echo "[+] Release draft created: $html_url" +fi + +echo '[+] Sending draft release URL to Matrix' + +msg_body=$(cat <Release pipeline for Substrate $version complete.
+Draft release created: $html_url +EOF +) +send_message "$(structure_message "$msg_body" "$formatted_msg_body")" "$MATRIX_ROOM_ID" "$MATRIX_ACCESS_TOKEN" + +echo "[+] Done! Maybe the release worked..." diff --git a/Cargo.lock b/Cargo.lock index b557048f9d36a3878df336e13402265a5e03025d..2bed3b2bb5f4d8b0154397e7c737ff60fe04a125 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,9 +61,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743ad5a418686aad3b87fd14c43badd828cf26e214a00f92a384291cf22e1811" +checksum = "d5e63fd144e18ba274ae7095c0197a870a7b9468abc801dd62f190d80817d2ec" dependencies = [ "memchr", ] @@ -146,7 +146,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d0864d84b8e07b145449be9a8537db86bf9de5ce03b913214694643b4743502" dependencies = [ - "quote 1.0.2", + "quote", "syn", ] @@ -242,9 +242,9 @@ checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" [[package]] name = "backtrace" -version = "0.3.43" +version = "0.3.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f80256bc78f67e7df7e36d77366f636ed976895d91fe2ab9efa3973e8fe8c4f" +checksum = "e4036b9bf40f3cf16aba72a3d65e8a520fc4bafcdc7079aea8f848c58c5b5536" dependencies = [ "backtrace-sys", "cfg-if", @@ -305,25 +305,26 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.49.4" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c07087f3d5731bf3fb375a81841b99597e25dc11bd3bc72d16d43adf6624a6e" +checksum = "99de13bb6361e01e493b3db7928085dcc474b7ba4f5481818e53a89d76b8393f" dependencies = [ "bitflags", "cexpr", "cfg-if", "clang-sys", "clap", - "env_logger 0.6.2", - "fxhash", + "env_logger 0.7.1", "lazy_static", + "lazycell", "log 0.4.8", "peeking_take_while", - "proc-macro2 0.4.30", - "quote 0.6.13", + "proc-macro2", + "quote", "regex", + "rustc-hash", "shlex", - "which 2.0.1", + "which", ] [[package]] @@ -422,29 +423,10 @@ dependencies = [ ] [[package]] -name = "browser-utils" -version = "0.8.0" -dependencies = [ - "chrono", - "clear_on_drop", - "console_error_panic_hook", - "console_log", - "futures 0.1.29", - "futures 0.3.4", - "futures-timer 3.0.1", - "js-sys", - "kvdb-web", - "libp2p", - "log 0.4.8", - "rand 0.6.5", - "rand 0.7.3", - "sc-chain-spec", - "sc-informant", - "sc-network", - "sc-service", - "wasm-bindgen", - "wasm-bindgen-futures", -] +name = "bs58" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c95ee6bba9d950218b6cc910cf62bc9e0a171d0f4537e3627b0f54d08549b188" [[package]] name = "bs58" @@ -591,7 +573,7 @@ dependencies = [ [[package]] name = "chain-spec-builder" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "ansi_term 0.12.1", "node-cli", @@ -610,7 +592,6 @@ dependencies = [ "js-sys", "num-integer", "num-traits", - "serde", "time", "wasm-bindgen", ] @@ -732,25 +713,23 @@ checksum = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b" [[package]] name = "cranelift-bforest" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd0f53d59dc9ab1c8ab68c991d8406b52b7a0aab0b15b05a3a6895579c4e5dd9" +version = "0.59.0" +source = "git+https://github.com/paritytech/wasmtime?branch=a-thread-safe-api#851887d84d03543f931f6312448d0dd5d8a9324e" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0381a794836fb994c47006465d46d46be072483b667f36013d993b9895117fee" +version = "0.59.0" +source = "git+https://github.com/paritytech/wasmtime?branch=a-thread-safe-api#851887d84d03543f931f6312448d0dd5d8a9324e" dependencies = [ "byteorder 1.3.4", "cranelift-bforest", "cranelift-codegen-meta", "cranelift-codegen-shared", "cranelift-entity", - "gimli 0.20.0", + "gimli", "log 0.4.8", "serde", "smallvec 1.2.0", @@ -760,9 +739,8 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "208c3c8d82bfef32a534c5020c6cfc3bc92f41388f1246b7bb98cf543331abaa" +version = "0.59.0" +source = "git+https://github.com/paritytech/wasmtime?branch=a-thread-safe-api#851887d84d03543f931f6312448d0dd5d8a9324e" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -770,24 +748,21 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea048c456a517e56fd6df8f0e3947922897e6e6f61fbc5eb557a36c7b8ff6394" +version = "0.59.0" +source = "git+https://github.com/paritytech/wasmtime?branch=a-thread-safe-api#851887d84d03543f931f6312448d0dd5d8a9324e" [[package]] name = "cranelift-entity" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8c7ed50812194c9e9de1fa39c77b39fc9ab48173d5e7ee88b25b6a8953e9b8" +version = "0.59.0" +source = "git+https://github.com/paritytech/wasmtime?branch=a-thread-safe-api#851887d84d03543f931f6312448d0dd5d8a9324e" dependencies = [ "serde", ] [[package]] name = "cranelift-frontend" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ceb931d9f919731df1b1ecdc716b5c66384b413a7f95909d1f45441ab9bef5" +version = "0.59.0" +source = "git+https://github.com/paritytech/wasmtime?branch=a-thread-safe-api#851887d84d03543f931f6312448d0dd5d8a9324e" dependencies = [ "cranelift-codegen", "log 0.4.8", @@ -797,9 +772,8 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "564ee82268bc25b914fcf331edfc2452f2d9ca34f976b187b4ca668beba250c8" +version = "0.59.0" +source = "git+https://github.com/paritytech/wasmtime?branch=a-thread-safe-api#851887d84d03543f931f6312448d0dd5d8a9324e" dependencies = [ "cranelift-codegen", "raw-cpuid", @@ -808,9 +782,8 @@ dependencies = [ [[package]] name = "cranelift-wasm" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de63e2271b374be5b07f359184e2126a08fb24d24a740cbc178b7e0107ddafa5" +version = "0.59.0" +source = "git+https://github.com/paritytech/wasmtime?branch=a-thread-safe-api#851887d84d03543f931f6312448d0dd5d8a9324e" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -818,7 +791,7 @@ dependencies = [ "log 0.4.8", "serde", "thiserror", - "wasmparser 0.48.2", + "wasmparser", ] [[package]] @@ -905,33 +878,36 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acec9a3b0b3559f15aee4f90746c4e5e293b701c0f7d3925d24e01645267b68c" +checksum = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061" dependencies = [ "crossbeam-utils", + "maybe-uninit", ] [[package]] name = "crossbeam-deque" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca" +checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" dependencies = [ "crossbeam-epoch", "crossbeam-utils", + "maybe-uninit", ] [[package]] name = "crossbeam-epoch" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" dependencies = [ - "autocfg 0.1.7", + "autocfg 1.0.0", "cfg-if", "crossbeam-utils", "lazy_static", + "maybe-uninit", "memoffset", "scopeguard", ] @@ -948,11 +924,11 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" dependencies = [ - "autocfg 0.1.7", + "autocfg 1.0.0", "cfg-if", "lazy_static", ] @@ -988,9 +964,9 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.6" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b5cadb6b25c77aeff80ba701712494213f4a8418fcda2ee11b6560c3ad0bf4c" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" dependencies = [ "memchr", ] @@ -1006,11 +982,11 @@ dependencies = [ [[package]] name = "ctor" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8ce37ad4184ab2ce004c33bf6379185d3b1c95801cab51026bd271bf68eedc" +checksum = "47c5e5ac752e18207b12e16b10631ae5f7f68f8805f335f9b817ead83d9ffce1" dependencies = [ - "quote 1.0.2", + "quote", "syn", ] @@ -1062,18 +1038,18 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f47ca1860a761136924ddd2422ba77b2ea54fe8cc75b9040804a0d9d32ad97" +checksum = "11c0346158a19b3627234e15596f5e465c360fcdb97d817bcb255e0510f5a788" [[package]] name = "derive_more" -version = "0.99.2" +version = "0.99.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2159be042979966de68315bce7034bb000c775f22e3e834e1c52ff78f041cae8" +checksum = "a806e96c59a76a5ba6e18735b6cf833344671e61e7863f2edb5c518ea2cac95c" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -1163,8 +1139,8 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecf634c5213044b8d54a46dd282cf5dd1f86bb5cb53e92c409cb4680a7fb9894" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -1242,6 +1218,33 @@ dependencies = [ "serde_json", ] +[[package]] +name = "ethbloom" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cfe1c169414b709cf28aa30c74060bdb830a03a8ba473314d079ac79d80a5f" +dependencies = [ + "crunchy", + "fixed-hash", + "impl-rlp", + "impl-serde 0.2.3", + "tiny-keccak 1.5.0", +] + +[[package]] +name = "ethereum-types" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba744248e3553a393143d5ebb68939fc3a4ec0c22a269682535f5ffe7fed728c" +dependencies = [ + "ethbloom", + "fixed-hash", + "impl-rlp", + "impl-serde 0.2.3", + "primitive-types", + "uint", +] + [[package]] name = "evm" version = "0.15.0" @@ -1315,9 +1318,9 @@ dependencies = [ [[package]] name = "failure" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9" +checksum = "b8529c2421efa3066a5cbd8063d2244603824daccb6936b079010bb2aa89464b" dependencies = [ "backtrace", "failure_derive", @@ -1325,12 +1328,12 @@ dependencies = [ [[package]] name = "failure_derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08" +checksum = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", "synstructure", ] @@ -1436,14 +1439,43 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "fork-tree" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "parity-scale-codec", ] +[[package]] +name = "frame-benchmarking" +version = "2.0.0-alpha.3" +dependencies = [ + "frame-support", + "frame-system", + "parity-scale-codec", + "sp-api", + "sp-io", + "sp-runtime", + "sp-runtime-interface", + "sp-std", +] + +[[package]] +name = "frame-benchmarking-cli" +version = "2.0.0-alpha.3" +dependencies = [ + "frame-benchmarking", + "parity-scale-codec", + "sc-cli", + "sc-client", + "sc-client-db", + "sc-executor", + "sc-service", + "sp-runtime", + "structopt", +] + [[package]] name = "frame-executive" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -1461,7 +1493,7 @@ dependencies = [ [[package]] name = "frame-metadata" -version = "11.0.0" +version = "11.0.0-alpha.3" dependencies = [ "parity-scale-codec", "serde", @@ -1471,7 +1503,7 @@ dependencies = [ [[package]] name = "frame-support" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "bitmask", "frame-metadata", @@ -1496,37 +1528,37 @@ dependencies = [ [[package]] name = "frame-support-procedural" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support-procedural-tools", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "frame-support-procedural-tools" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support-procedural-tools-derive", "proc-macro-crate", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "frame-support-procedural-tools-derive" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "frame-support-test" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "frame-support", "parity-scale-codec", @@ -1542,7 +1574,7 @@ dependencies = [ [[package]] name = "frame-system" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "criterion 0.2.11", "frame-support", @@ -1560,7 +1592,7 @@ dependencies = [ [[package]] name = "frame-system-rpc-runtime-api" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "parity-scale-codec", "sp-api", @@ -1719,8 +1751,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a5081aa3de1f7542a794a397cde100ed903b0630152d0973479018fd85423a7" dependencies = [ "proc-macro-hack", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -1744,9 +1776,9 @@ checksum = "a1de7508b218029b0f01662ed8f61b1c964b3ae99d6f25462d0f55a595109df6" [[package]] name = "futures-timer" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3de1a2b2a2a33d9e60e17980b60ee061eeaae96a5abe9121db0fdb9af167a1c5" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" dependencies = [ "gloo-timers", "send_wrapper 0.4.0", @@ -1797,15 +1829,6 @@ dependencies = [ "pin-project", ] -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder 1.3.4", -] - [[package]] name = "gcc" version = "0.3.55" @@ -1855,27 +1878,18 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "gimli" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162d18ae5f2e3b90a993d202f1ba17a5633c2484426f8bcae201f86194bacd00" -dependencies = [ - "arrayvec 0.4.12", - "byteorder 1.3.4", - "fallible-iterator", - "indexmap", - "stable_deref_trait", -] - [[package]] name = "gimli" version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81dd6190aad0f05ddbbf3245c54ed14ca4aa6dd32f22312b70d8f168c3e3e633" dependencies = [ + "arrayvec 0.5.1", "byteorder 1.3.4", + "fallible-iterator", "indexmap", + "smallvec 1.2.0", + "stable_deref_trait", ] [[package]] @@ -1927,34 +1941,6 @@ dependencies = [ "scroll", ] -[[package]] -name = "grafana-data-source" -version = "0.8.0" -dependencies = [ - "async-std", - "chrono", - "derive_more", - "futures-timer 3.0.1", - "futures-util", - "hyper 0.13.2", - "lazy_static", - "log 0.4.8", - "parking_lot 0.10.0", - "serde", - "serde_json", - "tokio 0.2.11", -] - -[[package]] -name = "grafana-data-source-test" -version = "2.0.0" -dependencies = [ - "futures 0.3.4", - "futures-timer 3.0.1", - "grafana-data-source", - "rand 0.7.3", -] - [[package]] name = "h2" version = "0.1.26" @@ -1988,7 +1974,7 @@ dependencies = [ "indexmap", "log 0.4.8", "slab", - "tokio 0.2.11", + "tokio 0.2.12", "tokio-util", ] @@ -2007,12 +1993,6 @@ dependencies = [ "crunchy", ] -[[package]] -name = "hashbrown" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1de41fb8dba9714efd92241565cdff73f78508c95697dd56787d3cba27e2353" - [[package]] name = "hashbrown" version = "0.6.3" @@ -2034,18 +2014,18 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff2656d88f158ce120947499e971d743c05dbcbed62e5bd2f38f1698bbc3772" +checksum = "1010591b26bbfe835e9faeabeb11866061cc7dcebffd56ad7d0942d0e61aefd8" dependencies = [ "libc", ] [[package]] name = "hex" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76cdda6bf525062a0c9e8f14ee2b37935c86b8efb6c8b69b3c83dfb518a914af" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" [[package]] name = "hex-literal" @@ -2214,7 +2194,7 @@ dependencies = [ "net2", "pin-project", "time", - "tokio 0.2.11", + "tokio 0.2.12", "tower-service", "want 0.3.0", ] @@ -2231,7 +2211,7 @@ dependencies = [ "hyper 0.13.2", "rustls", "rustls-native-certs", - "tokio 0.2.11", + "tokio 0.2.12", "tokio-rustls", "webpki", ] @@ -2313,8 +2293,8 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef5550a42e3740a0e71f909d4c861056a284060af885ae7aa6242820f920d9d" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -2436,8 +2416,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8609af8f63b626e8e211f52441fcdb6ec54f1a446606b10d5c89ae9bf8a20058" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -2618,9 +2598,9 @@ checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" [[package]] name = "libc" -version = "0.2.66" +version = "0.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +checksum = "eb147597cdf94ed43ab7a9038716637d2d1bf2bc571da995d0028dec06bd3018" [[package]] name = "libloading" @@ -2634,9 +2614,9 @@ dependencies = [ [[package]] name = "libp2p" -version = "0.16.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a58becf0b9585fcfbb8215bbe6e6ac187fcc180fd1026925ca180c845aa5a6e8" +checksum = "bba17ee9cac4bb89de5812159877d9b4f0a993bf41697a5a875940cd1eb71f24" dependencies = [ "bytes 0.5.4", "futures 0.3.4", @@ -2662,8 +2642,8 @@ dependencies = [ "libp2p-wasm-ext", "libp2p-websocket", "libp2p-yamux", - "parity-multiaddr", - "parity-multihash", + "parity-multiaddr 0.7.2", + "parity-multihash 0.2.3", "parking_lot 0.10.0", "pin-project", "smallvec 1.2.0", @@ -2677,17 +2657,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b874594c4b29de1a29f27871feba8e6cd13aa54a8a1e8f8c7cf3dfac5ca287c" dependencies = [ "asn1_der", - "bs58", + "bs58 0.3.0", "ed25519-dalek", "fnv", "futures 0.3.4", - "futures-timer 3.0.1", + "futures-timer 3.0.2", "lazy_static", "libsecp256k1", "log 0.4.8", "multistream-select", - "parity-multiaddr", - "parity-multihash", + "parity-multiaddr 0.7.2", + "parity-multihash 0.2.3", "parking_lot 0.10.0", "pin-project", "prost", @@ -2698,7 +2678,7 @@ dependencies = [ "sha2", "smallvec 1.2.0", "thiserror", - "unsigned-varint", + "unsigned-varint 0.3.1", "void", "zeroize 1.1.0", ] @@ -2709,7 +2689,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d472e9d522f588805c77801de10b957be84e10f019ca5f869fa1825b15ea9b" dependencies = [ - "quote 1.0.2", + "quote", "syn", ] @@ -2767,13 +2747,13 @@ dependencies = [ "libp2p-core", "libp2p-swarm", "log 0.4.8", - "lru 0.4.3", + "lru", "prost", "prost-build", "rand 0.7.3", "sha2", "smallvec 1.2.0", - "unsigned-varint", + "unsigned-varint 0.3.1", "wasm-timer", ] @@ -2795,9 +2775,9 @@ dependencies = [ [[package]] name = "libp2p-kad" -version = "0.16.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2efcff2af085e8181c421f68fe9c2b0a067379d146731925b3ac8f8e605c458" +checksum = "464dc8412978d40f0286be72ed9ab5e0e1386a4a06e7f174526739b5c3c1f041" dependencies = [ "arrayvec 0.5.1", "bytes 0.5.4", @@ -2808,14 +2788,14 @@ dependencies = [ "libp2p-core", "libp2p-swarm", "log 0.4.8", - "parity-multihash", + "parity-multihash 0.2.3", "prost", "prost-build", "rand 0.7.3", "sha2", "smallvec 1.2.0", "uint", - "unsigned-varint", + "unsigned-varint 0.3.1", "void", "wasm-timer", ] @@ -2855,16 +2835,16 @@ dependencies = [ "libp2p-core", "log 0.4.8", "parking_lot 0.10.0", - "unsigned-varint", + "unsigned-varint 0.3.1", ] [[package]] name = "libp2p-noise" -version = "0.16.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac7d33809afdf6794f09fdb2f9f94e1550ae230be5bae6430a078eb96fc9e5a6" +checksum = "b15a8a3d71f898beb6f854c8aae27aa1d198e0d1f2e49412261c2d90ef39675a" dependencies = [ - "curve25519-dalek 1.2.3", + "curve25519-dalek 2.0.0", "futures 0.3.4", "lazy_static", "libp2p-core", @@ -2875,7 +2855,7 @@ dependencies = [ "sha2", "snow", "static_assertions", - "x25519-dalek 0.5.2", + "x25519-dalek", "zeroize 1.1.0", ] @@ -2908,7 +2888,7 @@ dependencies = [ "prost", "prost-build", "rw-stream-sink", - "unsigned-varint", + "unsigned-varint 0.3.1", "void", ] @@ -2928,9 +2908,9 @@ dependencies = [ [[package]] name = "libp2p-secio" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec00eb9a3404ed76a0e14f637edcaa7f2b4a27a16884da4a56f2f21e166c2843" +checksum = "1219e9ecb4945d7331a05f5ffe96a1f6e28051bfa1223d4c60353c251de0354e" dependencies = [ "aes-ctr", "ctr", @@ -2958,9 +2938,9 @@ dependencies = [ [[package]] name = "libp2p-swarm" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1e9f4fb84a4bfe3d3a361c1fbcd4af017ba68f0a46a77bfbcc48bf8a456d6ef" +checksum = "275471e7c0e88ae004660866cd54f603bd8bd1f4caef541a27f50dd8640c4d4c" dependencies = [ "futures 0.3.4", "libp2p-core", @@ -2978,7 +2958,7 @@ checksum = "f9e80ad4e3535345f3d666554ce347d3100453775611c05c60786bf9a1747a10" dependencies = [ "async-std", "futures 0.3.4", - "futures-timer 3.0.1", + "futures-timer 3.0.2", "get_if_addrs", "ipnet", "libp2p-core", @@ -2999,9 +2979,9 @@ dependencies = [ [[package]] name = "libp2p-wasm-ext" -version = "0.16.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39703653caa36f4afd0def39cc49a3ac0fa1d4289ca1802e417af03e4f5ef950" +checksum = "923581c055bc4b8c5f42d4ce5ef43e52fe5216f1ea4bc26476cb8a966ce6220b" dependencies = [ "futures 0.3.4", "js-sys", @@ -3034,9 +3014,9 @@ dependencies = [ [[package]] name = "libp2p-yamux" -version = "0.16.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f72aa5a7273c29c6eaea09108a49feaefc7456164863f64f86a193f9e78a4b7f" +checksum = "9dac30de24ccde0e67f363d71a125c587bbe6589503f664947e9b084b68a34f1" dependencies = [ "futures 0.3.4", "libp2p-core", @@ -3047,9 +3027,9 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "6.2.4" +version = "6.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a0785e816e1e11e7599388a492c61ef80ddc2afc91e313e61662cce537809be" +checksum = "4e3b727e2dd20ec2fb7ed93f23d9fd5328a0871185485ebdaff007b47d3e27e4" dependencies = [ "bindgen", "cc", @@ -3127,22 +3107,13 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "lru" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8f669d42c72d18514dfca8115689c5f6370a17d980cb5bd777a67f404594c8" -dependencies = [ - "hashbrown 0.5.0", -] - [[package]] name = "lru" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0609345ddee5badacf857d4f547e0e5a2e987db77085c24cd887f73573a04237" dependencies = [ - "hashbrown 0.6.3", + "hashbrown", ] [[package]] @@ -3168,12 +3139,9 @@ checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" [[package]] name = "memchr" -version = "2.3.0" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223" -dependencies = [ - "libc", -] +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" [[package]] name = "memoffset" @@ -3192,7 +3160,7 @@ checksum = "198831fe8722331a395bc199a5d08efbc197497ef354cb4c77b969c02ffc0fc4" dependencies = [ "ahash", "hash-db", - "hashbrown 0.6.3", + "hashbrown", "parity-util-mem", ] @@ -3309,7 +3277,7 @@ dependencies = [ "log 0.4.8", "smallvec 1.2.0", "tokio-io", - "unsigned-varint", + "unsigned-varint 0.3.1", ] [[package]] @@ -3365,10 +3333,10 @@ dependencies = [ [[package]] name = "node-cli" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "assert_cmd", - "browser-utils", + "frame-benchmarking-cli", "frame-support", "frame-system", "futures 0.3.4", @@ -3411,6 +3379,7 @@ dependencies = [ "sc-tracing", "sc-transaction-pool", "serde", + "serde_json", "sp-authority-discovery", "sp-consensus", "sp-consensus-babe", @@ -3424,6 +3393,7 @@ dependencies = [ "sp-timestamp", "sp-transaction-pool", "structopt", + "substrate-browser-utils", "substrate-build-script-utils", "tempfile", "tracing", @@ -3434,9 +3404,10 @@ dependencies = [ [[package]] name = "node-executor" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "criterion 0.3.1", + "frame-benchmarking", "frame-support", "frame-system", "node-primitives", @@ -3466,7 +3437,7 @@ dependencies = [ [[package]] name = "node-inspect" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "derive_more", "log 0.4.8", @@ -3482,7 +3453,7 @@ dependencies = [ [[package]] name = "node-primitives" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "pretty_assertions", "sp-core", @@ -3492,7 +3463,7 @@ dependencies = [ [[package]] name = "node-rpc" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "jsonrpc-core", "node-primitives", @@ -3515,7 +3486,7 @@ dependencies = [ [[package]] name = "node-rpc-client" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "env_logger 0.7.1", "futures 0.1.29", @@ -3528,8 +3499,9 @@ dependencies = [ [[package]] name = "node-runtime" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ + "frame-benchmarking", "frame-executive", "frame-support", "frame-system", @@ -3589,7 +3561,7 @@ dependencies = [ [[package]] name = "node-template" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "futures 0.3.4", "log 0.4.8", @@ -3597,6 +3569,7 @@ dependencies = [ "sc-basic-authorship", "sc-cli", "sc-client", + "sc-client-api", "sc-consensus-aura", "sc-executor", "sc-finality-grandpa", @@ -3617,7 +3590,7 @@ dependencies = [ [[package]] name = "node-template-runtime" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-executive", "frame-support", @@ -3625,7 +3598,6 @@ dependencies = [ "pallet-aura", "pallet-balances", "pallet-grandpa", - "pallet-indices", "pallet-randomness-collective-flip", "pallet-sudo", "pallet-template", @@ -3650,7 +3622,7 @@ dependencies = [ [[package]] name = "node-testing" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "criterion 0.3.1", "frame-support", @@ -3689,13 +3661,13 @@ dependencies = [ "sp-runtime", "sp-timestamp", "substrate-test-client", - "tempdir", + "tempfile", "wabt", ] [[package]] name = "node-transaction-factory" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "log 0.4.8", "parity-scale-codec", @@ -3785,6 +3757,20 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea44a4fd660ab0f38434934ca0212e90fbeaaee54126ef20a3451c30c95bafae" +dependencies = [ + "flate2", + "goblin", + "parity-wasm 0.41.0", + "scroll", + "target-lexicon", + "uuid", +] + [[package]] name = "ole32-sys" version = "0.2.0" @@ -3860,16 +3846,16 @@ dependencies = [ [[package]] name = "owning_ref" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a4b8ea2179e6a2e27411d3bca09ca6dd630821cf6894c6c7c8467a8ee7ef13" +checksum = "6ff55baddef9e4ad00f88b6c743a2a8062d4c6ade126c2a528644b8e444d52ce" dependencies = [ "stable_deref_trait", ] [[package]] name = "pallet-assets" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -3883,7 +3869,7 @@ dependencies = [ [[package]] name = "pallet-aura" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -3905,7 +3891,7 @@ dependencies = [ [[package]] name = "pallet-authority-discovery" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -3923,7 +3909,7 @@ dependencies = [ [[package]] name = "pallet-authorship" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -3939,7 +3925,7 @@ dependencies = [ [[package]] name = "pallet-babe" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -3964,8 +3950,9 @@ dependencies = [ [[package]] name = "pallet-balances" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ + "frame-benchmarking", "frame-support", "frame-system", "pallet-transaction-payment", @@ -3977,9 +3964,23 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-benchmark" +version = "2.0.0-alpha.3" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "parity-scale-codec", + "serde", + "sp-io", + "sp-runtime", + "sp-std", +] + [[package]] name = "pallet-collective" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -3995,7 +3996,7 @@ dependencies = [ [[package]] name = "pallet-contracts" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "assert_matches", "frame-support", @@ -4020,7 +4021,7 @@ dependencies = [ [[package]] name = "pallet-contracts-primitives" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "parity-scale-codec", "sp-runtime", @@ -4029,7 +4030,7 @@ dependencies = [ [[package]] name = "pallet-contracts-rpc" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -4048,7 +4049,7 @@ dependencies = [ [[package]] name = "pallet-contracts-rpc-runtime-api" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "pallet-contracts-primitives", "parity-scale-codec", @@ -4059,7 +4060,7 @@ dependencies = [ [[package]] name = "pallet-democracy" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4076,7 +4077,7 @@ dependencies = [ [[package]] name = "pallet-elections" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4092,7 +4093,7 @@ dependencies = [ [[package]] name = "pallet-elections-phragmen" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4110,7 +4111,7 @@ dependencies = [ [[package]] name = "pallet-evm" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "evm", "frame-support", @@ -4130,8 +4131,9 @@ dependencies = [ [[package]] name = "pallet-example" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ + "frame-benchmarking", "frame-support", "frame-system", "pallet-balances", @@ -4143,9 +4145,24 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-example-offchain-worker" +version = "2.0.0-alpha.3" +dependencies = [ + "frame-support", + "frame-system", + "parity-scale-codec", + "serde", + "serde_json", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", +] + [[package]] name = "pallet-finality-tracker" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4162,7 +4179,7 @@ dependencies = [ [[package]] name = "pallet-generic-asset" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4176,7 +4193,7 @@ dependencies = [ [[package]] name = "pallet-grandpa" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4194,9 +4211,10 @@ dependencies = [ [[package]] name = "pallet-identity" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "enumflags2", + "frame-benchmarking", "frame-support", "frame-system", "pallet-balances", @@ -4210,7 +4228,7 @@ dependencies = [ [[package]] name = "pallet-im-online" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4228,7 +4246,7 @@ dependencies = [ [[package]] name = "pallet-indices" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4244,7 +4262,7 @@ dependencies = [ [[package]] name = "pallet-membership" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4258,7 +4276,7 @@ dependencies = [ [[package]] name = "pallet-nicks" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4273,7 +4291,7 @@ dependencies = [ [[package]] name = "pallet-offences" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4289,7 +4307,7 @@ dependencies = [ [[package]] name = "pallet-randomness-collective-flip" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4303,7 +4321,7 @@ dependencies = [ [[package]] name = "pallet-recovery" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "enumflags2", "frame-support", @@ -4319,7 +4337,7 @@ dependencies = [ [[package]] name = "pallet-scored-pool" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4334,7 +4352,7 @@ dependencies = [ [[package]] name = "pallet-session" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4354,7 +4372,7 @@ dependencies = [ [[package]] name = "pallet-society" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4370,7 +4388,7 @@ dependencies = [ [[package]] name = "pallet-staking" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4393,18 +4411,18 @@ dependencies = [ [[package]] name = "pallet-staking-reward-curve" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "sp-runtime", "syn", ] [[package]] name = "pallet-sudo" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4418,7 +4436,7 @@ dependencies = [ [[package]] name = "pallet-template" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4431,8 +4449,9 @@ dependencies = [ [[package]] name = "pallet-timestamp" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ + "frame-benchmarking", "frame-support", "frame-system", "impl-trait-for-tuples", @@ -4448,7 +4467,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4463,7 +4482,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -4480,7 +4499,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc-runtime-api" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "parity-scale-codec", @@ -4493,7 +4512,7 @@ dependencies = [ [[package]] name = "pallet-treasury" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4508,7 +4527,7 @@ dependencies = [ [[package]] name = "pallet-utility" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -4523,7 +4542,7 @@ dependencies = [ [[package]] name = "pallet-vesting" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "enumflags2", "frame-support", @@ -4545,6 +4564,24 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c276d76c5333b8c2579e02d49a06733a55b8282d2d9b13e8d53b6406bd7e30a" +[[package]] +name = "parity-multiaddr" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "045b3c7af871285146300da35b1932bb6e4639b66c7c98e85d06a32cbc4e8fa7" +dependencies = [ + "arrayref", + "bs58 0.2.5", + "byteorder 1.3.4", + "bytes 0.4.12", + "data-encoding", + "parity-multihash 0.1.3", + "percent-encoding 1.0.1", + "serde", + "unsigned-varint 0.2.3", + "url 1.7.2", +] + [[package]] name = "parity-multiaddr" version = "0.7.2" @@ -4552,17 +4589,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26df883298bc3f4e92528b4c5cc9f806b791955b136da3e5e939ed9de0fd958b" dependencies = [ "arrayref", - "bs58", + "bs58 0.3.0", "byteorder 1.3.4", "data-encoding", - "parity-multihash", + "parity-multihash 0.2.3", "percent-encoding 2.1.0", "serde", "static_assertions", - "unsigned-varint", + "unsigned-varint 0.3.1", "url 2.1.1", ] +[[package]] +name = "parity-multihash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3a17dc27848fd99e4f87eb0f8c9baba6ede0a6d555400c850ca45254ef4ce3" +dependencies = [ + "blake2", + "bytes 0.4.12", + "rand 0.6.5", + "sha-1", + "sha2", + "sha3", + "unsigned-varint 0.2.3", +] + [[package]] name = "parity-multihash" version = "0.2.3" @@ -4575,14 +4627,14 @@ dependencies = [ "sha-1", "sha2", "sha3", - "unsigned-varint", + "unsigned-varint 0.3.1", ] [[package]] name = "parity-scale-codec" -version = "1.1.2" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f747c06d9f3b2ad387ac881b9667298c81b1243aa9833f086e05996937c35507" +checksum = "f509c5e67ca0605ee17dcd3f91ef41cadd685c75a298fb6261b781a5acb3f910" dependencies = [ "arrayvec 0.5.1", "bitvec", @@ -4593,13 +4645,13 @@ dependencies = [ [[package]] name = "parity-scale-codec-derive" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34e513ff3e406f3ede6796dcdc83d0b32ffb86668cea1ccf7363118abeb00476" +checksum = "5a0ec292e92e8ec7c58e576adacc1e3f399c597c8f263c42f18420abe58e7245" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -4616,7 +4668,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef1476e40bf8f5c6776e9600983435821ca86eb9819d74a6207cca69d091406a" dependencies = [ "cfg-if", + "ethereum-types", + "hashbrown", "impl-trait-for-tuples", + "lru", "parity-util-mem-derive", "parking_lot 0.10.0", "primitive-types", @@ -4630,7 +4685,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f557c32c6d268a07c921471619c0295f5efad3a0e76d4f97a05c091a51d110b2" dependencies = [ - "proc-macro2 1.0.8", + "proc-macro2", "syn", "synstructure", ] @@ -4702,9 +4757,9 @@ dependencies = [ [[package]] name = "paste" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "423a519e1c6e828f1e73b720f9d9ed2fa643dce8a7737fb43235ce0b41eeaa49" +checksum = "63e1afe738d71b1ebab5f1207c055054015427dbfc7bbe9ee1266894156ec046" dependencies = [ "paste-impl", "proc-macro-hack", @@ -4712,13 +4767,13 @@ dependencies = [ [[package]] name = "paste-impl" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4214c9e912ef61bf42b81ba9a47e8aad1b2ffaf739ab162bf96d1e011f54e6c5" +checksum = "6d4dc4a7f6f743211c5aab239640a65091535d97d43d92a52bca435a640892bb" dependencies = [ "proc-macro-hack", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -4781,8 +4836,8 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "385322a45f2ecf3410c68d2a549a4a2685e8051d0f278e39743ff4e451cb9b3f" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -4830,9 +4885,9 @@ checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" [[package]] name = "predicates" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9bfe52247e5cc9b2f943682a85a5549fb9662245caf094504e69a2f03fe64d4" +checksum = "1188bf092c81c18228c383b190c069a8a613c18a046ffa9fdfc0f5fc8fb2da8a" dependencies = [ "difference", "predicates-core", @@ -4890,25 +4945,25 @@ dependencies = [ [[package]] name = "proc-macro-error" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875077759af22fa20b610ad4471d8155b321c89c3f2785526c9839b099be4e0a" +checksum = "052b3c9af39c7e5e94245f820530487d19eb285faedcb40e0c3275132293f242" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "rustversion", "syn", ] [[package]] name = "proc-macro-error-attr" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5717d9fa2664351a01ed73ba5ef6df09c01a521cb42cb65a061432a826f3c7a" +checksum = "d175bef481c7902e63e3165627123fff3502f06ac043d3ef42d08c1246da9253" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "rustversion", "syn", "syn-mid", @@ -4920,8 +4975,8 @@ version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -4933,20 +4988,25 @@ checksum = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" [[package]] name = "proc-macro2" -version = "0.4.30" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +checksum = "6c09721c6781493a2a492a96b5a5bf19b65917fe6728884e7c44dd0c60ca3435" dependencies = [ - "unicode-xid 0.1.0", + "unicode-xid", ] [[package]] -name = "proc-macro2" -version = "1.0.8" +name = "prometheus" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acb317c6ff86a4e579dfa00fc5e6cca91ecbb4e7eb2df0468805b674eb88548" +checksum = "5567486d5778e2c6455b1b90ff1c558f29e751fc018130fa182e15828e728af1" dependencies = [ - "unicode-xid 0.2.0", + "cfg-if", + "fnv", + "lazy_static", + "protobuf", + "quick-error", + "spin", ] [[package]] @@ -4974,7 +5034,7 @@ dependencies = [ "prost", "prost-types", "tempfile", - "which 3.1.0", + "which", ] [[package]] @@ -4985,8 +5045,8 @@ checksum = "537aa19b95acde10a12fec4301466386f757403de4cd4e5b4fa78fb5ecb18f72" dependencies = [ "anyhow", "itertools", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -5000,6 +5060,12 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6686ddd96a8dbe2687b5f2a687b2cfb520854010ec480f2d74c32e7c9873d3c5" + [[package]] name = "pwasm-utils" version = "0.12.0" @@ -5030,32 +5096,23 @@ dependencies = [ ] [[package]] -name = "quicksink" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8461ef7445f61fd72d8dcd0629ce724b9131b3c2eb36e83a5d3d4161c127530" -dependencies = [ - "futures-core", - "futures-sink", - "pin-project-lite", -] - -[[package]] -name = "quote" -version = "0.6.13" +name = "quicksink" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +checksum = "a8461ef7445f61fd72d8dcd0629ce724b9131b3c2eb36e83a5d3d4161c127530" dependencies = [ - "proc-macro2 0.4.30", + "futures-core", + "futures-sink", + "pin-project-lite", ] [[package]] name = "quote" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" +checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" dependencies = [ - "proc-macro2 1.0.8", + "proc-macro2", ] [[package]] @@ -5462,8 +5519,8 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3bba175698996010c4f6dce5e7f173b6eb781fce25d2cfc45e27091ce0b79f6" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -5530,13 +5587,13 @@ dependencies = [ [[package]] name = "sc-authority-discovery" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ - "bytes 0.4.12", + "bytes 0.5.4", "derive_more", "env_logger 0.7.1", "futures 0.3.4", - "futures-timer 3.0.1", + "futures-timer 3.0.2", "libp2p", "log 0.4.8", "parity-scale-codec", @@ -5559,14 +5616,13 @@ dependencies = [ [[package]] name = "sc-basic-authorship" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "futures 0.3.4", "log 0.4.8", "parity-scale-codec", "parking_lot 0.10.0", "sc-block-builder", - "sc-client", "sc-client-api", "sc-telemetry", "sc-transaction-pool", @@ -5583,7 +5639,7 @@ dependencies = [ [[package]] name = "sc-block-builder" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "parity-scale-codec", "sc-client-api", @@ -5598,7 +5654,7 @@ dependencies = [ [[package]] name = "sc-chain-spec" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "impl-trait-for-tuples", "sc-chain-spec-derive", @@ -5612,17 +5668,17 @@ dependencies = [ [[package]] name = "sc-chain-spec-derive" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "sc-cli" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "ansi_term 0.12.1", "app_dirs", @@ -5653,14 +5709,15 @@ dependencies = [ "sp-runtime", "sp-state-machine", "structopt", + "substrate-prometheus-endpoint", "tempfile", "time", - "tokio 0.2.11", + "tokio 0.2.12", ] [[package]] name = "sc-client" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "derive_more", "env_logger 0.7.1", @@ -5690,6 +5747,7 @@ dependencies = [ "sp-std", "sp-trie", "sp-version", + "substrate-prometheus-endpoint", "substrate-test-runtime-client", "tempfile", "tracing", @@ -5697,7 +5755,7 @@ dependencies = [ [[package]] name = "sc-client-api" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "derive_more", "fnv", @@ -5720,6 +5778,7 @@ dependencies = [ "sp-runtime", "sp-state-machine", "sp-std", + "sp-storage", "sp-test-primitives", "sp-transaction-pool", "sp-trie", @@ -5728,7 +5787,7 @@ dependencies = [ [[package]] name = "sc-client-db" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "env_logger 0.7.1", "hash-db", @@ -5753,19 +5812,19 @@ dependencies = [ "sp-runtime", "sp-state-machine", "sp-trie", + "substrate-prometheus-endpoint", "substrate-test-runtime-client", "tempfile", ] [[package]] name = "sc-consensus-aura" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "derive_more", "env_logger 0.7.1", - "futures 0.1.29", "futures 0.3.4", - "futures-timer 3.0.1", + "futures-timer 3.0.2", "log 0.4.8", "parity-scale-codec", "parking_lot 0.10.0", @@ -5793,19 +5852,17 @@ dependencies = [ "sp-version", "substrate-test-runtime-client", "tempfile", - "tokio 0.1.22", ] [[package]] name = "sc-consensus-babe" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "derive_more", "env_logger 0.7.1", "fork-tree", - "futures 0.1.29", "futures 0.3.4", - "futures-timer 3.0.1", + "futures-timer 3.0.2", "log 0.4.8", "merlin", "num-bigint", @@ -5844,12 +5901,11 @@ dependencies = [ "sp-version", "substrate-test-runtime-client", "tempfile", - "tokio 0.1.22", ] [[package]] name = "sc-consensus-babe-rpc" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "derive_more", "futures 0.3.4", @@ -5874,7 +5930,7 @@ dependencies = [ [[package]] name = "sc-consensus-epochs" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "fork-tree", "parity-scale-codec", @@ -5886,7 +5942,7 @@ dependencies = [ [[package]] name = "sc-consensus-manual-seal" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "derive_more", "env_logger 0.7.1", @@ -5909,12 +5965,12 @@ dependencies = [ "substrate-test-runtime-client", "substrate-test-runtime-transaction-pool", "tempfile", - "tokio 0.2.11", + "tokio 0.2.12", ] [[package]] name = "sc-consensus-pow" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "derive_more", "futures 0.3.4", @@ -5934,10 +5990,10 @@ dependencies = [ [[package]] name = "sc-consensus-slots" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "futures 0.3.4", - "futures-timer 3.0.1", + "futures-timer 3.0.2", "log 0.4.8", "parity-scale-codec", "parking_lot 0.10.0", @@ -5955,7 +6011,7 @@ dependencies = [ [[package]] name = "sc-consensus-uncles" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "log 0.4.8", "sc-client-api", @@ -5968,7 +6024,7 @@ dependencies = [ [[package]] name = "sc-executor" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "assert_matches", "derive_more", @@ -5987,6 +6043,7 @@ dependencies = [ "sp-externalities", "sp-io", "sp-panic-handler", + "sp-runtime", "sp-runtime-interface", "sp-serializer", "sp-state-machine", @@ -6001,7 +6058,7 @@ dependencies = [ [[package]] name = "sc-executor-common" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "derive_more", "log 0.4.8", @@ -6016,7 +6073,7 @@ dependencies = [ [[package]] name = "sc-executor-wasmi" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "log 0.4.8", "parity-scale-codec", @@ -6031,37 +6088,37 @@ dependencies = [ [[package]] name = "sc-executor-wasmtime" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "assert_matches", "log 0.4.8", "parity-scale-codec", "parity-wasm 0.41.0", "sc-executor-common", + "scoped-tls", "sp-allocator", "sp-core", "sp-runtime-interface", "sp-wasm-interface", - "wasmi", "wasmtime", ] [[package]] name = "sc-finality-grandpa" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "assert_matches", "env_logger 0.7.1", "finality-grandpa", "fork-tree", - "futures 0.1.29", "futures 0.3.4", - "futures-timer 3.0.1", + "futures-timer 3.0.2", "log 0.4.8", "parity-scale-codec", "parking_lot 0.10.0", "pin-project", "rand 0.7.3", + "sc-block-builder", "sc-client", "sc-client-api", "sc-keystore", @@ -6082,14 +6139,15 @@ dependencies = [ "sp-keyring", "sp-runtime", "sp-state-machine", + "substrate-prometheus-endpoint", "substrate-test-runtime-client", "tempfile", - "tokio 0.1.22", + "tokio 0.2.12", ] [[package]] name = "sc-informant" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "ansi_term 0.12.1", "futures 0.3.4", @@ -6105,7 +6163,7 @@ dependencies = [ [[package]] name = "sc-keystore" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "derive_more", "hex", @@ -6120,7 +6178,7 @@ dependencies = [ [[package]] name = "sc-network" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "assert_matches", "async-std", @@ -6133,16 +6191,17 @@ dependencies = [ "fnv", "fork-tree", "futures 0.3.4", - "futures-timer 3.0.1", + "futures-timer 3.0.2", "futures_codec", "libp2p", "linked-hash-map", "linked_hash_set", "log 0.4.8", - "lru 0.4.3", + "lru", "nohash-hasher", "parity-scale-codec", "parking_lot 0.10.0", + "pin-project", "prost", "prost-build", "quickcheck", @@ -6165,12 +6224,12 @@ dependencies = [ "sp-keyring", "sp-runtime", "sp-test-primitives", - "substrate-test-client", + "substrate-prometheus-endpoint", "substrate-test-runtime", "substrate-test-runtime-client", "tempfile", "thiserror", - "unsigned-varint", + "unsigned-varint 0.3.1", "void", "wasm-timer", "zeroize 1.1.0", @@ -6178,14 +6237,13 @@ dependencies = [ [[package]] name = "sc-network-gossip" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ - "futures 0.1.29", "futures 0.3.4", - "futures-timer 3.0.1", + "futures-timer 3.0.2", "libp2p", "log 0.4.8", - "lru 0.1.17", + "lru", "parking_lot 0.10.0", "sc-network", "sp-runtime", @@ -6194,12 +6252,11 @@ dependencies = [ [[package]] name = "sc-network-test" -version = "0.8.0" +version = "0.8.0-dev" dependencies = [ "env_logger 0.7.1", - "futures 0.1.29", "futures 0.3.4", - "futures-timer 3.0.1", + "futures-timer 3.0.2", "libp2p", "log 0.4.8", "parking_lot 0.10.0", @@ -6216,18 +6273,17 @@ dependencies = [ "substrate-test-runtime", "substrate-test-runtime-client", "tempfile", - "tokio 0.1.22", ] [[package]] name = "sc-offchain" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "bytes 0.5.4", "env_logger 0.7.1", "fnv", "futures 0.3.4", - "futures-timer 3.0.1", + "futures-timer 3.0.2", "hyper 0.13.2", "hyper-rustls", "log 0.4.8", @@ -6247,12 +6303,12 @@ dependencies = [ "sp-transaction-pool", "substrate-test-runtime-client", "threadpool", - "tokio 0.2.11", + "tokio 0.2.12", ] [[package]] name = "sc-peerset" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "futures 0.3.4", "libp2p", @@ -6264,7 +6320,7 @@ dependencies = [ [[package]] name = "sc-rpc" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "assert_matches", "futures 0.1.29", @@ -6301,7 +6357,7 @@ dependencies = [ [[package]] name = "sc-rpc-api" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "derive_more", "futures 0.3.4", @@ -6323,7 +6379,7 @@ dependencies = [ [[package]] name = "sc-rpc-server" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "jsonrpc-core", "jsonrpc-http-server", @@ -6337,7 +6393,7 @@ dependencies = [ [[package]] name = "sc-runtime-test" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "sp-allocator", "sp-core", @@ -6350,18 +6406,17 @@ dependencies = [ [[package]] name = "sc-service" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "derive_more", "exit-future", "futures 0.1.29", "futures 0.3.4", "futures-diagnose", - "futures-timer 3.0.1", - "grafana-data-source", + "futures-timer 3.0.2", "lazy_static", "log 0.4.8", - "parity-multiaddr", + "parity-multiaddr 0.5.0", "parity-scale-codec", "parity-util-mem", "parking_lot 0.10.0", @@ -6393,17 +6448,17 @@ dependencies = [ "sp-runtime", "sp-session", "sp-transaction-pool", + "substrate-prometheus-endpoint", "substrate-test-runtime-client", "sysinfo", "target_info", - "tokio 0.2.11", "tracing", "wasm-timer", ] [[package]] name = "sc-service-test" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "env_logger 0.7.1", "fdlimit", @@ -6423,22 +6478,25 @@ dependencies = [ [[package]] name = "sc-state-db" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "env_logger 0.7.1", "log 0.4.8", "parity-scale-codec", + "parity-util-mem", + "parity-util-mem-derive", "parking_lot 0.10.0", + "sc-client-api", "sp-core", ] [[package]] name = "sc-telemetry" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "bytes 0.5.4", "futures 0.3.4", - "futures-timer 3.0.1", + "futures-timer 3.0.2", "libp2p", "log 0.4.8", "parking_lot 0.10.0", @@ -6455,10 +6513,9 @@ dependencies = [ [[package]] name = "sc-tracing" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "erased-serde", - "grafana-data-source", "log 0.4.8", "parking_lot 0.10.0", "sc-telemetry", @@ -6471,7 +6528,7 @@ dependencies = [ [[package]] name = "sc-transaction-graph" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "assert_matches", "criterion 0.3.1", @@ -6493,7 +6550,7 @@ dependencies = [ [[package]] name = "sc-transaction-pool" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "derive_more", "futures 0.3.4", @@ -6544,10 +6601,16 @@ dependencies = [ ] [[package]] -name = "scopeguard" +name = "scoped-tls" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" +checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scroll" @@ -6564,8 +6627,8 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8584eea9b9ff42825b46faf46a8c24d2cff13ec152fa2a50df788b87c07ee28" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -6658,16 +6721,16 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "serde_json" -version = "1.0.47" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15913895b61e0be854afd32fd4163fcd2a3df34142cf2cb961b310ce694cbf90" +checksum = "9371ade75d4c2d6cb154141b9752cf3781ec9c05e0e5cf35060e1e70ee7b9c25" dependencies = [ "itoa", "ryu", @@ -6788,8 +6851,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a945ec7f7ce853e89ffa36be1e27dce9a43e82ff9093bf3461c30d5da74ed11b" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -6823,7 +6886,7 @@ dependencies = [ "rustc_version", "sha2", "subtle 2.2.2", - "x25519-dalek 0.6.0", + "x25519-dalek", ] [[package]] @@ -6854,7 +6917,7 @@ checksum = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3" [[package]] name = "sp-allocator" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "derive_more", "log 0.4.8", @@ -6865,7 +6928,7 @@ dependencies = [ [[package]] name = "sp-api" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "hash-db", "parity-scale-codec", @@ -6880,22 +6943,23 @@ dependencies = [ [[package]] name = "sp-api-proc-macro" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "blake2-rfc", "proc-macro-crate", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "sp-api-test" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "criterion 0.3.1", "parity-scale-codec", "rustversion", + "sc-block-builder", "sp-api", "sp-blockchain", "sp-consensus", @@ -6908,7 +6972,7 @@ dependencies = [ [[package]] name = "sp-application-crypto" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "parity-scale-codec", "serde", @@ -6919,7 +6983,7 @@ dependencies = [ [[package]] name = "sp-application-crypto-test" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "sp-api", "sp-application-crypto", @@ -6930,7 +6994,7 @@ dependencies = [ [[package]] name = "sp-arithmetic" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "criterion 0.3.1", "integer-sqrt", @@ -6945,7 +7009,7 @@ dependencies = [ [[package]] name = "sp-authority-discovery" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "parity-scale-codec", "sp-api", @@ -6956,7 +7020,7 @@ dependencies = [ [[package]] name = "sp-authorship" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "parity-scale-codec", "sp-inherents", @@ -6966,7 +7030,7 @@ dependencies = [ [[package]] name = "sp-block-builder" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "parity-scale-codec", "sp-api", @@ -6977,11 +7041,11 @@ dependencies = [ [[package]] name = "sp-blockchain" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "derive_more", "log 0.4.8", - "lru 0.4.3", + "lru", "parity-scale-codec", "parking_lot 0.10.0", "sp-block-builder", @@ -6992,12 +7056,12 @@ dependencies = [ [[package]] name = "sp-consensus" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "derive_more", "futures 0.3.4", "futures-diagnose", - "futures-timer 3.0.1", + "futures-timer 3.0.2", "libp2p", "log 0.4.8", "parity-scale-codec", @@ -7014,7 +7078,7 @@ dependencies = [ [[package]] name = "sp-consensus-aura" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "parity-scale-codec", "sp-api", @@ -7027,7 +7091,7 @@ dependencies = [ [[package]] name = "sp-consensus-babe" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "parity-scale-codec", "schnorrkel", @@ -7042,7 +7106,7 @@ dependencies = [ [[package]] name = "sp-consensus-pow" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "parity-scale-codec", "sp-api", @@ -7053,7 +7117,7 @@ dependencies = [ [[package]] name = "sp-core" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "base58", "blake2-rfc", @@ -7097,16 +7161,16 @@ dependencies = [ [[package]] name = "sp-debug-derive" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "sp-externalities" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "environmental", "sp-std", @@ -7115,7 +7179,7 @@ dependencies = [ [[package]] name = "sp-finality-grandpa" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "parity-scale-codec", "serde", @@ -7127,7 +7191,7 @@ dependencies = [ [[package]] name = "sp-finality-tracker" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "parity-scale-codec", "sp-inherents", @@ -7136,7 +7200,7 @@ dependencies = [ [[package]] name = "sp-inherents" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "derive_more", "parity-scale-codec", @@ -7147,7 +7211,7 @@ dependencies = [ [[package]] name = "sp-io" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "hash-db", "libsecp256k1", @@ -7164,7 +7228,7 @@ dependencies = [ [[package]] name = "sp-keyring" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "lazy_static", "sp-core", @@ -7174,7 +7238,7 @@ dependencies = [ [[package]] name = "sp-offchain" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "sp-api", "sp-runtime", @@ -7182,7 +7246,7 @@ dependencies = [ [[package]] name = "sp-panic-handler" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "backtrace", "log 0.4.8", @@ -7190,7 +7254,7 @@ dependencies = [ [[package]] name = "sp-phragmen" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "rand 0.7.3", "serde", @@ -7202,7 +7266,7 @@ dependencies = [ [[package]] name = "sp-rpc" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "serde", "serde_json", @@ -7211,8 +7275,9 @@ dependencies = [ [[package]] name = "sp-runtime" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ + "hash256-std-hasher", "impl-trait-for-tuples", "log 0.4.8", "parity-scale-codec", @@ -7231,7 +7296,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "parity-scale-codec", "primitive-types", @@ -7250,22 +7315,22 @@ dependencies = [ [[package]] name = "sp-runtime-interface-proc-macro" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "Inflector", "proc-macro-crate", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "sp-runtime-interface-test" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "sc-executor", - "sp-core", "sp-io", + "sp-runtime", "sp-runtime-interface", "sp-runtime-interface-test-wasm", "sp-state-machine", @@ -7273,7 +7338,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface-test-wasm" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "sp-core", "sp-io", @@ -7284,7 +7349,7 @@ dependencies = [ [[package]] name = "sp-sandbox" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "assert_matches", "parity-scale-codec", @@ -7298,7 +7363,7 @@ dependencies = [ [[package]] name = "sp-serializer" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "serde", "serde_json", @@ -7306,7 +7371,7 @@ dependencies = [ [[package]] name = "sp-session" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "sp-api", "sp-core", @@ -7316,7 +7381,7 @@ dependencies = [ [[package]] name = "sp-staking" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "parity-scale-codec", "sp-runtime", @@ -7325,7 +7390,7 @@ dependencies = [ [[package]] name = "sp-state-machine" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ "hash-db", "hex-literal", @@ -7337,6 +7402,7 @@ dependencies = [ "sp-core", "sp-externalities", "sp-panic-handler", + "sp-runtime", "sp-trie", "trie-db", "trie-root", @@ -7344,11 +7410,11 @@ dependencies = [ [[package]] name = "sp-std" -version = "2.0.0" +version = "2.0.0-alpha.3" [[package]] name = "sp-storage" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "impl-serde 0.2.3", "serde", @@ -7358,7 +7424,7 @@ dependencies = [ [[package]] name = "sp-test-primitives" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "parity-scale-codec", "parity-util-mem", @@ -7370,7 +7436,7 @@ dependencies = [ [[package]] name = "sp-timestamp" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -7383,7 +7449,7 @@ dependencies = [ [[package]] name = "sp-transaction-pool" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "derive_more", "futures 0.3.4", @@ -7396,7 +7462,7 @@ dependencies = [ [[package]] name = "sp-trie" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "criterion 0.2.11", "hash-db", @@ -7404,6 +7470,7 @@ dependencies = [ "memory-db", "parity-scale-codec", "sp-core", + "sp-runtime", "sp-std", "trie-bench", "trie-db", @@ -7413,7 +7480,7 @@ dependencies = [ [[package]] name = "sp-version" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "impl-serde 0.2.3", "parity-scale-codec", @@ -7424,7 +7491,7 @@ dependencies = [ [[package]] name = "sp-wasm-interface" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -7502,8 +7569,8 @@ checksum = "095064aa1f5b94d14e635d0a5684cf140c43ae40a0fd990708d38f5d669e5f64" dependencies = [ "heck", "proc-macro-error", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -7523,14 +7590,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0054a7df764039a6cd8592b9de84be4bec368ff081d203a7d5371cbfa8e65c81" dependencies = [ "heck", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] [[package]] name = "subkey" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "clap", "derive_more", @@ -7570,13 +7637,38 @@ dependencies = [ "sha2", ] +[[package]] +name = "substrate-browser-utils" +version = "0.8.0-alpha.3" +dependencies = [ + "chrono", + "clear_on_drop", + "console_error_panic_hook", + "console_log", + "futures 0.1.29", + "futures 0.3.4", + "futures-timer 3.0.2", + "js-sys", + "kvdb-web", + "libp2p", + "log 0.4.8", + "rand 0.6.5", + "rand 0.7.3", + "sc-chain-spec", + "sc-informant", + "sc-network", + "sc-service", + "wasm-bindgen", + "wasm-bindgen-futures", +] + [[package]] name = "substrate-build-script-utils" -version = "2.0.0" +version = "2.0.0-alpha.3" [[package]] name = "substrate-frame-rpc-support" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "frame-support", "frame-system", @@ -7587,12 +7679,12 @@ dependencies = [ "sc-rpc-api", "serde", "sp-storage", - "tokio 0.1.22", + "tokio 0.2.12", ] [[package]] name = "substrate-frame-rpc-system" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ "env_logger 0.7.1", "frame-system-rpc-runtime-api", @@ -7613,9 +7705,22 @@ dependencies = [ "substrate-test-runtime-client", ] +[[package]] +name = "substrate-prometheus-endpoint" +version = "0.8.0-alpha.3" +dependencies = [ + "async-std", + "derive_more", + "futures-util", + "hyper 0.13.2", + "log 0.4.8", + "prometheus", + "tokio 0.2.12", +] + [[package]] name = "substrate-test-client" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "futures 0.3.4", "hash-db", @@ -7634,7 +7739,7 @@ dependencies = [ [[package]] name = "substrate-test-runtime" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "cfg-if", "frame-executive", @@ -7675,7 +7780,7 @@ dependencies = [ [[package]] name = "substrate-test-runtime-client" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "futures 0.3.4", "parity-scale-codec", @@ -7692,7 +7797,7 @@ dependencies = [ [[package]] name = "substrate-test-runtime-transaction-pool" -version = "2.0.0" +version = "2.0.0-dev" dependencies = [ "derive_more", "futures 0.3.4", @@ -7707,7 +7812,7 @@ dependencies = [ [[package]] name = "substrate-test-utils" -version = "2.0.0" +version = "2.0.0-alpha.3" [[package]] name = "substrate-wasm-builder" @@ -7717,6 +7822,7 @@ dependencies = [ "build-helper", "cargo_metadata", "fs2", + "itertools", "tempfile", "toml", "walkdir", @@ -7741,13 +7847,13 @@ checksum = "7c65d530b10ccaeac294f349038a597e435b18fb456aadd0840a623f83b9e941" [[package]] name = "syn" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6f3550d8dff9ef7dc34d384ac6f107e5d31c8f57d9f28e0081503f547ac8f5" +checksum = "123bd9499cfb380418d509322d7a6d52e5315f064fe4b3ad18a53d6b92c07859" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", - "unicode-xid 0.2.0", + "proc-macro2", + "quote", + "unicode-xid", ] [[package]] @@ -7756,8 +7862,8 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -7767,10 +7873,10 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", - "unicode-xid 0.2.0", + "unicode-xid", ] [[package]] @@ -7804,16 +7910,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c63f48baada5c52e65a29eef93ab4f8982681b67f9e8d29c7b05abcfec2b9ffe" -[[package]] -name = "tempdir" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" -dependencies = [ - "rand 0.4.6", - "remove_dir_all", -] - [[package]] name = "tempfile" version = "3.1.0" @@ -7844,8 +7940,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a605baa797821796a751f4a959e1206079b24a4b7e1ed302b7d785d81a9276c9" dependencies = [ "lazy_static", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", "version_check 0.9.1", ] @@ -7861,21 +7957,21 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205684fd018ca14432b12cce6ea3d46763311a571c3d294e71ba3f01adcf1aad" +checksum = "ee14bf8e6767ab4c687c9e8bc003879e042a96fd67a3ba5934eadb6536bef4db" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57e4d2e50ca050ed44fb58309bdce3efa79948f84f9993ad1978de5eebdce5a7" +checksum = "a7b51e1fbc44b5a0840be594fbc0f960be09050f2617e61e6aa43bef97cd3ef4" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -7910,9 +8006,9 @@ dependencies = [ [[package]] name = "tiny-bip39" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd1fb03fe8e07d17cd851a624a9fff74642a997b67fbd1ccd77533241640d92" +checksum = "a6848cd8f566953ce1e8faeba12ee23cbdbb0437754792cd857d44628b5685e3" dependencies = [ "failure", "hmac", @@ -7921,6 +8017,7 @@ dependencies = [ "rand 0.7.3", "rustc-hash", "sha2", + "unicode-normalization", ] [[package]] @@ -7977,9 +8074,9 @@ dependencies = [ [[package]] name = "tokio" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fdd17989496f49cdc57978c96f0c9fe5e4a58a8bddc6813c449a4624f6a030b" +checksum = "b34bee1facdc352fba10c9c58b654e6ecb6a2250167772bf86071f7c5f2f5061" dependencies = [ "bytes 0.5.4", "fnv", @@ -8074,12 +8171,12 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4b1e7ed7d5d4c2af3d999904b0eebe76544897cdbfb2b9684bed2174ab20f7c" +checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", ] @@ -8110,7 +8207,7 @@ checksum = "141afec0978abae6573065a48882c6bae44c5cc61db9b511ac4abf6a09bfd9cc" dependencies = [ "futures-core", "rustls", - "tokio 0.2.11", + "tokio 0.2.12", "webpki", ] @@ -8233,7 +8330,7 @@ dependencies = [ "futures-sink", "log 0.4.8", "pin-project-lite", - "tokio 0.2.11", + "tokio 0.2.12", ] [[package]] @@ -8253,9 +8350,9 @@ checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" [[package]] name = "tracing" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e213bd24252abeb86a0b7060e02df677d367ce6cb772cef17e9214b8390a8d3" +checksum = "1721cc8cf7d770cc4257872507180f35a4797272f5962f24c806af9e7faf52ab" dependencies = [ "cfg-if", "tracing-attributes", @@ -8264,19 +8361,19 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cfd395def5a60236e187e1ff905cb55668a59f29928dec05e6e1b1fd2ac1f3" +checksum = "7fbad39da2f9af1cae3016339ad7f2c7a9e870f12e8fd04c4fd7ef35b30c0d2b" dependencies = [ - "quote 1.0.2", + "quote", "syn", ] [[package]] name = "tracing-core" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13a46f11e372b8bd4b4398ea54353412fdd7fd42a8370c7e543e218cf7661978" +checksum = "0aa83a9a47081cd522c09c81b31aec2c9273424976f922ad61c053b58350b715" dependencies = [ "lazy_static", ] @@ -8316,7 +8413,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de9222c50cc325855621271157c973da27a0dcd26fa06f8edf81020bd2333df0" dependencies = [ "hash-db", - "hashbrown 0.6.3", + "hashbrown", "log 0.4.8", "rustc-hex", "smallvec 1.2.0", @@ -8349,9 +8446,9 @@ checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" [[package]] name = "trybuild" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f5b3f750c701725331ac78e389b5d143b7d25f6b6ffffd0d419759a9063ac5f" +checksum = "26ff1b18659a2218332848d76ad1c867ce4c6ee37b085e6bc8de9a6d11401220" dependencies = [ "glob 0.3.0", "lazy_static", @@ -8455,23 +8552,24 @@ checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" [[package]] name = "unicode-xid" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" +checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" [[package]] -name = "unicode-xid" -version = "0.2.0" +name = "unsigned-varint" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +checksum = "a7f0023a96687fe169081e8adce3f65e3874426b7886e9234d490af2dc077959" [[package]] name = "unsigned-varint" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c689459fbaeb50e56c6749275f084decfd02194ac5852e6617d95d0d3cf02eaf" +checksum = "3b7ffb36714206d2f5f05d61a2bc350415c642f2c54433f0ebf829afbe41d570" dependencies = [ "bytes 0.5.4", + "futures 0.3.4", "futures_codec", ] @@ -8503,6 +8601,12 @@ dependencies = [ "percent-encoding 2.1.0", ] +[[package]] +name = "uuid" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11" + [[package]] name = "vcpkg" version = "0.2.8" @@ -8624,8 +8728,8 @@ dependencies = [ "bumpalo", "lazy_static", "log 0.4.8", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", "wasm-bindgen-shared", ] @@ -8648,7 +8752,7 @@ version = "0.2.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "574094772ce6921576fb6f2e3f7497b8a76273b6db092be18fc48a082de09dc3" dependencies = [ - "quote 1.0.2", + "quote", "wasm-bindgen-macro-support", ] @@ -8658,8 +8762,8 @@ version = "0.2.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e85031354f25eaebe78bb7db1c3d86140312a911a106b2e29f9cc440ce3e7668" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -8680,8 +8784,8 @@ dependencies = [ "anyhow", "heck", "log 0.4.8", - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", "wasm-bindgen-backend", "weedle", @@ -8740,21 +8844,14 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.48.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "073da89bf1c84db000dd68ce660c1b4a08e3a2d28fd1e3394ab9e7abdde4a0f8" - -[[package]] -name = "wasmparser" -version = "0.51.1" +version = "0.51.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e41b27a1677fe28c115de49efca55dabb14f7fece2c32947ffb9b1064fe5bd4" +checksum = "aeb1956b19469d1c5e63e459d29e7b5aa0f558d9f16fcef09736f8a265e6c10a" [[package]] name = "wasmtime" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5614d964c3e7d07a13b59aca66103c52656bd80430f0d86dc7eeb3af4f03d4a2" +version = "0.12.0" +source = "git+https://github.com/paritytech/wasmtime?branch=a-thread-safe-api#851887d84d03543f931f6312448d0dd5d8a9324e" dependencies = [ "anyhow", "backtrace", @@ -8764,9 +8861,10 @@ dependencies = [ "region", "rustc-demangle", "target-lexicon", - "wasmparser 0.51.1", + "wasmparser", "wasmtime-environ", "wasmtime-jit", + "wasmtime-profiling", "wasmtime-runtime", "wat", "winapi 0.3.8", @@ -8774,25 +8872,23 @@ dependencies = [ [[package]] name = "wasmtime-debug" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feb5900275b4ef0b621ce725b9d5660b12825d7f7d79b392b97baf089ffab8c0" +version = "0.12.0" +source = "git+https://github.com/paritytech/wasmtime?branch=a-thread-safe-api#851887d84d03543f931f6312448d0dd5d8a9324e" dependencies = [ "anyhow", "faerie", - "gimli 0.19.0", + "gimli", "more-asserts", "target-lexicon", "thiserror", - "wasmparser 0.51.1", + "wasmparser", "wasmtime-environ", ] [[package]] name = "wasmtime-environ" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04661851e133fb11691c4a0f92a705766b4bbf7afc06811f949e295cc8414fc" +version = "0.12.0" +source = "git+https://github.com/paritytech/wasmtime?branch=a-thread-safe-api#851887d84d03543f931f6312448d0dd5d8a9324e" dependencies = [ "anyhow", "base64 0.11.0", @@ -8812,16 +8908,15 @@ dependencies = [ "sha2", "thiserror", "toml", - "wasmparser 0.51.1", + "wasmparser", "winapi 0.3.8", "zstd", ] [[package]] name = "wasmtime-jit" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d451353764ce55c9bb6a8b260063cfc209b7adadd277a9a872ab4563a69e357c" +version = "0.12.0" +source = "git+https://github.com/paritytech/wasmtime?branch=a-thread-safe-api#851887d84d03543f931f6312448d0dd5d8a9324e" dependencies = [ "anyhow", "cfg-if", @@ -8834,46 +8929,63 @@ dependencies = [ "region", "target-lexicon", "thiserror", - "wasmparser 0.51.1", + "wasmparser", "wasmtime-debug", "wasmtime-environ", + "wasmtime-profiling", "wasmtime-runtime", "winapi 0.3.8", ] +[[package]] +name = "wasmtime-profiling" +version = "0.12.0" +source = "git+https://github.com/paritytech/wasmtime?branch=a-thread-safe-api#851887d84d03543f931f6312448d0dd5d8a9324e" +dependencies = [ + "gimli", + "goblin", + "lazy_static", + "libc", + "object", + "scroll", + "serde", + "target-lexicon", +] + [[package]] name = "wasmtime-runtime" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dbd4fc114b828cae3e405fed413df4b3814d87a92ea029640cec9ba41f0c162" +version = "0.12.0" +source = "git+https://github.com/paritytech/wasmtime?branch=a-thread-safe-api#851887d84d03543f931f6312448d0dd5d8a9324e" dependencies = [ "backtrace", "cc", "cfg-if", "indexmap", + "lazy_static", "libc", "memoffset", "more-asserts", "region", "thiserror", "wasmtime-environ", + "wasmtime-profiling", "winapi 0.3.8", ] [[package]] name = "wast" -version = "7.0.0" +version = "9.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12a729d076deb29c8509fa71f2d427729f9394f9496844ed8fcab152f35d163d" +checksum = "ee7b16105405ca2aa2376ba522d8d4b1a11604941dd3bb7df9fd2ece60f8d16a" dependencies = [ "leb128", ] [[package]] name = "wat" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5795e34a4b39893653dec97e644fac85c31398e0ce1abecc48967aac83d9e8ce" +checksum = "56173f7f4fb59aebe35a7e71423845e1c6c7144bfb56362d497931b6b3bed0f6" dependencies = [ "wast", ] @@ -8969,16 +9081,6 @@ dependencies = [ "nom", ] -[[package]] -name = "which" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164" -dependencies = [ - "failure", - "libc", -] - [[package]] name = "which" version = "3.1.0" @@ -9059,17 +9161,6 @@ dependencies = [ "winapi-build", ] -[[package]] -name = "x25519-dalek" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee1585dc1484373cbc1cee7aafda26634665cf449436fd6e24bfd1fad230538" -dependencies = [ - "clear_on_drop", - "curve25519-dalek 1.2.3", - "rand_core 0.3.1", -] - [[package]] name = "x25519-dalek" version = "0.6.0" @@ -9089,9 +9180,9 @@ checksum = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57" [[package]] name = "yamux" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d73295bc9d9acf89dd9336b3b5f5b57731ee72b587857dd4312721a0196b48e5" +checksum = "f03098897b734bd943ab23f6aa9f98aafd72a88516deedd66f9d564c57bf2f19" dependencies = [ "bytes 0.5.4", "futures 0.3.4", @@ -9123,8 +9214,8 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de251eec69fc7c1bc3923403d18ececb929380e016afe103da75f396704f8ca2" dependencies = [ - "proc-macro2 1.0.8", - "quote 1.0.2", + "proc-macro2", + "quote", "syn", "synstructure", ] diff --git a/Cargo.toml b/Cargo.toml index a42a8e24d0f48e5412d18b93a2aa5ade5f5b0737..0459bc8ebbd6abd658db6295fefc4c699291207e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,15 +53,16 @@ members = [ "client/telemetry", "client/transaction-pool", "client/transaction-pool/graph", + "utils/prometheus", "utils/wasm-builder-runner", - "utils/grafana-data-source", - "utils/grafana-data-source/test", "frame/assets", "frame/aura", "frame/authority-discovery", "frame/authorship", "frame/babe", "frame/balances", + "frame/benchmarking", + "frame/benchmark", "frame/collective", "frame/contracts", "frame/contracts/rpc", @@ -71,6 +72,7 @@ members = [ "frame/elections", "frame/evm", "frame/example", + "frame/example-offchain-worker", "frame/executive", "frame/finality-tracker", "frame/generic-asset", @@ -156,6 +158,7 @@ members = [ "utils/browser", "utils/build-script-utils", "utils/fork-tree", + "utils/frame/benchmarking-cli", "utils/frame/rpc/support", "utils/frame/rpc/system", "utils/wasm-builder", diff --git a/bin/node-template/README.md b/bin/node-template/README.md index c411dbeef5bcccf9e05d64ddee2405cb5237c678..4ae60478fc72b77f32664b4225c023c36ff5f1af 100644 --- a/bin/node-template/README.md +++ b/bin/node-template/README.md @@ -1,6 +1,6 @@ # Substrate Node Template -A new SRML-based Substrate node, ready for hacking. +A new FRAME-based Substrate node, ready for hacking. ## Build diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index 9ad4a0e8a55ad5ad82a07252f1d62fc12e8ce101..1e8c3fad2e3c0e2f1a8cef0f2d7fff57f2b45ff3 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -1,10 +1,12 @@ [package] name = "node-template" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Anonymous"] edition = "2018" license = "Unlicense" build = "build.rs" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [[bin]] name = "node-template" @@ -14,25 +16,26 @@ futures = "0.3.1" log = "0.4.8" structopt = "0.3.8" -sc-cli = { version = "0.8.0", path = "../../../client/cli" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sc-executor = { version = "0.8", path = "../../../client/executor" } -sc-service = { version = "0.8", path = "../../../client/service" } -sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } -sc-transaction-pool = { version = "2.0.0", path = "../../../client/transaction-pool" } -sp-transaction-pool = { version = "2.0.0", path = "../../../primitives/transaction-pool" } -sc-network = { version = "0.8", path = "../../../client/network" } -sc-consensus-aura = { version = "0.8", path = "../../../client/consensus/aura" } -sp-consensus-aura = { version = "0.8", path = "../../../primitives/consensus/aura" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } -grandpa = { version = "0.8", package = "sc-finality-grandpa", path = "../../../client/finality-grandpa" } -grandpa-primitives = { version = "2.0.0", package = "sp-finality-grandpa", path = "../../../primitives/finality-grandpa" } -sc-client = { version = "0.8", path = "../../../client/" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sc-basic-authorship = { path = "../../../client/basic-authorship" } +sc-cli = { version = "0.8.0-alpha.2", path = "../../../client/cli" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sc-executor = { version = "0.8.0-alpha.2", path = "../../../client/executor" } +sc-service = { version = "0.8.0-alpha.2", path = "../../../client/service" } +sp-inherents = { version = "2.0.0-alpha.2", path = "../../../primitives/inherents" } +sc-transaction-pool = { version = "2.0.0-alpha.2", path = "../../../client/transaction-pool" } +sp-transaction-pool = { version = "2.0.0-alpha.2", path = "../../../primitives/transaction-pool" } +sc-network = { version = "0.8.0-alpha.2", path = "../../../client/network" } +sc-consensus-aura = { version = "0.8.0-alpha.2", path = "../../../client/consensus/aura" } +sp-consensus-aura = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/aura" } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/common" } +grandpa = { version = "0.8.0-alpha.2", package = "sc-finality-grandpa", path = "../../../client/finality-grandpa" } +grandpa-primitives = { version = "2.0.0-alpha.2", package = "sp-finality-grandpa", path = "../../../primitives/finality-grandpa" } +sc-client = { version = "0.8.0-alpha.2", path = "../../../client/" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../../../client/api" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +sc-basic-authorship = { path = "../../../client/basic-authorship" , version = "0.8.0-alpha.2"} -node-template-runtime = { version = "2.0.0", path = "../runtime" } +node-template-runtime = { version = "2.0.0-alpha.2", path = "../runtime" } [build-dependencies] vergen = "3.0.4" -build-script-utils = { version = "2.0.0", package = "substrate-build-script-utils", path = "../../../utils/build-script-utils" } +build-script-utils = { version = "2.0.0-alpha.2", package = "substrate-build-script-utils", path = "../../../utils/build-script-utils" } diff --git a/bin/node-template/node/src/chain_spec.rs b/bin/node-template/node/src/chain_spec.rs index 9bdfea3b7820d5ed80e5912d65de7e91a3b14f9a..64b84005072fda218c5b866e8ec8a1898e27df18 100644 --- a/bin/node-template/node/src/chain_spec.rs +++ b/bin/node-template/node/src/chain_spec.rs @@ -1,7 +1,7 @@ use sp_core::{Pair, Public, sr25519}; use node_template_runtime::{ AccountId, AuraConfig, BalancesConfig, GenesisConfig, GrandpaConfig, - IndicesConfig, SudoConfig, SystemConfig, WASM_BINARY, Signature + SudoConfig, SystemConfig, WASM_BINARY, Signature }; use sp_consensus_aura::sr25519::{AuthorityId as AuraId}; use grandpa_primitives::{AuthorityId as GrandpaId}; @@ -127,21 +127,18 @@ fn testnet_genesis(initial_authorities: Vec<(AuraId, GrandpaId)>, code: WASM_BINARY.to_vec(), changes_trie_config: Default::default(), }), - indices: Some(IndicesConfig { - indices: vec![], - }), balances: Some(BalancesConfig { balances: endowed_accounts.iter().cloned().map(|k|(k, 1 << 60)).collect(), }), - sudo: Some(SudoConfig { - key: root_key, - }), aura: Some(AuraConfig { authorities: initial_authorities.iter().map(|x| (x.0.clone())).collect(), }), grandpa: Some(GrandpaConfig { authorities: initial_authorities.iter().map(|x| (x.1.clone(), 1)).collect(), }), + sudo: Some(SudoConfig { + key: root_key, + }), } } diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index e7e386703deee0a7894dfc5bf931c6c41fa89438..0f4c301dbff5b7defe0565bf71e85b21a6418867 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -15,32 +15,35 @@ // along with Substrate. If not, see . use sp_consensus_aura::sr25519::{AuthorityPair as AuraPair}; -use sc_cli::{VersionInfo, error}; +use sc_cli::VersionInfo; use crate::service; use crate::chain_spec; use crate::cli::Cli; /// Parse and run command line arguments -pub fn run(version: VersionInfo) -> error::Result<()> { +pub fn run(version: VersionInfo) -> sc_cli::Result<()> { let opt = sc_cli::from_args::(&version); - let config = sc_service::Configuration::new(&version); + let mut config = sc_service::Configuration::from_version(&version); match opt.subcommand { - Some(subcommand) => sc_cli::run_subcommand( - config, - subcommand, - chain_spec::load_spec, - |config: _| Ok(new_full_start!(config).0), - &version, - ), - None => sc_cli::run( - config, - opt.run, - service::new_light, - service::new_full, - chain_spec::load_spec, - &version, - ) + Some(subcommand) => { + subcommand.init(&version)?; + subcommand.update_config(&mut config, chain_spec::load_spec, &version)?; + subcommand.run( + config, + |config: _| Ok(new_full_start!(config).0), + ) + }, + None => { + opt.run.init(&version)?; + opt.run.update_config(&mut config, chain_spec::load_spec, &version)?; + opt.run.run( + config, + service::new_light, + service::new_full, + &version, + ) + }, } } diff --git a/bin/node-template/node/src/main.rs b/bin/node-template/node/src/main.rs index 9d0a57d77a851d551e9a8e087365a8d8cb470977..91b2c257e0cd733f7e87d00e02857101810a2fb0 100644 --- a/bin/node-template/node/src/main.rs +++ b/bin/node-template/node/src/main.rs @@ -7,10 +7,8 @@ mod service; mod cli; mod command; -pub use sc_cli::{VersionInfo, error}; - -fn main() -> Result<(), error::Error> { - let version = VersionInfo { +fn main() -> sc_cli::Result<()> { + let version = sc_cli::VersionInfo { name: "Substrate Node", commit: env!("VERGEN_SHA_SHORT"), version: env!("CARGO_PKG_VERSION"), diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 5466f3e919c0baa4bed2b83622fdf35d5987e5b8..f289ff58549dbc354bfdb706f8edf1659f6b044a 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -3,14 +3,14 @@ use std::sync::Arc; use std::time::Duration; use sc_client::LongestChain; +use sc_client_api::ExecutorProvider; use node_template_runtime::{self, GenesisConfig, opaque::Block, RuntimeApi}; use sc_service::{error::{Error as ServiceError}, AbstractService, Configuration, ServiceBuilder}; use sp_inherents::InherentDataProviders; -use sc_network::{construct_simple_protocol}; use sc_executor::native_executor_instance; pub use sc_executor::NativeExecutor; use sp_consensus_aura::sr25519::{AuthorityPair as AuraPair}; -use grandpa::{self, FinalityProofProvider as GrandpaFinalityProofProvider}; +use grandpa::{self, FinalityProofProvider as GrandpaFinalityProofProvider, StorageAndProofProvider}; // Our native executor instance. native_executor_instance!( @@ -19,17 +19,13 @@ native_executor_instance!( node_template_runtime::native_version, ); -construct_simple_protocol! { - /// Demo protocol attachment for substrate. - pub struct NodeProtocol where Block = Block { } -} - /// Starts a `ServiceBuilder` for a full service. /// /// Use this macro if you don't actually need the full service, but just the builder in order to /// be able to perform chain operations. macro_rules! new_full_start { ($config:expr) => {{ + use std::sync::Arc; let mut import_setup = None; let inherent_data_providers = sp_inherents::InherentDataProviders::new(); @@ -43,27 +39,24 @@ macro_rules! new_full_start { let pool_api = sc_transaction_pool::FullChainApi::new(client.clone()); Ok(sc_transaction_pool::BasicPool::new(config, std::sync::Arc::new(pool_api))) })? - .with_import_queue(|_config, client, mut select_chain, transaction_pool| { + .with_import_queue(|_config, client, mut select_chain, _transaction_pool| { let select_chain = select_chain.take() .ok_or_else(|| sc_service::Error::SelectChainRequired)?; let (grandpa_block_import, grandpa_link) = - grandpa::block_import::<_, _, _, node_template_runtime::RuntimeApi, _>( - client.clone(), &*client, select_chain - )?; + grandpa::block_import(client.clone(), &(client.clone() as Arc<_>), select_chain)?; let aura_block_import = sc_consensus_aura::AuraBlockImport::<_, _, _, AuraPair>::new( grandpa_block_import.clone(), client.clone(), ); - let import_queue = sc_consensus_aura::import_queue::<_, _, _, AuraPair, _>( - sc_consensus_aura::SlotDuration::get_or_compute(&*client)?, + let import_queue = sc_consensus_aura::import_queue::<_, _, _, AuraPair>( + sc_consensus_aura::slot_duration(&*client)?, aura_block_import, Some(Box::new(grandpa_block_import.clone())), None, client, inherent_data_providers.clone(), - Some(transaction_pool), )?; import_setup = Some((grandpa_block_import, grandpa_link)); @@ -95,17 +88,19 @@ pub fn new_full(config: Configuration) import_setup.take() .expect("Link Half and Block Import are present for Full Services or setup failed before. qed"); - let service = builder.with_network_protocol(|_| Ok(NodeProtocol::new()))? - .with_finality_proof_provider(|client, backend| - Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, client)) as _) - )? + let service = builder + .with_finality_proof_provider(|client, backend| { + // GenesisAuthoritySetProvider is implemented for StorageAndProofProvider + let provider = client as Arc>; + Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, provider)) as _) + })? .build()?; if participates_in_consensus { - let proposer = sc_basic_authorship::ProposerFactory { - client: service.client(), - transaction_pool: service.transaction_pool(), - }; + let proposer = sc_basic_authorship::ProposerFactory::new( + service.client(), + service.transaction_pool() + ); let client = service.client(); let select_chain = service.select_chain() @@ -115,7 +110,7 @@ pub fn new_full(config: Configuration) sp_consensus::CanAuthorWithNativeVersion::new(client.executor().clone()); let aura = sc_consensus_aura::start_aura::<_, _, _, _, _, AuraPair, _, _, _>( - sc_consensus_aura::SlotDuration::get_or_compute(&*client)?, + sc_consensus_aura::slot_duration(&*client)?, client, select_chain, block_import, @@ -145,44 +140,41 @@ pub fn new_full(config: Configuration) gossip_duration: Duration::from_millis(333), justification_period: 512, name: Some(name), - observer_enabled: true, + observer_enabled: false, keystore, is_authority, }; - match (is_authority, disable_grandpa) { - (false, false) => { - // start the lightweight GRANDPA observer - service.spawn_task("grandpa-observer", grandpa::run_grandpa_observer( - grandpa_config, - grandpa_link, - service.network(), - service.on_exit(), - )?); - }, - (true, false) => { - // start the full GRANDPA voter - let voter_config = grandpa::GrandpaParams { - config: grandpa_config, - link: grandpa_link, - network: service.network(), - inherent_data_providers: inherent_data_providers.clone(), - on_exit: service.on_exit(), - telemetry_on_connect: Some(service.telemetry_on_connect_stream()), - voting_rule: grandpa::VotingRulesBuilder::default().build(), - }; - - // the GRANDPA voter task is considered infallible, i.e. - // if it fails we take down the service with it. - service.spawn_essential_task("grandpa", grandpa::run_grandpa_voter(voter_config)?); - }, - (_, true) => { - grandpa::setup_disabled_grandpa( - service.client(), - &inherent_data_providers, - service.network(), - )?; - }, + let enable_grandpa = !disable_grandpa; + if enable_grandpa { + // start the full GRANDPA voter + // NOTE: non-authorities could run the GRANDPA observer protocol, but at + // this point the full voter should provide better guarantees of block + // and vote data availability than the observer. The observer has not + // been tested extensively yet and having most nodes in a network run it + // could lead to finality stalls. + let grandpa_config = grandpa::GrandpaParams { + config: grandpa_config, + link: grandpa_link, + network: service.network(), + inherent_data_providers: inherent_data_providers.clone(), + telemetry_on_connect: Some(service.telemetry_on_connect_stream()), + voting_rule: grandpa::VotingRulesBuilder::default().build(), + prometheus_registry: service.prometheus_registry() + }; + + // the GRANDPA voter task is considered infallible, i.e. + // if it fails we take down the service with it. + service.spawn_essential_task( + "grandpa-voter", + grandpa::run_grandpa_voter(grandpa_config)? + ); + } else { + grandpa::setup_disabled_grandpa( + service.client(), + &inherent_data_providers, + service.network(), + )?; } Ok(service) @@ -212,28 +204,31 @@ pub fn new_light(config: Configuration) let fetch_checker = fetcher .map(|fetcher| fetcher.checker().clone()) .ok_or_else(|| "Trying to start light import queue without active fetch checker")?; - let grandpa_block_import = grandpa::light_block_import::<_, _, _, RuntimeApi>( - client.clone(), backend, &*client.clone(), Arc::new(fetch_checker), + let grandpa_block_import = grandpa::light_block_import( + client.clone(), + backend, + &(client.clone() as Arc<_>), + Arc::new(fetch_checker), )?; let finality_proof_import = grandpa_block_import.clone(); let finality_proof_request_builder = finality_proof_import.create_finality_proof_request_builder(); - let import_queue = sc_consensus_aura::import_queue::<_, _, _, AuraPair, ()>( - sc_consensus_aura::SlotDuration::get_or_compute(&*client)?, + let import_queue = sc_consensus_aura::import_queue::<_, _, _, AuraPair>( + sc_consensus_aura::slot_duration(&*client)?, grandpa_block_import, None, Some(Box::new(finality_proof_import)), client, inherent_data_providers.clone(), - None, )?; Ok((import_queue, finality_proof_request_builder)) })? - .with_network_protocol(|_| Ok(NodeProtocol::new()))? - .with_finality_proof_provider(|client, backend| - Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, client)) as _) - )? + .with_finality_proof_provider(|client, backend| { + // GenesisAuthoritySetProvider is implemented for StorageAndProofProvider + let provider = client as Arc>; + Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, provider)) as _) + })? .build() } diff --git a/bin/node-template/pallets/template/Cargo.toml b/bin/node-template/pallets/template/Cargo.toml index 8ea3f3adabc52c2c87802b2f407d188f14bd202c..b39fcc1dae4d5bdb3b71edfaac02c55783879e29 100644 --- a/bin/node-template/pallets/template/Cargo.toml +++ b/bin/node-template/pallets/template/Cargo.toml @@ -2,43 +2,47 @@ authors = ['Anonymous'] edition = '2018' name = 'pallet-template' -version = '2.0.0' +version = "2.0.0-alpha.3" +license = "Unlicense" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet template" [dependencies] -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } safe-mix = { default-features = false, version = '1.0.0' } [dependencies.frame-support] default-features = false -version = '2.0.0' +version = "2.0.0-alpha.2" path = "../../../../frame/support" [dependencies.system] default-features = false package = 'frame-system' -version = '2.0.0' +version = "2.0.0-alpha.2" path = "../../../../frame/system" - [dev-dependencies.sp-core] default-features = false -version = '2.0.0' +version = "2.0.0-alpha.2" path = "../../../../primitives/core" [dev-dependencies.sp-io] default-features = false -version = '2.0.0' +version = "2.0.0-alpha.2" path = "../../../../primitives/io" [dev-dependencies.sp-runtime] default-features = false -version = '2.0.0' +version = "2.0.0-alpha.2" path = "../../../../primitives/runtime" + [features] default = ['std'] std = [ - 'codec/std', - 'frame-support/std', - 'safe-mix/std', - 'system/std' + 'codec/std', + 'frame-support/std', + 'safe-mix/std', + 'system/std' ] diff --git a/bin/node-template/pallets/template/src/lib.rs b/bin/node-template/pallets/template/src/lib.rs index aa4d2cbc994d398046b6a6e7668c512c1b793a7c..34e055bf546b0b57a247a8cb083a41de7d064508 100644 --- a/bin/node-template/pallets/template/src/lib.rs +++ b/bin/node-template/pallets/template/src/lib.rs @@ -1,12 +1,12 @@ #![cfg_attr(not(feature = "std"), no_std)] -/// A runtime module template with necessary imports +/// A FRAME pallet template with necessary imports /// Feel free to remove or edit this file as needed. /// If you change the name of this file, make sure to update its references in runtime/src/lib.rs /// If you remove this file, you can remove those references -/// For more guidance on Substrate modules, see the example module +/// For more guidance on Substrate FRAME, see the example pallet /// https://github.com/paritytech/substrate/blob/master/frame/example/src/lib.rs use frame_support::{decl_module, decl_storage, decl_event, decl_error, dispatch}; diff --git a/bin/node-template/pallets/template/src/mock.rs b/bin/node-template/pallets/template/src/mock.rs index b3c1098db6743189ec8fcad258459f2e013126e8..2ea81ffb456261122b234c6ae3c36f52eaa9c430 100644 --- a/bin/node-template/pallets/template/src/mock.rs +++ b/bin/node-template/pallets/template/src/mock.rs @@ -11,9 +11,9 @@ impl_outer_origin! { pub enum Origin for Test {} } -// For testing the module, we construct most of a mock runtime. This means +// For testing the pallet, we construct most of a mock runtime. This means // first constructing a configuration type (`Test`) which `impl`s each of the -// configuration traits of modules we want to use. +// configuration traits of pallets we want to use. #[derive(Clone, Eq, PartialEq)] pub struct Test; parameter_types! { @@ -41,7 +41,7 @@ impl system::Trait for Test { type ModuleToIndex = (); type AccountData = (); type OnNewAccount = (); - type OnReapAccount = (); + type OnKilledAccount = (); } impl Trait for Test { type Event = (); diff --git a/bin/node-template/runtime/Cargo.toml b/bin/node-template/runtime/Cargo.toml index ddecb0e4cff4c11da2ddf6e7973c1c57e082d1e1..9268dd8c05035f4dd8022702e9792f421ab556c5 100644 --- a/bin/node-template/runtime/Cargo.toml +++ b/bin/node-template/runtime/Cargo.toml @@ -1,42 +1,43 @@ [package] name = "node-template-runtime" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Anonymous"] edition = "2018" license = "Unlicense" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } -aura = { version = "2.0.0", default-features = false, package = "pallet-aura", path = "../../../frame/aura" } -balances = { version = "2.0.0", default-features = false, package = "pallet-balances", path = "../../../frame/balances" } -frame-support = { version = "2.0.0", default-features = false, path = "../../../frame/support" } -grandpa = { version = "2.0.0", default-features = false, package = "pallet-grandpa", path = "../../../frame/grandpa" } -indices = { version = "2.0.0", default-features = false, package = "pallet-indices", path = "../../../frame/indices" } -randomness-collective-flip = { version = "2.0.0", default-features = false, package = "pallet-randomness-collective-flip", path = "../../../frame/randomness-collective-flip" } -sudo = { version = "2.0.0", default-features = false, package = "pallet-sudo", path = "../../../frame/sudo" } -system = { version = "2.0.0", default-features = false, package = "frame-system", path = "../../../frame/system" } -timestamp = { version = "2.0.0", default-features = false, package = "pallet-timestamp", path = "../../../frame/timestamp" } -transaction-payment = { version = "2.0.0", default-features = false, package = "pallet-transaction-payment", path = "../../../frame/transaction-payment" } -frame-executive = { version = "2.0.0", default-features = false, path = "../../../frame/executive" } +aura = { version = "2.0.0-alpha.2", default-features = false, package = "pallet-aura", path = "../../../frame/aura" } +balances = { version = "2.0.0-alpha.2", default-features = false, package = "pallet-balances", path = "../../../frame/balances" } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/support" } +grandpa = { version = "2.0.0-alpha.2", default-features = false, package = "pallet-grandpa", path = "../../../frame/grandpa" } +randomness-collective-flip = { version = "2.0.0-alpha.2", default-features = false, package = "pallet-randomness-collective-flip", path = "../../../frame/randomness-collective-flip" } +sudo = { version = "2.0.0-alpha.2", default-features = false, package = "pallet-sudo", path = "../../../frame/sudo" } +system = { version = "2.0.0-alpha.2", default-features = false, package = "frame-system", path = "../../../frame/system" } +timestamp = { version = "2.0.0-alpha.2", default-features = false, package = "pallet-timestamp", path = "../../../frame/timestamp" } +transaction-payment = { version = "2.0.0-alpha.2", default-features = false, package = "pallet-transaction-payment", path = "../../../frame/transaction-payment" } +frame-executive = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/executive" } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-api = { version = "2.0.0", default-features = false, path = "../../../primitives/api" } -sp-block-builder = { path = "../../../primitives/block-builder", default-features = false} -sp-consensus-aura = { version = "0.8", default-features = false, path = "../../../primitives/consensus/aura" } -sp-core = { version = "2.0.0", default-features = false, path = "../../../primitives/core" } -sp-inherents = { path = "../../../primitives/inherents", default-features = false} -sp-io = { version = "2.0.0", default-features = false, path = "../../../primitives/io" } -sp-offchain = { version = "2.0.0", default-features = false, path = "../../../primitives/offchain" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../../primitives/runtime" } -sp-session = { version = "2.0.0", default-features = false, path = "../../../primitives/session" } -sp-std = { version = "2.0.0", default-features = false, path = "../../../primitives/std" } -sp-transaction-pool = { version = "2.0.0", default-features = false, path = "../../../primitives/transaction-pool" } -sp-version = { version = "2.0.0", default-features = false, path = "../../../primitives/version" } +sp-api = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/api" } +sp-block-builder = { path = "../../../primitives/block-builder", default-features = false, version = "2.0.0-alpha.2"} +sp-consensus-aura = { version = "0.8.0-alpha.2", default-features = false, path = "../../../primitives/consensus/aura" } +sp-core = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/core" } +sp-inherents = { path = "../../../primitives/inherents", default-features = false, version = "2.0.0-alpha.2"} +sp-io = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/io" } +sp-offchain = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/offchain" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/runtime" } +sp-session = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/session" } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/std" } +sp-transaction-pool = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/transaction-pool" } +sp-version = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/version" } -template = { version = "2.0.0", default-features = false, path = "../pallets/template", package = "pallet-template" } +template = { version = "2.0.0-alpha.2", default-features = false, path = "../pallets/template", package = "pallet-template" } [build-dependencies] -wasm-builder-runner = { version = "1.0.4", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } +wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } [features] default = ["std"] @@ -47,7 +48,6 @@ std = [ "frame-executive/std", "frame-support/std", "grandpa/std", - "indices/std", "randomness-collective-flip/std", "serde", "sp-api/std", diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index f3ff911f16b1ce65218fb06dbad4ae39b7f90526..53ef4fc9b30e0e576eb97301fab94521eac1a458 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -15,7 +15,7 @@ use sp_runtime::{ impl_opaque_keys, MultiSignature, }; use sp_runtime::traits::{ - BlakeTwo256, Block as BlockT, StaticLookup, Verify, ConvertInto, IdentifyAccount + BlakeTwo256, Block as BlockT, IdentityLookup, Verify, ConvertInto, IdentifyAccount }; use sp_api::impl_runtime_apis; use sp_consensus_aura::sr25519::AuthorityId as AuraId; @@ -132,7 +132,7 @@ impl system::Trait for Runtime { /// The aggregated dispatch type that is available for extrinsics. type Call = Call; /// The lookup mechanism to get account ID from whatever is passed in dispatchers. - type Lookup = Indices; + type Lookup = IdentityLookup; /// The index type for storing how many extrinsics an account has signed. type Index = Index; /// The index type for blocks. @@ -164,7 +164,7 @@ impl system::Trait for Runtime { /// What to do if a new account is created. type OnNewAccount = (); /// What to do if an account is fully reaped from the system. - type OnReapAccount = Balances; + type OnKilledAccount = (); /// The data to be stored in an account. type AccountData = balances::AccountData; } @@ -177,23 +177,6 @@ impl grandpa::Trait for Runtime { type Event = Event; } -parameter_types! { - /// How much an index costs. - pub const IndexDeposit: u128 = 100; -} - -impl indices::Trait for Runtime { - /// The type for recording indexing into the account enumeration. If this ever overflows, there - /// will be problems! - type AccountIndex = AccountIndex; - /// The ubiquitous event type. - type Event = Event; - /// The currency type. - type Currency = Balances; - /// How much an index costs. - type Deposit = IndexDeposit; -} - parameter_types! { pub const MinimumPeriod: u64 = SLOT_DURATION / 2; } @@ -252,21 +235,20 @@ construct_runtime!( UncheckedExtrinsic = UncheckedExtrinsic { System: system::{Module, Call, Config, Storage, Event}, + RandomnessCollectiveFlip: randomness_collective_flip::{Module, Call, Storage}, Timestamp: timestamp::{Module, Call, Storage, Inherent}, Aura: aura::{Module, Config, Inherent(Timestamp)}, Grandpa: grandpa::{Module, Call, Storage, Config, Event}, - Indices: indices::{Module, Call, Storage, Event, Config}, Balances: balances::{Module, Call, Storage, Config, Event}, TransactionPayment: transaction_payment::{Module, Storage}, Sudo: sudo::{Module, Call, Config, Storage, Event}, // Used for the module template in `./template.rs` TemplateModule: template::{Module, Call, Storage, Event}, - RandomnessCollectiveFlip: randomness_collective_flip::{Module, Call, Storage}, } ); /// The address format for describing accounts. -pub type Address = ::Source; +pub type Address = AccountId; /// Block header type as expected by this runtime. pub type Header = generic::Header; /// Block type as expected by this runtime. diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 079e8d13e2d690eba49701253cfddc7efa29b8fb..e18b6b228e65553543891d6e5cbcd6fb6094c752 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -1,12 +1,14 @@ [package] name = "node-cli" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] description = "Substrate node implementation in Rust." build = "build.rs" edition = "2018" license = "GPL-3.0" default-run = "substrate" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [package.metadata.wasm-pack.profile.release] # `wasm-opt` has some problems on linux, see @@ -29,7 +31,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] # third-party dependencies -codec = { package = "parity-scale-codec", version = "1.0.6" } +codec = { package = "parity-scale-codec", version = "1.2.0" } serde = { version = "1.0.102", features = ["derive"] } futures = { version = "0.3.1", features = ["compat"] } hex-literal = "0.2.1" @@ -40,81 +42,84 @@ structopt = { version = "0.3.8", optional = true } tracing = "0.1.10" # primitives -sp-authority-discovery = { version = "2.0.0", path = "../../../primitives/authority-discovery" } -sp-consensus-babe = { version = "0.8", path = "../../../primitives/consensus/babe" } -grandpa-primitives = { version = "2.0.0", package = "sp-finality-grandpa", path = "../../../primitives/finality-grandpa" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-timestamp = { version = "2.0.0", default-features = false, path = "../../../primitives/timestamp" } -sp-finality-tracker = { version = "2.0.0", default-features = false, path = "../../../primitives/finality-tracker" } -sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } -sp-keyring = { version = "2.0.0", path = "../../../primitives/keyring" } -sp-io = { version = "2.0.0", path = "../../../primitives/io" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } +sp-authority-discovery = { version = "2.0.0-alpha.2", path = "../../../primitives/authority-discovery" } +sp-consensus-babe = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/babe" } +grandpa-primitives = { version = "2.0.0-alpha.2", package = "sp-finality-grandpa", path = "../../../primitives/finality-grandpa" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +sp-timestamp = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/timestamp" } +sp-finality-tracker = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/finality-tracker" } +sp-inherents = { version = "2.0.0-alpha.2", path = "../../../primitives/inherents" } +sp-keyring = { version = "2.0.0-alpha.2", path = "../../../primitives/keyring" } +sp-io = { version = "2.0.0-alpha.2", path = "../../../primitives/io" } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/common" } # client dependencies -sc-client-api = { version = "2.0.0", path = "../../../client/api" } -sc-client = { version = "0.8", path = "../../../client/" } -sc-chain-spec = { version = "2.0.0", path = "../../../client/chain-spec" } -sc-transaction-pool = { version = "2.0.0", path = "../../../client/transaction-pool" } -sp-transaction-pool = { version = "2.0.0", path = "../../../primitives/transaction-pool" } -sc-network = { version = "0.8", path = "../../../client/network" } -sc-consensus-babe = { version = "0.8", path = "../../../client/consensus/babe" } -grandpa = { version = "0.8", package = "sc-finality-grandpa", path = "../../../client/finality-grandpa" } -sc-client-db = { version = "0.8", default-features = false, path = "../../../client/db" } -sc-offchain = { version = "2.0.0", path = "../../../client/offchain" } -sc-rpc = { version = "2.0.0", path = "../../../client/rpc" } -sc-basic-authorship = { version = "0.8", path = "../../../client/basic-authorship" } -sc-service = { version = "0.8", default-features = false, path = "../../../client/service" } -sc-tracing = { version = "2.0.0", path = "../../../client/tracing" } -sc-telemetry = { version = "2.0.0", path = "../../../client/telemetry" } -sc-authority-discovery = { version = "0.8", path = "../../../client/authority-discovery" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../../../client/api" } +sc-client = { version = "0.8.0-alpha.2", path = "../../../client/" } +sc-chain-spec = { version = "2.0.0-alpha.2", path = "../../../client/chain-spec" } +sc-transaction-pool = { version = "2.0.0-alpha.2", path = "../../../client/transaction-pool" } +sp-transaction-pool = { version = "2.0.0-alpha.2", path = "../../../primitives/transaction-pool" } +sc-network = { version = "0.8.0-alpha.2", path = "../../../client/network" } +sc-consensus-babe = { version = "0.8.0-alpha.2", path = "../../../client/consensus/babe" } +grandpa = { version = "0.8.0-alpha.2", package = "sc-finality-grandpa", path = "../../../client/finality-grandpa" } +sc-client-db = { version = "0.8.0-alpha.2", default-features = false, path = "../../../client/db" } +sc-offchain = { version = "2.0.0-alpha.2", path = "../../../client/offchain" } +sc-rpc = { version = "2.0.0-alpha.2", path = "../../../client/rpc" } +sc-basic-authorship = { version = "0.8.0-alpha.2", path = "../../../client/basic-authorship" } +sc-service = { version = "0.8.0-alpha.2", default-features = false, path = "../../../client/service" } +sc-tracing = { version = "2.0.0-alpha.2", path = "../../../client/tracing" } +sc-telemetry = { version = "2.0.0-alpha.2", path = "../../../client/telemetry" } +sc-authority-discovery = { version = "0.8.0-alpha.2", path = "../../../client/authority-discovery" } # frame dependencies -pallet-indices = { version = "2.0.0", path = "../../../frame/indices" } -pallet-timestamp = { version = "2.0.0", default-features = false, path = "../../../frame/timestamp" } -pallet-contracts = { version = "2.0.0", path = "../../../frame/contracts" } -frame-system = { version = "2.0.0", path = "../../../frame/system" } -pallet-balances = { version = "2.0.0", path = "../../../frame/balances" } -pallet-transaction-payment = { version = "2.0.0", path = "../../../frame/transaction-payment" } -frame-support = { version = "2.0.0", default-features = false, path = "../../../frame/support" } -pallet-im-online = { version = "2.0.0", default-features = false, path = "../../../frame/im-online" } -pallet-authority-discovery = { version = "2.0.0", path = "../../../frame/authority-discovery" } +pallet-indices = { version = "2.0.0-alpha.2", path = "../../../frame/indices" } +pallet-timestamp = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/timestamp" } +pallet-contracts = { version = "2.0.0-alpha.2", path = "../../../frame/contracts" } +frame-system = { version = "2.0.0-alpha.2", path = "../../../frame/system" } +pallet-balances = { version = "2.0.0-alpha.2", path = "../../../frame/balances" } +pallet-transaction-payment = { version = "2.0.0-alpha.2", path = "../../../frame/transaction-payment" } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/support" } +pallet-im-online = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/im-online" } +pallet-authority-discovery = { version = "2.0.0-alpha.2", path = "../../../frame/authority-discovery" } # node-specific dependencies -node-runtime = { version = "2.0.0", path = "../runtime" } -node-rpc = { version = "2.0.0", path = "../rpc" } -node-primitives = { version = "2.0.0", path = "../primitives" } -node-executor = { version = "2.0.0", path = "../executor" } +node-runtime = { version = "2.0.0-alpha.2", path = "../runtime" } +node-rpc = { version = "2.0.0-alpha.2", path = "../rpc" } +node-primitives = { version = "2.0.0-alpha.2", path = "../primitives" } +node-executor = { version = "2.0.0-alpha.2", path = "../executor" } # CLI-specific dependencies -sc-cli = { version = "0.8.0", optional = true, path = "../../../client/cli" } -node-transaction-factory = { version = "0.8.0", optional = true, path = "../transaction-factory" } -node-inspect = { version = "0.8.0", optional = true, path = "../inspect" } +sc-cli = { version = "0.8.0-alpha.2", optional = true, path = "../../../client/cli" } +frame-benchmarking-cli = { version = "2.0.0-alpha.2", optional = true, path = "../../../utils/frame/benchmarking-cli" } +node-transaction-factory = { version = "0.8.0-alpha.2", optional = true, path = "../transaction-factory" } +node-inspect = { version = "0.8.0-alpha.2", optional = true, path = "../inspect" } # WASM-specific dependencies wasm-bindgen = { version = "0.2.57", optional = true } wasm-bindgen-futures = { version = "0.4.7", optional = true } -browser-utils = { path = "../../../utils/browser", optional = true } +browser-utils = { package = "substrate-browser-utils", path = "../../../utils/browser", optional = true, version = "0.8.0-alpha.2" } [dev-dependencies] -sc-keystore = { version = "2.0.0", path = "../../../client/keystore" } -sc-consensus-babe = { version = "0.8", features = ["test-helpers"], path = "../../../client/consensus/babe" } -sc-consensus-epochs = { version = "0.8", path = "../../../client/consensus/epochs" } -sc-service-test = { version = "2.0.0", path = "../../../client/service/test" } +sc-keystore = { version = "2.0.0-alpha.2", path = "../../../client/keystore" } +sc-consensus-babe = { version = "0.8.0-alpha.2", features = ["test-helpers"], path = "../../../client/consensus/babe" } +sc-consensus-epochs = { version = "0.8.0-alpha.2", path = "../../../client/consensus/epochs" } +sc-service-test = { version = "2.0.0-dev", path = "../../../client/service/test" } futures = "0.3.1" tempfile = "3.1.0" assert_cmd = "0.12" nix = "0.17" +serde_json = "1.0" [build-dependencies] -build-script-utils = { version = "2.0.0", package = "substrate-build-script-utils", path = "../../../utils/build-script-utils" } +build-script-utils = { version = "2.0.0-alpha.2", package = "substrate-build-script-utils", path = "../../../utils/build-script-utils" } structopt = { version = "0.3.8", optional = true } -node-transaction-factory = { version = "0.8.0", optional = true, path = "../transaction-factory" } -node-inspect = { version = "0.8.0", optional = true, path = "../inspect" } +node-transaction-factory = { version = "0.8.0-alpha.2", optional = true, path = "../transaction-factory" } +node-inspect = { version = "0.8.0-alpha.2", optional = true, path = "../inspect" } +frame-benchmarking-cli = { version = "2.0.0-alpha.2", optional = true, path = "../../../utils/frame/benchmarking-cli" } [build-dependencies.sc-cli] -version = "0.8.0" +version = "0.8.0-alpha.2" package = "sc-cli" path = "../../../client/cli" optional = true @@ -135,6 +140,7 @@ cli = [ "node-inspect", "node-transaction-factory", "sc-cli", + "frame-benchmarking-cli", "sc-service/rocksdb", "structopt", "vergen", @@ -145,3 +151,4 @@ wasmtime = [ "sc-cli/wasmtime", "sc-service/wasmtime", ] +runtime-benchmarks = [ "node-runtime/runtime-benchmarks" ] diff --git a/bin/node/cli/bin/main.rs b/bin/node/cli/bin/main.rs index e951c04710b92ff387b2f5fd5ba3c144b68af05c..8c4412667baceec56904864413178cbc88001497 100644 --- a/bin/node/cli/bin/main.rs +++ b/bin/node/cli/bin/main.rs @@ -18,10 +18,8 @@ #![warn(missing_docs)] -use sc_cli::VersionInfo; - -fn main() -> Result<(), sc_cli::error::Error> { - let version = VersionInfo { +fn main() -> sc_cli::Result<()> { + let version = sc_cli::VersionInfo { name: "Substrate Node", commit: env!("VERGEN_SHA_SHORT"), version: env!("CARGO_PKG_VERSION"), diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index 5cdfb5cab86f591343a18596aa5ee49c9f3d3ee7..af24db704c379e4ca11faf0a359e03afb04167f8 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -243,11 +243,10 @@ pub fn testnet_genesis( }), pallet_session: Some(SessionConfig { keys: initial_authorities.iter().map(|x| { - (x.0.clone(), session_keys(x.2.clone(), x.3.clone(), x.4.clone(), x.5.clone())) + (x.0.clone(), x.0.clone(), session_keys(x.2.clone(), x.3.clone(), x.4.clone(), x.5.clone())) }).collect::>(), }), pallet_staking: Some(StakingConfig { - current_era: 0, validator_count: initial_authorities.len() as u32 * 2, minimum_validator_count: initial_authorities.len() as u32, stakers: initial_authorities.iter().map(|x| { diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index 7dcd02699d664c9c44d31cc2152e3f208007a277..b6db9c3deb7e31f8f8da62d30b6971bf572b30d1 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -19,11 +19,6 @@ use structopt::StructOpt; /// An overarching CLI command definition. #[derive(Clone, Debug, StructOpt)] -#[structopt(settings = &[ - structopt::clap::AppSettings::GlobalVersion, - structopt::clap::AppSettings::ArgsNegateSubcommands, - structopt::clap::AppSettings::SubcommandsNegateReqs, -])] pub struct Cli { /// Possible subcommand with parameters. #[structopt(subcommand)] @@ -53,6 +48,13 @@ pub enum Subcommand { about = "Decode given block or extrinsic using current native runtime." )] Inspect(node_inspect::cli::InspectCmd), + + /// The custom benchmark subcommmand benchmarking runtime pallets. + #[structopt( + name = "benchmark", + about = "Benchmark runtime pallets." + )] + Benchmark(frame_benchmarking_cli::BenchmarkCmd), } /// The `factory` command used to generate transactions. diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index ba0f2785c1f00ba25241353cf0f2eb2fc0b92e69..dfdf5533f2b343e5f3fbe372a4b3d058f9c97f07 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -14,13 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use sc_cli::{VersionInfo, error}; +use sc_cli::VersionInfo; use sc_service::{Roles as ServiceRoles}; use node_transaction_factory::RuntimeAdapter; use crate::{Cli, service, ChainSpec, load_spec, Subcommand, factory_impl::FactoryState}; /// Parse command line arguments into service configuration. -pub fn run(args: I, version: VersionInfo) -> error::Result<()> +pub fn run(args: I, version: VersionInfo) -> sc_cli::Result<()> where I: Iterator, T: Into + Clone, @@ -28,42 +28,52 @@ where let args: Vec<_> = args.collect(); let opt = sc_cli::from_iter::(args.clone(), &version); - let mut config = sc_service::Configuration::new(&version); + let mut config = sc_service::Configuration::from_version(&version); match opt.subcommand { - None => sc_cli::run( - config, - opt.run, - service::new_light, - service::new_full, - load_spec, - &version, - ), + None => { + opt.run.init(&version)?; + opt.run.update_config(&mut config, load_spec, &version)?; + opt.run.run( + config, + service::new_light, + service::new_full, + &version, + ) + }, Some(Subcommand::Inspect(cmd)) => { - cmd.init(&mut config, load_spec, &version)?; + cmd.init(&version)?; + cmd.update_config(&mut config, load_spec, &version)?; let client = sc_service::new_full_client::< - node_runtime::Block,node_runtime::RuntimeApi, node_executor::Executor, _, _, + node_runtime::Block, node_runtime::RuntimeApi, node_executor::Executor, _, _, >(&config)?; let inspect = node_inspect::Inspector::::new(client); cmd.run(inspect) }, + Some(Subcommand::Benchmark(cmd)) => { + cmd.init(&version)?; + cmd.update_config(&mut config, load_spec, &version)?; + + cmd.run::<_, _, node_runtime::Block, node_executor::Executor>(config) + }, Some(Subcommand::Factory(cli_args)) => { - sc_cli::init(&cli_args.shared_params, &version)?; - sc_cli::init_config(&mut config, &cli_args.shared_params, &version, load_spec)?; - sc_cli::fill_import_params( + cli_args.shared_params.init(&version)?; + cli_args.shared_params.update_config(&mut config, load_spec, &version)?; + cli_args.import_params.update_config( &mut config, - &cli_args.import_params, ServiceRoles::FULL, cli_args.shared_params.dev, )?; - sc_cli::fill_config_keystore_in_memory(&mut config)?; + config.use_in_memory_keystore()?; match ChainSpec::from(config.expect_chain_spec().id()) { Some(ref c) if c == &ChainSpec::Development || c == &ChainSpec::LocalTestnet => {}, - _ => panic!("Factory is only supported for development and local testnet."), + _ => return Err( + "Factory is only supported for development and local testnet.".into() + ), } // Setup tracing. @@ -72,7 +82,9 @@ where cli_args.import_params.tracing_receiver.into(), tracing_targets ); if let Err(e) = tracing::subscriber::set_global_default(subscriber) { - panic!("Unable to set global default subscriber {}", e); + return Err( + format!("Unable to set global default subscriber {}", e).into() + ); } } @@ -91,12 +103,13 @@ where Ok(()) }, - Some(Subcommand::Base(subcommand)) => sc_cli::run_subcommand( - config, - subcommand, - load_spec, - |config: service::NodeConfiguration| Ok(new_full_start!(config).0), - &version, - ), + Some(Subcommand::Base(subcommand)) => { + subcommand.init(&version)?; + subcommand.update_config(&mut config, load_spec, &version)?; + subcommand.run( + config, + |config: service::NodeConfiguration| Ok(new_full_start!(config).0), + ) + }, } } diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index a882483a44a3dd8411eb6892a71e0cb50a506fbd..332c47ea132ea3dc176a07a20d34a7d2e62d0d74 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -22,7 +22,7 @@ use std::sync::Arc; use sc_consensus_babe; use sc_client::{self, LongestChain}; -use grandpa::{self, FinalityProofProvider as GrandpaFinalityProofProvider}; +use grandpa::{self, FinalityProofProvider as GrandpaFinalityProofProvider, StorageAndProofProvider}; use node_executor; use node_primitives::Block; use node_runtime::{GenesisConfig, RuntimeApi}; @@ -30,7 +30,6 @@ use sc_service::{ AbstractService, ServiceBuilder, config::Configuration, error::{Error as ServiceError}, }; use sp_inherents::InherentDataProviders; -use sc_network::construct_simple_protocol; use sc_service::{Service, NetworkStatus}; use sc_client::{Client, LocalCallExecutor}; @@ -40,17 +39,13 @@ use node_executor::NativeExecutor; use sc_network::NetworkService; use sc_offchain::OffchainWorkers; -construct_simple_protocol! { - /// Demo protocol attachment for substrate. - pub struct NodeProtocol where Block = Block { } -} - /// Starts a `ServiceBuilder` for a full service. /// /// Use this macro if you don't actually need the full service, but just the builder in order to /// be able to perform chain operations. macro_rules! new_full_start { ($config:expr) => {{ + use std::sync::Arc; type RpcExtension = jsonrpc_core::IoHandler; let mut import_setup = None; let inherent_data_providers = sp_inherents::InherentDataProviders::new(); @@ -70,7 +65,7 @@ macro_rules! new_full_start { .ok_or_else(|| sc_service::Error::SelectChainRequired)?; let (grandpa_block_import, grandpa_link) = grandpa::block_import( client.clone(), - &*client, + &(client.clone() as Arc<_>), select_chain, )?; let justification_import = grandpa_block_import.clone(); @@ -79,7 +74,6 @@ macro_rules! new_full_start { sc_consensus_babe::Config::get_or_compute(&*client)?, grandpa_block_import, client.clone(), - client.clone(), )?; let import_queue = sc_consensus_babe::import_queue( @@ -87,7 +81,6 @@ macro_rules! new_full_start { block_import.clone(), Some(Box::new(justification_import)), None, - client.clone(), client, inherent_data_providers.clone(), )?; @@ -124,6 +117,7 @@ macro_rules! new_full { ($config:expr, $with_startup_data: expr) => {{ use futures::prelude::*; use sc_network::Event; + use sc_client_api::ExecutorProvider; let ( is_authority, @@ -146,10 +140,12 @@ macro_rules! new_full { let (builder, mut import_setup, inherent_data_providers) = new_full_start!($config); - let service = builder.with_network_protocol(|_| Ok(crate::service::NodeProtocol::new()))? - .with_finality_proof_provider(|client, backend| - Ok(Arc::new(grandpa::FinalityProofProvider::new(backend, client)) as _) - )? + let service = builder + .with_finality_proof_provider(|client, backend| { + // GenesisAuthoritySetProvider is implemented for StorageAndProofProvider + let provider = client as Arc>; + Ok(Arc::new(grandpa::FinalityProofProvider::new(backend, provider)) as _) + })? .build()?; let (block_import, grandpa_link, babe_link) = import_setup.take() @@ -158,10 +154,10 @@ macro_rules! new_full { ($with_startup_data)(&block_import, &babe_link); if participates_in_consensus { - let proposer = sc_basic_authorship::ProposerFactory { - client: service.client(), - transaction_pool: service.transaction_pool(), - }; + let proposer = sc_basic_authorship::ProposerFactory::new( + service.client(), + service.transaction_pool() + ); let client = service.client(); let select_chain = service.select_chain() @@ -215,46 +211,41 @@ macro_rules! new_full { gossip_duration: std::time::Duration::from_millis(333), justification_period: 512, name: Some(name), - observer_enabled: true, + observer_enabled: false, keystore, is_authority, }; - match (is_authority, disable_grandpa) { - (false, false) => { - // start the lightweight GRANDPA observer - service.spawn_task("grandpa-observer", grandpa::run_grandpa_observer( - config, - grandpa_link, - service.network(), - service.on_exit(), - )?); - }, - (true, false) => { - // start the full GRANDPA voter - let grandpa_config = grandpa::GrandpaParams { - config: config, - link: grandpa_link, - network: service.network(), - inherent_data_providers: inherent_data_providers.clone(), - on_exit: service.on_exit(), - telemetry_on_connect: Some(service.telemetry_on_connect_stream()), - voting_rule: grandpa::VotingRulesBuilder::default().build(), - }; - // the GRANDPA voter task is considered infallible, i.e. - // if it fails we take down the service with it. - service.spawn_essential_task( - "grandpa-voter", - grandpa::run_grandpa_voter(grandpa_config)? - ); - }, - (_, true) => { - grandpa::setup_disabled_grandpa( - service.client(), - &inherent_data_providers, - service.network(), - )?; - }, + let enable_grandpa = !disable_grandpa; + if enable_grandpa { + // start the full GRANDPA voter + // NOTE: non-authorities could run the GRANDPA observer protocol, but at + // this point the full voter should provide better guarantees of block + // and vote data availability than the observer. The observer has not + // been tested extensively yet and having most nodes in a network run it + // could lead to finality stalls. + let grandpa_config = grandpa::GrandpaParams { + config, + link: grandpa_link, + network: service.network(), + inherent_data_providers: inherent_data_providers.clone(), + telemetry_on_connect: Some(service.telemetry_on_connect_stream()), + voting_rule: grandpa::VotingRulesBuilder::default().build(), + prometheus_registry: service.prometheus_registry(), + }; + + // the GRANDPA voter task is considered infallible, i.e. + // if it fails we take down the service with it. + service.spawn_essential_task( + "grandpa-voter", + grandpa::run_grandpa_voter(grandpa_config)? + ); + } else { + grandpa::setup_disabled_grandpa( + service.client(), + &inherent_data_providers, + service.network(), + )?; } Ok((service, inherent_data_providers)) @@ -264,20 +255,15 @@ macro_rules! new_full { }} } -#[allow(dead_code)] type ConcreteBlock = node_primitives::Block; -#[allow(dead_code)] type ConcreteClient = Client< Backend, - LocalCallExecutor, - NativeExecutor>, + LocalCallExecutor, NativeExecutor>, ConcreteBlock, node_runtime::RuntimeApi >; -#[allow(dead_code)] type ConcreteBackend = Backend; -#[allow(dead_code)] type ConcreteTransactionPool = sc_transaction_pool::BasicPool< sc_transaction_pool::FullChainApi, ConcreteBlock @@ -294,7 +280,7 @@ pub fn new_full(config: NodeConfiguration) ConcreteClient, LongestChain, NetworkStatus, - NetworkService::Hash>, + NetworkService::Hash>, ConcreteTransactionPool, OffchainWorkers< ConcreteClient, @@ -331,10 +317,10 @@ pub fn new_light(config: NodeConfiguration) let fetch_checker = fetcher .map(|fetcher| fetcher.checker().clone()) .ok_or_else(|| "Trying to start light import queue without active fetch checker")?; - let grandpa_block_import = grandpa::light_block_import::<_, _, _, RuntimeApi>( + let grandpa_block_import = grandpa::light_block_import( client.clone(), backend, - &*client, + &(client.clone() as Arc<_>), Arc::new(fetch_checker), )?; @@ -346,7 +332,6 @@ pub fn new_light(config: NodeConfiguration) sc_consensus_babe::Config::get_or_compute(&*client)?, grandpa_block_import, client.clone(), - client.clone(), )?; let import_queue = sc_consensus_babe::import_queue( @@ -355,16 +340,16 @@ pub fn new_light(config: NodeConfiguration) None, Some(Box::new(finality_proof_import)), client.clone(), - client, inherent_data_providers.clone(), )?; Ok((import_queue, finality_proof_request_builder)) })? - .with_network_protocol(|_| Ok(NodeProtocol::new()))? - .with_finality_proof_provider(|client, backend| - Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, client)) as _) - )? + .with_finality_proof_provider(|client, backend| { + // GenesisAuthoritySetProvider is implemented for StorageAndProofProvider + let provider = client as Arc>; + Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, provider)) as _) + })? .with_rpc_extensions(|builder,| -> Result { @@ -504,7 +489,7 @@ mod tests { |config| { let mut setup_handles = None; new_full!(config, | - block_import: &sc_consensus_babe::BabeBlockImport<_, _, Block, _, _, _>, + block_import: &sc_consensus_babe::BabeBlockImport, babe_link: &sc_consensus_babe::BabeLink, | { setup_handles = Some((block_import.clone(), babe_link.clone())); @@ -521,10 +506,10 @@ mod tests { let parent_header = service.client().header(&parent_id).unwrap().unwrap(); let parent_hash = parent_header.hash(); let parent_number = *parent_header.number(); - let mut proposer_factory = sc_basic_authorship::ProposerFactory { - client: service.client(), - transaction_pool: service.transaction_pool(), - }; + let mut proposer_factory = sc_basic_authorship::ProposerFactory::new( + service.client(), + service.transaction_pool() + ); let epoch = babe_link.epoch_changes().lock().epoch_for_child_of( descendent_query(&*service.client()), diff --git a/bin/node/cli/tests/build_spec_works.rs b/bin/node/cli/tests/build_spec_works.rs new file mode 100644 index 0000000000000000000000000000000000000000..2eca71a5b5978de1e1de20873b6584a7d5c703b0 --- /dev/null +++ b/bin/node/cli/tests/build_spec_works.rs @@ -0,0 +1,37 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use assert_cmd::cargo::cargo_bin; +use std::process::Command; +use tempfile::tempdir; + +#[test] +fn build_spec_works() { + let base_path = tempdir().expect("could not create a temp dir"); + + let output = Command::new(cargo_bin("substrate")) + .args(&["build-spec", "--dev", "-d"]) + .arg(base_path.path()) + .output() + .unwrap(); + assert!(output.status.success()); + + // Make sure that the `dev` chain folder exists, but the `db` doesn't + assert!(base_path.path().join("chains/dev/").exists()); + assert!(!base_path.path().join("chains/dev/db").exists()); + + let _value: serde_json::Value = serde_json::from_slice(output.stdout.as_slice()).unwrap(); +} diff --git a/utils/grafana-data-source/test/src/main.rs b/bin/node/cli/tests/check_block_works.rs similarity index 52% rename from utils/grafana-data-source/test/src/main.rs rename to bin/node/cli/tests/check_block_works.rs index 53deaffc3beb6088acc71bfee202bfbae41f0ef4..6bfb82a8bfafb84c1b041b87dc093e2481193ec5 100644 --- a/utils/grafana-data-source/test/src/main.rs +++ b/bin/node/cli/tests/check_block_works.rs @@ -1,4 +1,4 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// Copyright 2020 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Substrate is free software: you can redistribute it and/or modify @@ -14,31 +14,25 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use grafana_data_source::{run_server, record_metrics}; -use std::time::Duration; -use rand::Rng; -use futures::{future::join, executor}; +#![cfg(unix)] -async fn randomness() { - loop { - futures_timer::Delay::new(Duration::from_secs(1)).await; +use assert_cmd::cargo::cargo_bin; +use std::process::Command; +use tempfile::tempdir; - let random = rand::thread_rng().gen_range(0.0, 1000.0); +mod common; - let result = record_metrics!( - "random data" => random, - "random^2" => random * random, - ); +#[test] +fn check_block_works() { + let base_path = tempdir().expect("could not create a temp dir"); - if let Err(error) = result { - eprintln!("{}", error); - } - } -} + common::run_dev_node_for_a_while(base_path.path()); -fn main() { - executor::block_on(join( - run_server("127.0.0.1:9955".parse().unwrap()), - randomness() - )).0.unwrap(); + let status = Command::new(cargo_bin("substrate")) + .args(&["check-block", "--dev", "--pruning", "archive", "-d"]) + .arg(base_path.path()) + .arg("1") + .status() + .unwrap(); + assert!(status.success()); } diff --git a/bin/node/cli/tests/common.rs b/bin/node/cli/tests/common.rs index 4044f69d08168e4dda1d526ff9b1c4e5efbd32b8..34e371195c16b20e216a9426f238ce6b78bcd257 100644 --- a/bin/node/cli/tests/common.rs +++ b/bin/node/cli/tests/common.rs @@ -14,21 +14,53 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use std::{process::{Child, ExitStatus}, thread, time::Duration}; +#![cfg(unix)] +#![allow(dead_code)] + +use std::{process::{Child, ExitStatus}, thread, time::Duration, path::Path}; +use assert_cmd::cargo::cargo_bin; +use std::{convert::TryInto, process::Command}; +use nix::sys::signal::{kill, Signal::SIGINT}; +use nix::unistd::Pid; /// Wait for the given `child` the given number of `secs`. /// /// Returns the `Some(exit status)` or `None` if the process did not finish in the given time. pub fn wait_for(child: &mut Child, secs: usize) -> Option { - for _ in 0..secs { + for i in 0..secs { match child.try_wait().unwrap() { - Some(status) => return Some(status), + Some(status) => { + if i > 5 { + eprintln!("Child process took {} seconds to exit gracefully", i); + } + return Some(status) + }, None => thread::sleep(Duration::from_secs(1)), } } - eprintln!("Took to long to exit. Killing..."); + eprintln!("Took too long to exit (> {} seconds). Killing...", secs); let _ = child.kill(); child.wait().unwrap(); None } + +/// Run the node for a while (30 seconds) +pub fn run_dev_node_for_a_while(base_path: &Path) { + let mut cmd = Command::new(cargo_bin("substrate")); + + let mut cmd = cmd + .args(&["--dev"]) + .arg("-d") + .arg(base_path) + .spawn() + .unwrap(); + + // Let it produce some blocks. + thread::sleep(Duration::from_secs(30)); + assert!(cmd.try_wait().unwrap().is_none(), "the process should still be running"); + + // Stop the process + kill(Pid::from_raw(cmd.id().try_into().unwrap()), SIGINT).unwrap(); + assert!(wait_for(&mut cmd, 40).map(|x| x.success()).unwrap_or_default()); +} diff --git a/bin/node/cli/tests/factory.rs b/bin/node/cli/tests/factory.rs new file mode 100644 index 0000000000000000000000000000000000000000..2930cd52e2e16e497d197849a9833d492c3868d7 --- /dev/null +++ b/bin/node/cli/tests/factory.rs @@ -0,0 +1,40 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#![cfg(unix)] + +use assert_cmd::cargo::cargo_bin; +use std::process::{Command, Stdio}; +use tempfile::tempdir; + +mod common; + +#[test] +fn factory_works() { + let base_path = tempdir().expect("could not create a temp dir"); + + let status = Command::new(cargo_bin("substrate")) + .stdout(Stdio::null()) + .args(&["factory", "--dev", "-d"]) + .arg(base_path.path()) + .status() + .unwrap(); + assert!(status.success()); + + // Make sure that the `dev` chain folder exists & `db` + assert!(base_path.path().join("chains/dev/").exists()); + assert!(base_path.path().join("chains/dev/db").exists()); +} diff --git a/bin/node/cli/tests/import_export_and_revert_work.rs b/bin/node/cli/tests/import_export_and_revert_work.rs new file mode 100644 index 0000000000000000000000000000000000000000..131265e3b4ab9bd868b9728bca6cda558d2982f2 --- /dev/null +++ b/bin/node/cli/tests/import_export_and_revert_work.rs @@ -0,0 +1,59 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#![cfg(unix)] + +use assert_cmd::cargo::cargo_bin; +use std::{process::Command, fs}; +use tempfile::tempdir; + +mod common; + +#[test] +fn import_export_and_revert_work() { + let base_path = tempdir().expect("could not create a temp dir"); + let exported_blocks = base_path.path().join("exported_blocks"); + + common::run_dev_node_for_a_while(base_path.path()); + + let status = Command::new(cargo_bin("substrate")) + .args(&["export-blocks", "--dev", "--pruning", "archive", "-d"]) + .arg(base_path.path()) + .arg(&exported_blocks) + .status() + .unwrap(); + assert!(status.success()); + + let metadata = fs::metadata(&exported_blocks).unwrap(); + assert!(metadata.len() > 0, "file exported_blocks should not be empty"); + + let _ = fs::remove_dir_all(base_path.path().join("db")); + + let status = Command::new(cargo_bin("substrate")) + .args(&["import-blocks", "--dev", "--pruning", "archive", "-d"]) + .arg(base_path.path()) + .arg(&exported_blocks) + .status() + .unwrap(); + assert!(status.success()); + + let status = Command::new(cargo_bin("substrate")) + .args(&["revert", "--dev", "--pruning", "archive", "-d"]) + .arg(base_path.path()) + .status() + .unwrap(); + assert!(status.success()); +} diff --git a/bin/node/cli/tests/inspect_works.rs b/bin/node/cli/tests/inspect_works.rs new file mode 100644 index 0000000000000000000000000000000000000000..441b08ccf46da1b1ef70ec49a7a46015d94f3c4c --- /dev/null +++ b/bin/node/cli/tests/inspect_works.rs @@ -0,0 +1,38 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#![cfg(unix)] + +use assert_cmd::cargo::cargo_bin; +use std::process::Command; +use tempfile::tempdir; + +mod common; + +#[test] +fn inspect_works() { + let base_path = tempdir().expect("could not create a temp dir"); + + common::run_dev_node_for_a_while(base_path.path()); + + let status = Command::new(cargo_bin("substrate")) + .args(&["inspect", "--dev", "--pruning", "archive", "-d"]) + .arg(base_path.path()) + .args(&["block", "1"]) + .status() + .unwrap(); + assert!(status.success()); +} diff --git a/bin/node/cli/tests/purge_chain_works.rs b/bin/node/cli/tests/purge_chain_works.rs index e6b71db9910487a4e55f2fdf117e48c833b120bb..020259d0c595a57a01b8a9b113522d826e1de912 100644 --- a/bin/node/cli/tests/purge_chain_works.rs +++ b/bin/node/cli/tests/purge_chain_works.rs @@ -15,39 +15,27 @@ // along with Substrate. If not, see . use assert_cmd::cargo::cargo_bin; -use std::{convert::TryInto, process::Command, thread, time::Duration, fs, path::PathBuf}; +use std::process::Command; +use tempfile::tempdir; mod common; #[test] #[cfg(unix)] fn purge_chain_works() { - use nix::sys::signal::{kill, Signal::SIGINT}; - use nix::unistd::Pid; + let base_path = tempdir().expect("could not create a temp dir"); - let base_path = "purge_chain_test"; - - let _ = fs::remove_dir_all(base_path); - let mut cmd = Command::new(cargo_bin("substrate")) - .args(&["--dev", "-d", base_path]) - .spawn() - .unwrap(); - - // Let it produce some blocks. - thread::sleep(Duration::from_secs(30)); - assert!(cmd.try_wait().unwrap().is_none(), "the process should still be running"); - - // Stop the process - kill(Pid::from_raw(cmd.id().try_into().unwrap()), SIGINT).unwrap(); - assert!(common::wait_for(&mut cmd, 30).map(|x| x.success()).unwrap_or_default()); + common::run_dev_node_for_a_while(base_path.path()); let status = Command::new(cargo_bin("substrate")) - .args(&["purge-chain", "--dev", "-d", base_path, "-y"]) + .args(&["purge-chain", "--dev", "-d"]) + .arg(base_path.path()) + .arg("-y") .status() .unwrap(); assert!(status.success()); // Make sure that the `dev` chain folder exists, but the `db` is deleted. - assert!(PathBuf::from(base_path).join("chains/dev/").exists()); - assert!(!PathBuf::from(base_path).join("chains/dev/db").exists()); + assert!(base_path.path().join("chains/dev/").exists()); + assert!(!base_path.path().join("chains/dev/db").exists()); } diff --git a/bin/node/cli/tests/running_the_node_and_interrupt.rs b/bin/node/cli/tests/running_the_node_and_interrupt.rs index dbb57bdd21ab8de8add8f4faf4057aea2157fdaf..67efedccbe771a65e892b5dbe94bac702d3c01ce 100644 --- a/bin/node/cli/tests/running_the_node_and_interrupt.rs +++ b/bin/node/cli/tests/running_the_node_and_interrupt.rs @@ -15,7 +15,8 @@ // along with Substrate. If not, see . use assert_cmd::cargo::cargo_bin; -use std::{convert::TryInto, process::Command, thread, time::Duration, fs}; +use std::{convert::TryInto, process::Command, thread, time::Duration}; +use tempfile::tempdir; mod common; @@ -26,13 +27,14 @@ fn running_the_node_works_and_can_be_interrupted() { use nix::unistd::Pid; fn run_command_and_kill(signal: Signal) { - let _ = fs::remove_dir_all("interrupt_test"); + let base_path = tempdir().expect("could not create a temp dir"); let mut cmd = Command::new(cargo_bin("substrate")) - .args(&["--dev", "-d", "interrupt_test"]) + .args(&["--dev", "-d"]) + .arg(base_path.path()) .spawn() .unwrap(); - thread::sleep(Duration::from_secs(30)); + thread::sleep(Duration::from_secs(20)); assert!(cmd.try_wait().unwrap().is_none(), "the process should still be running"); kill(Pid::from_raw(cmd.id().try_into().unwrap()), signal).unwrap(); assert_eq!( diff --git a/bin/node/executor/Cargo.toml b/bin/node/executor/Cargo.toml index 1d894e39fa978599d973824c2513dd96dabb62fd..c8ef75a5a2a9ae64c3949c0bd250262796ef0874 100644 --- a/bin/node/executor/Cargo.toml +++ b/bin/node/executor/Cargo.toml @@ -1,39 +1,42 @@ [package] name = "node-executor" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] description = "Substrate node implementation in Rust." edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -codec = { package = "parity-scale-codec", version = "1.0.0" } -node-primitives = { version = "2.0.0", path = "../primitives" } -node-runtime = { version = "2.0.0", path = "../runtime" } -sc-executor = { version = "0.8", path = "../../../client/executor" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-io = { version = "2.0.0", path = "../../../primitives/io" } -sp-state-machine = { version = "0.8", path = "../../../primitives/state-machine" } -sp-trie = { version = "2.0.0", path = "../../../primitives/trie" } +codec = { package = "parity-scale-codec", version = "1.2.0" } +node-primitives = { version = "2.0.0-alpha.2", path = "../primitives" } +node-runtime = { version = "2.0.0-alpha.2", path = "../runtime" } +sc-executor = { version = "0.8.0-alpha.2", path = "../../../client/executor" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-io = { version = "2.0.0-alpha.2", path = "../../../primitives/io" } +sp-state-machine = { version = "0.8.0-alpha.2", path = "../../../primitives/state-machine" } +sp-trie = { version = "2.0.0-alpha.2", path = "../../../primitives/trie" } trie-root = "0.16.0" +frame-benchmarking = { version = "2.0.0-alpha.2", path = "../../../frame/benchmarking" } [dev-dependencies] criterion = "0.3.0" -frame-support = { version = "2.0.0", path = "../../../frame/support" } -frame-system = { version = "2.0.0", path = "../../../frame/system" } -node-testing = { version = "2.0.0", path = "../testing" } -pallet-balances = { version = "2.0.0", path = "../../../frame/balances" } -pallet-contracts = { version = "2.0.0", path = "../../../frame/contracts" } -pallet-grandpa = { version = "2.0.0", path = "../../../frame/grandpa" } -pallet-im-online = { version = "2.0.0", path = "../../../frame/im-online" } -pallet-indices = { version = "2.0.0", path = "../../../frame/indices" } -pallet-session = { version = "2.0.0", path = "../../../frame/session" } -pallet-timestamp = { version = "2.0.0", path = "../../../frame/timestamp" } -pallet-transaction-payment = { version = "2.0.0", path = "../../../frame/transaction-payment" } -pallet-treasury = { version = "2.0.0", path = "../../../frame/treasury" } -sp-application-crypto = { version = "2.0.0", path = "../../../primitives/application-crypto" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -substrate-test-client = { version = "2.0.0", path = "../../../test-utils/client" } +frame-support = { version = "2.0.0-alpha.2", path = "../../../frame/support" } +frame-system = { version = "2.0.0-alpha.2", path = "../../../frame/system" } +node-testing = { version = "2.0.0-alpha.2", path = "../testing" } +pallet-balances = { version = "2.0.0-alpha.2", path = "../../../frame/balances" } +pallet-contracts = { version = "2.0.0-alpha.2", path = "../../../frame/contracts" } +pallet-grandpa = { version = "2.0.0-alpha.2", path = "../../../frame/grandpa" } +pallet-im-online = { version = "2.0.0-alpha.2", path = "../../../frame/im-online" } +pallet-indices = { version = "2.0.0-alpha.2", path = "../../../frame/indices" } +pallet-session = { version = "2.0.0-alpha.2", path = "../../../frame/session" } +pallet-timestamp = { version = "2.0.0-alpha.2", path = "../../../frame/timestamp" } +pallet-transaction-payment = { version = "2.0.0-alpha.2", path = "../../../frame/transaction-payment" } +pallet-treasury = { version = "2.0.0-alpha.2", path = "../../../frame/treasury" } +sp-application-crypto = { version = "2.0.0-alpha.2", path = "../../../primitives/application-crypto" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +substrate-test-client = { version = "2.0.0-dev", path = "../../../test-utils/client" } wabt = "0.9.2" [features] diff --git a/bin/node/executor/benches/bench.rs b/bin/node/executor/benches/bench.rs index 034c7c6759e112a7db29ab63cf7c01a81f35340a..ef0cdf445a561c813a3059602e3aef41106f4ec3 100644 --- a/bin/node/executor/benches/bench.rs +++ b/bin/node/executor/benches/bench.rs @@ -23,12 +23,13 @@ use node_runtime::{ }; use node_runtime::constants::currency::*; use node_testing::keyring::*; -use sp_core::{Blake2Hasher, NativeOrEncoded, NeverNativeValue}; +use sp_core::{NativeOrEncoded, NeverNativeValue}; use sp_core::storage::well_known_keys; use sp_core::traits::CodeExecutor; use frame_support::Hashable; use sp_state_machine::TestExternalities as CoreTestExternalities; use sc_executor::{NativeExecutor, RuntimeInfo, WasmExecutionMethod, Externalities}; +use sp_runtime::traits::BlakeTwo256; criterion_group!(benches, bench_execute_block); criterion_main!(benches); @@ -54,7 +55,7 @@ fn sign(xt: CheckedExtrinsic) -> UncheckedExtrinsic { node_testing::keyring::sign(xt, VERSION, GENESIS_HASH) } -fn new_test_ext(genesis_config: &GenesisConfig) -> TestExternalities { +fn new_test_ext(genesis_config: &GenesisConfig) -> TestExternalities { let mut test_ext = TestExternalities::new_with_code( COMPACT_CODE, genesis_config.build_storage().unwrap(), @@ -76,7 +77,7 @@ fn construct_block( let extrinsics = extrinsics.into_iter().map(sign).collect::>(); // calculate the header fields that we can. - let extrinsics_root = Layout::::ordered_trie_root( + let extrinsics_root = Layout::::ordered_trie_root( extrinsics.iter().map(Encode::encode) ).to_fixed_bytes() .into(); @@ -90,7 +91,7 @@ fn construct_block( }; // execute the block to get the real header. - executor.call::<_, NeverNativeValue, fn() -> _>( + executor.call:: _>( ext, "Core_initialize_block", &header.encode(), @@ -99,7 +100,7 @@ fn construct_block( ).0.unwrap(); for i in extrinsics.iter() { - executor.call::<_, NeverNativeValue, fn() -> _>( + executor.call:: _>( ext, "BlockBuilder_apply_extrinsic", &i.encode(), @@ -108,7 +109,7 @@ fn construct_block( ).0.unwrap(); } - let header = match executor.call::<_, NeverNativeValue, fn() -> _>( + let header = match executor.call:: _>( ext, "BlockBuilder_finalize_block", &[0u8;0], @@ -165,7 +166,7 @@ fn bench_execute_block(c: &mut Criterion) { // Get the runtime version to initialize the runtimes cache. { let mut test_ext = new_test_ext(&genesis_config); - executor.runtime_version(&mut test_ext.ext()); + executor.runtime_version(&mut test_ext.ext()).unwrap(); } let blocks = test_blocks(&genesis_config, &executor); @@ -174,7 +175,7 @@ fn bench_execute_block(c: &mut Criterion) { || new_test_ext(&genesis_config), |test_ext| { for block in blocks.iter() { - executor.call::<_, NeverNativeValue, fn() -> _>( + executor.call:: _>( &mut test_ext.ext(), "Core_execute_block", &block.0, diff --git a/bin/node/executor/src/lib.rs b/bin/node/executor/src/lib.rs index 72f40b7c1f0cd6c1984b39a914ba841452708d27..bcc7f48507398cd0ea15319460f8d13e6058f2ec 100644 --- a/bin/node/executor/src/lib.rs +++ b/bin/node/executor/src/lib.rs @@ -26,5 +26,5 @@ native_executor_instance!( pub Executor, node_runtime::api::dispatch, node_runtime::native_version, - sp_io::benchmarking::HostFunctions, + frame_benchmarking::benchmarking::HostFunctions, ); diff --git a/bin/node/executor/tests/basic.rs b/bin/node/executor/tests/basic.rs index 100bdf3fe60ee8a90a557e4b174c6e76cc712b60..1ee0a17c81120ca32a7a9af2d704ce97699ee454 100644 --- a/bin/node/executor/tests/basic.rs +++ b/bin/node/executor/tests/basic.rs @@ -21,13 +21,11 @@ use frame_support::{ weights::{GetDispatchInfo, DispatchInfo, DispatchClass}, }; use sp_core::{ - Blake2Hasher, NeverNativeValue, map, - traits::Externalities, - storage::{well_known_keys, Storage}, + NeverNativeValue, map, traits::Externalities, storage::{well_known_keys, Storage}, }; use sp_runtime::{ ApplyExtrinsicResult, Fixed64, - traits::{Hash as HashT, Convert}, + traits::{Hash as HashT, Convert, BlakeTwo256}, transaction_validity::InvalidTransaction, }; use pallet_contracts::ContractAddressFor; @@ -93,7 +91,6 @@ fn changes_trie_block() -> (Vec, Hash) { ) } - /// block 1 and 2 must be created together to ensure transactions are only signed once (since they /// are not guaranteed to be deterministic) and to ensure that the correct state is propagated /// from block1's execution to block2 to derive the correct storage_root. @@ -161,10 +158,10 @@ fn block_with_size(time: u64, nonce: u32, size: usize) -> (Vec, Hash) { #[test] fn panic_execution_with_foreign_code_gives_error() { - let mut t = TestExternalities::::new_with_code(BLOATY_CODE, Storage { + let mut t = TestExternalities::::new_with_code(BLOATY_CODE, Storage { top: map![ >::hashed_key_for(alice()) => { - (69u128, 0u128, 0u128, 0u128).encode() + (69u128, 0u8, 0u128, 0u128, 0u128).encode() }, >::hashed_key().to_vec() => { 69_u128.encode() @@ -197,10 +194,10 @@ fn panic_execution_with_foreign_code_gives_error() { #[test] fn bad_extrinsic_with_native_equivalent_code_gives_error() { - let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { + let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { top: map![ >::hashed_key_for(alice()) => { - (0u32, 69u128, 0u128, 0u128, 0u128).encode() + (0u32, 0u8, 69u128, 0u128, 0u128, 0u128).encode() }, >::hashed_key().to_vec() => { 69_u128.encode() @@ -233,10 +230,10 @@ fn bad_extrinsic_with_native_equivalent_code_gives_error() { #[test] fn successful_execution_with_native_equivalent_code_gives_ok() { - let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { + let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { top: map![ >::hashed_key_for(alice()) => { - (0u32, 111 * DOLLARS, 0u128, 0u128, 0u128).encode() + (0u32, 0u8, 111 * DOLLARS, 0u128, 0u128, 0u128).encode() }, >::hashed_key().to_vec() => { (111 * DOLLARS).encode() @@ -275,10 +272,10 @@ fn successful_execution_with_native_equivalent_code_gives_ok() { #[test] fn successful_execution_with_foreign_code_gives_ok() { - let mut t = TestExternalities::::new_with_code(BLOATY_CODE, Storage { + let mut t = TestExternalities::::new_with_code(BLOATY_CODE, Storage { top: map![ >::hashed_key_for(alice()) => { - (0u32, 111 * DOLLARS, 0u128, 0u128, 0u128).encode() + (0u32, 0u8, 111 * DOLLARS, 0u128, 0u128, 0u128).encode() }, >::hashed_key().to_vec() => { (111 * DOLLARS).encode() @@ -347,7 +344,7 @@ fn full_native_block_import_works() { }, EventRecord { phase: Phase::ApplyExtrinsic(1), - event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(1984800000000)), + event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(fees * 8 / 10)), topics: vec![], }, EventRecord { @@ -381,13 +378,14 @@ fn full_native_block_import_works() { ).0.unwrap(); t.execute_with(|| { + let fees = transfer_fee(&xt(), fm); assert_eq!( Balances::total_balance(&alice()), - alice_last_known_balance - 10 * DOLLARS - transfer_fee(&xt(), fm), + alice_last_known_balance - 10 * DOLLARS - fees, ); assert_eq!( Balances::total_balance(&bob()), - 179 * DOLLARS - transfer_fee(&xt(), fm), + 179 * DOLLARS - fees, ); let events = vec![ EventRecord { @@ -399,7 +397,7 @@ fn full_native_block_import_works() { }, EventRecord { phase: Phase::ApplyExtrinsic(1), - event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(1984788199392)), + event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(fees * 8 / 10)), topics: vec![], }, EventRecord { @@ -422,7 +420,7 @@ fn full_native_block_import_works() { }, EventRecord { phase: Phase::ApplyExtrinsic(2), - event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(1984788199392)), + event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(fees * 8 / 10)), topics: vec![], }, EventRecord { @@ -699,7 +697,7 @@ fn native_big_block_import_fails_on_fallback() { #[test] fn panic_execution_gives_error() { - let mut t = TestExternalities::::new_with_code(BLOATY_CODE, Storage { + let mut t = TestExternalities::::new_with_code(BLOATY_CODE, Storage { top: map![ >::hashed_key().to_vec() => { 0_u128.encode() @@ -730,10 +728,10 @@ fn panic_execution_gives_error() { #[test] fn successful_execution_gives_ok() { - let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { + let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { top: map![ >::hashed_key_for(alice()) => { - (0u32, 111 * DOLLARS, 0u128, 0u128, 0u128).encode() + (0u32, 0u8, 111 * DOLLARS, 0u128, 0u128, 0u128).encode() }, >::hashed_key().to_vec() => { (111 * DOLLARS).encode() diff --git a/bin/node/executor/tests/common.rs b/bin/node/executor/tests/common.rs index 090d2ee5d4ace1b188641536e0732033fa6d0f34..34f3034208a08eb6d69cdda4929d99a77b4da0a1 100644 --- a/bin/node/executor/tests/common.rs +++ b/bin/node/executor/tests/common.rs @@ -17,11 +17,8 @@ use codec::{Encode, Decode}; use frame_support::Hashable; use sp_state_machine::TestExternalities as CoreTestExternalities; -use sp_core::{ - Blake2Hasher, NeverNativeValue, NativeOrEncoded, - traits::CodeExecutor, -}; -use sp_runtime::{ApplyExtrinsicResult, traits::Header as HeaderT}; +use sp_core::{NeverNativeValue, NativeOrEncoded, traits::CodeExecutor}; +use sp_runtime::{ApplyExtrinsicResult, traits::{Header as HeaderT, BlakeTwo256}}; use sc_executor::{NativeExecutor, WasmExecutionMethod}; use sc_executor::error::Result; @@ -67,14 +64,14 @@ pub fn executor_call< R:Decode + Encode + PartialEq, NC: FnOnce() -> std::result::Result + std::panic::UnwindSafe >( - t: &mut TestExternalities, + t: &mut TestExternalities, method: &str, data: &[u8], use_native: bool, native_call: Option, ) -> (Result>, bool) { let mut t = t.ext(); - executor().call::<_, R, NC>( + executor().call::( &mut t, method, data, @@ -83,7 +80,7 @@ pub fn executor_call< ) } -pub fn new_test_ext(code: &[u8], support_changes_trie: bool) -> TestExternalities { +pub fn new_test_ext(code: &[u8], support_changes_trie: bool) -> TestExternalities { let mut ext = TestExternalities::new_with_code( code, node_testing::genesis::config(support_changes_trie, Some(code)).build_storage().unwrap(), @@ -97,7 +94,7 @@ pub fn new_test_ext(code: &[u8], support_changes_trie: bool) -> TestExternalitie /// `extrinsics` must be a list of valid extrinsics, i.e. none of the extrinsics for example /// can report `ExhaustResources`. Otherwise, this function panics. pub fn construct_block( - env: &mut TestExternalities, + env: &mut TestExternalities, number: BlockNumber, parent_hash: Hash, extrinsics: Vec, @@ -109,7 +106,7 @@ pub fn construct_block( // calculate the header fields that we can. let extrinsics_root = - Layout::::ordered_trie_root(extrinsics.iter().map(Encode::encode)) + Layout::::ordered_trie_root(extrinsics.iter().map(Encode::encode)) .to_fixed_bytes() .into(); diff --git a/bin/node/executor/tests/fees.rs b/bin/node/executor/tests/fees.rs index ba303a6feb6ff05698f59de15485f65d9beb999c..ea9d740a05ef86e77fa923c8863d55a984cdbca7 100644 --- a/bin/node/executor/tests/fees.rs +++ b/bin/node/executor/tests/fees.rs @@ -20,18 +20,12 @@ use frame_support::{ traits::Currency, weights::GetDispatchInfo, }; -use sp_core::{ - Blake2Hasher, NeverNativeValue, map, - storage::Storage, -}; -use sp_runtime::{ - Fixed64, - traits::Convert, -}; +use sp_core::{NeverNativeValue, map, storage::Storage}; +use sp_runtime::{Fixed64, Perbill, traits::{Convert, BlakeTwo256}}; use node_runtime::{ - CheckedExtrinsic, Call, Runtime, Balances, - TransactionPayment, TransactionBaseFee, TransactionByteFee, - WeightFeeCoefficient, constants::currency::*, + CheckedExtrinsic, Call, Runtime, Balances, TransactionPayment, TransactionBaseFee, + TransactionByteFee, WeightFeeCoefficient, + constants::currency::*, }; use node_runtime::impls::LinearWeightToFee; use node_primitives::Balance; @@ -60,12 +54,12 @@ fn fee_multiplier_increases_and_decreases_on_big_weight() { GENESIS_HASH.into(), vec![ CheckedExtrinsic { - signed: None, - function: Call::Timestamp(pallet_timestamp::Call::set(42 * 1000)), + signed: None, + function: Call::Timestamp(pallet_timestamp::Call::set(42 * 1000)), }, CheckedExtrinsic { signed: Some((charlie(), signed_extra(0, 0))), - function: Call::System(frame_system::Call::fill_block()), + function: Call::System(frame_system::Call::fill_block(Perbill::from_percent(90))), } ] ); @@ -77,8 +71,8 @@ fn fee_multiplier_increases_and_decreases_on_big_weight() { block1.1.clone(), vec![ CheckedExtrinsic { - signed: None, - function: Call::Timestamp(pallet_timestamp::Call::set(52 * 1000)), + signed: None, + function: Call::Timestamp(pallet_timestamp::Call::set(52 * 1000)), }, CheckedExtrinsic { signed: Some((charlie(), signed_extra(1, 0))), @@ -87,7 +81,11 @@ fn fee_multiplier_increases_and_decreases_on_big_weight() { ] ); - println!("++ Block 1 size: {} / Block 2 size {}", block1.0.encode().len(), block2.0.encode().len()); + println!( + "++ Block 1 size: {} / Block 2 size {}", + block1.0.encode().len(), + block2.0.encode().len(), + ); // execute a big block. executor_call:: _>( @@ -132,13 +130,13 @@ fn transaction_fee_is_correct_ultimate() { // - 1 MILLICENTS in substrate node. // - 1 milli-dot based on current polkadot runtime. // (this baed on assigning 0.1 CENT to the cheapest tx with `weight = 100`) - let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { + let mut t = TestExternalities::::new_with_code(COMPACT_CODE, Storage { top: map![ >::hashed_key_for(alice()) => { - (0u32, 100 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS).encode() + (0u32, 0u8, 100 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS).encode() }, >::hashed_key_for(bob()) => { - (0u32, 10 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS).encode() + (0u32, 0u8, 10 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS).encode() }, >::hashed_key().to_vec() => { (110 * DOLLARS).encode() diff --git a/bin/node/executor/tests/submit_transaction.rs b/bin/node/executor/tests/submit_transaction.rs index 90e64c41369ee9f110f87978b380f381794ad751..f06f9ff4665b886f7d92ec6140be7e090dce696b 100644 --- a/bin/node/executor/tests/submit_transaction.rs +++ b/bin/node/executor/tests/submit_transaction.rs @@ -167,8 +167,9 @@ fn submitted_transaction_should_be_valid() { // add balance to the account let author = extrinsic.signature.clone().unwrap().0; let address = Indices::lookup(author).unwrap(); - let account = pallet_balances::AccountData { free: 5_000_000_000_000, ..Default::default() }; - >::insert(&address, (0u32, account)); + let data = pallet_balances::AccountData { free: 5_000_000_000_000, ..Default::default() }; + let account = frame_system::AccountInfo { nonce: 0u32, refcount: 0u8, data }; + >::insert(&address, account); // check validity let res = Executive::validate_transaction(extrinsic); diff --git a/bin/node/inspect/Cargo.toml b/bin/node/inspect/Cargo.toml index cbce7e4589698b75e7c0c0c01cb48c896628cef2..022f4d0ca49f5252cedce67ea6750a4c5aae8a18 100644 --- a/bin/node/inspect/Cargo.toml +++ b/bin/node/inspect/Cargo.toml @@ -1,17 +1,20 @@ [package] name = "node-inspect" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -codec = { package = "parity-scale-codec", version = "1.0.0" } +codec = { package = "parity-scale-codec", version = "1.2.0" } derive_more = "0.99" log = "0.4.8" -sc-cli = { version = "0.8.0", path = "../../../client/cli" } -sc-client-api = { version = "2.0.0", path = "../../../client/api" } -sc-service = { version = "0.8", default-features = false, path = "../../../client/service" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } +sc-cli = { version = "0.8.0-alpha.2", path = "../../../client/cli" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../../../client/api" } +sc-service = { version = "0.8.0-alpha.2", default-features = false, path = "../../../client/service" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../../primitives/blockchain" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } structopt = "0.3.8" diff --git a/bin/node/inspect/src/cli.rs b/bin/node/inspect/src/cli.rs index 27afcfff919ee276220f20871a0140f6c2aa326c..5d51bd5848f17d308933fdcb69d429504fa8445e 100644 --- a/bin/node/inspect/src/cli.rs +++ b/bin/node/inspect/src/cli.rs @@ -16,12 +16,8 @@ //! Structs to easily compose inspect sub-command for CLI. -use std::{ - fmt::Debug, - str::FromStr, -}; -use crate::{Inspector, PrettyPrinter}; -use sc_cli::{ImportParams, SharedParams, error}; +use std::fmt::Debug; +use sc_cli::{ImportParams, SharedParams}; use structopt::StructOpt; /// The `inspect` command used to print decoded chain data. @@ -64,177 +60,3 @@ pub enum InspectSubCmd { input: String, }, } - -impl InspectCmd { - /// Parse CLI arguments and initialize given config. - pub fn init( - &self, - config: &mut sc_service::config::Configuration, - spec_factory: impl FnOnce(&str) -> Result>, String>, - version: &sc_cli::VersionInfo, - ) -> error::Result<()> where - G: sc_service::RuntimeGenesis, - E: sc_service::ChainSpecExtension, - { - sc_cli::init_config(config, &self.shared_params, version, spec_factory)?; - // make sure to configure keystore - sc_cli::fill_config_keystore_in_memory(config)?; - // and all import params (especially pruning that has to match db meta) - sc_cli::fill_import_params( - config, - &self.import_params, - sc_service::Roles::FULL, - self.shared_params.dev, - )?; - Ok(()) - } - - /// Run the inspect command, passing the inspector. - pub fn run( - self, - inspect: Inspector, - ) -> error::Result<()> where - B: sp_runtime::traits::Block, - B::Hash: FromStr, - P: PrettyPrinter, - { - match self.command { - InspectSubCmd::Block { input } => { - let input = input.parse()?; - let res = inspect.block(input) - .map_err(|e| format!("{}", e))?; - println!("{}", res); - Ok(()) - }, - InspectSubCmd::Extrinsic { input } => { - let input = input.parse()?; - let res = inspect.extrinsic(input) - .map_err(|e| format!("{}", e))?; - println!("{}", res); - Ok(()) - }, - } - } -} - - -/// A block to retrieve. -#[derive(Debug, Clone, PartialEq)] -pub enum BlockAddress { - /// Get block by hash. - Hash(Hash), - /// Get block by number. - Number(Number), - /// Raw SCALE-encoded bytes. - Bytes(Vec), -} - -impl FromStr for BlockAddress { - type Err = String; - - fn from_str(s: &str) -> Result { - // try to parse hash first - if let Ok(hash) = s.parse() { - return Ok(Self::Hash(hash)) - } - - // then number - if let Ok(number) = s.parse() { - return Ok(Self::Number(number)) - } - - // then assume it's bytes (hex-encoded) - sp_core::bytes::from_hex(s) - .map(Self::Bytes) - .map_err(|e| format!( - "Given string does not look like hash or number. It could not be parsed as bytes either: {}", - e - )) - } -} - -/// An extrinsic address to decode and print out. -#[derive(Debug, Clone, PartialEq)] -pub enum ExtrinsicAddress { - /// Extrinsic as part of existing block. - Block(BlockAddress, usize), - /// Raw SCALE-encoded extrinsic bytes. - Bytes(Vec), -} - -impl FromStr for ExtrinsicAddress { - type Err = String; - - fn from_str(s: &str) -> Result { - // first try raw bytes - if let Ok(bytes) = sp_core::bytes::from_hex(s).map(Self::Bytes) { - return Ok(bytes) - } - - // split by a bunch of different characters - let mut it = s.split(|c| c == '.' || c == ':' || c == ' '); - let block = it.next() - .expect("First element of split iterator is never empty; qed") - .parse()?; - - let index = it.next() - .ok_or_else(|| format!("Extrinsic index missing: example \"5:0\""))? - .parse() - .map_err(|e| format!("Invalid index format: {}", e))?; - - Ok(Self::Block(block, index)) - } -} - - -#[cfg(test)] -mod tests { - use super::*; - use sp_core::hash::H160 as Hash; - - #[test] - fn should_parse_block_strings() { - type BlockAddress = super::BlockAddress; - - let b0 = BlockAddress::from_str("3BfC20f0B9aFcAcE800D73D2191166FF16540258"); - let b1 = BlockAddress::from_str("1234"); - let b2 = BlockAddress::from_str("0"); - let b3 = BlockAddress::from_str("0x0012345f"); - - - assert_eq!(b0, Ok(BlockAddress::Hash( - "3BfC20f0B9aFcAcE800D73D2191166FF16540258".parse().unwrap() - ))); - assert_eq!(b1, Ok(BlockAddress::Number(1234))); - assert_eq!(b2, Ok(BlockAddress::Number(0))); - assert_eq!(b3, Ok(BlockAddress::Bytes(vec![0, 0x12, 0x34, 0x5f]))); - } - - #[test] - fn should_parse_extrinsic_address() { - type BlockAddress = super::BlockAddress; - type ExtrinsicAddress = super::ExtrinsicAddress; - - let e0 = ExtrinsicAddress::from_str("1234"); - let b0 = ExtrinsicAddress::from_str("3BfC20f0B9aFcAcE800D73D2191166FF16540258:5"); - let b1 = ExtrinsicAddress::from_str("1234:0"); - let b2 = ExtrinsicAddress::from_str("0 0"); - let b3 = ExtrinsicAddress::from_str("0x0012345f"); - - - assert_eq!(e0, Err("Extrinsic index missing: example \"5:0\"".into())); - assert_eq!(b0, Ok(ExtrinsicAddress::Block( - BlockAddress::Hash("3BfC20f0B9aFcAcE800D73D2191166FF16540258".parse().unwrap()), - 5 - ))); - assert_eq!(b1, Ok(ExtrinsicAddress::Block( - BlockAddress::Number(1234), - 0 - ))); - assert_eq!(b2, Ok(ExtrinsicAddress::Block( - BlockAddress::Number(0), - 0 - ))); - assert_eq!(b3, Ok(ExtrinsicAddress::Bytes(vec![0, 0x12, 0x34, 0x5f]))); - } -} diff --git a/bin/node/inspect/src/command.rs b/bin/node/inspect/src/command.rs new file mode 100644 index 0000000000000000000000000000000000000000..71e70e3e44fd4cda6ac398a385d1785c765079b9 --- /dev/null +++ b/bin/node/inspect/src/command.rs @@ -0,0 +1,204 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Command ran by the CLI + +use std::{ + fmt::Debug, + str::FromStr, +}; + +use crate::cli::{InspectCmd, InspectSubCmd}; +use crate::{Inspector, PrettyPrinter}; + +impl InspectCmd { + /// Initialize + pub fn init(&self, version: &sc_cli::VersionInfo) -> sc_cli::Result<()> { + self.shared_params.init(version) + } + + /// Parse CLI arguments and initialize given config. + pub fn update_config( + &self, + mut config: &mut sc_service::config::Configuration, + spec_factory: impl FnOnce(&str) -> Result>, String>, + version: &sc_cli::VersionInfo, + ) -> sc_cli::Result<()> where + G: sc_service::RuntimeGenesis, + E: sc_service::ChainSpecExtension, + { + self.shared_params.update_config(config, spec_factory, version)?; + + // make sure to configure keystore + config.use_in_memory_keystore()?; + + // and all import params (especially pruning that has to match db meta) + self.import_params.update_config( + &mut config, + sc_service::Roles::FULL, + self.shared_params.dev, + )?; + + Ok(()) + } + + /// Run the inspect command, passing the inspector. + pub fn run( + self, + inspect: Inspector, + ) -> sc_cli::Result<()> where + B: sp_runtime::traits::Block, + B::Hash: FromStr, + P: PrettyPrinter, + { + match self.command { + InspectSubCmd::Block { input } => { + let input = input.parse()?; + let res = inspect.block(input) + .map_err(|e| format!("{}", e))?; + println!("{}", res); + Ok(()) + }, + InspectSubCmd::Extrinsic { input } => { + let input = input.parse()?; + let res = inspect.extrinsic(input) + .map_err(|e| format!("{}", e))?; + println!("{}", res); + Ok(()) + }, + } + } +} + +/// A block to retrieve. +#[derive(Debug, Clone, PartialEq)] +pub enum BlockAddress { + /// Get block by hash. + Hash(Hash), + /// Get block by number. + Number(Number), + /// Raw SCALE-encoded bytes. + Bytes(Vec), +} + +impl FromStr for BlockAddress { + type Err = String; + + fn from_str(s: &str) -> Result { + // try to parse hash first + if let Ok(hash) = s.parse() { + return Ok(Self::Hash(hash)) + } + + // then number + if let Ok(number) = s.parse() { + return Ok(Self::Number(number)) + } + + // then assume it's bytes (hex-encoded) + sp_core::bytes::from_hex(s) + .map(Self::Bytes) + .map_err(|e| format!( + "Given string does not look like hash or number. It could not be parsed as bytes either: {}", + e + )) + } +} + +/// An extrinsic address to decode and print out. +#[derive(Debug, Clone, PartialEq)] +pub enum ExtrinsicAddress { + /// Extrinsic as part of existing block. + Block(BlockAddress, usize), + /// Raw SCALE-encoded extrinsic bytes. + Bytes(Vec), +} + +impl FromStr for ExtrinsicAddress { + type Err = String; + + fn from_str(s: &str) -> Result { + // first try raw bytes + if let Ok(bytes) = sp_core::bytes::from_hex(s).map(Self::Bytes) { + return Ok(bytes) + } + + // split by a bunch of different characters + let mut it = s.split(|c| c == '.' || c == ':' || c == ' '); + let block = it.next() + .expect("First element of split iterator is never empty; qed") + .parse()?; + + let index = it.next() + .ok_or_else(|| format!("Extrinsic index missing: example \"5:0\""))? + .parse() + .map_err(|e| format!("Invalid index format: {}", e))?; + + Ok(Self::Block(block, index)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use sp_core::hash::H160 as Hash; + + #[test] + fn should_parse_block_strings() { + type BlockAddress = super::BlockAddress; + + let b0 = BlockAddress::from_str("3BfC20f0B9aFcAcE800D73D2191166FF16540258"); + let b1 = BlockAddress::from_str("1234"); + let b2 = BlockAddress::from_str("0"); + let b3 = BlockAddress::from_str("0x0012345f"); + + + assert_eq!(b0, Ok(BlockAddress::Hash( + "3BfC20f0B9aFcAcE800D73D2191166FF16540258".parse().unwrap() + ))); + assert_eq!(b1, Ok(BlockAddress::Number(1234))); + assert_eq!(b2, Ok(BlockAddress::Number(0))); + assert_eq!(b3, Ok(BlockAddress::Bytes(vec![0, 0x12, 0x34, 0x5f]))); + } + + #[test] + fn should_parse_extrinsic_address() { + type BlockAddress = super::BlockAddress; + type ExtrinsicAddress = super::ExtrinsicAddress; + + let e0 = ExtrinsicAddress::from_str("1234"); + let b0 = ExtrinsicAddress::from_str("3BfC20f0B9aFcAcE800D73D2191166FF16540258:5"); + let b1 = ExtrinsicAddress::from_str("1234:0"); + let b2 = ExtrinsicAddress::from_str("0 0"); + let b3 = ExtrinsicAddress::from_str("0x0012345f"); + + + assert_eq!(e0, Err("Extrinsic index missing: example \"5:0\"".into())); + assert_eq!(b0, Ok(ExtrinsicAddress::Block( + BlockAddress::Hash("3BfC20f0B9aFcAcE800D73D2191166FF16540258".parse().unwrap()), + 5 + ))); + assert_eq!(b1, Ok(ExtrinsicAddress::Block( + BlockAddress::Number(1234), + 0 + ))); + assert_eq!(b2, Ok(ExtrinsicAddress::Block( + BlockAddress::Number(0), + 0 + ))); + assert_eq!(b3, Ok(ExtrinsicAddress::Bytes(vec![0, 0x12, 0x34, 0x5f]))); + } +} diff --git a/bin/node/inspect/src/lib.rs b/bin/node/inspect/src/lib.rs index 5c4e18c0a74a643b4625b616b61c284281171e74..cd32f08e9fe59309bc3f4daf33035a763c1b242b 100644 --- a/bin/node/inspect/src/lib.rs +++ b/bin/node/inspect/src/lib.rs @@ -23,6 +23,7 @@ #![warn(missing_docs)] pub mod cli; +pub mod command; use std::{ fmt, @@ -37,8 +38,10 @@ use sp_runtime::{ traits::{Block, HashFor, NumberFor, Hash} }; +use command::{BlockAddress, ExtrinsicAddress}; + /// A helper type for a generic block input. -pub type BlockAddressFor = cli::BlockAddress< +pub type BlockAddressFor = BlockAddress< as Hash>::Output, NumberFor >; @@ -148,10 +151,10 @@ impl> Inspector fn get_block(&self, input: BlockAddressFor) -> Result { Ok(match input { - cli::BlockAddress::Bytes(bytes) => { + BlockAddress::Bytes(bytes) => { TBlock::decode(&mut &*bytes)? }, - cli::BlockAddress::Number(number) => { + BlockAddress::Number(number) => { let id = BlockId::number(number); let not_found = format!("Could not find block {:?}", id); let body = self.chain.block_body(&id)? @@ -160,7 +163,7 @@ impl> Inspector .ok_or_else(|| Error::NotFound(not_found.clone()))?; TBlock::new(header, body) }, - cli::BlockAddress::Hash(hash) => { + BlockAddress::Hash(hash) => { let id = BlockId::hash(hash); let not_found = format!("Could not find block {:?}", id); let body = self.chain.block_body(&id)? @@ -175,7 +178,7 @@ impl> Inspector /// Get a pretty-printed extrinsic. pub fn extrinsic( &self, - input: cli::ExtrinsicAddress< as Hash>::Output, NumberFor>, + input: ExtrinsicAddress< as Hash>::Output, NumberFor>, ) -> Result { struct ExtrinsicPrinter<'a, A: Block, B>(A::Extrinsic, &'a B); impl<'a, A: Block, B: PrettyPrinter> fmt::Display for ExtrinsicPrinter<'a, A, B> { @@ -185,7 +188,7 @@ impl> Inspector } let ext = match input { - cli::ExtrinsicAddress::Block(block, index) => { + ExtrinsicAddress::Block(block, index) => { let block = self.get_block(block)?; block.extrinsics() .get(index) @@ -194,7 +197,7 @@ impl> Inspector "Could not find extrinsic {} in block {:?}", index, block )))? }, - cli::ExtrinsicAddress::Bytes(bytes) => { + ExtrinsicAddress::Bytes(bytes) => { TBlock::Extrinsic::decode(&mut &*bytes)? } }; diff --git a/bin/node/primitives/Cargo.toml b/bin/node/primitives/Cargo.toml index 5fc6ce8f101bdb807481917dcd65e679ab851cb5..cb271b987dba2e99c725258713d633863ff81e8b 100644 --- a/bin/node/primitives/Cargo.toml +++ b/bin/node/primitives/Cargo.toml @@ -1,16 +1,18 @@ [package] name = "node-primitives" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sp-core = { version = "2.0.0", default-features = false, path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/runtime" } [dev-dependencies] -sp-serializer = { version = "2.0.0", path = "../../../primitives/serializer" } +sp-serializer = { version = "2.0.0-alpha.2", path = "../../../primitives/serializer" } pretty_assertions = "0.6.1" [features] diff --git a/bin/node/rpc-client/Cargo.toml b/bin/node/rpc-client/Cargo.toml index 0d8106dceed7fff28e6d7bf1f7006a77362af920..8b37aff291341bd52cc11503fc4d37597769411c 100644 --- a/bin/node/rpc-client/Cargo.toml +++ b/bin/node/rpc-client/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "node-rpc-client" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] env_logger = "0.7.0" @@ -11,5 +13,5 @@ futures = "0.1.29" hyper = "0.12.35" jsonrpc-core-client = { version = "14.0.3", features = ["http", "ws"] } log = "0.4.8" -node-primitives = { version = "2.0.0", path = "../primitives" } -sc-rpc = { version = "2.0.0", path = "../../../client/rpc" } +node-primitives = { version = "2.0.0-alpha.2", path = "../primitives" } +sc-rpc = { version = "2.0.0-alpha.2", path = "../../../client/rpc" } diff --git a/bin/node/rpc/Cargo.toml b/bin/node/rpc/Cargo.toml index aefc9222f781da744c647e0969a337f9950b5d0d..1155eab304f8ae953207565d13f9434a4d27bb50 100644 --- a/bin/node/rpc/Cargo.toml +++ b/bin/node/rpc/Cargo.toml @@ -1,25 +1,27 @@ [package] name = "node-rpc" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sc-client = { version = "0.8", path = "../../../client/" } +sc-client = { version = "0.8.0-alpha.2", path = "../../../client/" } jsonrpc-core = "14.0.3" -node-primitives = { version = "2.0.0", path = "../primitives" } -node-runtime = { version = "2.0.0", path = "../runtime" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0", path = "../../../primitives/api" } -pallet-contracts-rpc = { version = "0.8", path = "../../../frame/contracts/rpc/" } -pallet-transaction-payment-rpc = { version = "2.0.0", path = "../../../frame/transaction-payment/rpc/" } -substrate-frame-rpc-system = { version = "2.0.0", path = "../../../utils/frame/rpc/system" } -sp-transaction-pool = { version = "2.0.0", path = "../../../primitives/transaction-pool" } -sc-consensus-babe = { version = "0.8", path = "../../../client/consensus/babe" } -sc-consensus-babe-rpc = { version = "0.8", path = "../../../client/consensus/babe/rpc" } -sp-consensus-babe = { version = "0.8", path = "../../../primitives/consensus/babe" } -sc-keystore = { version = "2.0.0", path = "../../../client/keystore" } -sc-consensus-epochs = { version = "0.8", path = "../../../client/consensus/epochs" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } +node-primitives = { version = "2.0.0-alpha.2", path = "../primitives" } +node-runtime = { version = "2.0.0-alpha.2", path = "../runtime" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-alpha.2", path = "../../../primitives/api" } +pallet-contracts-rpc = { version = "0.8.0-alpha.2", path = "../../../frame/contracts/rpc/" } +pallet-transaction-payment-rpc = { version = "2.0.0-alpha.2", path = "../../../frame/transaction-payment/rpc/" } +substrate-frame-rpc-system = { version = "2.0.0-alpha.2", path = "../../../utils/frame/rpc/system" } +sp-transaction-pool = { version = "2.0.0-alpha.2", path = "../../../primitives/transaction-pool" } +sc-consensus-babe = { version = "0.8.0-alpha.2", path = "../../../client/consensus/babe" } +sc-consensus-babe-rpc = { version = "0.8.0-alpha.2", path = "../../../client/consensus/babe/rpc" } +sp-consensus-babe = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/babe" } +sc-keystore = { version = "2.0.0-alpha.2", path = "../../../client/keystore" } +sc-consensus-epochs = { version = "0.8.0-alpha.2", path = "../../../client/consensus/epochs" } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/common" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../../primitives/blockchain" } diff --git a/bin/node/rpc/src/lib.rs b/bin/node/rpc/src/lib.rs index 16e5446bb1591c56f7e0cb66c9e580f4deb06883..4e1cfa56733a7bbbbb50fd944241410465d488c1 100644 --- a/bin/node/rpc/src/lib.rs +++ b/bin/node/rpc/src/lib.rs @@ -23,9 +23,9 @@ //! need some strong assumptions about the particular runtime. //! //! The RPCs available in this crate however can make some assumptions -//! about how the runtime is constructed and what `SRML` modules +//! about how the runtime is constructed and what FRAME pallets //! are part of it. Therefore all node-runtime-specific RPCs can -//! be placed here or imported from corresponding `SRML` RPC definitions. +//! be placed here or imported from corresponding FRAME RPC definitions. #![warn(missing_docs)] diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 3f8e8b6731477e4cbaabaf4e3012bb139d259e6d..0d65cf5339536c088f94243cbcb4ac80c17f0f64 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -1,77 +1,80 @@ [package] name = "node-runtime" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] # third-party dependencies -codec = { package = "parity-scale-codec", version = "1.0.6", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } integer-sqrt = { version = "0.1.2" } rustc-hex = { version = "2.0", optional = true } serde = { version = "1.0.102", optional = true } # primitives -sp-authority-discovery = { version = "2.0.0", default-features = false, path = "../../../primitives/authority-discovery" } -sp-consensus-babe = { version = "0.8", default-features = false, path = "../../../primitives/consensus/babe" } -sp-block-builder = { path = "../../../primitives/block-builder", default-features = false} -sp-inherents = { version = "2.0.0", default-features = false, path = "../../../primitives/inherents" } -node-primitives = { version = "2.0.0", default-features = false, path = "../primitives" } -sp-offchain = { version = "2.0.0", default-features = false, path = "../../../primitives/offchain" } -sp-core = { version = "2.0.0", default-features = false, path = "../../../primitives/core" } -sp-std = { version = "2.0.0", default-features = false, path = "../../../primitives/std" } -sp-api = { version = "2.0.0", default-features = false, path = "../../../primitives/api" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../../primitives/runtime" } -sp-staking = { version = "2.0.0", default-features = false, path = "../../../primitives/staking" } -sp-keyring = { version = "2.0.0", optional = true, path = "../../../primitives/keyring" } -sp-session = { version = "2.0.0", default-features = false, path = "../../../primitives/session" } -sp-transaction-pool = { version = "2.0.0", default-features = false, path = "../../../primitives/transaction-pool" } -sp-version = { version = "2.0.0", default-features = false, path = "../../../primitives/version" } +sp-authority-discovery = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/authority-discovery" } +sp-consensus-babe = { version = "0.8.0-alpha.2", default-features = false, path = "../../../primitives/consensus/babe" } +sp-block-builder = { path = "../../../primitives/block-builder", default-features = false, version = "2.0.0-alpha.2"} +sp-inherents = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/inherents" } +node-primitives = { version = "2.0.0-alpha.2", default-features = false, path = "../primitives" } +sp-offchain = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/offchain" } +sp-core = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/core" } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/std" } +sp-api = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/api" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/runtime" } +sp-staking = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/staking" } +sp-keyring = { version = "2.0.0-alpha.2", optional = true, path = "../../../primitives/keyring" } +sp-session = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/session" } +sp-transaction-pool = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/transaction-pool" } +sp-version = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/version" } # frame dependencies -frame-executive = { version = "2.0.0", default-features = false, path = "../../../frame/executive" } -frame-support = { version = "2.0.0", default-features = false, path = "../../../frame/support" } -frame-system = { version = "2.0.0", default-features = false, path = "../../../frame/system" } -frame-system-rpc-runtime-api = { version = "2.0.0", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } -pallet-authority-discovery = { version = "2.0.0", default-features = false, path = "../../../frame/authority-discovery" } -pallet-authorship = { version = "2.0.0", default-features = false, path = "../../../frame/authorship" } -pallet-babe = { version = "2.0.0", default-features = false, path = "../../../frame/babe" } -pallet-balances = { version = "2.0.0", default-features = false, path = "../../../frame/balances" } -pallet-collective = { version = "2.0.0", default-features = false, path = "../../../frame/collective" } -pallet-contracts = { version = "2.0.0", default-features = false, path = "../../../frame/contracts" } -pallet-contracts-primitives = { version = "2.0.0", default-features = false, path = "../../../frame/contracts/common/" } -pallet-contracts-rpc-runtime-api = { version = "0.8.0", default-features = false, path = "../../../frame/contracts/rpc/runtime-api/" } -pallet-democracy = { version = "2.0.0", default-features = false, path = "../../../frame/democracy" } -pallet-elections-phragmen = { version = "2.0.0", default-features = false, path = "../../../frame/elections-phragmen" } -pallet-finality-tracker = { version = "2.0.0", default-features = false, path = "../../../frame/finality-tracker" } -pallet-grandpa = { version = "2.0.0", default-features = false, path = "../../../frame/grandpa" } -pallet-im-online = { version = "2.0.0", default-features = false, path = "../../../frame/im-online" } -pallet-indices = { version = "2.0.0", default-features = false, path = "../../../frame/indices" } -pallet-identity = { version = "2.0.0", default-features = false, path = "../../../frame/identity" } -pallet-membership = { version = "2.0.0", default-features = false, path = "../../../frame/membership" } -pallet-offences = { version = "2.0.0", default-features = false, path = "../../../frame/offences" } -pallet-randomness-collective-flip = { version = "2.0.0", default-features = false, path = "../../../frame/randomness-collective-flip" } -pallet-recovery = { version = "2.0.0", default-features = false, path = "../../../frame/recovery" } -pallet-session = { version = "2.0.0", features = ["historical"], path = "../../../frame/session", default-features = false } -pallet-staking = { version = "2.0.0", features = ["migrate"], path = "../../../frame/staking", default-features = false } -pallet-staking-reward-curve = { version = "2.0.0", path = "../../../frame/staking/reward-curve" } -pallet-sudo = { version = "2.0.0", default-features = false, path = "../../../frame/sudo" } -pallet-society = { version = "2.0.0", default-features = false, path = "../../../frame/society" } -pallet-timestamp = { version = "2.0.0", default-features = false, path = "../../../frame/timestamp" } -pallet-treasury = { version = "2.0.0", default-features = false, path = "../../../frame/treasury" } -pallet-utility = { version = "2.0.0", default-features = false, path = "../../../frame/utility" } -pallet-transaction-payment = { version = "2.0.0", default-features = false, path = "../../../frame/transaction-payment" } -pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" } -pallet-vesting = { version = "2.0.0", default-features = false, path = "../../../frame/vesting" } +frame-executive = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/executive" } +frame-benchmarking = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/benchmarking", optional = true } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/support" } +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/system" } +frame-system-rpc-runtime-api = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } +pallet-authority-discovery = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/authority-discovery" } +pallet-authorship = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/authorship" } +pallet-babe = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/babe" } +pallet-balances = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/balances" } +pallet-collective = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/collective" } +pallet-contracts = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/contracts" } +pallet-contracts-primitives = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/contracts/common/" } +pallet-contracts-rpc-runtime-api = { version = "0.8.0-alpha.2", default-features = false, path = "../../../frame/contracts/rpc/runtime-api/" } +pallet-democracy = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/democracy" } +pallet-elections-phragmen = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/elections-phragmen" } +pallet-finality-tracker = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/finality-tracker" } +pallet-grandpa = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/grandpa" } +pallet-im-online = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/im-online" } +pallet-indices = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/indices" } +pallet-identity = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/identity" } +pallet-membership = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/membership" } +pallet-offences = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/offences" } +pallet-randomness-collective-flip = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/randomness-collective-flip" } +pallet-recovery = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/recovery" } +pallet-session = { version = "2.0.0-alpha.2", features = ["historical"], path = "../../../frame/session", default-features = false } +pallet-staking = { version = "2.0.0-alpha.2", features = ["migrate"], path = "../../../frame/staking", default-features = false } +pallet-staking-reward-curve = { version = "2.0.0-alpha.2", path = "../../../frame/staking/reward-curve" } +pallet-sudo = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/sudo" } +pallet-society = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/society" } +pallet-timestamp = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/timestamp" } +pallet-treasury = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/treasury" } +pallet-utility = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/utility" } +pallet-transaction-payment = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/transaction-payment" } +pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" } +pallet-vesting = { version = "2.0.0-alpha.2", default-features = false, path = "../../../frame/vesting" } [build-dependencies] -wasm-builder-runner = { version = "1.0.4", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } +wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } [dev-dependencies] -sp-io = { version = "2.0.0", path = "../../../primitives/io" } +sp-io = { version = "2.0.0-alpha.2", path = "../../../primitives/io" } [features] default = ["std"] @@ -115,6 +118,7 @@ std = [ "sp-session/std", "pallet-sudo/std", "frame-support/std", + "frame-benchmarking/std", "frame-system-rpc-runtime-api/std", "frame-system/std", "pallet-timestamp/std", @@ -128,3 +132,9 @@ std = [ "pallet-recovery/std", "pallet-vesting/std", ] +runtime-benchmarks = [ + "frame-benchmarking", + "pallet-timestamp/runtime-benchmarks", + "pallet-identity/runtime-benchmarks", + "pallet-balances/runtime-benchmarks", +] diff --git a/bin/node/runtime/src/constants.rs b/bin/node/runtime/src/constants.rs index b2c880c08bbf8098e16154e99f3a797a6d27eec8..bf12492f8db029dc22b519dd52af74adcad99a7f 100644 --- a/bin/node/runtime/src/constants.rs +++ b/bin/node/runtime/src/constants.rs @@ -38,7 +38,7 @@ pub mod time { /// a slot being empty). /// This value is only used indirectly to define the unit constants below /// that are expressed in blocks. The rest of the code should use - /// `SLOT_DURATION` instead (like the timestamp module for calculating the + /// `SLOT_DURATION` instead (like the Timestamp pallet for calculating the /// minimum period). /// /// If using BABE with secondary slots (default) then all of the slots will diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index e8c0a103e652e912db5d6c98733416c119d4dccf..d14ecda843560f3efa36ac10c5b7a5087a2da0e2 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -31,14 +31,14 @@ pub use node_primitives::{AccountId, Signature}; use node_primitives::{AccountIndex, Balance, BlockNumber, Hash, Index, Moment}; use sp_api::impl_runtime_apis; use sp_runtime::{ - Permill, Perbill, Percent, ApplyExtrinsicResult, BenchmarkResults, + Permill, Perbill, Percent, ApplyExtrinsicResult, RuntimeString, impl_opaque_keys, generic, create_runtime_str, }; use sp_runtime::curve::PiecewiseLinear; use sp_runtime::transaction_validity::TransactionValidity; use sp_runtime::traits::{ self, BlakeTwo256, Block as BlockT, StaticLookup, SaturatedConversion, - ConvertInto, OpaqueKeys, Benchmarking, + ConvertInto, OpaqueKeys, }; use sp_version::RuntimeVersion; #[cfg(any(feature = "std", test))] @@ -68,6 +68,7 @@ use impls::{CurrencyToVoteHandler, Author, LinearWeightToFee, TargetedFeeAdjustm /// Constant values used within the runtime. pub mod constants; use constants::{time::*, currency::*}; +use frame_system::Trait; // Make the WASM binary available. #[cfg(feature = "std")] @@ -82,7 +83,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // and set impl_version to 0. If only runtime // implementation changes and behavior does not, then leave spec_version as // is and increment impl_version. - spec_version: 223, + spec_version: 231, impl_version: 0, apis: RUNTIME_API_VERSIONS, }; @@ -132,7 +133,7 @@ impl frame_system::Trait for Runtime { type ModuleToIndex = ModuleToIndex; type AccountData = pallet_balances::AccountData; type OnNewAccount = (); - type OnReapAccount = (Balances, Staking, Contracts, Session, Recovery); + type OnKilledAccount = (); } parameter_types! { @@ -271,6 +272,7 @@ parameter_types! { pub const BondingDuration: pallet_staking::EraIndex = 24 * 28; pub const SlashDeferDuration: pallet_staking::EraIndex = 24 * 7; // 1/4 the bonding duration. pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; + pub const MaxNominatorRewardedPerValidator: u32 = 64; } impl pallet_staking::Trait for Runtime { @@ -288,6 +290,7 @@ impl pallet_staking::Trait for Runtime { type SlashCancelOrigin = pallet_collective::EnsureProportionAtLeast<_3, _4, AccountId, CouncilCollective>; type SessionInterface = Self; type RewardCurve = RewardCurve; + type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; } parameter_types! { @@ -330,11 +333,16 @@ impl pallet_democracy::Trait for Runtime { type Slash = Treasury; } +parameter_types! { + pub const CouncilMotionDuration: BlockNumber = 5 * DAYS; +} + type CouncilCollective = pallet_collective::Instance1; impl pallet_collective::Trait for Runtime { type Origin = Origin; type Proposal = Call; type Event = Event; + type MotionDuration = CouncilMotionDuration; } parameter_types! { @@ -360,11 +368,16 @@ impl pallet_elections_phragmen::Trait for Runtime { type TermDuration = TermDuration; } +parameter_types! { + pub const TechnicalMotionDuration: BlockNumber = 5 * DAYS; +} + type TechnicalCollective = pallet_collective::Instance2; impl pallet_collective::Trait for Runtime { type Origin = Origin; type Proposal = Call; type Event = Event; + type MotionDuration = TechnicalMotionDuration; } impl pallet_membership::Trait for Runtime { @@ -373,6 +386,7 @@ impl pallet_membership::Trait for Runtime { type RemoveOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; type SwapOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; type ResetOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; + type PrimeOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; type MembershipInitialized = TechnicalCommittee; type MembershipChanged = TechnicalCommittee; } @@ -590,10 +604,15 @@ impl pallet_society::Trait for Runtime { type ChallengePeriod = ChallengePeriod; } +parameter_types! { + pub const MinVestedTransfer: Balance = 100 * DOLLARS; +} + impl pallet_vesting::Trait for Runtime { type Event = Event; type Currency = Balances; type BlockNumberToBalance = ConvertInto; + type MinVestedTransfer = MinVestedTransfer; } construct_runtime!( @@ -818,28 +837,48 @@ impl_runtime_apis! { } } - impl crate::Benchmark for Runtime { - fn dispatch_benchmark(module: Vec, extrinsic: Vec, steps: u32, repeat: u32) - -> Option> - { - match module.as_slice() { - b"pallet-balances" | b"balances" => Balances::run_benchmark(extrinsic, steps, repeat).ok(), - b"pallet-identity" | b"identity" => Identity::run_benchmark(extrinsic, steps, repeat).ok(), - b"pallet-timestamp" | b"timestamp" => Timestamp::run_benchmark(extrinsic, steps, repeat).ok(), - _ => return None, - } + #[cfg(feature = "runtime-benchmarks")] + impl frame_benchmarking::Benchmark for Runtime { + fn dispatch_benchmark( + module: Vec, + extrinsic: Vec, + lowest_range_values: Vec, + highest_range_values: Vec, + steps: Vec, + repeat: u32, + ) -> Result, RuntimeString> { + use frame_benchmarking::Benchmarking; + + let result = match module.as_slice() { + b"pallet-balances" | b"balances" => Balances::run_benchmark( + extrinsic, + lowest_range_values, + highest_range_values, + steps, + repeat, + ), + b"pallet-identity" | b"identity" => Identity::run_benchmark( + extrinsic, + lowest_range_values, + highest_range_values, + steps, + repeat, + ), + b"pallet-timestamp" | b"timestamp" => Timestamp::run_benchmark( + extrinsic, + lowest_range_values, + highest_range_values, + steps, + repeat, + ), + _ => Err("Benchmark not found for this pallet."), + }; + + result.map_err(|e| e.into()) } } } -sp_api::decl_runtime_apis! { - pub trait Benchmark - { - fn dispatch_benchmark(module: Vec, extrinsic: Vec, steps: u32, repeat: u32) - -> Option>; - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/bin/node/testing/Cargo.toml b/bin/node/testing/Cargo.toml index 840b2d0fefcd7258815cfd0ccb79c3dabee15946..4496047b50cff4be14a0435ffa993bbd0d232d29 100644 --- a/bin/node/testing/Cargo.toml +++ b/bin/node/testing/Cargo.toml @@ -1,53 +1,56 @@ [package] name = "node-testing" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] description = "Test utilities for Substrate node." edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +publish = true [dependencies] -pallet-balances = { version = "2.0.0", path = "../../../frame/balances" } -sc-client = { version = "0.8", path = "../../../client/" } -sc-client-db = { version = "0.8", path = "../../../client/db/", features = ["kvdb-rocksdb"] } -sc-client-api = { version = "2.0", path = "../../../client/api/" } -codec = { package = "parity-scale-codec", version = "1.0.0" } -pallet-contracts = { version = "2.0.0", path = "../../../frame/contracts" } -pallet-grandpa = { version = "2.0.0", path = "../../../frame/grandpa" } -pallet-indices = { version = "2.0.0", path = "../../../frame/indices" } -sp-keyring = { version = "2.0.0", path = "../../../primitives/keyring" } -node-executor = { version = "2.0.0", path = "../executor" } -node-primitives = { version = "2.0.0", path = "../primitives" } -node-runtime = { version = "2.0.0", path = "../runtime" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-io = { version = "2.0.0", path = "../../../primitives/io" } -frame-support = { version = "2.0.0", path = "../../../frame/support" } -pallet-session = { version = "2.0.0", path = "../../../frame/session" } -pallet-society = { version = "2.0.0", path = "../../../frame/society" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -pallet-staking = { version = "2.0.0", path = "../../../frame/staking" } -sc-executor = { version = "0.8", path = "../../../client/executor", features = ["wasmtime"] } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } -frame-system = { version = "2.0.0", path = "../../../frame/system" } -substrate-test-client = { version = "2.0.0", path = "../../../test-utils/client" } -pallet-timestamp = { version = "2.0.0", path = "../../../frame/timestamp" } -pallet-transaction-payment = { version = "2.0.0", path = "../../../frame/transaction-payment" } -pallet-treasury = { version = "2.0.0", path = "../../../frame/treasury" } +pallet-balances = { version = "2.0.0-alpha.2", path = "../../../frame/balances" } +sc-client = { version = "0.8.0-alpha.2", path = "../../../client/" } +sc-client-db = { version = "0.8.0-alpha.2", path = "../../../client/db/", features = ["kvdb-rocksdb"] } +sc-client-api = { version = "2.0.0-alpha.2", path = "../../../client/api/" } +codec = { package = "parity-scale-codec", version = "1.2.0" } +pallet-contracts = { version = "2.0.0-alpha.2", path = "../../../frame/contracts" } +pallet-grandpa = { version = "2.0.0-alpha.2", path = "../../../frame/grandpa" } +pallet-indices = { version = "2.0.0-alpha.2", path = "../../../frame/indices" } +sp-keyring = { version = "2.0.0-alpha.2", path = "../../../primitives/keyring" } +node-executor = { version = "2.0.0-alpha.2", path = "../executor" } +node-primitives = { version = "2.0.0-alpha.2", path = "../primitives" } +node-runtime = { version = "2.0.0-alpha.2", path = "../runtime" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-io = { version = "2.0.0-alpha.2", path = "../../../primitives/io" } +frame-support = { version = "2.0.0-alpha.2", path = "../../../frame/support" } +pallet-session = { version = "2.0.0-alpha.2", path = "../../../frame/session" } +pallet-society = { version = "2.0.0-alpha.2", path = "../../../frame/society" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +pallet-staking = { version = "2.0.0-alpha.2", path = "../../../frame/staking" } +sc-executor = { version = "0.8.0-alpha.2", path = "../../../client/executor", features = ["wasmtime"] } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/common" } +frame-system = { version = "2.0.0-alpha.2", path = "../../../frame/system" } +substrate-test-client = { version = "2.0.0-dev", path = "../../../test-utils/client" } +pallet-timestamp = { version = "2.0.0-alpha.2", path = "../../../frame/timestamp" } +pallet-transaction-payment = { version = "2.0.0-alpha.2", path = "../../../frame/transaction-payment" } +pallet-treasury = { version = "2.0.0-alpha.2", path = "../../../frame/treasury" } wabt = "0.9.2" -sp-api = { version = "2.0.0", path = "../../../primitives/api" } -sp-finality-tracker = { version = "2.0.0", default-features = false, path = "../../../primitives/finality-tracker" } -sp-timestamp = { version = "2.0.0", default-features = false, path = "../../../primitives/timestamp" } -sp-block-builder = { version = "2.0.0", path = "../../../primitives/block-builder" } -sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } +sp-api = { version = "2.0.0-alpha.2", path = "../../../primitives/api" } +sp-finality-tracker = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/finality-tracker" } +sp-timestamp = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/timestamp" } +sp-block-builder = { version = "2.0.0-alpha.2", path = "../../../primitives/block-builder" } +sp-inherents = { version = "2.0.0-alpha.2", path = "../../../primitives/inherents" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../../primitives/blockchain" } log = "0.4.8" -tempdir = "0.3" +tempfile = "3.1.0" fs_extra = "1" [dev-dependencies] criterion = "0.3.0" -sc-cli = { version = "0.8.0", path = "../../../client/cli" } -sc-service = { version = "0.8.0", path = "../../../client/service", features = ["rocksdb"] } +sc-cli = { version = "0.8.0-alpha.2", path = "../../../client/cli" } +sc-service = { version = "0.8.0-alpha.2", path = "../../../client/service", features = ["rocksdb"] } [[bench]] name = "import" diff --git a/bin/node/testing/benches/import.rs b/bin/node/testing/benches/import.rs index f8cbbec79d5c2ba11bdccb5d379633f7aae3ea04..79cb71b164371e3bd6bb29c0f58ce2efa28a59af 100644 --- a/bin/node/testing/benches/import.rs +++ b/bin/node/testing/benches/import.rs @@ -28,16 +28,23 @@ //! to much configuring - just block full of randomized transactions. //! It is not supposed to measure runtime modules weight correctness +use std::fmt; use node_testing::bench::{BenchDb, Profile}; +use node_primitives::Block; use sp_runtime::generic::BlockId; use criterion::{Criterion, criterion_group, criterion_main}; use sc_client_api::backend::Backend; criterion_group!( name = benches; - config = Criterion::default().sample_size(50).warm_up_time(std::time::Duration::from_secs(20)); + config = Criterion::default().sample_size(20).warm_up_time(std::time::Duration::from_secs(20)); targets = bench_block_import ); +criterion_group!( + name = wasm_size; + config = Criterion::default().sample_size(10); + targets = bench_wasm_size_import +); criterion_group!( name = profile; config = Criterion::default().sample_size(10); @@ -139,3 +146,60 @@ fn profile_block_import(c: &mut Criterion) { }, ); } + +struct Setup { + db: BenchDb, + block: Block, +} + +struct SetupIterator { + current: usize, + finish: usize, + multiplier: usize, +} + +impl SetupIterator { + fn new(current: usize, finish: usize, multiplier: usize) -> Self { + SetupIterator { current, finish, multiplier } + } +} + +impl Iterator for SetupIterator { + type Item = Setup; + + fn next(&mut self) -> Option { + if self.current >= self.finish { return None } + + self.current += 1; + + let size = self.current * self.multiplier; + let mut db = BenchDb::new(size); + let block = db.generate_block(size); + Some(Setup { db, block }) + } +} + +impl fmt::Debug for Setup { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Setup: {} tx/block", self.block.extrinsics.len()) + } +} + +fn bench_wasm_size_import(c: &mut Criterion) { + sc_cli::init_logger(""); + + c.bench_function_over_inputs("wasm_size_import", + move |bencher, setup| { + bencher.iter_batched( + || { + setup.db.create_context(Profile::Wasm) + }, + |mut context| { + context.import_block(setup.block.clone()); + }, + criterion::BatchSize::PerIteration, + ); + }, + SetupIterator::new(5, 15, 50), + ); +} diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index 5653ba77016462ea9fd17e95c9a982223a5f4a3b..19906dd6a1bf5638bc8c07ae8e0693d00d1108a2 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -81,7 +81,7 @@ pub struct BenchDb { impl Clone for BenchDb { fn clone(&self) -> Self { let keyring = self.keyring.clone(); - let dir = tempdir::TempDir::new("sub-bench").expect("temp dir creation failed"); + let dir = tempfile::tempdir().expect("temp dir creation failed"); let seed_dir = self.directory_guard.0.path(); @@ -120,7 +120,7 @@ impl BenchDb { pub fn new(keyring_length: usize) -> Self { let keyring = BenchKeyring::new(keyring_length); - let dir = tempdir::TempDir::new("sub-bench").expect("temp dir creation failed"); + let dir = tempfile::tempdir().expect("temp dir creation failed"); log::trace!( target: "bench-logistics", "Created seed db at {}", @@ -155,6 +155,7 @@ impl BenchDb { None, None, ExecutionExtensions::new(profile.into_execution_strategies(), None), + None, ).expect("Should not fail"); (client, backend) @@ -357,7 +358,7 @@ impl Profile { } } -struct Guard(tempdir::TempDir); +struct Guard(tempfile::TempDir); impl Guard { fn path(&self) -> &Path { diff --git a/bin/node/testing/src/genesis.rs b/bin/node/testing/src/genesis.rs index 9c163ea6153b2f11bf871883b570d1a7acd05e18..e35059e0c6ffd300cb2941e4322e22c8bbf832ee 100644 --- a/bin/node/testing/src/genesis.rs +++ b/bin/node/testing/src/genesis.rs @@ -69,22 +69,21 @@ pub fn config_endowed( }), pallet_session: Some(SessionConfig { keys: vec![ - (alice(), to_session_keys( + (dave(), alice(), to_session_keys( &Ed25519Keyring::Alice, &Sr25519Keyring::Alice, )), - (bob(), to_session_keys( + (eve(), bob(), to_session_keys( &Ed25519Keyring::Bob, &Sr25519Keyring::Bob, )), - (charlie(), to_session_keys( + (ferdie(), charlie(), to_session_keys( &Ed25519Keyring::Charlie, &Sr25519Keyring::Charlie, )), ] }), pallet_staking: Some(StakingConfig { - current_era: 0, stakers: vec![ (dave(), alice(), 111 * DOLLARS, pallet_staking::StakerStatus::Validator), (eve(), bob(), 100 * DOLLARS, pallet_staking::StakerStatus::Validator), diff --git a/bin/node/transaction-factory/Cargo.toml b/bin/node/transaction-factory/Cargo.toml index cd40e2f0d939db732339659aeea02f184b062096..8de968baa64f55c7076fb5da014cc518632f93d8 100644 --- a/bin/node/transaction-factory/Cargo.toml +++ b/bin/node/transaction-factory/Cargo.toml @@ -1,20 +1,22 @@ [package] name = "node-transaction-factory" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sp-block-builder = { version = "2.0.0", path = "../../../primitives/block-builder" } -sc-cli = { version = "0.8.0", path = "../../../client/cli" } -sc-client-api = { version = "2.0.0", path = "../../../client/api" } -sc-client = { version = "0.8", path = "../../../client" } -codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } +sp-block-builder = { version = "2.0.0-alpha.2", path = "../../../primitives/block-builder" } +sc-cli = { version = "0.8.0-alpha.2", path = "../../../client/cli" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../../../client/api" } +sc-client = { version = "0.8.0-alpha.2", path = "../../../client" } +codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/common" } log = "0.4.8" -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-api = { version = "2.0.0", path = "../../../primitives/api" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sc-service = { version = "0.8", path = "../../../client/service" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-api = { version = "2.0.0-alpha.2", path = "../../../primitives/api" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +sc-service = { version = "0.8.0-alpha.2", path = "../../../client/service" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../../primitives/blockchain" } diff --git a/bin/node/transaction-factory/src/lib.rs b/bin/node/transaction-factory/src/lib.rs index 7dafeed40616eaf74997fef2c10624d8e5275910..a4c001145ac7eb6bc99d7fa91740fe4d265650e3 100644 --- a/bin/node/transaction-factory/src/lib.rs +++ b/bin/node/transaction-factory/src/lib.rs @@ -83,7 +83,7 @@ pub fn factory( mut factory_state: RA, client: &Arc>, select_chain: &Sc, -) -> sc_cli::error::Result<()> +) -> sc_cli::Result<()> where Block: BlockT, Exec: sc_client::CallExecutor + Send + Sync + Clone, @@ -97,7 +97,7 @@ where RA: RuntimeAdapter, Block::Hash: From, { - let best_header: Result<::Header, sc_cli::error::Error> = + let best_header: Result<::Header, sc_cli::Error> = select_chain.best_chain().map_err(|e| format!("{:?}", e).into()); let mut best_hash = best_header?.hash(); let mut best_block_id = BlockId::::hash(best_hash); diff --git a/bin/utils/chain-spec-builder/Cargo.toml b/bin/utils/chain-spec-builder/Cargo.toml index 8f122a35a02e67a63acdd3bf7f551d779f668d9d..5e51c4358e513d8c6a3ec5164d3aba0520d3f405 100644 --- a/bin/utils/chain-spec-builder/Cargo.toml +++ b/bin/utils/chain-spec-builder/Cargo.toml @@ -1,15 +1,17 @@ [package] name = "chain-spec-builder" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] ansi_term = "0.12.1" -sc-keystore = { version = "2.0.0", path = "../../../client/keystore" } -node-cli = { version = "2.0.0", path = "../../node/cli" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } +sc-keystore = { version = "2.0.0-alpha.2", path = "../../../client/keystore" } +node-cli = { version = "2.0.0-alpha.2", path = "../../node/cli" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } rand = "0.7.2" structopt = "0.3.8" diff --git a/bin/utils/subkey/Cargo.toml b/bin/utils/subkey/Cargo.toml index 8267366d7fe02d782086fa05674fe191dcf000f8..252978a3a98cbc143139b9cdfba1169a1ebc6de9 100644 --- a/bin/utils/subkey/Cargo.toml +++ b/bin/utils/subkey/Cargo.toml @@ -1,16 +1,18 @@ [package] name = "subkey" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] futures = "0.1.29" -sp-core = { version = "*", path = "../../../primitives/core" } -node-runtime = { version = "*", path = "../../node/runtime" } -node-primitives = { version = "*", path = "../../node/primitives" } -sp-runtime = { version = "*", path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +node-runtime = { version = "2.0.0-alpha.2", path = "../../node/runtime" } +node-primitives = { version = "2.0.0-alpha.2", path = "../../node/primitives" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } rand = "0.7.2" clap = "2.33.0" tiny-bip39 = "0.7" @@ -18,17 +20,17 @@ rustc-hex = "2.0.1" substrate-bip39 = "0.3.1" hex = "0.4.0" hex-literal = "0.2.1" -codec = { package = "parity-scale-codec", version = "1.0.0" } -frame-system = { version = "2.0.0", path = "../../../frame/system" } -pallet-balances = { version = "2.0.0", path = "../../../frame/balances" } -pallet-transaction-payment = { version = "2.0.0", path = "../../../frame/transaction-payment" } +codec = { package = "parity-scale-codec", version = "1.2.0" } +frame-system = { version = "2.0.0-alpha.2", path = "../../../frame/system" } +pallet-balances = { version = "2.0.0-alpha.2", path = "../../../frame/balances" } +pallet-transaction-payment = { version = "2.0.0-alpha.2", path = "../../../frame/transaction-payment" } rpassword = "4.0.1" itertools = "0.8.2" derive_more = { version = "0.99.2" } -sc-rpc = { version = "2.0.0", path = "../../../client/rpc" } +sc-rpc = { version = "2.0.0-alpha.2", path = "../../../client/rpc" } jsonrpc-core-client = { version = "14.0.3", features = ["http"] } hyper = "0.12.35" -libp2p = "0.16.0" +libp2p = "0.16.2" serde_json = "1.0" [features] diff --git a/client/Cargo.toml b/client/Cargo.toml index c89fe88145d148a6d64e70dcae4adac7eb97021d..61199f04da970bbefb4f494e2d46a5824538d499 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -1,41 +1,45 @@ [package] name = "sc-client" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Substrate Client and associated logic." [dependencies] -sc-block-builder = { version = "0.8", path = "block-builder" } -sc-client-api = { version = "2.0.0", path = "api" } -codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } -sp-consensus = { version = "0.8", path = "../primitives/consensus/common" } +sc-block-builder = { version = "0.8.0-alpha.2", path = "block-builder" } +sc-client-api = { version = "2.0.0-alpha.2", path = "api" } +codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +sp-consensus = { version = "0.8.0-alpha.2", path = "../primitives/consensus/common" } derive_more = { version = "0.99.2" } -sc-executor = { version = "0.8", path = "executor" } -sp-externalities = { version = "0.8.0", path = "../primitives/externalities" } +sc-executor = { version = "0.8.0-alpha.2", path = "executor" } +sp-externalities = { version = "0.8.0-alpha.2", path = "../primitives/externalities" } fnv = { version = "1.0.6" } futures = { version = "0.3.1", features = ["compat"] } hash-db = { version = "0.15.2" } hex-literal = { version = "0.2.1" } -sp-inherents = { version = "2.0.0", path = "../primitives/inherents" } -sp-keyring = { version = "2.0.0", path = "../primitives/keyring" } +sp-inherents = { version = "2.0.0-alpha.2", path = "../primitives/inherents" } +sp-keyring = { version = "2.0.0-alpha.2", path = "../primitives/keyring" } kvdb = "0.4.0" log = { version = "0.4.8" } parking_lot = "0.10.0" -sp-core = { version = "2.0.0", path = "../primitives/core" } -sp-std = { version = "2.0.0", path = "../primitives/std" } -sp-version = { version = "2.0.0", path = "../primitives/version" } -sp-api = { version = "2.0.0", path = "../primitives/api" } -sp-runtime = { version = "2.0.0", path = "../primitives/runtime" } -sp-blockchain = { version = "2.0.0", path = "../primitives/blockchain" } -sp-state-machine = { version = "0.8", path = "../primitives/state-machine" } -sc-telemetry = { version = "2.0.0", path = "telemetry" } -sp-trie = { version = "2.0.0", path = "../primitives/trie" } +sp-core = { version = "2.0.0-alpha.2", path = "../primitives/core" } +sp-std = { version = "2.0.0-alpha.2", path = "../primitives/std" } +sp-version = { version = "2.0.0-alpha.2", path = "../primitives/version" } +sp-api = { version = "2.0.0-alpha.2", path = "../primitives/api" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../primitives/runtime" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../primitives/blockchain" } +sp-state-machine = { version = "0.8.0-alpha.2", path = "../primitives/state-machine" } +sc-telemetry = { version = "2.0.0-alpha.2", path = "telemetry" } +sp-trie = { version = "2.0.0-alpha.2", path = "../primitives/trie" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.8.0-alpha.2", path = "../utils/prometheus" } tracing = "0.1.10" [dev-dependencies] env_logger = "0.7.0" tempfile = "3.1.0" -substrate-test-runtime-client = { version = "2.0.0", path = "../test-utils/runtime/client" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../test-utils/runtime/client" } kvdb-memorydb = "0.4.0" -sp-panic-handler = { version = "2.0.0", path = "../primitives/panic-handler" } +sp-panic-handler = { version = "2.0.0-alpha.2", path = "../primitives/panic-handler" } diff --git a/client/api/Cargo.toml b/client/api/Cargo.toml index 27a40c4d94cfbd70a77bfeac94dc770c264b7268..7ceb12eaf60bb780bca061539852bc1a42c8819f 100644 --- a/client/api/Cargo.toml +++ b/client/api/Cargo.toml @@ -1,35 +1,41 @@ [package] name = "sc-client-api" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Substrate client interfaces." +documentation = "https://docs.rs/sc-client-api" + [dependencies] -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-consensus = { version = "0.8", path = "../../primitives/consensus/common" } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../primitives/consensus/common" } derive_more = { version = "0.99.2" } -sc-executor = { version = "0.8", path = "../executor" } -sp-externalities = { version = "0.8.0", path = "../../primitives/externalities" } +sc-executor = { version = "0.8.0-alpha.2", path = "../executor" } +sp-externalities = { version = "0.8.0-alpha.2", path = "../../primitives/externalities" } fnv = { version = "1.0.6" } futures = { version = "0.3.1" } hash-db = { version = "0.15.2", default-features = false } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../primitives/blockchain" } hex-literal = { version = "0.2.1" } -sp-inherents = { version = "2.0.0", default-features = false, path = "../../primitives/inherents" } -sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } +sp-inherents = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/inherents" } +sp-keyring = { version = "2.0.0-alpha.2", path = "../../primitives/keyring" } kvdb = "0.4.0" log = { version = "0.4.8" } parking_lot = "0.10.0" -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-version = { version = "2.0.0", default-features = false, path = "../../primitives/version" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -sp-state-machine = { version = "0.8", path = "../../primitives/state-machine" } -sc-telemetry = { version = "2.0.0", path = "../telemetry" } -sp-trie = { version = "2.0.0", path = "../../primitives/trie" } -sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } +sp-core = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/std" } +sp-version = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/version" } +sp-api = { version = "2.0.0-alpha.2", path = "../../primitives/api" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/runtime" } +sp-state-machine = { version = "0.8.0-alpha.2", path = "../../primitives/state-machine" } +sc-telemetry = { version = "2.0.0-alpha.2", path = "../telemetry" } +sp-trie = { version = "2.0.0-alpha.2", path = "../../primitives/trie" } +sp-storage = { version = "2.0.0-alpha.2", path = "../../primitives/storage" } +sp-transaction-pool = { version = "2.0.0-alpha.2", path = "../../primitives/transaction-pool" } [dev-dependencies] -sp-test-primitives = { version = "2.0.0", path = "../../primitives/test-primitives" } +sp-test-primitives = { version = "2.0.0-dev", path = "../../primitives/test-primitives" } diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index a389af5671b327cb21f7454d31848fe85d645e15..d10e62cc54920e56509ddbb06a51f38296b417c1 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -21,11 +21,12 @@ use std::collections::HashMap; use sp_core::ChangesTrieConfigurationRange; use sp_core::offchain::OffchainStorage; use sp_runtime::{generic::BlockId, Justification, Storage}; -use sp_runtime::traits::{Block as BlockT, NumberFor, HasherFor}; +use sp_runtime::traits::{Block as BlockT, NumberFor, HashFor}; use sp_state_machine::{ ChangesTrieState, ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction, StorageCollection, ChildStorageCollection, }; +use sp_storage::{StorageData, StorageKey, ChildInfo}; use crate::{ blockchain::{ Backend as BlockchainBackend, well_known_cache_keys @@ -38,12 +39,13 @@ use sp_consensus::BlockOrigin; use parking_lot::RwLock; pub use sp_state_machine::Backend as StateBackend; +use std::marker::PhantomData; /// Extracts the state backend type for the given backend. pub type StateBackendFor = >::State; /// Extracts the transaction for the given state backend. -pub type TransactionForSB = >>::Transaction; +pub type TransactionForSB = >>::Transaction; /// Extracts the transaction for the given backend. pub type TransactionFor = TransactionForSB, Block>; @@ -111,7 +113,7 @@ impl NewBlockState { /// Keeps hold if the inserted block state and data. pub trait BlockImportOperation { /// Associated state backend type. - type State: StateBackend>; + type State: StateBackend>; /// Returns pending state. /// @@ -149,7 +151,7 @@ pub trait BlockImportOperation { /// Inject changes trie data into the database. fn update_changes_trie( &mut self, - update: ChangesTrieTransaction, NumberFor>, + update: ChangesTrieTransaction, NumberFor>, ) -> sp_blockchain::Result<()>; /// Insert auxiliary keys. @@ -169,6 +171,15 @@ pub trait BlockImportOperation { fn mark_head(&mut self, id: BlockId) -> sp_blockchain::Result<()>; } +/// Interface for performing operations on the backend. +pub trait LockImportRun> { + /// Lock the import lock, and run operations inside. + fn lock_import_and_run(&self, f: F) -> Result + where + F: FnOnce(&mut ClientImportOperation) -> Result, + Err: From; +} + /// Finalize Facilities pub trait Finalizer> { /// Mark all blocks up to given as finalized in operation. @@ -228,6 +239,123 @@ pub trait AuxStore { fn get_aux(&self, key: &[u8]) -> sp_blockchain::Result>>; } +/// An `Iterator` that iterates keys in a given block under a prefix. +pub struct KeyIterator<'a, State, Block> { + state: State, + prefix: Option<&'a StorageKey>, + current_key: Vec, + _phantom: PhantomData, +} + +impl <'a, State, Block> KeyIterator<'a, State, Block> { + /// create a KeyIterator instance + pub fn new(state: State, prefix: Option<&'a StorageKey>, current_key: Vec) -> Self { + Self { + state, + prefix, + current_key, + _phantom: PhantomData, + } + } +} + +impl<'a, State, Block> Iterator for KeyIterator<'a, State, Block> where + Block: BlockT, + State: StateBackend>, +{ + type Item = StorageKey; + + fn next(&mut self) -> Option { + let next_key = self.state + .next_storage_key(&self.current_key) + .ok() + .flatten()?; + // this terminates the iterator the first time it fails. + if let Some(prefix) = self.prefix { + if !next_key.starts_with(&prefix.0[..]) { + return None; + } + } + self.current_key = next_key.clone(); + Some(StorageKey(next_key)) + } +} +/// Provides acess to storage primitives +pub trait StorageProvider> { + /// Given a `BlockId` and a key, return the value under the key in that block. + fn storage(&self, id: &BlockId, key: &StorageKey) -> sp_blockchain::Result>; + + /// Given a `BlockId` and a key prefix, return the matching storage keys in that block. + fn storage_keys(&self, id: &BlockId, key_prefix: &StorageKey) -> sp_blockchain::Result>; + + /// Given a `BlockId` and a key, return the value under the hash in that block. + fn storage_hash(&self, id: &BlockId, key: &StorageKey) -> sp_blockchain::Result>; + + /// Given a `BlockId` and a key prefix, return the matching child storage keys and values in that block. + fn storage_pairs( + &self, + id: &BlockId, + key_prefix: &StorageKey + ) -> sp_blockchain::Result>; + + /// Given a `BlockId` and a key prefix, return a `KeyIterator` iterates matching storage keys in that block. + fn storage_keys_iter<'a>( + &self, + id: &BlockId, + prefix: Option<&'a StorageKey>, + start_key: Option<&StorageKey> + ) -> sp_blockchain::Result>; + + /// Given a `BlockId`, a key and a child storage key, return the value under the key in that block. + fn child_storage( + &self, + id: &BlockId, + storage_key: &StorageKey, + child_info: ChildInfo, + key: &StorageKey + ) -> sp_blockchain::Result>; + + /// Given a `BlockId`, a key prefix, and a child storage key, return the matching child storage keys. + fn child_storage_keys( + &self, + id: &BlockId, + child_storage_key: &StorageKey, + child_info: ChildInfo, + key_prefix: &StorageKey + ) -> sp_blockchain::Result>; + + /// Given a `BlockId`, a key and a child storage key, return the hash under the key in that block. + fn child_storage_hash( + &self, + id: &BlockId, + storage_key: &StorageKey, + child_info: ChildInfo, + key: &StorageKey + ) -> sp_blockchain::Result>; + + /// Get longest range within [first; last] that is possible to use in `key_changes` + /// and `key_changes_proof` calls. + /// Range could be shortened from the beginning if some changes tries have been pruned. + /// Returns Ok(None) if changes tries are not supported. + fn max_key_changes_range( + &self, + first: NumberFor, + last: BlockId, + ) -> sp_blockchain::Result, BlockId)>>; + + /// Get pairs of (block, extrinsic) where key has been changed at given blocks range. + /// Works only for runtimes that are supporting changes tries. + /// + /// Changes are returned in descending order (i.e. last block comes first). + fn key_changes( + &self, + first: NumberFor, + last: BlockId, + storage_key: Option<&StorageKey>, + key: &StorageKey + ) -> sp_blockchain::Result, u32)>>; +} + /// Client backend. /// /// Manages the data layer. @@ -244,7 +372,7 @@ pub trait Backend: AuxStore + Send + Sync { /// Associated blockchain backend type. type Blockchain: BlockchainBackend; /// Associated state backend type. - type State: StateBackend> + Send; + type State: StateBackend> + Send; /// Offchain workers local storage. type OffchainStorage: OffchainStorage; @@ -292,11 +420,6 @@ pub trait Backend: AuxStore + Send + Sync { /// Returns state backend with post-state of given block. fn state_at(&self, block: BlockId) -> sp_blockchain::Result; - /// Destroy state and save any useful data, such as cache. - fn destroy_state(&self, _state: Self::State) -> sp_blockchain::Result<()> { - Ok(()) - } - /// Attempts to revert the chain by `n` blocks. If `revert_finalized` is set /// it will attempt to revert past any finalized block, this is unsafe and /// can potentially leave the node in an inconsistent state. @@ -335,10 +458,10 @@ pub trait Backend: AuxStore + Send + Sync { /// Changes trie storage that supports pruning. pub trait PrunableStateChangesTrieStorage: - StateChangesTrieStorage, NumberFor> + StateChangesTrieStorage, NumberFor> { /// Get reference to StateChangesTrieStorage. - fn storage(&self) -> &dyn StateChangesTrieStorage, NumberFor>; + fn storage(&self) -> &dyn StateChangesTrieStorage, NumberFor>; /// Get configuration at given block. fn configuration_at(&self, at: &BlockId) -> sp_blockchain::Result< ChangesTrieConfigurationRange, Block::Hash> @@ -368,7 +491,7 @@ pub trait RemoteBackend: Backend { pub fn changes_tries_state_at_block<'a, Block: BlockT>( block: &BlockId, maybe_storage: Option<&'a dyn PrunableStateChangesTrieStorage>, -) -> sp_blockchain::Result, NumberFor>>> { +) -> sp_blockchain::Result, NumberFor>>> { let storage = match maybe_storage { Some(storage) => storage, None => return Ok(None), diff --git a/client/api/src/call_executor.rs b/client/api/src/call_executor.rs index aff38a26aadd6e4e784471ddd4ba0efe9cb17f36..f39d7971578967724c638f21842e2fa82aed55f5 100644 --- a/client/api/src/call_executor.rs +++ b/client/api/src/call_executor.rs @@ -19,7 +19,7 @@ use std::{panic::UnwindSafe, result, cell::RefCell}; use codec::{Encode, Decode}; use sp_runtime::{ - generic::BlockId, traits::{Block as BlockT, HasherFor}, + generic::BlockId, traits::{Block as BlockT, HashFor}, }; use sp_state_machine::{ OverlayedChanges, ExecutionManager, ExecutionStrategy, StorageProof, @@ -29,6 +29,18 @@ use sp_externalities::Extensions; use sp_core::NativeOrEncoded; use sp_api::{ProofRecorder, InitializeBlock, StorageTransactionCache}; +use crate::execution_extensions::ExecutionExtensions; + +/// Executor Provider +pub trait ExecutorProvider { + /// executor instance + type Executor: CallExecutor; + /// Get call executor reference. + fn executor(&self) -> &Self::Executor; + + /// Get a reference to the execution extensions. + fn execution_extensions(&self) -> &ExecutionExtensions; +} /// Method call executor. pub trait CallExecutor { @@ -89,7 +101,7 @@ pub trait CallExecutor { /// Execute a call to a contract on top of given state, gathering execution proof. /// /// No changes are made. - fn prove_at_state>>( + fn prove_at_state>>( &self, mut state: S, overlay: &mut OverlayedChanges, @@ -107,9 +119,9 @@ pub trait CallExecutor { /// Execute a call to a contract on top of given trie state, gathering execution proof. /// /// No changes are made. - fn prove_at_trie_state>>( + fn prove_at_trie_state>>( &self, - trie_state: &sp_state_machine::TrieBackend>, + trie_state: &sp_state_machine::TrieBackend>, overlay: &mut OverlayedChanges, method: &str, call_data: &[u8] diff --git a/client/api/src/client.rs b/client/api/src/client.rs index 7503ce4a79e4627eb9fac7527e3670cfb35c23f3..22503732be4e98bddc150cad0bd08ea142e6d772 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -21,7 +21,7 @@ use futures::channel::mpsc; use sp_core::storage::StorageKey; use sp_runtime::{ traits::{Block as BlockT, NumberFor}, - generic::BlockId + generic::{BlockId, SignedBlock} }; use sp_consensus::BlockOrigin; @@ -76,9 +76,13 @@ pub trait BlockchainEvents { /// Fetch block body by ID. pub trait BlockBody { /// Get block body by ID. Returns `None` if the body is not stored. - fn block_body(&self, + fn block_body( + &self, id: &BlockId ) -> sp_blockchain::Result::Extrinsic>>>; + + /// Get full block by id. + fn block(&self, id: &BlockId) -> sp_blockchain::Result>>; } /// Provide a list of potential uncle headers for a given block. @@ -97,13 +101,56 @@ pub struct ClientInfo { pub usage: Option, } +/// A wrapper to store the size of some memory. +#[derive(Default, Clone, Debug, Copy)] +pub struct MemorySize(usize); + +impl MemorySize { + /// Creates `Self` from the given `bytes` size. + pub fn from_bytes(bytes: usize) -> Self { + Self(bytes) + } + + /// Returns the memory size as bytes. + pub fn as_bytes(self) -> usize { + self.0 + } +} + +impl fmt::Display for MemorySize { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.0 < 1024 { + write!(f, "{} bytes", self.0) + } else if self.0 < 1024 * 1024 { + write!(f, "{:.2} KiB", self.0 as f64 / 1024f64) + } else if self.0 < 1024 * 1024 * 1024 { + write!(f, "{:.2} MiB", self.0 as f64 / (1024f64 * 1024f64)) + } else { + write!(f, "{:.2} GiB", self.0 as f64 / (1024f64 * 1024f64 * 1024f64)) + } + } +} + +/// Memory statistics for state db. +#[derive(Default, Clone, Debug)] +pub struct StateDbMemoryInfo { + /// Memory usage of the non-canonical overlay + pub non_canonical: MemorySize, + /// Memory usage of the pruning window. + pub pruning: Option, + /// Memory usage of the pinned blocks. + pub pinned: MemorySize, +} + /// Memory statistics for client instance. #[derive(Default, Clone, Debug)] pub struct MemoryInfo { /// Size of state cache. - pub state_cache: usize, + pub state_cache: MemorySize, /// Size of backend database cache. - pub database_cache: usize, + pub database_cache: MemorySize, + /// Size of the state db. + pub state_db: StateDbMemoryInfo, } /// I/O statistics for client instance. @@ -144,10 +191,16 @@ pub struct UsageInfo { impl fmt::Display for UsageInfo { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, - "caches: ({} state, {} db overlay), i/o: ({} tx, {} write, {} read, {} avg tx, {}/{} key cache reads/total, {} key writes)", + write!( + f, + "caches: ({} state, {} db overlay), \ + state db: ({} non-canonical, {} pruning, {} pinned), \ + i/o: ({} tx, {} write, {} read, {} avg tx, {}/{} key cache reads/total, {} key writes)", self.memory.state_cache, self.memory.database_cache, + self.memory.state_db.non_canonical, + self.memory.state_db.pruning.unwrap_or_default(), + self.memory.state_db.pinned, self.io.transactions, self.io.bytes_written, self.io.bytes_read, diff --git a/client/api/src/lib.rs b/client/api/src/lib.rs index 69d0c94ac2b8bb9ef0bd71661c76a922b8fba977..66f51e75c793782123757be13a24c23ede1f96fa 100644 --- a/client/api/src/lib.rs +++ b/client/api/src/lib.rs @@ -23,6 +23,7 @@ pub mod client; pub mod execution_extensions; pub mod light; pub mod notifications; +pub mod proof_provider; pub use sp_blockchain as blockchain; pub use backend::*; @@ -31,6 +32,7 @@ pub use call_executor::*; pub use client::*; pub use light::*; pub use notifications::*; +pub use proof_provider::*; pub use sp_state_machine::{StorageProof, ExecutionStrategy}; diff --git a/client/api/src/proof_provider.rs b/client/api/src/proof_provider.rs new file mode 100644 index 0000000000000000000000000000000000000000..2d9876f7ad278dd7067199f9090122d5c9c073ab --- /dev/null +++ b/client/api/src/proof_provider.rs @@ -0,0 +1,71 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . +//! Proof utilities +use sp_runtime::{ + generic::BlockId, + traits::{Block as BlockT}, +}; +use crate::{StorageProof, ChangesProof}; +use sp_storage::{ChildInfo, StorageKey}; + +/// Interface for providing block proving utilities. +pub trait ProofProvider { + /// Reads storage value at a given block + key, returning read proof. + fn read_proof( + &self, + id: &BlockId, + keys: &mut dyn Iterator, + ) -> sp_blockchain::Result; + + /// Reads child storage value at a given block + storage_key + key, returning + /// read proof. + fn read_child_proof( + &self, + id: &BlockId, + storage_key: &[u8], + child_info: ChildInfo, + keys: &mut dyn Iterator, + ) -> sp_blockchain::Result; + + /// Execute a call to a contract on top of state in a block of given hash + /// AND returning execution proof. + /// + /// No changes are made. + fn execution_proof( + &self, + id: &BlockId, + method: &str, + call_data: &[u8], + ) -> sp_blockchain::Result<(Vec, StorageProof)>; + /// Reads given header and generates CHT-based header proof. + fn header_proof(&self, id: &BlockId) -> sp_blockchain::Result<(Block::Header, StorageProof)>; + + /// Get proof for computation of (block, extrinsic) pairs where key has been changed at given blocks range. + /// `min` is the hash of the first block, which changes trie root is known to the requester - when we're using + /// changes tries from ascendants of this block, we should provide proofs for changes tries roots + /// `max` is the hash of the last block known to the requester - we can't use changes tries from descendants + /// of this block. + /// Works only for runtimes that are supporting changes tries. + fn key_changes_proof( + &self, + first: Block::Hash, + last: Block::Hash, + min: Block::Hash, + max: Block::Hash, + storage_key: Option<&StorageKey>, + key: &StorageKey, + ) -> sp_blockchain::Result>; +} diff --git a/client/authority-discovery/Cargo.toml b/client/authority-discovery/Cargo.toml index 248304d137018b0d0fa6e86490ac1f4e6016db72..0c3b739bb9deb9ca582d88e49821ad15cae593cf 100644 --- a/client/authority-discovery/Cargo.toml +++ b/client/authority-discovery/Cargo.toml @@ -1,36 +1,39 @@ [package] name = "sc-authority-discovery" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Substrate authority discovery." [build-dependencies] prost-build = "0.6.1" [dependencies] -bytes = "0.4.12" -codec = { package = "parity-scale-codec", default-features = false, version = "1.0.3" } +bytes = "0.5.0" +codec = { package = "parity-scale-codec", default-features = false, version = "1.2.0" } derive_more = "0.99.2" futures = "0.3.1" futures-timer = "3.0.1" -libp2p = { version = "0.16.0", default-features = false, features = ["secp256k1", "libp2p-websocket"] } +libp2p = { version = "0.16.2", default-features = false, features = ["secp256k1", "libp2p-websocket"] } log = "0.4.8" prost = "0.6.1" rand = "0.7.2" -sc-client-api = { version = "2.0.0", path = "../api" } -sc-keystore = { version = "2.0.0", path = "../keystore" } -sc-network = { version = "0.8", path = "../network" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../api" } +sc-keystore = { version = "2.0.0-alpha.2", path = "../keystore" } +sc-network = { version = "0.8.0-alpha.2", path = "../network" } serde_json = "1.0.41" -sp-authority-discovery = { version = "2.0.0", path = "../../primitives/authority-discovery" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } +sp-authority-discovery = { version = "2.0.0-alpha.2", path = "../../primitives/authority-discovery" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../primitives/blockchain" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } +sp-api = { version = "2.0.0-alpha.2", path = "../../primitives/api" } [dev-dependencies] env_logger = "0.7.0" quickcheck = "0.9.0" -sc-peerset = { version = "2.0.0", path = "../peerset" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client"} +sc-peerset = { version = "2.0.0-alpha.2", path = "../peerset" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client"} diff --git a/client/authority-discovery/src/lib.rs b/client/authority-discovery/src/lib.rs index 6260ac9a85b12ec0af93227048e8279438161c11..92dcc264502f03612463b44d128148a2aeddebe3 100644 --- a/client/authority-discovery/src/lib.rs +++ b/client/authority-discovery/src/lib.rs @@ -60,7 +60,6 @@ use libp2p::Multiaddr; use log::{debug, error, log_enabled, warn}; use prost::Message; use sc_client_api::blockchain::HeaderBackend; -use sc_network::specialization::NetworkSpecialization; use sc_network::{DhtEvent, ExHashT, NetworkStateInfo}; use sp_authority_discovery::{AuthorityDiscoveryApi, AuthorityId, AuthoritySignature, AuthorityPair}; use sp_core::crypto::{key_types, Pair}; @@ -477,10 +476,9 @@ pub trait NetworkProvider: NetworkStateInfo { fn get_value(&self, key: &libp2p::kad::record::Key); } -impl NetworkProvider for sc_network::NetworkService +impl NetworkProvider for sc_network::NetworkService where B: BlockT + 'static, - S: NetworkSpecialization, H: ExHashT, { fn set_priority_group( diff --git a/client/authority-discovery/src/tests.rs b/client/authority-discovery/src/tests.rs index 77ed6a1d948e2897ea331a5727d49e1d7e416200..55ee6c7aac13ace754865a6ae32de814a29eb4a3 100644 --- a/client/authority-discovery/src/tests.rs +++ b/client/authority-discovery/src/tests.rs @@ -215,7 +215,7 @@ impl ApiExt for RuntimeApi { fn into_storage_changes( &self, _: &Self::StateBackend, - _: Option<&sp_api::ChangesTrieState, sp_api::NumberFor>>, + _: Option<&sp_api::ChangesTrieState, sp_api::NumberFor>>, _: ::Hash, ) -> std::result::Result, String> where Self: Sized diff --git a/client/basic-authorship/Cargo.toml b/client/basic-authorship/Cargo.toml index 6a013f7a74fce889d329bc930da20c8aca02cbd9..7503221f6556072b8b45d69b5b1f420397638a9c 100644 --- a/client/basic-authorship/Cargo.toml +++ b/client/basic-authorship/Cargo.toml @@ -1,28 +1,30 @@ [package] name = "sc-basic-authorship" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Basic implementation of block-authoring logic." [dependencies] log = "0.4.8" futures = "0.3.1" -codec = { package = "parity-scale-codec", version = "1.0.0" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } -sc-client = { version = "0.8", path = "../" } -sc-client-api = { version = "2.0.0", path = "../api" } -sp-consensus = { version = "0.8", path = "../../primitives/consensus/common" } -sp-inherents = { version = "2.0.0", path = "../../primitives/inherents" } -sc-telemetry = { version = "2.0.0", path = "../telemetry" } -sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } -sc-block-builder = { version = "0.8", path = "../block-builder" } +codec = { package = "parity-scale-codec", version = "1.2.0" } +sp-api = { version = "2.0.0-alpha.2", path = "../../primitives/api" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../primitives/blockchain" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../api" } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../primitives/consensus/common" } +sp-inherents = { version = "2.0.0-alpha.2", path = "../../primitives/inherents" } +sc-telemetry = { version = "2.0.0-alpha.2", path = "../telemetry" } +sp-transaction-pool = { version = "2.0.0-alpha.2", path = "../../primitives/transaction-pool" } +sc-block-builder = { version = "0.8.0-alpha.2", path = "../block-builder" } tokio-executor = { version = "0.2.0-alpha.6", features = ["blocking"] } [dev-dependencies] -sc-transaction-pool = { version = "2.0.0", path = "../../client/transaction-pool" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } +sc-transaction-pool = { version = "2.0.0-alpha.2", path = "../../client/transaction-pool" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } parking_lot = "0.10.0" diff --git a/client/basic-authorship/src/basic_authorship.rs b/client/basic-authorship/src/basic_authorship.rs index a99453544e5f63e3a6a6e5d62845b465d4bc86d8..41216c2b5442116ca8eac2bcbabeaafbba940254 100644 --- a/client/basic-authorship/src/basic_authorship.rs +++ b/client/basic-authorship/src/basic_authorship.rs @@ -19,48 +19,59 @@ // FIXME #1021 move this into sp-consensus use std::{time, sync::Arc}; -use sc_client_api::{CallExecutor, backend}; -use sc_client::Client as SubstrateClient; +use sc_client_api::backend; use codec::Decode; use sp_consensus::{evaluation, Proposal, RecordProof}; use sp_inherents::InherentData; use log::{error, info, debug, trace}; use sp_core::ExecutionContext; use sp_runtime::{ - traits::{Block as BlockT, Hash as HashT, Header as HeaderT, DigestFor, BlakeTwo256}, generic::BlockId, + traits::{Block as BlockT, Hash as HashT, Header as HeaderT, DigestFor, BlakeTwo256}, }; use sp_transaction_pool::{TransactionPool, InPoolTransaction}; use sc_telemetry::{telemetry, CONSENSUS_INFO}; -use sc_block_builder::BlockBuilderApi; +use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider}; use sp_api::{ProvideRuntimeApi, ApiExt}; use futures::prelude::*; +use sp_blockchain::{HeaderBackend, ApplyExtrinsicFailed}; +use std::marker::PhantomData; /// Proposer factory. -pub struct ProposerFactory where A: TransactionPool { +pub struct ProposerFactory { /// The client instance. - pub client: Arc, + client: Arc, /// The transaction pool. - pub transaction_pool: Arc, + transaction_pool: Arc, + /// phantom member to pin the `Backend` type. + _phantom: PhantomData, +} + +impl ProposerFactory { + pub fn new(client: Arc, transaction_pool: Arc) -> Self { + ProposerFactory { + client, + transaction_pool, + _phantom: PhantomData, + } + } } -impl ProposerFactory, A> +impl ProposerFactory where A: TransactionPool + 'static, B: backend::Backend + Send + Sync + 'static, - E: CallExecutor + Send + Sync + Clone + 'static, Block: BlockT, - RA: Send + Sync + 'static, - SubstrateClient: ProvideRuntimeApi, - as ProvideRuntimeApi>::Api: - BlockBuilderApi + - ApiExt>, + C: BlockBuilderProvider + HeaderBackend + ProvideRuntimeApi + + Send + Sync + 'static, + C::Api: ApiExt> + + BlockBuilderApi, { pub fn init_with_now( &mut self, parent_header: &::Header, now: Box time::Instant + Send + Sync>, - ) -> Proposer, A> { + ) -> Proposer { let parent_hash = parent_header.hash(); let id = BlockId::hash(parent_hash); @@ -75,6 +86,7 @@ impl ProposerFactory, A> parent_number: *parent_header.number(), transaction_pool: self.transaction_pool.clone(), now, + _phantom: PhantomData, }), }; @@ -82,21 +94,19 @@ impl ProposerFactory, A> } } -impl sp_consensus::Environment for - ProposerFactory, A> +impl sp_consensus::Environment for + ProposerFactory where A: TransactionPool + 'static, B: backend::Backend + Send + Sync + 'static, - E: CallExecutor + Send + Sync + Clone + 'static, Block: BlockT, - RA: Send + Sync + 'static, - SubstrateClient: ProvideRuntimeApi, - as ProvideRuntimeApi>::Api: - BlockBuilderApi + - ApiExt>, + C: BlockBuilderProvider + HeaderBackend + ProvideRuntimeApi + + Send + Sync + 'static, + C::Api: ApiExt> + + BlockBuilderApi, { type CreateProposer = future::Ready>; - type Proposer = Proposer, A>; + type Proposer = Proposer; type Error = sp_blockchain::Error; fn init( @@ -108,32 +118,31 @@ impl sp_consensus::Environment for } /// The proposer logic. -pub struct Proposer { - inner: Arc>, +pub struct Proposer { + inner: Arc>, } /// Proposer inner, to wrap parameters under Arc. -struct ProposerInner { +struct ProposerInner { client: Arc, parent_hash: ::Hash, parent_id: BlockId, parent_number: <::Header as HeaderT>::Number, transaction_pool: Arc, now: Box time::Instant + Send + Sync>, + _phantom: PhantomData, } -impl sp_consensus::Proposer for - Proposer, A> +impl sp_consensus::Proposer for + Proposer where A: TransactionPool + 'static, B: backend::Backend + Send + Sync + 'static, - E: CallExecutor + Send + Sync + Clone + 'static, Block: BlockT, - RA: Send + Sync + 'static, - SubstrateClient: ProvideRuntimeApi, - as ProvideRuntimeApi>::Api: - BlockBuilderApi + - ApiExt>, + C: BlockBuilderProvider + HeaderBackend + ProvideRuntimeApi + + Send + Sync + 'static, + C::Api: ApiExt> + + BlockBuilderApi, { type Transaction = backend::TransactionFor; type Proposal = tokio_executor::blocking::Blocking< @@ -157,16 +166,15 @@ impl sp_consensus::Proposer for } } -impl ProposerInner, A> where - A: TransactionPool, - B: sc_client_api::backend::Backend + Send + Sync + 'static, - E: CallExecutor + Send + Sync + Clone + 'static, - Block: BlockT, - RA: Send + Sync + 'static, - SubstrateClient: ProvideRuntimeApi, - as ProvideRuntimeApi>::Api: - BlockBuilderApi + - ApiExt>, +impl ProposerInner + where + A: TransactionPool, + B: backend::Backend + Send + Sync + 'static, + Block: BlockT, + C: BlockBuilderProvider + HeaderBackend + ProvideRuntimeApi + + Send + Sync + 'static, + C::Api: ApiExt> + + BlockBuilderApi, { fn propose_with( &self, @@ -222,7 +230,7 @@ impl ProposerInner, Ok(()) => { debug!("[{:?}] Pushed to the block.", pending_tx_hash); } - Err(sp_blockchain::Error::ApplyExtrinsicFailed(sp_blockchain::ApplyExtrinsicFailed::Validity(e))) + Err(sp_blockchain::Error::ApplyExtrinsicFailed(ApplyExtrinsicFailed::Validity(e))) if e.exhausted_resources() => { if is_first { debug!("[{:?}] Invalid transaction: FullBlock on empty block", pending_tx_hash); @@ -238,6 +246,13 @@ impl ProposerInner, break; } } + Err(e) if skipped > 0 => { + trace!( + "[{:?}] Ignoring invalid transaction when skipping: {}", + pending_tx_hash, + e + ); + } Err(e) => { debug!("[{:?}] Invalid transaction: {}", pending_tx_hash, e); unqueue_invalid.push(pending_tx_hash); @@ -284,10 +299,10 @@ mod tests { use super::*; use parking_lot::Mutex; - use sp_consensus::Proposer; + use sp_consensus::{BlockOrigin, Proposer}; use substrate_test_runtime_client::{ - runtime::{Extrinsic, Transfer}, AccountKeyring, DefaultTestClientBuilderExt, - TestClientBuilderExt, + prelude::*, + runtime::{Extrinsic, Transfer}, }; use sc_transaction_pool::{BasicPool, FullChainApi}; use sp_api::Core; @@ -315,10 +330,7 @@ mod tests { txpool.submit_at(&BlockId::number(0), vec![extrinsic(0), extrinsic(1)]) ).unwrap(); - let mut proposer_factory = ProposerFactory { - client: client.clone(), - transaction_pool: txpool.clone(), - }; + let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone()); let cell = Mutex::new(time::Instant::now()); let mut proposer = proposer_factory.init_with_now( @@ -359,10 +371,7 @@ mod tests { txpool.submit_at(&BlockId::number(0), vec![extrinsic(0)]), ).unwrap(); - let mut proposer_factory = ProposerFactory { - client: client.clone(), - transaction_pool: txpool.clone(), - }; + let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone()); let mut proposer = proposer_factory.init_with_now( &client.header(&block_id).unwrap().unwrap(), @@ -393,4 +402,73 @@ mod tests { storage_changes.transaction_storage_root, ); } + + #[test] + fn should_not_remove_invalid_transactions_when_skipping() { + // given + let mut client = Arc::new(substrate_test_runtime_client::new()); + let txpool = Arc::new( + BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone()))).0 + ); + + futures::executor::block_on( + txpool.submit_at(&BlockId::number(0), vec![ + extrinsic(0), + extrinsic(1), + Transfer { + amount: Default::default(), + nonce: 2, + from: AccountKeyring::Alice.into(), + to: Default::default(), + }.into_resources_exhausting_tx(), + extrinsic(3), + Transfer { + amount: Default::default(), + nonce: 4, + from: AccountKeyring::Alice.into(), + to: Default::default(), + }.into_resources_exhausting_tx(), + extrinsic(5), + extrinsic(6), + ]) + ).unwrap(); + + let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone()); + let mut propose_block = | + client: &TestClient, + number, + expected_block_extrinsics, + expected_pool_transactions, + | { + let mut proposer = proposer_factory.init_with_now( + &client.header(&BlockId::number(number)).unwrap().unwrap(), + Box::new(move || time::Instant::now()), + ); + + // when + let deadline = time::Duration::from_secs(9); + let block = futures::executor::block_on( + proposer.propose(Default::default(), Default::default(), deadline, RecordProof::No) + ).map(|r| r.block).unwrap(); + + // then + // block should have some extrinsics although we have some more in the pool. + assert_eq!(block.extrinsics().len(), expected_block_extrinsics); + assert_eq!(txpool.ready().count(), expected_pool_transactions); + + block + }; + + // let's create one block and import it + let block = propose_block(&client, 0, 2, 7); + client.import(BlockOrigin::Own, block).unwrap(); + + // now let's make sure that we can still make some progress + + // This is most likely incorrect, and caused by #5139 + let tx_remaining = 0; + let block = propose_block(&client, 1, 2, tx_remaining); + client.import(BlockOrigin::Own, block).unwrap(); + } } + diff --git a/client/basic-authorship/src/lib.rs b/client/basic-authorship/src/lib.rs index e9087c89e07ed373ec2cb932b37b2b6822f570e3..5ec0bc6f9a520c5d8292ce21cc219b5a8a658bab 100644 --- a/client/basic-authorship/src/lib.rs +++ b/client/basic-authorship/src/lib.rs @@ -28,10 +28,7 @@ //! # let client = Arc::new(substrate_test_runtime_client::new()); //! # let txpool = Arc::new(BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone()))).0); //! // The first step is to create a `ProposerFactory`. -//! let mut proposer_factory = ProposerFactory { -//! client: client.clone(), -//! transaction_pool: txpool.clone(), -//! }; +//! let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone()); //! //! // From this factory, we create a `Proposer`. //! let proposer = proposer_factory.init( diff --git a/client/block-builder/Cargo.toml b/client/block-builder/Cargo.toml index 383a931b2f8998288a362713a6a24aacaeff70c9..745669c033e3870d27cb4b349bb2aee05187c9cd 100644 --- a/client/block-builder/Cargo.toml +++ b/client/block-builder/Cargo.toml @@ -1,17 +1,21 @@ [package] name = "sc-block-builder" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Substrate block builder" + [dependencies] -sp-state-machine = { version = "0.8", path = "../../primitives/state-machine" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } -sp-consensus = { version = "0.8.0", path = "../../primitives/consensus/common" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-block-builder = { version = "2.0.0", path = "../../primitives/block-builder" } -sc-client-api = { version = "2.0.0", path = "../api" } -codec = { package = "parity-scale-codec", version = "1.0.6", features = ["derive"] } +sp-state-machine = { version = "0.8.0-alpha.2", path = "../../primitives/state-machine" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } +sp-api = { version = "2.0.0-alpha.2", path = "../../primitives/api" } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../primitives/consensus/common" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../primitives/blockchain" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +sp-block-builder = { version = "2.0.0-alpha.2", path = "../../primitives/block-builder" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../api" } +codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index 26bc9ecea8deac6ecd5f3988878555183cc26aa7..2666fd9cd713999f863729b50d9afbdcdb5ec9f6 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -28,9 +28,7 @@ use codec::Encode; use sp_runtime::{ generic::BlockId, - traits::{ - Header as HeaderT, Hash, Block as BlockT, HashFor, DigestFor, NumberFor, One, HasherFor, - }, + traits::{Header as HeaderT, Hash, Block as BlockT, HashFor, DigestFor, NumberFor, One}, }; use sp_blockchain::{ApplyExtrinsicFailed, Error}; use sp_core::ExecutionContext; @@ -47,7 +45,7 @@ use sc_client_api::backend; /// backend to get the state of the block. Furthermore an optional `proof` is included which /// can be used to proof that the build block contains the expected data. The `proof` will /// only be set when proof recording was activated. -pub struct BuiltBlock>> { +pub struct BuiltBlock>> { /// The actual block that was build. pub block: Block, /// The changes that need to be applied to the backend to get the state of the build block. @@ -56,13 +54,34 @@ pub struct BuiltBlock, } -impl>> BuiltBlock { +impl>> BuiltBlock { /// Convert into the inner values. pub fn into_inner(self) -> (Block, StorageChanges, Option) { (self.block, self.storage_changes, self.proof) } } +/// Block builder provider +pub trait BlockBuilderProvider + where + Block: BlockT, + B: backend::Backend, + Self: Sized, + RA: ProvideRuntimeApi, +{ + /// Create a new block, built on top of `parent`. + /// + /// When proof recording is enabled, all accessed trie nodes are saved. + /// These recorded trie nodes can be used by a third party to proof the + /// output of this block builder without having access to the full storage. + fn new_block_at>( + &self, + parent: &BlockId, + inherent_digests: DigestFor, + record_proof: R, + ) -> sp_blockchain::Result>; +} + /// Utility for building new (valid) blocks from a stream of extrinsics. pub struct BlockBuilder<'a, Block: BlockT, A: ProvideRuntimeApi, B> { extrinsics: Vec, @@ -121,7 +140,7 @@ where backend, }) } - + /// Push onto the block's list of extrinsics. /// /// This will ensure the extrinsic can be validly executed (by executing it). @@ -131,8 +150,7 @@ where /// Push onto the block's list of extrinsics. /// - /// This will treat incoming extrinsic `xt` as untrusted and perform additional checks - /// (currenty checking signature). + /// This will treat incoming extrinsic `xt` as trusted and skip signature check (for signed transactions). pub fn push_trusted(&mut self, xt: ::Extrinsic) -> Result<(), ApiErrorFor> { self.push_internal(xt, true) } @@ -141,60 +159,36 @@ where let block_id = &self.block_id; let extrinsics = &mut self.extrinsics; - if self + let use_trusted = skip_signature && self .api .has_api_with::>, _>( block_id, - |version| version < 4, - )? - { - // Run compatibility fallback for v3. - self.api.map_api_result(|api| { - #[allow(deprecated)] - match api.apply_extrinsic_before_version_4_with_context( + |version| version >= 5, + )?; + + self.api.map_api_result(|api| { + let apply_result = if use_trusted { + api.apply_trusted_extrinsic_with_context( block_id, ExecutionContext::BlockConstruction, xt.clone(), - )? { - Ok(_) => { - extrinsics.push(xt); - Ok(()) - } - Err(e) => Err(ApplyExtrinsicFailed::from(e).into()), - } - }) - } else { - let use_trusted = skip_signature && self - .api - .has_api_with::>, _>( + )? + } else { + api.apply_extrinsic_with_context( block_id, - |version| version >= 5, - )?; - - self.api.map_api_result(|api| { - let apply_result = if use_trusted { - api.apply_trusted_extrinsic_with_context( - block_id, - ExecutionContext::BlockConstruction, - xt.clone(), - )? - } else { - api.apply_extrinsic_with_context( - block_id, - ExecutionContext::BlockConstruction, - xt.clone(), - )? - }; - - match apply_result { - Ok(_) => { - extrinsics.push(xt); - Ok(()) - } - Err(tx_validity) => Err(ApplyExtrinsicFailed::Validity(tx_validity).into()), + ExecutionContext::BlockConstruction, + xt.clone(), + )? + }; + + match apply_result { + Ok(_) => { + extrinsics.push(xt); + Ok(()) } - }) - } + Err(tx_validity) => Err(ApplyExtrinsicFailed::Validity(tx_validity).into()), + } + }) } /// Consume the builder to build a valid `Block` containing all pushed extrinsics. @@ -230,17 +224,11 @@ where &state, changes_trie_state.as_ref(), parent_hash, - ); - - // We need to destroy the state, before we check if `storage_changes` is `Ok(_)` - { - let _lock = self.backend.get_import_lock().read(); - self.backend.destroy_state(state)?; - } + )?; Ok(BuiltBlock { block: ::new(header, self.extrinsics), - storage_changes: storage_changes?, + storage_changes, proof, }) } diff --git a/client/chain-spec/Cargo.toml b/client/chain-spec/Cargo.toml index 222914145e160cbc71e9e7007a7e1f70c01d1380..10d4fc2622480c70dbbd938e3210b1421c3e0f84 100644 --- a/client/chain-spec/Cargo.toml +++ b/client/chain-spec/Cargo.toml @@ -1,16 +1,19 @@ [package] name = "sc-chain-spec" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Substrate chain configurations." [dependencies] -sc-chain-spec-derive = { version = "2.0.0", path = "./derive" } +sc-chain-spec-derive = { version = "2.0.0-alpha.2", path = "./derive" } impl-trait-for-tuples = "0.1.3" -sc-network = { version = "0.8", path = "../network" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } +sc-network = { version = "0.8.0-alpha.2", path = "../network" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sc-telemetry = { version = "2.0.0", path = "../telemetry" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } +sc-telemetry = { version = "2.0.0-alpha.2", path = "../telemetry" } diff --git a/client/chain-spec/derive/Cargo.toml b/client/chain-spec/derive/Cargo.toml index 566948883bccb18890de62e54ae3a3547737357c..df0e2f92b2e94a2bfbf47d5fd63214925919c14d 100644 --- a/client/chain-spec/derive/Cargo.toml +++ b/client/chain-spec/derive/Cargo.toml @@ -1,9 +1,12 @@ [package] name = "sc-chain-spec-derive" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Macros to derive chain spec extension traits implementation." [lib] proc-macro = true @@ -11,7 +14,7 @@ proc-macro = true [dependencies] proc-macro-crate = "0.1.4" proc-macro2 = "1.0.6" -quote = "1.0.2" +quote = "1.0.3" syn = "1.0.7" [dev-dependencies] diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index ab9c851bdde5fb862188b871bbf35b063db4188e..a7e5738fc4bd8883b70697640f09111e5910901e 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -117,8 +117,8 @@ struct ChildRawStorage { #[serde(deny_unknown_fields)] /// Storage content for genesis block. struct RawGenesis { - pub top: GenesisStorage, - pub children: HashMap, + top: GenesisStorage, + children: HashMap, } #[derive(Serialize, Deserialize)] @@ -134,14 +134,14 @@ enum Genesis { #[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] struct ClientSpec { - pub name: String, - pub id: String, - pub boot_nodes: Vec, - pub telemetry_endpoints: Option, - pub protocol_id: Option, - pub properties: Option, + name: String, + id: String, + boot_nodes: Vec, + telemetry_endpoints: Option, + protocol_id: Option, + properties: Option, #[serde(flatten)] - pub extensions: E, + extensions: E, // Never used, left only for backward compatibility. consensus_engine: (), #[serde(skip_serializing)] diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml index e176894d64c078dff802e49875c6a5321b1c6a18..588aa134ab172e3ce8988e43bac5b9a72049b5ff 100644 --- a/client/cli/Cargo.toml +++ b/client/cli/Cargo.toml @@ -1,10 +1,12 @@ [package] name = "sc-cli" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] description = "Substrate CLI interface." edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] clap = "2.33.0" @@ -21,20 +23,21 @@ tokio = { version = "0.2.9", features = [ "signal", "rt-core", "rt-threaded" ] } futures = "0.3.1" fdlimit = "0.1.1" serde_json = "1.0.41" -sc-informant = { version = "0.8", path = "../informant" } -sp-panic-handler = { version = "2.0.0", path = "../../primitives/panic-handler" } -sc-client-api = { version = "2.0.0", path = "../api" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } -sc-network = { version = "0.8", path = "../network" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sc-service = { version = "0.8", default-features = false, path = "../service" } -sp-state-machine = { version = "0.8", path = "../../primitives/state-machine" } -sc-telemetry = { version = "2.0.0", path = "../telemetry" } -sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } +sc-informant = { version = "0.8.0-alpha.2", path = "../informant" } +sp-panic-handler = { version = "2.0.0-alpha.2", path = "../../primitives/panic-handler" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../api" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../primitives/blockchain" } +sc-network = { version = "0.8.0-alpha.2", path = "../network" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +sc-service = { version = "0.8.0-alpha.2", default-features = false, path = "../service" } +sp-state-machine = { version = "0.8.0-alpha.2", path = "../../primitives/state-machine" } +sc-telemetry = { version = "2.0.0-alpha.2", path = "../telemetry" } +substrate-prometheus-endpoint = { path = "../../utils/prometheus" , version = "0.8.0-alpha.2"} +sp-keyring = { version = "2.0.0-alpha.2", path = "../../primitives/keyring" } names = "0.11.0" structopt = "0.3.8" -sc-tracing = { version = "2.0.0", path = "../tracing" } +sc-tracing = { version = "2.0.0-alpha.2", path = "../tracing" } chrono = "0.4.10" parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } diff --git a/client/cli/src/execution_strategy.rs b/client/cli/src/arg_enums.rs similarity index 51% rename from client/cli/src/execution_strategy.rs rename to client/cli/src/arg_enums.rs index 888d7b6c4a09680103c15b44e27b67b700f6505c..384087bec0dbe4538c7227e36a6dbce7d789586b 100644 --- a/client/cli/src/execution_strategy.rs +++ b/client/cli/src/arg_enums.rs @@ -14,10 +14,74 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +// NOTE: we allow missing docs here because arg_enum! creates the function variants without doc #![allow(missing_docs)] use structopt::clap::arg_enum; +arg_enum! { + /// How to execute Wasm runtime code + #[allow(missing_docs)] + #[derive(Debug, Clone, Copy)] + pub enum WasmExecutionMethod { + // Uses an interpreter. + Interpreted, + // Uses a compiled runtime. + Compiled, + } +} + +impl WasmExecutionMethod { + /// Returns list of variants that are not disabled by feature flags. + pub fn enabled_variants() -> Vec<&'static str> { + Self::variants() + .iter() + .cloned() + .filter(|&name| cfg!(feature = "wasmtime") || name != "Compiled") + .collect() + } +} + +impl Into for WasmExecutionMethod { + fn into(self) -> sc_service::config::WasmExecutionMethod { + match self { + WasmExecutionMethod::Interpreted => sc_service::config::WasmExecutionMethod::Interpreted, + #[cfg(feature = "wasmtime")] + WasmExecutionMethod::Compiled => sc_service::config::WasmExecutionMethod::Compiled, + #[cfg(not(feature = "wasmtime"))] + WasmExecutionMethod::Compiled => panic!( + "Substrate must be compiled with \"wasmtime\" feature for compiled Wasm execution" + ), + } + } +} + +arg_enum! { + #[allow(missing_docs)] + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub enum TracingReceiver { + Log, + Telemetry, + } +} + +impl Into for TracingReceiver { + fn into(self) -> sc_tracing::TracingReceiver { + match self { + TracingReceiver::Log => sc_tracing::TracingReceiver::Log, + TracingReceiver::Telemetry => sc_tracing::TracingReceiver::Telemetry, + } + } +} + +arg_enum! { + #[allow(missing_docs)] + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub enum NodeKeyType { + Ed25519 + } +} + arg_enum! { /// How to execute blocks #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -33,6 +97,17 @@ arg_enum! { } } +impl Into for ExecutionStrategy { + fn into(self) -> sc_client_api::ExecutionStrategy { + match self { + ExecutionStrategy::Native => sc_client_api::ExecutionStrategy::NativeWhenPossible, + ExecutionStrategy::Wasm => sc_client_api::ExecutionStrategy::AlwaysWasm, + ExecutionStrategy::Both => sc_client_api::ExecutionStrategy::Both, + ExecutionStrategy::NativeElseWasm => sc_client_api::ExecutionStrategy::NativeElseWasm, + } + } +} + impl ExecutionStrategy { /// Returns the variant as `'&static str`. pub fn as_str(&self) -> &'static str { diff --git a/client/cli/src/commands/build_spec_cmd.rs b/client/cli/src/commands/build_spec_cmd.rs new file mode 100644 index 0000000000000000000000000000000000000000..9b71207efabf8de381d7066b284356e6c2663c11 --- /dev/null +++ b/client/cli/src/commands/build_spec_cmd.rs @@ -0,0 +1,104 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use structopt::StructOpt; +use log::info; +use sc_network::config::build_multiaddr; +use sc_service::{Configuration, ChainSpecExtension, RuntimeGenesis, ChainSpec}; + +use crate::error; +use crate::VersionInfo; +use crate::params::SharedParams; +use crate::params::NodeKeyParams; + +/// The `build-spec` command used to build a specification. +#[derive(Debug, StructOpt, Clone)] +pub struct BuildSpecCmd { + /// Force raw genesis storage output. + #[structopt(long = "raw")] + pub raw: bool, + + /// Disable adding the default bootnode to the specification. + /// + /// By default the `/ip4/127.0.0.1/tcp/30333/p2p/NODE_PEER_ID` bootnode is added to the + /// specification when no bootnode exists. + #[structopt(long = "disable-default-bootnode")] + pub disable_default_bootnode: bool, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub node_key_params: NodeKeyParams, +} + +impl BuildSpecCmd { + /// Run the build-spec command + pub fn run( + self, + config: Configuration, + ) -> error::Result<()> + where + G: RuntimeGenesis, + E: ChainSpecExtension, + { + info!("Building chain spec"); + let mut spec = config.expect_chain_spec().clone(); + let raw_output = self.raw; + + if spec.boot_nodes().is_empty() && !self.disable_default_bootnode { + let keys = config.network.node_key.into_keypair()?; + let peer_id = keys.public().into_peer_id(); + let addr = build_multiaddr![ + Ip4([127, 0, 0, 1]), + Tcp(30333u16), + P2p(peer_id) + ]; + spec.add_boot_node(addr) + } + + let json = sc_service::chain_ops::build_spec(spec, raw_output)?; + + print!("{}", json); + + Ok(()) + } + + /// Update and prepare a `Configuration` with command line parameters + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + self.shared_params.update_config(&mut config, spec_factory, version)?; + + let net_config_path = config + .in_chain_config_dir(crate::commands::DEFAULT_NETWORK_CONFIG_PATH) + .expect("We provided a base_path"); + + self.node_key_params.update_config(&mut config, Some(&net_config_path))?; + + Ok(()) + } +} + diff --git a/client/cli/src/commands/check_block_cmd.rs b/client/cli/src/commands/check_block_cmd.rs new file mode 100644 index 0000000000000000000000000000000000000000..1036be16de4ebcde466710781f4cfecb283216eb --- /dev/null +++ b/client/cli/src/commands/check_block_cmd.rs @@ -0,0 +1,105 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::fmt::Debug; +use std::str::FromStr; +use structopt::StructOpt; +use sc_service::{ + Configuration, ChainSpecExtension, RuntimeGenesis, ServiceBuilderCommand, Roles, ChainSpec, +}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; +use sp_runtime::generic::BlockId; + +use crate::error; +use crate::VersionInfo; +use crate::runtime::run_until_exit; +use crate::params::SharedParams; +use crate::params::ImportParams; + +/// The `check-block` command used to validate blocks. +#[derive(Debug, StructOpt, Clone)] +pub struct CheckBlockCmd { + /// Block hash or number + #[structopt(value_name = "HASH or NUMBER")] + pub input: String, + + /// The default number of 64KB pages to ever allocate for Wasm execution. + /// + /// Don't alter this unless you know what you're doing. + #[structopt(long = "default-heap-pages", value_name = "COUNT")] + pub default_heap_pages: Option, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub import_params: ImportParams, +} + +impl CheckBlockCmd { + /// Run the check-block command + pub fn run( + self, + config: Configuration, + builder: B, + ) -> error::Result<()> + where + B: FnOnce(Configuration) -> Result, + G: RuntimeGenesis, + E: ChainSpecExtension, + BC: ServiceBuilderCommand + Unpin, + BB: sp_runtime::traits::Block + Debug, + <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, + ::Hash: std::str::FromStr, + { + let input = if self.input.starts_with("0x") { &self.input[2..] } else { &self.input[..] }; + let block_id = match FromStr::from_str(input) { + Ok(hash) => BlockId::hash(hash), + Err(_) => match self.input.parse::() { + Ok(n) => BlockId::number((n as u32).into()), + Err(_) => return Err(error::Error::Input("Invalid hash or number specified".into())), + } + }; + + let start = std::time::Instant::now(); + run_until_exit(config, |config| { + Ok(builder(config)?.check_block(block_id)) + })?; + println!("Completed in {} ms.", start.elapsed().as_millis()); + + Ok(()) + } + + /// Update and prepare a `Configuration` with command line parameters + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + self.shared_params.update_config(&mut config, spec_factory, version)?; + self.import_params.update_config(&mut config, Roles::FULL, self.shared_params.dev)?; + config.use_in_memory_keystore()?; + + Ok(()) + } +} diff --git a/client/cli/src/commands/export_blocks_cmd.rs b/client/cli/src/commands/export_blocks_cmd.rs new file mode 100644 index 0000000000000000000000000000000000000000..21195cccd46648aa7c1efabe332e596168b9c098 --- /dev/null +++ b/client/cli/src/commands/export_blocks_cmd.rs @@ -0,0 +1,117 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::io; +use std::fs; +use std::path::PathBuf; +use std::fmt::Debug; +use log::info; +use structopt::StructOpt; +use sc_service::{ + Configuration, ChainSpecExtension, RuntimeGenesis, ServiceBuilderCommand, ChainSpec, + config::DatabaseConfig, Roles, +}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; + +use crate::error; +use crate::VersionInfo; +use crate::runtime::run_until_exit; +use crate::params::{SharedParams, BlockNumber, PruningParams}; + +/// The `export-blocks` command used to export blocks. +#[derive(Debug, StructOpt, Clone)] +pub struct ExportBlocksCmd { + /// Output file name or stdout if unspecified. + #[structopt(parse(from_os_str))] + pub output: Option, + + /// Specify starting block number. + /// + /// Default is 1. + #[structopt(long = "from", value_name = "BLOCK")] + pub from: Option, + + /// Specify last block number. + /// + /// Default is best block. + #[structopt(long = "to", value_name = "BLOCK")] + pub to: Option, + + /// Use binary output rather than JSON. + #[structopt(long = "binary", value_name = "BOOL", parse(try_from_str), default_value("false"))] + pub binary: bool, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub pruning_params: PruningParams, +} + +impl ExportBlocksCmd { + /// Run the export-blocks command + pub fn run( + self, + config: Configuration, + builder: B, + ) -> error::Result<()> + where + B: FnOnce(Configuration) -> Result, + G: RuntimeGenesis, + E: ChainSpecExtension, + BC: ServiceBuilderCommand + Unpin, + BB: sp_runtime::traits::Block + Debug, + <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, + ::Hash: std::str::FromStr, + { + if let DatabaseConfig::Path { ref path, .. } = config.expect_database() { + info!("DB path: {}", path.display()); + } + let from = self.from.as_ref().and_then(|f| f.parse().ok()).unwrap_or(1); + let to = self.to.as_ref().and_then(|t| t.parse().ok()); + + let binary = self.binary; + + let file: Box = match &self.output { + Some(filename) => Box::new(fs::File::create(filename)?), + None => Box::new(io::stdout()), + }; + + run_until_exit(config, |config| { + Ok(builder(config)?.export_blocks(file, from.into(), to, binary)) + }) + } + + /// Update and prepare a `Configuration` with command line parameters + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + self.shared_params.update_config(&mut config, spec_factory, version)?; + self.pruning_params.update_config(&mut config, Roles::FULL, true)?; + config.use_in_memory_keystore()?; + + Ok(()) + } +} diff --git a/client/cli/src/commands/import_blocks_cmd.rs b/client/cli/src/commands/import_blocks_cmd.rs new file mode 100644 index 0000000000000000000000000000000000000000..60a57ab78d1b1b3563d4f3a047f3be70152820eb --- /dev/null +++ b/client/cli/src/commands/import_blocks_cmd.rs @@ -0,0 +1,107 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::fmt::Debug; +use std::io::{Read, Seek, self}; +use std::fs; +use std::path::PathBuf; +use structopt::StructOpt; +use sc_service::{ + Configuration, ChainSpecExtension, RuntimeGenesis, ServiceBuilderCommand, ChainSpec, Roles, +}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; + +use crate::error; +use crate::VersionInfo; +use crate::runtime::run_until_exit; +use crate::params::SharedParams; +use crate::params::ImportParams; + +/// The `import-blocks` command used to import blocks. +#[derive(Debug, StructOpt, Clone)] +pub struct ImportBlocksCmd { + /// Input file or stdin if unspecified. + #[structopt(parse(from_os_str))] + pub input: Option, + + /// The default number of 64KB pages to ever allocate for Wasm execution. + /// + /// Don't alter this unless you know what you're doing. + #[structopt(long = "default-heap-pages", value_name = "COUNT")] + pub default_heap_pages: Option, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub import_params: ImportParams, +} + +/// Internal trait used to cast to a dynamic type that implements Read and Seek. +trait ReadPlusSeek: Read + Seek {} + +impl ReadPlusSeek for T {} + +impl ImportBlocksCmd { + /// Run the import-blocks command + pub fn run( + self, + config: Configuration, + builder: B, + ) -> error::Result<()> + where + B: FnOnce(Configuration) -> Result, + G: RuntimeGenesis, + E: ChainSpecExtension, + BC: ServiceBuilderCommand + Unpin, + BB: sp_runtime::traits::Block + Debug, + <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, + ::Hash: std::str::FromStr, + { + let file: Box = match &self.input { + Some(filename) => Box::new(fs::File::open(filename)?), + None => { + let mut buffer = Vec::new(); + io::stdin().read_to_end(&mut buffer)?; + Box::new(io::Cursor::new(buffer)) + }, + }; + + run_until_exit(config, |config| { + Ok(builder(config)?.import_blocks(file, false)) + }) + } + + /// Update and prepare a `Configuration` with command line parameters + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + self.shared_params.update_config(&mut config, spec_factory, version)?; + self.import_params.update_config(&mut config, Roles::FULL, self.shared_params.dev)?; + config.use_in_memory_keystore()?; + + Ok(()) + } +} diff --git a/client/cli/src/commands/mod.rs b/client/cli/src/commands/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..e9f991c7458837a97926e7b47520ba331bacc93d --- /dev/null +++ b/client/cli/src/commands/mod.rs @@ -0,0 +1,145 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +mod runcmd; +mod export_blocks_cmd; +mod build_spec_cmd; +mod import_blocks_cmd; +mod check_block_cmd; +mod revert_cmd; +mod purge_chain_cmd; + +use std::fmt::Debug; +use structopt::StructOpt; + +use sc_service::{ + Configuration, ChainSpecExtension, RuntimeGenesis, ServiceBuilderCommand, ChainSpec, +}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; + +use crate::error; +use crate::VersionInfo; +use crate::params::SharedParams; + +pub use crate::commands::runcmd::RunCmd; +pub use crate::commands::export_blocks_cmd::ExportBlocksCmd; +pub use crate::commands::build_spec_cmd::BuildSpecCmd; +pub use crate::commands::import_blocks_cmd::ImportBlocksCmd; +pub use crate::commands::check_block_cmd::CheckBlockCmd; +pub use crate::commands::revert_cmd::RevertCmd; +pub use crate::commands::purge_chain_cmd::PurgeChainCmd; + +/// default sub directory to store network config +const DEFAULT_NETWORK_CONFIG_PATH : &'static str = "network"; + +/// All core commands that are provided by default. +/// +/// The core commands are split into multiple subcommands and `Run` is the default subcommand. From +/// the CLI user perspective, it is not visible that `Run` is a subcommand. So, all parameters of +/// `Run` are exported as main executable parameters. +#[derive(Debug, Clone, StructOpt)] +pub enum Subcommand { + /// Build a spec.json file, outputing to stdout. + BuildSpec(build_spec_cmd::BuildSpecCmd), + + /// Export blocks to a file. + ExportBlocks(export_blocks_cmd::ExportBlocksCmd), + + /// Import blocks from file. + ImportBlocks(import_blocks_cmd::ImportBlocksCmd), + + /// Validate a single block. + CheckBlock(check_block_cmd::CheckBlockCmd), + + /// Revert chain to the previous state. + Revert(revert_cmd::RevertCmd), + + /// Remove the whole chain data. + PurgeChain(purge_chain_cmd::PurgeChainCmd), +} + +impl Subcommand { + /// Get the shared parameters of a `CoreParams` command + pub fn get_shared_params(&self) -> &SharedParams { + use Subcommand::*; + + match self { + BuildSpec(params) => ¶ms.shared_params, + ExportBlocks(params) => ¶ms.shared_params, + ImportBlocks(params) => ¶ms.shared_params, + CheckBlock(params) => ¶ms.shared_params, + Revert(params) => ¶ms.shared_params, + PurgeChain(params) => ¶ms.shared_params, + } + } + + /// Run any `CoreParams` command + pub fn run( + self, + config: Configuration, + builder: B, + ) -> error::Result<()> + where + B: FnOnce(Configuration) -> Result, + G: RuntimeGenesis, + E: ChainSpecExtension, + BC: ServiceBuilderCommand + Unpin, + BB: sp_runtime::traits::Block + Debug, + <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, + ::Hash: std::str::FromStr, + { + match self { + Subcommand::BuildSpec(cmd) => cmd.run(config), + Subcommand::ExportBlocks(cmd) => cmd.run(config, builder), + Subcommand::ImportBlocks(cmd) => cmd.run(config, builder), + Subcommand::CheckBlock(cmd) => cmd.run(config, builder), + Subcommand::PurgeChain(cmd) => cmd.run(config), + Subcommand::Revert(cmd) => cmd.run(config, builder), + } + } + + /// Update and prepare a `Configuration` with command line parameters + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + match self { + Subcommand::BuildSpec(cmd) => cmd.update_config(&mut config, spec_factory, version), + Subcommand::ExportBlocks(cmd) => cmd.update_config(&mut config, spec_factory, version), + Subcommand::ImportBlocks(cmd) => cmd.update_config(&mut config, spec_factory, version), + Subcommand::CheckBlock(cmd) => cmd.update_config(&mut config, spec_factory, version), + Subcommand::PurgeChain(cmd) => cmd.update_config(&mut config, spec_factory, version), + Subcommand::Revert(cmd) => cmd.update_config(&mut config, spec_factory, version), + } + } + + /// Initialize substrate. This must be done only once. + /// + /// This method: + /// + /// 1. Set the panic handler + /// 2. Raise the FD limit + /// 3. Initialize the logger + pub fn init(&self, version: &VersionInfo) -> error::Result<()> { + self.get_shared_params().init(version) + } +} diff --git a/client/cli/src/commands/purge_chain_cmd.rs b/client/cli/src/commands/purge_chain_cmd.rs new file mode 100644 index 0000000000000000000000000000000000000000..b7c559e5cc359eec154a250a4c41cb2b8f5bd70d --- /dev/null +++ b/client/cli/src/commands/purge_chain_cmd.rs @@ -0,0 +1,106 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::fmt::Debug; +use std::io::{Write, self}; +use std::fs; +use structopt::StructOpt; +use sc_service::{ + Configuration, ChainSpecExtension, RuntimeGenesis, ChainSpec, + config::{DatabaseConfig}, +}; + +use crate::error; +use crate::VersionInfo; +use crate::params::SharedParams; + +/// The `purge-chain` command used to remove the whole chain. +#[derive(Debug, StructOpt, Clone)] +pub struct PurgeChainCmd { + /// Skip interactive prompt by answering yes automatically. + #[structopt(short = "y")] + pub yes: bool, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, +} + +impl PurgeChainCmd { + /// Run the purge command + pub fn run( + self, + config: Configuration, + ) -> error::Result<()> + where + G: RuntimeGenesis, + E: ChainSpecExtension, + { + let db_path = match config.expect_database() { + DatabaseConfig::Path { path, .. } => path, + _ => { + eprintln!("Cannot purge custom database implementation"); + return Ok(()); + } + }; + + if !self.yes { + print!("Are you sure to remove {:?}? [y/N]: ", &db_path); + io::stdout().flush().expect("failed to flush stdout"); + + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + let input = input.trim(); + + match input.chars().nth(0) { + Some('y') | Some('Y') => {}, + _ => { + println!("Aborted"); + return Ok(()); + }, + } + } + + match fs::remove_dir_all(&db_path) { + Ok(_) => { + println!("{:?} removed.", &db_path); + Ok(()) + }, + Err(ref err) if err.kind() == io::ErrorKind::NotFound => { + eprintln!("{:?} did not exist.", &db_path); + Ok(()) + }, + Err(err) => Result::Err(err.into()) + } + } + + /// Update and prepare a `Configuration` with command line parameters + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + self.shared_params.update_config(&mut config, spec_factory, version)?; + config.use_in_memory_keystore()?; + + Ok(()) + } +} diff --git a/client/cli/src/commands/revert_cmd.rs b/client/cli/src/commands/revert_cmd.rs new file mode 100644 index 0000000000000000000000000000000000000000..f0c534898effd33ac33b979c322a98d78b3877cd --- /dev/null +++ b/client/cli/src/commands/revert_cmd.rs @@ -0,0 +1,83 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::fmt::Debug; +use structopt::StructOpt; +use sc_service::{ + Configuration, ChainSpecExtension, RuntimeGenesis, ServiceBuilderCommand, ChainSpec, Roles, +}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; + +use crate::error; +use crate::VersionInfo; +use crate::params::{BlockNumber, SharedParams, PruningParams}; + +/// The `revert` command used revert the chain to a previous state. +#[derive(Debug, StructOpt, Clone)] +pub struct RevertCmd { + /// Number of blocks to revert. + #[structopt(default_value = "256")] + pub num: BlockNumber, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub pruning_params: PruningParams, +} + +impl RevertCmd { + /// Run the revert command + pub fn run( + self, + config: Configuration, + builder: B, + ) -> error::Result<()> + where + B: FnOnce(Configuration) -> Result, + G: RuntimeGenesis, + E: ChainSpecExtension, + BC: ServiceBuilderCommand + Unpin, + BB: sp_runtime::traits::Block + Debug, + <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, + ::Hash: std::str::FromStr, + { + let blocks = self.num.parse()?; + builder(config)?.revert_chain(blocks)?; + + Ok(()) + } + + /// Update and prepare a `Configuration` with command line parameters + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + self.shared_params.update_config(&mut config, spec_factory, version)?; + self.pruning_params.update_config(&mut config, Roles::FULL, true)?; + config.use_in_memory_keystore()?; + + Ok(()) + } +} diff --git a/client/cli/src/commands/runcmd.rs b/client/cli/src/commands/runcmd.rs new file mode 100644 index 0000000000000000000000000000000000000000..2a070b27b8b8518f7037271ca30230c23605b160 --- /dev/null +++ b/client/cli/src/commands/runcmd.rs @@ -0,0 +1,736 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::path::PathBuf; +use std::net::SocketAddr; +use std::fs; +use log::info; +use structopt::{StructOpt, clap::arg_enum}; +use names::{Generator, Name}; +use regex::Regex; +use chrono::prelude::*; +use sc_service::{ + AbstractService, Configuration, ChainSpecExtension, RuntimeGenesis, ChainSpec, Roles, + config::{KeystoreConfig, PrometheusConfig}, +}; +use sc_telemetry::TelemetryEndpoints; + +use crate::VersionInfo; +use crate::error; +use crate::params::ImportParams; +use crate::params::SharedParams; +use crate::params::NetworkConfigurationParams; +use crate::params::TransactionPoolParams; +use crate::runtime::run_service_until_exit; + +/// The maximum number of characters for a node name. +const NODE_NAME_MAX_LENGTH: usize = 32; + +/// default sub directory for the key store +const DEFAULT_KEYSTORE_CONFIG_PATH : &'static str = "keystore"; + +arg_enum! { + /// Whether off-chain workers are enabled. + #[allow(missing_docs)] + #[derive(Debug, Clone)] + pub enum OffchainWorkerEnabled { + Always, + Never, + WhenValidating, + } +} + +/// The `run` command used to run a node. +#[derive(Debug, StructOpt, Clone)] +pub struct RunCmd { + /// Enable validator mode. + /// + /// The node will be started with the authority role and actively + /// participate in any consensus task that it can (e.g. depending on + /// availability of local keys). + #[structopt( + long = "validator", + conflicts_with_all = &[ "sentry" ] + )] + pub validator: bool, + + /// Enable sentry mode. + /// + /// The node will be started with the authority role and participate in + /// consensus tasks as an "observer", it will never actively participate + /// regardless of whether it could (e.g. keys are available locally). This + /// mode is useful as a secure proxy for validators (which would run + /// detached from the network), since we want this node to participate in + /// the full consensus protocols in order to have all needed consensus data + /// available to relay to private nodes. + #[structopt( + long = "sentry", + conflicts_with_all = &[ "validator", "light" ] + )] + pub sentry: bool, + + /// Disable GRANDPA voter when running in validator mode, otherwise disables the GRANDPA observer. + #[structopt(long = "no-grandpa")] + pub no_grandpa: bool, + + /// Experimental: Run in light client mode. + #[structopt(long = "light", conflicts_with = "sentry")] + pub light: bool, + + /// Listen to all RPC interfaces. + /// + /// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use a RPC proxy + /// server to filter out dangerous methods. More details: https://github.com/paritytech/substrate/wiki/Public-RPC. + /// Use `--unsafe-rpc-external` to suppress the warning if you understand the risks. + #[structopt(long = "rpc-external")] + pub rpc_external: bool, + + /// Listen to all RPC interfaces. + /// + /// Same as `--rpc-external`. + #[structopt(long = "unsafe-rpc-external")] + pub unsafe_rpc_external: bool, + + /// Listen to all Websocket interfaces. + /// + /// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use a RPC proxy + /// server to filter out dangerous methods. More details: https://github.com/paritytech/substrate/wiki/Public-RPC. + /// Use `--unsafe-ws-external` to suppress the warning if you understand the risks. + #[structopt(long = "ws-external")] + pub ws_external: bool, + + /// Listen to all Websocket interfaces. + /// + /// Same as `--ws-external` but doesn't warn you about it. + #[structopt(long = "unsafe-ws-external")] + pub unsafe_ws_external: bool, + + /// Listen to all Prometheus data source interfaces. + /// + /// Default is local. + #[structopt(long = "prometheus-external")] + pub prometheus_external: bool, + + /// Specify HTTP RPC server TCP port. + #[structopt(long = "rpc-port", value_name = "PORT")] + pub rpc_port: Option, + + /// Specify WebSockets RPC server TCP port. + #[structopt(long = "ws-port", value_name = "PORT")] + pub ws_port: Option, + + /// Maximum number of WS RPC server connections. + #[structopt(long = "ws-max-connections", value_name = "COUNT")] + pub ws_max_connections: Option, + + /// Specify browser Origins allowed to access the HTTP & WS RPC servers. + /// + /// A comma-separated list of origins (protocol://domain or special `null` + /// value). Value of `all` will disable origin validation. Default is to + /// allow localhost and https://polkadot.js.org origins. When running in + /// --dev mode the default is to allow all origins. + #[structopt(long = "rpc-cors", value_name = "ORIGINS", parse(try_from_str = parse_cors))] + pub rpc_cors: Option, + + /// Specify Prometheus data source server TCP Port. + #[structopt(long = "prometheus-port", value_name = "PORT")] + pub prometheus_port: Option, + + /// Do not expose a Prometheus metric endpoint. + /// + /// Prometheus metric endpoint is enabled by default. + #[structopt(long = "no-prometheus")] + pub no_prometheus: bool, + + /// The human-readable name for this node. + /// + /// The node name will be reported to the telemetry server, if enabled. + #[structopt(long = "name", value_name = "NAME")] + pub name: Option, + + /// Disable connecting to the Substrate telemetry server. + /// + /// Telemetry is on by default on global chains. + #[structopt(long = "no-telemetry")] + pub no_telemetry: bool, + + /// The URL of the telemetry server to connect to. + /// + /// This flag can be passed multiple times as a mean to specify multiple + /// telemetry endpoints. Verbosity levels range from 0-9, with 0 denoting + /// the least verbosity. If no verbosity level is specified the default is + /// 0. + #[structopt(long = "telemetry-url", value_name = "URL VERBOSITY", parse(try_from_str = parse_telemetry_endpoints))] + pub telemetry_endpoints: Vec<(String, u8)>, + + /// Should execute offchain workers on every block. + /// + /// By default it's only enabled for nodes that are authoring new blocks. + #[structopt( + long = "offchain-worker", + value_name = "ENABLED", + possible_values = &OffchainWorkerEnabled::variants(), + case_insensitive = true, + default_value = "WhenValidating" + )] + pub offchain_worker: OffchainWorkerEnabled, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub import_params: ImportParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub network_config: NetworkConfigurationParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub pool_config: TransactionPoolParams, + + /// Shortcut for `--name Alice --validator` with session keys for `Alice` added to keystore. + #[structopt(long, conflicts_with_all = &["bob", "charlie", "dave", "eve", "ferdie", "one", "two"])] + pub alice: bool, + + /// Shortcut for `--name Bob --validator` with session keys for `Bob` added to keystore. + #[structopt(long, conflicts_with_all = &["alice", "charlie", "dave", "eve", "ferdie", "one", "two"])] + pub bob: bool, + + /// Shortcut for `--name Charlie --validator` with session keys for `Charlie` added to keystore. + #[structopt(long, conflicts_with_all = &["alice", "bob", "dave", "eve", "ferdie", "one", "two"])] + pub charlie: bool, + + /// Shortcut for `--name Dave --validator` with session keys for `Dave` added to keystore. + #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "eve", "ferdie", "one", "two"])] + pub dave: bool, + + /// Shortcut for `--name Eve --validator` with session keys for `Eve` added to keystore. + #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "ferdie", "one", "two"])] + pub eve: bool, + + /// Shortcut for `--name Ferdie --validator` with session keys for `Ferdie` added to keystore. + #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "one", "two"])] + pub ferdie: bool, + + /// Shortcut for `--name One --validator` with session keys for `One` added to keystore. + #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "ferdie", "two"])] + pub one: bool, + + /// Shortcut for `--name Two --validator` with session keys for `Two` added to keystore. + #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "ferdie", "one"])] + pub two: bool, + + /// Enable authoring even when offline. + #[structopt(long = "force-authoring")] + pub force_authoring: bool, + + /// Specify custom keystore path. + #[structopt(long = "keystore-path", value_name = "PATH", parse(from_os_str))] + pub keystore_path: Option, + + /// Use interactive shell for entering the password used by the keystore. + #[structopt( + long = "password-interactive", + conflicts_with_all = &[ "password", "password-filename" ] + )] + pub password_interactive: bool, + + /// Password used by the keystore. + #[structopt( + long = "password", + conflicts_with_all = &[ "password-interactive", "password-filename" ] + )] + pub password: Option, + + /// File that contains the password used by the keystore. + #[structopt( + long = "password-filename", + value_name = "PATH", + parse(from_os_str), + conflicts_with_all = &[ "password-interactive", "password" ] + )] + pub password_filename: Option +} + +impl RunCmd { + /// Get the `Sr25519Keyring` matching one of the flag + pub fn get_keyring(&self) -> Option { + use sp_keyring::Sr25519Keyring::*; + + if self.alice { Some(Alice) } + else if self.bob { Some(Bob) } + else if self.charlie { Some(Charlie) } + else if self.dave { Some(Dave) } + else if self.eve { Some(Eve) } + else if self.ferdie { Some(Ferdie) } + else if self.one { Some(One) } + else if self.two { Some(Two) } + else { None } + } + + /// Update and prepare a `Configuration` with command line parameters of `RunCmd` and `VersionInfo` + pub fn update_config( + &self, + mut config: &mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<()> + where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + self.shared_params.update_config(&mut config, spec_factory, version)?; + + let password = if self.password_interactive { + #[cfg(not(target_os = "unknown"))] + { + Some(input_keystore_password()?.into()) + } + #[cfg(target_os = "unknown")] + None + } else if let Some(ref file) = self.password_filename { + Some(fs::read_to_string(file).map_err(|e| format!("{}", e))?.into()) + } else if let Some(ref password) = self.password { + Some(password.clone().into()) + } else { + None + }; + + let path = self.keystore_path.clone().or( + config.in_chain_config_dir(DEFAULT_KEYSTORE_CONFIG_PATH) + ); + + config.keystore = KeystoreConfig::Path { + path: path.ok_or_else(|| "No `base_path` provided to create keystore path!".to_string())?, + password, + }; + + let keyring = self.get_keyring(); + let is_dev = self.shared_params.dev; + let is_light = self.light; + let is_authority = (self.validator || self.sentry || is_dev || keyring.is_some()) + && !is_light; + let role = + if is_light { + sc_service::Roles::LIGHT + } else if is_authority { + sc_service::Roles::AUTHORITY + } else { + sc_service::Roles::FULL + }; + + self.import_params.update_config(&mut config, role, is_dev)?; + + config.name = match (self.name.as_ref(), keyring) { + (Some(name), _) => name.to_string(), + (_, Some(keyring)) => keyring.to_string(), + (None, None) => generate_node_name(), + }; + if let Err(msg) = is_node_name_valid(&config.name) { + return Err(error::Error::Input( + format!("Invalid node name '{}'. Reason: {}. If unsure, use none.", + config.name, + msg, + ) + )); + } + + // set sentry mode (i.e. act as an authority but **never** actively participate) + config.sentry_mode = self.sentry; + + config.offchain_worker = match (&self.offchain_worker, role) { + (OffchainWorkerEnabled::WhenValidating, sc_service::Roles::AUTHORITY) => true, + (OffchainWorkerEnabled::Always, _) => true, + (OffchainWorkerEnabled::Never, _) => false, + (OffchainWorkerEnabled::WhenValidating, _) => false, + }; + + config.roles = role; + config.disable_grandpa = self.no_grandpa; + + let client_id = config.client_id(); + let network_path = config + .in_chain_config_dir(crate::commands::DEFAULT_NETWORK_CONFIG_PATH) + .expect("We provided a basepath"); + self.network_config.update_config( + &mut config, + network_path, + client_id, + is_dev, + )?; + + self.pool_config.update_config(&mut config)?; + + config.dev_key_seed = keyring + .map(|a| format!("//{}", a)).or_else(|| { + if is_dev && !is_light { + Some("//Alice".into()) + } else { + None + } + }); + + if config.rpc_http.is_none() || self.rpc_port.is_some() { + let rpc_interface: &str = interface_str(self.rpc_external, self.unsafe_rpc_external, self.validator)?; + config.rpc_http = Some(parse_address(&format!("{}:{}", rpc_interface, 9933), self.rpc_port)?); + } + if config.rpc_ws.is_none() || self.ws_port.is_some() { + let ws_interface: &str = interface_str(self.ws_external, self.unsafe_ws_external, self.validator)?; + config.rpc_ws = Some(parse_address(&format!("{}:{}", ws_interface, 9944), self.ws_port)?); + } + + config.rpc_ws_max_connections = self.ws_max_connections; + config.rpc_cors = self.rpc_cors.clone().unwrap_or_else(|| if is_dev { + log::warn!("Running in --dev mode, RPC CORS has been disabled."); + Cors::All + } else { + Cors::List(vec![ + "http://localhost:*".into(), + "http://127.0.0.1:*".into(), + "https://localhost:*".into(), + "https://127.0.0.1:*".into(), + "https://polkadot.js.org".into(), + ]) + }).into(); + + // Override telemetry + if self.no_telemetry { + config.telemetry_endpoints = None; + } else if !self.telemetry_endpoints.is_empty() { + config.telemetry_endpoints = Some( + TelemetryEndpoints::new(self.telemetry_endpoints.clone()) + ); + } + + // Override prometheus + if self.no_prometheus { + config.prometheus_config = None; + } else if config.prometheus_config.is_none() { + let prometheus_interface: &str = if self.prometheus_external { "0.0.0.0" } else { "127.0.0.1" }; + config.prometheus_config = Some(PrometheusConfig::new_with_default_registry( + parse_address(&format!("{}:{}", prometheus_interface, 9615), self.prometheus_port)?, + )); + } + + config.tracing_targets = self.import_params.tracing_targets.clone().into(); + config.tracing_receiver = self.import_params.tracing_receiver.clone().into(); + + // Imply forced authoring on --dev + config.force_authoring = self.shared_params.dev || self.force_authoring; + + Ok(()) + } + + /// Run the command that runs the node + pub fn run( + self, + config: Configuration, + new_light: FNL, + new_full: FNF, + version: &VersionInfo, + ) -> error::Result<()> + where + G: RuntimeGenesis, + E: ChainSpecExtension, + FNL: FnOnce(Configuration) -> Result, + FNF: FnOnce(Configuration) -> Result, + SL: AbstractService + Unpin, + SF: AbstractService + Unpin, + { + info!("{}", version.name); + info!(" version {}", config.full_version()); + info!(" by {}, {}-{}", version.author, version.copyright_start_year, Local::today().year()); + info!("Chain specification: {}", config.expect_chain_spec().name()); + info!("Node name: {}", config.name); + info!("Roles: {}", config.display_role()); + + match config.roles { + Roles::LIGHT => run_service_until_exit( + config, + new_light, + ), + _ => run_service_until_exit( + config, + new_full, + ), + } + } + + /// Initialize substrate. This must be done only once. + /// + /// This method: + /// + /// 1. Set the panic handler + /// 2. Raise the FD limit + /// 3. Initialize the logger + pub fn init(&self, version: &VersionInfo) -> error::Result<()> { + self.shared_params.init(version) + } +} + +/// Check whether a node name is considered as valid +pub fn is_node_name_valid(_name: &str) -> Result<(), &str> { + let name = _name.to_string(); + if name.chars().count() >= NODE_NAME_MAX_LENGTH { + return Err("Node name too long"); + } + + let invalid_chars = r"[\\.@]"; + let re = Regex::new(invalid_chars).unwrap(); + if re.is_match(&name) { + return Err("Node name should not contain invalid chars such as '.' and '@'"); + } + + let invalid_patterns = r"(https?:\\/+)?(www)+"; + let re = Regex::new(invalid_patterns).unwrap(); + if re.is_match(&name) { + return Err("Node name should not contain urls"); + } + + Ok(()) +} + +#[cfg(not(target_os = "unknown"))] +fn input_keystore_password() -> Result { + rpassword::read_password_from_tty(Some("Keystore password: ")) + .map_err(|e| format!("{:?}", e)) +} + +fn generate_node_name() -> String { + let result = loop { + let node_name = Generator::with_naming(Name::Numbered).next().unwrap(); + let count = node_name.chars().count(); + + if count < NODE_NAME_MAX_LENGTH { + break node_name + } + }; + + result +} + +fn parse_address( + address: &str, + port: Option, +) -> Result { + let mut address: SocketAddr = address.parse().map_err( + |_| format!("Invalid address: {}", address) + )?; + if let Some(port) = port { + address.set_port(port); + } + + Ok(address) +} + +fn interface_str( + is_external: bool, + is_unsafe_external: bool, + is_validator: bool, +) -> Result<&'static str, error::Error> { + if is_external && is_validator { + return Err(error::Error::Input("--rpc-external and --ws-external options shouldn't be \ + used if the node is running as a validator. Use `--unsafe-rpc-external` if you understand \ + the risks. See the options description for more information.".to_owned())); + } + + if is_external || is_unsafe_external { + log::warn!("It isn't safe to expose RPC publicly without a proxy server that filters \ + available set of RPC methods."); + + Ok("0.0.0.0") + } else { + Ok("127.0.0.1") + } +} + +/// Default to verbosity level 0, if none is provided. +fn parse_telemetry_endpoints(s: &str) -> Result<(String, u8), Box> { + let pos = s.find(' '); + match pos { + None => { + Ok((s.to_owned(), 0)) + }, + Some(pos_) => { + let verbosity = s[pos_ + 1..].parse()?; + let url = s[..pos_].parse()?; + Ok((url, verbosity)) + } + } +} + +/// CORS setting +/// +/// The type is introduced to overcome `Option>` +/// handling of `structopt`. +#[derive(Clone, Debug)] +pub enum Cors { + /// All hosts allowed + All, + /// Only hosts on the list are allowed. + List(Vec), +} + +impl From for Option> { + fn from(cors: Cors) -> Self { + match cors { + Cors::All => None, + Cors::List(list) => Some(list), + } + } +} + +/// Parse cors origins +fn parse_cors(s: &str) -> Result> { + let mut is_all = false; + let mut origins = Vec::new(); + for part in s.split(',') { + match part { + "all" | "*" => { + is_all = true; + break; + }, + other => origins.push(other.to_owned()), + } + } + + Ok(if is_all { Cors::All } else { Cors::List(origins) }) +} + +#[cfg(test)] +mod tests { + use super::*; + use sc_service::config::DatabaseConfig; + + const TEST_VERSION_INFO: &'static VersionInfo = &VersionInfo { + name: "node-test", + version: "0.1.0", + commit: "some_commit", + executable_name: "node-test", + description: "description", + author: "author", + support_url: "http://example.org", + copyright_start_year: 2020, + }; + + #[test] + fn tests_node_name_good() { + assert!(is_node_name_valid("short name").is_ok()); + } + + #[test] + fn tests_node_name_bad() { + assert!(is_node_name_valid("long names are not very cool for the ui").is_err()); + assert!(is_node_name_valid("Dots.not.Ok").is_err()); + assert!(is_node_name_valid("http://visit.me").is_err()); + assert!(is_node_name_valid("https://visit.me").is_err()); + assert!(is_node_name_valid("www.visit.me").is_err()); + assert!(is_node_name_valid("email@domain").is_err()); + } + + #[test] + fn keystore_path_is_generated_correctly() { + let chain_spec = ChainSpec::from_genesis( + "test", + "test-id", + || (), + Vec::new(), + None, + None, + None, + None::<()>, + ); + + for keystore_path in vec![None, Some("/keystore/path")] { + let args: Vec<&str> = vec![]; + let mut cli = RunCmd::from_iter(args); + cli.keystore_path = keystore_path.clone().map(PathBuf::from); + + let mut config = Configuration::default(); + config.config_dir = Some(PathBuf::from("/test/path")); + config.chain_spec = Some(chain_spec.clone()); + let chain_spec = chain_spec.clone(); + cli.update_config(&mut config, move |_| Ok(Some(chain_spec)), TEST_VERSION_INFO).unwrap(); + + let expected_path = match keystore_path { + Some(path) => PathBuf::from(path), + None => PathBuf::from("/test/path/chains/test-id/keystore"), + }; + + assert_eq!(expected_path, config.keystore.path().unwrap().to_owned()); + } + } + + #[test] + fn ensure_load_spec_provide_defaults() { + let chain_spec = ChainSpec::from_genesis( + "test", + "test-id", + || (), + vec!["boo".to_string()], + Some(TelemetryEndpoints::new(vec![("foo".to_string(), 42)])), + None, + None, + None::<()>, + ); + + let args: Vec<&str> = vec![]; + let cli = RunCmd::from_iter(args); + + let mut config = Configuration::from_version(TEST_VERSION_INFO); + cli.update_config(&mut config, |_| Ok(Some(chain_spec)), TEST_VERSION_INFO).unwrap(); + + assert!(config.chain_spec.is_some()); + assert!(!config.network.boot_nodes.is_empty()); + assert!(config.telemetry_endpoints.is_some()); + } + + #[test] + fn ensure_update_config_for_running_node_provides_defaults() { + let chain_spec = ChainSpec::from_genesis( + "test", + "test-id", + || (), + vec![], + None, + None, + None, + None::<()>, + ); + + let args: Vec<&str> = vec![]; + let cli = RunCmd::from_iter(args); + + let mut config = Configuration::from_version(TEST_VERSION_INFO); + cli.init(&TEST_VERSION_INFO).unwrap(); + cli.update_config(&mut config, |_| Ok(Some(chain_spec)), TEST_VERSION_INFO).unwrap(); + + assert!(config.config_dir.is_some()); + assert!(config.database.is_some()); + if let Some(DatabaseConfig::Path { ref cache_size, .. }) = config.database { + assert!(cache_size.is_some()); + } else { + panic!("invalid config.database variant"); + } + assert!(!config.name.is_empty()); + assert!(config.network.config_path.is_some()); + assert!(!config.network.listen_addresses.is_empty()); + } +} diff --git a/client/cli/src/error.rs b/client/cli/src/error.rs index 074cb353c3a0a6d2a8cbd7653a156e1a169de04c..edc1adecc762c5f70c11958d1b6f9b0b93863904 100644 --- a/client/cli/src/error.rs +++ b/client/cli/src/error.rs @@ -49,6 +49,12 @@ impl std::convert::From for Error { } } +impl std::convert::From<&str> for Error { + fn from(s: &str) -> Error { + Error::Input(s.to_string()) + } +} + impl std::error::Error for Error { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { diff --git a/client/cli/src/lib.rs b/client/cli/src/lib.rs index 7495ad8e75690b10d68da914c029c6fd711c25cf..e28edebd60def53ba34497d416a9bea7919ef983 100644 --- a/client/cli/src/lib.rs +++ b/client/cli/src/lib.rs @@ -19,127 +19,33 @@ #![warn(missing_docs)] #![warn(unused_extern_crates)] -#[macro_use] -mod traits; mod params; -mod execution_strategy; -pub mod error; +mod arg_enums; +mod error; mod runtime; -mod node_key; - -use sc_client_api::execution_extensions::ExecutionStrategies; -use sc_service::{ - config::{Configuration, DatabaseConfig, KeystoreConfig}, - ServiceBuilderCommand, - RuntimeGenesis, ChainSpecExtension, PruningMode, ChainSpec, - AbstractService, Roles as ServiceRoles, -}; +mod commands; + pub use sc_service::config::VersionInfo; -use sc_network::{ - self, - multiaddr::Protocol, - config::{ - NetworkConfiguration, TransportConfig, NonReservedPeerMode, - }, -}; - -use std::{ - io::Write, iter, fmt::Debug, fs, - net::{Ipv4Addr, SocketAddr}, path::PathBuf, -}; + +use std::io::Write; use regex::Regex; -use structopt::{StructOpt, clap}; +use structopt::{StructOpt, clap::{self, AppSettings}}; pub use structopt; -use params::{ - NetworkConfigurationParams, TransactionPoolParams, Cors, -}; -pub use params::{ - SharedParams, ImportParams, ExecutionStrategy, Subcommand, RunCmd, BuildSpecCmd, - ExportBlocksCmd, ImportBlocksCmd, CheckBlockCmd, PurgeChainCmd, RevertCmd, - BenchmarkCmd, -}; -pub use traits::GetSharedParams; -use app_dirs::{AppInfo, AppDataType}; +pub use params::*; +pub use commands::*; +pub use arg_enums::*; +pub use error::*; use log::info; use lazy_static::lazy_static; -use sc_telemetry::TelemetryEndpoints; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; pub use crate::runtime::{run_until_exit, run_service_until_exit}; -use execution_strategy::*; -use names::{Generator, Name}; -use chrono::prelude::*; - -/// default sub directory to store network config -const DEFAULT_NETWORK_CONFIG_PATH : &'static str = "network"; -/// default sub directory to store database -const DEFAULT_DB_CONFIG_PATH : &'static str = "db"; -/// default sub directory for the key store -const DEFAULT_KEYSTORE_CONFIG_PATH : &'static str = "keystore"; - -/// The maximum number of characters for a node name. -const NODE_NAME_MAX_LENGTH: usize = 32; - -fn get_chain_key(cli: &SharedParams) -> String { - match cli.chain { - Some(ref chain) => chain.clone(), - None => if cli.dev { "dev".into() } else { "".into() } - } -} - -fn generate_node_name() -> String { - let result = loop { - let node_name = Generator::with_naming(Name::Numbered).next().unwrap(); - let count = node_name.chars().count(); - - if count < NODE_NAME_MAX_LENGTH { - break node_name - } - }; - - result -} - -/// Load spec to `Configuration` from shared params and spec factory. -pub fn load_spec<'a, G, E, F>( - mut config: &'a mut Configuration, - cli: &SharedParams, - factory: F, -) -> error::Result<&'a ChainSpec> where - G: RuntimeGenesis, - E: ChainSpecExtension, - F: FnOnce(&str) -> Result>, String>, -{ - let chain_key = get_chain_key(cli); - let spec = match factory(&chain_key)? { - Some(spec) => spec, - None => ChainSpec::from_json_file(PathBuf::from(chain_key))? - }; - - config.network.boot_nodes = spec.boot_nodes().to_vec(); - config.telemetry_endpoints = spec.telemetry_endpoints().clone(); - - config.chain_spec = Some(spec); - - Ok(config.chain_spec.as_ref().unwrap()) -} - -fn base_path(cli: &SharedParams, version: &VersionInfo) -> PathBuf { - cli.base_path.clone() - .unwrap_or_else(|| - app_dirs::get_app_root( - AppDataType::UserData, - &AppInfo { - name: version.executable_name, - author: version.author - } - ).expect("app directories exist on all supported platforms; qed") - ) -} /// Helper function used to parse the command line arguments. This is the equivalent of -/// `structopt`'s `from_args()` except that it takes a `VersionInfo` argument to provide the name of -/// the application, author, "about" and version. +/// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of +/// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. +/// +/// To allow running the node without subcommand, tt also sets a few more settings: +/// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. /// /// Gets the struct from the command line arguments. Print the /// error message and quit the program in case of failure. @@ -152,7 +58,10 @@ where /// Helper function used to parse the command line arguments. This is the equivalent of /// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of -/// the application, author, "about" and version. +/// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. +/// +/// To allow running the node without subcommand, tt also sets a few more settings: +/// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. /// /// Gets the struct from any iterator such as a `Vec` of your making. /// Print the error message and quit the program in case of failure. @@ -174,14 +83,22 @@ where .name(version.executable_name) .author(version.author) .about(version.description) - .version(full_version.as_str()); + .version(full_version.as_str()) + .settings(&[ + AppSettings::GlobalVersion, + AppSettings::ArgsNegateSubcommands, + AppSettings::SubcommandsNegateReqs, + ]); T::from_clap(&app.get_matches_from(iter)) } /// Helper function used to parse the command line arguments. This is the equivalent of -/// `structopt`'s `try_from_iter()` except that it takes a `VersionInfo` argument to provide the -/// name of the application, author, "about" and version. +/// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of +/// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. +/// +/// To allow running the node without subcommand, tt also sets a few more settings: +/// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. /// /// Gets the struct from any iterator such as a `Vec` of your making. /// Print the error message and quit the program in case of failure. @@ -215,53 +132,6 @@ where Ok(T::from_clap(&matches)) } -/// A helper function that initializes and runs the node -pub fn run( - mut config: Configuration, - run_cmd: RunCmd, - new_light: FNL, - new_full: FNF, - spec_factory: F, - version: &VersionInfo, -) -> error::Result<()> -where - F: FnOnce(&str) -> Result>, String>, - FNL: FnOnce(Configuration) -> Result, - FNF: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - SL: AbstractService + Unpin, - SF: AbstractService + Unpin, -{ - init(&run_cmd.shared_params, version)?; - init_config(&mut config, &run_cmd.shared_params, version, spec_factory)?; - run_cmd.run(config, new_light, new_full, version) -} - -/// A helper function that initializes and runs any of the subcommand variants of `CoreParams`. -pub fn run_subcommand( - mut config: Configuration, - subcommand: Subcommand, - spec_factory: F, - builder: B, - version: &VersionInfo, -) -> error::Result<()> -where - F: FnOnce(&str) -> Result>, String>, - B: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - BC: ServiceBuilderCommand + Unpin, - BB: sp_runtime::traits::Block + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, -{ - let shared_params = subcommand.get_shared_params(); - init(shared_params, version)?; - init_config(&mut config, shared_params, version, spec_factory)?; - subcommand.run(config, builder) -} - /// Initialize substrate. This must be done only once. /// /// This method: @@ -269,7 +139,7 @@ where /// 1. Set the panic handler /// 2. Raise the FD limit /// 3. Initialize the logger -pub fn init(shared_params: &SharedParams, version: &VersionInfo) -> error::Result<()> { +pub fn init(logger_pattern: &str, version: &VersionInfo) -> error::Result<()> { let full_version = sc_service::config::full_version_from_strs( version.version, version.commit @@ -277,425 +147,11 @@ pub fn init(shared_params: &SharedParams, version: &VersionInfo) -> error::Resul sp_panic_handler::set(version.support_url, &full_version); fdlimit::raise_fd_limit(); - init_logger(shared_params.log.as_ref().map(|v| v.as_ref()).unwrap_or("")); - - Ok(()) -} - -/// Initialize the given `config`. -/// -/// This will load the chain spec, set the `config_dir` and the `database_dir`. -pub fn init_config( - config: &mut Configuration, - shared_params: &SharedParams, - version: &VersionInfo, - spec_factory: F, -) -> error::Result<()> where - F: FnOnce(&str) -> Result>, String>, - G: RuntimeGenesis, - E: ChainSpecExtension, -{ - load_spec(config, shared_params, spec_factory)?; - - if config.config_dir.is_none() { - config.config_dir = Some(base_path(&shared_params, version)); - } - - if config.database.is_none() { - config.database = Some(DatabaseConfig::Path { - path: config - .in_chain_config_dir(DEFAULT_DB_CONFIG_PATH) - .expect("We provided a base_path/config_dir."), - cache_size: None, - }); - } - - Ok(()) -} - -/// Run the node -/// -/// Builds and runs either a full or a light node, depending on the `role` within the `Configuration`. -pub fn run_node( - config: Configuration, - new_light: FNL, - new_full: FNF, - version: &VersionInfo, -) -> error::Result<()> -where - FNL: FnOnce(Configuration) -> Result, - FNF: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - SL: AbstractService + Unpin, - SF: AbstractService + Unpin, -{ - info!("{}", version.name); - info!(" version {}", config.full_version()); - info!(" by {}, {}-{}", version.author, version.copyright_start_year, Local::today().year()); - info!("Chain specification: {}", config.expect_chain_spec().name()); - info!("Node name: {}", config.name); - info!("Roles: {}", display_role(&config)); - - match config.roles { - ServiceRoles::LIGHT => run_service_until_exit( - config, - new_light, - ), - _ => run_service_until_exit( - config, - new_full, - ), - } -} - -/// Returns a string displaying the node role, special casing the sentry mode -/// (returning `SENTRY`), since the node technically has an `AUTHORITY` role but -/// doesn't participate. -pub fn display_role(config: &Configuration) -> String { - if config.sentry_mode { - "SENTRY".to_string() - } else { - format!("{:?}", config.roles) - } -} - -/// Fill the given `PoolConfiguration` by looking at the cli parameters. -fn fill_transaction_pool_configuration( - options: &mut Configuration, - params: TransactionPoolParams, -) -> error::Result<()> { - // ready queue - options.transaction_pool.ready.count = params.pool_limit; - options.transaction_pool.ready.total_bytes = params.pool_kbytes * 1024; - - // future queue - let factor = 10; - options.transaction_pool.future.count = params.pool_limit / factor; - options.transaction_pool.future.total_bytes = params.pool_kbytes * 1024 / factor; + init_logger(logger_pattern); Ok(()) } -/// Fill the given `NetworkConfiguration` by looking at the cli parameters. -fn fill_network_configuration( - cli: NetworkConfigurationParams, - config_path: PathBuf, - config: &mut NetworkConfiguration, - client_id: String, - is_dev: bool, -) -> error::Result<()> { - config.boot_nodes.extend(cli.bootnodes.into_iter()); - config.config_path = Some(config_path.to_string_lossy().into()); - config.net_config_path = config.config_path.clone(); - - config.reserved_nodes.extend(cli.reserved_nodes.into_iter()); - if cli.reserved_only { - config.non_reserved_mode = NonReservedPeerMode::Deny; - } - - config.sentry_nodes.extend(cli.sentry_nodes.into_iter()); - - for addr in cli.listen_addr.iter() { - let addr = addr.parse().ok().ok_or(error::Error::InvalidListenMultiaddress)?; - config.listen_addresses.push(addr); - } - - if config.listen_addresses.is_empty() { - let port = match cli.port { - Some(port) => port, - None => 30333, - }; - - config.listen_addresses = vec![ - iter::once(Protocol::Ip4(Ipv4Addr::new(0, 0, 0, 0))) - .chain(iter::once(Protocol::Tcp(port))) - .collect() - ]; - } - - config.client_version = client_id; - config.node_key = node_key::node_key_config(cli.node_key_params, &config.net_config_path)?; - - config.in_peers = cli.in_peers; - config.out_peers = cli.out_peers; - - config.transport = TransportConfig::Normal { - enable_mdns: !is_dev && !cli.no_mdns, - allow_private_ipv4: !cli.no_private_ipv4, - wasm_external_transport: None, - use_yamux_flow_control: cli.use_yamux_flow_control - }; - - config.max_parallel_downloads = cli.max_parallel_downloads; - - Ok(()) -} - -#[cfg(not(target_os = "unknown"))] -fn input_keystore_password() -> Result { - rpassword::read_password_from_tty(Some("Keystore password: ")) - .map_err(|e| format!("{:?}", e)) -} - -/// Use in memory keystore config when it is not required at all. -pub fn fill_config_keystore_in_memory(config: &mut sc_service::Configuration) - -> Result<(), String> -{ - match &mut config.keystore { - cfg @ KeystoreConfig::None => { *cfg = KeystoreConfig::InMemory; Ok(()) }, - _ => Err("Keystore config specified when it should not be!".into()), - } -} - -/// Fill the password field of the given config instance. -fn fill_config_keystore_password_and_path( - config: &mut sc_service::Configuration, - cli: &RunCmd, -) -> Result<(), String> { - let password = if cli.password_interactive { - #[cfg(not(target_os = "unknown"))] - { - Some(input_keystore_password()?.into()) - } - #[cfg(target_os = "unknown")] - None - } else if let Some(ref file) = cli.password_filename { - Some(fs::read_to_string(file).map_err(|e| format!("{}", e))?.into()) - } else if let Some(ref password) = cli.password { - Some(password.clone().into()) - } else { - None - }; - - let path = cli.keystore_path.clone().or( - config.in_chain_config_dir(DEFAULT_KEYSTORE_CONFIG_PATH) - ); - - config.keystore = KeystoreConfig::Path { - path: path.ok_or_else(|| "No `base_path` provided to create keystore path!")?, - password, - }; - - Ok(()) -} - -/// Put block import CLI params into `config` object. -pub fn fill_import_params( - config: &mut Configuration, - cli: &ImportParams, - role: sc_service::Roles, - is_dev: bool, -) -> error::Result<()> -where - G: RuntimeGenesis, -{ - if let Some(DatabaseConfig::Path { ref mut cache_size, .. }) = config.database { - *cache_size = Some(cli.database_cache_size); - } - - config.state_cache_size = cli.state_cache_size; - - // by default we disable pruning if the node is an authority (i.e. - // `ArchiveAll`), otherwise we keep state for the last 256 blocks. if the - // node is an authority and pruning is enabled explicitly, then we error - // unless `unsafe_pruning` is set. - config.pruning = match &cli.pruning { - Some(ref s) if s == "archive" => PruningMode::ArchiveAll, - None if role == sc_service::Roles::AUTHORITY => PruningMode::ArchiveAll, - None => PruningMode::default(), - Some(s) => { - if role == sc_service::Roles::AUTHORITY && !cli.unsafe_pruning { - return Err(error::Error::Input( - "Validators should run with state pruning disabled (i.e. archive). \ - You can ignore this check with `--unsafe-pruning`.".to_string() - )); - } - - PruningMode::keep_blocks(s.parse() - .map_err(|_| error::Error::Input("Invalid pruning mode specified".to_string()))? - ) - }, - }; - - config.wasm_method = cli.wasm_method.into(); - - let exec = &cli.execution_strategies; - let exec_all_or = |strat: ExecutionStrategy, default: ExecutionStrategy| { - exec.execution.unwrap_or(if strat == default && is_dev { - ExecutionStrategy::Native - } else { - strat - }).into() - }; - - config.execution_strategies = ExecutionStrategies { - syncing: exec_all_or(exec.execution_syncing, DEFAULT_EXECUTION_SYNCING), - importing: exec_all_or(exec.execution_import_block, DEFAULT_EXECUTION_IMPORT_BLOCK), - block_construction: - exec_all_or(exec.execution_block_construction, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION), - offchain_worker: - exec_all_or(exec.execution_offchain_worker, DEFAULT_EXECUTION_OFFCHAIN_WORKER), - other: exec_all_or(exec.execution_other, DEFAULT_EXECUTION_OTHER), - }; - Ok(()) -} - -/// Update and prepare a `Configuration` with command line parameters of `RunCmd` and `VersionInfo` -pub fn update_config_for_running_node( - mut config: &mut Configuration, - cli: RunCmd, -) -> error::Result<()> -where - G: RuntimeGenesis, -{ - fill_config_keystore_password_and_path(&mut config, &cli)?; - - let keyring = cli.get_keyring(); - let is_dev = cli.shared_params.dev; - let is_light = cli.light; - let is_authority = (cli.validator || cli.sentry || is_dev || keyring.is_some()) - && !is_light; - let role = - if is_light { - sc_service::Roles::LIGHT - } else if is_authority { - sc_service::Roles::AUTHORITY - } else { - sc_service::Roles::FULL - }; - - fill_import_params(&mut config, &cli.import_params, role, is_dev)?; - - config.name = match (cli.name.as_ref(), keyring) { - (Some(name), _) => name.to_string(), - (_, Some(keyring)) => keyring.to_string(), - (None, None) => generate_node_name(), - }; - if let Err(msg) = node_key::is_node_name_valid(&config.name) { - return Err(error::Error::Input( - format!("Invalid node name '{}'. Reason: {}. If unsure, use none.", - config.name, - msg, - ) - )); - } - - // set sentry mode (i.e. act as an authority but **never** actively participate) - config.sentry_mode = cli.sentry; - - config.offchain_worker = match (cli.offchain_worker, role) { - (params::OffchainWorkerEnabled::WhenValidating, sc_service::Roles::AUTHORITY) => true, - (params::OffchainWorkerEnabled::Always, _) => true, - (params::OffchainWorkerEnabled::Never, _) => false, - (params::OffchainWorkerEnabled::WhenValidating, _) => false, - }; - - config.roles = role; - config.disable_grandpa = cli.no_grandpa; - - let client_id = config.client_id(); - fill_network_configuration( - cli.network_config, - config.in_chain_config_dir(DEFAULT_NETWORK_CONFIG_PATH).expect("We provided a basepath"), - &mut config.network, - client_id, - is_dev, - )?; - - fill_transaction_pool_configuration(&mut config, cli.pool_config)?; - - config.dev_key_seed = keyring - .map(|a| format!("//{}", a)).or_else(|| { - if is_dev && !is_light { - Some("//Alice".into()) - } else { - None - } - }); - - if config.rpc_http.is_none() || cli.rpc_port.is_some() { - let rpc_interface: &str = interface_str(cli.rpc_external, cli.unsafe_rpc_external, cli.validator)?; - config.rpc_http = Some(parse_address(&format!("{}:{}", rpc_interface, 9933), cli.rpc_port)?); - } - if config.rpc_ws.is_none() || cli.ws_port.is_some() { - let ws_interface: &str = interface_str(cli.ws_external, cli.unsafe_ws_external, cli.validator)?; - config.rpc_ws = Some(parse_address(&format!("{}:{}", ws_interface, 9944), cli.ws_port)?); - } - - if config.grafana_port.is_none() || cli.grafana_port.is_some() { - let grafana_interface: &str = if cli.grafana_external { "0.0.0.0" } else { "127.0.0.1" }; - config.grafana_port = Some( - parse_address(&format!("{}:{}", grafana_interface, 9955), cli.grafana_port)? - ); - } - - config.rpc_ws_max_connections = cli.ws_max_connections; - config.rpc_cors = cli.rpc_cors.unwrap_or_else(|| if is_dev { - log::warn!("Running in --dev mode, RPC CORS has been disabled."); - Cors::All - } else { - Cors::List(vec![ - "http://localhost:*".into(), - "http://127.0.0.1:*".into(), - "https://localhost:*".into(), - "https://127.0.0.1:*".into(), - "https://polkadot.js.org".into(), - "https://substrate-ui.parity.io".into(), - ]) - }).into(); - - // Override telemetry - if cli.no_telemetry { - config.telemetry_endpoints = None; - } else if !cli.telemetry_endpoints.is_empty() { - config.telemetry_endpoints = Some(TelemetryEndpoints::new(cli.telemetry_endpoints)); - } - - config.tracing_targets = cli.import_params.tracing_targets.into(); - config.tracing_receiver = cli.import_params.tracing_receiver.into(); - - // Imply forced authoring on --dev - config.force_authoring = cli.shared_params.dev || cli.force_authoring; - - Ok(()) -} - -fn interface_str( - is_external: bool, - is_unsafe_external: bool, - is_validator: bool, -) -> Result<&'static str, error::Error> { - if is_external && is_validator { - return Err(error::Error::Input("--rpc-external and --ws-external options shouldn't be \ - used if the node is running as a validator. Use `--unsafe-rpc-external` if you understand \ - the risks. See the options description for more information.".to_owned())); - } - - if is_external || is_unsafe_external { - log::warn!("It isn't safe to expose RPC publicly without a proxy server that filters \ - available set of RPC methods."); - - Ok("0.0.0.0") - } else { - Ok("127.0.0.1") - } -} - -fn parse_address( - address: &str, - port: Option, -) -> Result { - let mut address: SocketAddr = address.parse().map_err( - |_| format!("Invalid address: {}", address) - )?; - if let Some(port) = port { - address.set_port(port); - } - - Ok(address) -} - /// Initialize the logger pub fn init_logger(pattern: &str) { use ansi_term::Colour; @@ -765,116 +221,3 @@ fn kill_color(s: &str) -> String { } RE.replace_all(s, "").to_string() } - -#[cfg(test)] -mod tests { - use super::*; - - const TEST_VERSION_INFO: &'static VersionInfo = &VersionInfo { - name: "node-test", - version: "0.1.0", - commit: "some_commit", - executable_name: "node-test", - description: "description", - author: "author", - support_url: "http://example.org", - copyright_start_year: 2020, - }; - - #[test] - fn keystore_path_is_generated_correctly() { - let chain_spec = ChainSpec::from_genesis( - "test", - "test-id", - || (), - Vec::new(), - None, - None, - None, - None::<()>, - ); - - for keystore_path in vec![None, Some("/keystore/path")] { - let args: Vec<&str> = vec![]; - let mut run_cmds = RunCmd::from_iter(args); - run_cmds.keystore_path = keystore_path.clone().map(PathBuf::from); - - let mut node_config = Configuration::default(); - node_config.config_dir = Some(PathBuf::from("/test/path")); - node_config.chain_spec = Some(chain_spec.clone()); - update_config_for_running_node( - &mut node_config, - run_cmds.clone(), - ).unwrap(); - - let expected_path = match keystore_path { - Some(path) => PathBuf::from(path), - None => PathBuf::from("/test/path/chains/test-id/keystore"), - }; - - assert_eq!(expected_path, node_config.keystore.path().unwrap().to_owned()); - } - } - - #[test] - fn ensure_load_spec_provide_defaults() { - let chain_spec = ChainSpec::from_genesis( - "test", - "test-id", - || (), - vec!["boo".to_string()], - Some(TelemetryEndpoints::new(vec![("foo".to_string(), 42)])), - None, - None, - None::<()>, - ); - - let args: Vec<&str> = vec![]; - let cli = RunCmd::from_iter(args); - - let mut config = Configuration::new(TEST_VERSION_INFO); - load_spec(&mut config, &cli.shared_params, |_| Ok(Some(chain_spec))).unwrap(); - - assert!(config.chain_spec.is_some()); - assert!(!config.network.boot_nodes.is_empty()); - assert!(config.telemetry_endpoints.is_some()); - } - - #[test] - fn ensure_update_config_for_running_node_provides_defaults() { - let chain_spec = ChainSpec::from_genesis( - "test", - "test-id", - || (), - vec![], - None, - None, - None, - None::<()>, - ); - - let args: Vec<&str> = vec![]; - let cli = RunCmd::from_iter(args); - - let mut config = Configuration::new(TEST_VERSION_INFO); - init(&cli.shared_params, &TEST_VERSION_INFO).unwrap(); - init_config( - &mut config, - &cli.shared_params, - &TEST_VERSION_INFO, - |_| Ok(Some(chain_spec)), - ).unwrap(); - update_config_for_running_node(&mut config, cli).unwrap(); - - assert!(config.config_dir.is_some()); - assert!(config.database.is_some()); - if let Some(DatabaseConfig::Path { ref cache_size, .. }) = config.database { - assert!(cache_size.is_some()); - } else { - panic!("invalid config.database variant"); - } - assert!(!config.name.is_empty()); - assert!(config.network.config_path.is_some()); - assert!(!config.network.listen_addresses.is_empty()); - } -} diff --git a/client/cli/src/node_key.rs b/client/cli/src/node_key.rs deleted file mode 100644 index 4401481ca56ce1ca171cc8ab9c9ade5ee5874ee7..0000000000000000000000000000000000000000 --- a/client/cli/src/node_key.rs +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2017-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -use sc_network::{ - self, - config::{ - NodeKeyConfig, - }, -}; -use sp_core::H256; -use regex::Regex; -use std::{path::{Path, PathBuf}, str::FromStr}; -use crate::error; -use crate::params::{NodeKeyParams, NodeKeyType}; - -/// The file name of the node's Ed25519 secret key inside the chain-specific -/// network config directory, if neither `--node-key` nor `--node-key-file` -/// is specified in combination with `--node-key-type=ed25519`. -const NODE_KEY_ED25519_FILE: &str = "secret_ed25519"; - -/// Check whether a node name is considered as valid -pub fn is_node_name_valid(_name: &str) -> Result<(), &str> { - let name = _name.to_string(); - if name.chars().count() >= crate::NODE_NAME_MAX_LENGTH { - return Err("Node name too long"); - } - - let invalid_chars = r"[\\.@]"; - let re = Regex::new(invalid_chars).unwrap(); - if re.is_match(&name) { - return Err("Node name should not contain invalid chars such as '.' and '@'"); - } - - let invalid_patterns = r"(https?:\\/+)?(www)+"; - let re = Regex::new(invalid_patterns).unwrap(); - if re.is_match(&name) { - return Err("Node name should not contain urls"); - } - - Ok(()) -} - -/// Create a `NodeKeyConfig` from the given `NodeKeyParams` in the context -/// of an optional network config storage directory. -pub fn node_key_config

(params: NodeKeyParams, net_config_dir: &Option

) - -> error::Result -where - P: AsRef -{ - match params.node_key_type { - NodeKeyType::Ed25519 => - params.node_key.as_ref().map(parse_ed25519_secret).unwrap_or_else(|| - Ok(params.node_key_file - .or_else(|| net_config_file(net_config_dir, NODE_KEY_ED25519_FILE)) - .map(sc_network::config::Secret::File) - .unwrap_or(sc_network::config::Secret::New))) - .map(NodeKeyConfig::Ed25519) - } -} - -/// Create an error caused by an invalid node key argument. -fn invalid_node_key(e: impl std::fmt::Display) -> error::Error { - error::Error::Input(format!("Invalid node key: {}", e)) -} - -/// Parse a Ed25519 secret key from a hex string into a `sc_network::Secret`. -fn parse_ed25519_secret(hex: &String) -> error::Result { - H256::from_str(&hex).map_err(invalid_node_key).and_then(|bytes| - sc_network::config::identity::ed25519::SecretKey::from_bytes(bytes) - .map(sc_network::config::Secret::Input) - .map_err(invalid_node_key)) -} - -fn net_config_file

(net_config_dir: &Option

, name: &str) -> Option -where - P: AsRef -{ - net_config_dir.as_ref().map(|d| d.as_ref().join(name)) -} - -#[cfg(test)] -mod tests { - use sc_network::config::identity::ed25519; - use super::*; - - #[test] - fn tests_node_name_good() { - assert!(is_node_name_valid("short name").is_ok()); - } - - #[test] - fn tests_node_name_bad() { - assert!(is_node_name_valid("long names are not very cool for the ui").is_err()); - assert!(is_node_name_valid("Dots.not.Ok").is_err()); - assert!(is_node_name_valid("http://visit.me").is_err()); - assert!(is_node_name_valid("https://visit.me").is_err()); - assert!(is_node_name_valid("www.visit.me").is_err()); - assert!(is_node_name_valid("email@domain").is_err()); - } - - #[test] - fn test_node_key_config_input() { - fn secret_input(net_config_dir: Option) -> error::Result<()> { - NodeKeyType::variants().iter().try_for_each(|t| { - let node_key_type = NodeKeyType::from_str(t).unwrap(); - let sk = match node_key_type { - NodeKeyType::Ed25519 => ed25519::SecretKey::generate().as_ref().to_vec() - }; - let params = NodeKeyParams { - node_key_type, - node_key: Some(format!("{:x}", H256::from_slice(sk.as_ref()))), - node_key_file: None - }; - node_key_config(params, &net_config_dir).and_then(|c| match c { - NodeKeyConfig::Ed25519(sc_network::config::Secret::Input(ref ski)) - if node_key_type == NodeKeyType::Ed25519 && - &sk[..] == ski.as_ref() => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) - }) - }) - } - - assert!(secret_input(None).is_ok()); - assert!(secret_input(Some("x".to_string())).is_ok()); - } - - #[test] - fn test_node_key_config_file() { - fn secret_file(net_config_dir: Option) -> error::Result<()> { - NodeKeyType::variants().iter().try_for_each(|t| { - let node_key_type = NodeKeyType::from_str(t).unwrap(); - let tmp = tempfile::Builder::new().prefix("alice").tempdir()?; - let file = tmp.path().join(format!("{}_mysecret", t)).to_path_buf(); - let params = NodeKeyParams { - node_key_type, - node_key: None, - node_key_file: Some(file.clone()) - }; - node_key_config(params, &net_config_dir).and_then(|c| match c { - NodeKeyConfig::Ed25519(sc_network::config::Secret::File(ref f)) - if node_key_type == NodeKeyType::Ed25519 && f == &file => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) - }) - }) - } - - assert!(secret_file(None).is_ok()); - assert!(secret_file(Some("x".to_string())).is_ok()); - } - - #[test] - fn test_node_key_config_default() { - fn with_def_params(f: F) -> error::Result<()> - where - F: Fn(NodeKeyParams) -> error::Result<()> - { - NodeKeyType::variants().iter().try_for_each(|t| { - let node_key_type = NodeKeyType::from_str(t).unwrap(); - f(NodeKeyParams { - node_key_type, - node_key: None, - node_key_file: None - }) - }) - } - - fn no_config_dir() -> error::Result<()> { - with_def_params(|params| { - let typ = params.node_key_type; - node_key_config::(params, &None) - .and_then(|c| match c { - NodeKeyConfig::Ed25519(sc_network::config::Secret::New) - if typ == NodeKeyType::Ed25519 => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) - }) - }) - } - - fn some_config_dir(net_config_dir: String) -> error::Result<()> { - with_def_params(|params| { - let dir = PathBuf::from(net_config_dir.clone()); - let typ = params.node_key_type; - node_key_config(params, &Some(net_config_dir.clone())) - .and_then(move |c| match c { - NodeKeyConfig::Ed25519(sc_network::config::Secret::File(ref f)) - if typ == NodeKeyType::Ed25519 && - f == &dir.join(NODE_KEY_ED25519_FILE) => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) - }) - }) - } - - assert!(no_config_dir().is_ok()); - assert!(some_config_dir("x".to_string()).is_ok()); - } -} diff --git a/client/cli/src/params.rs b/client/cli/src/params.rs deleted file mode 100644 index d6c437f668b2c39ed496b6f9f220f050785058c7..0000000000000000000000000000000000000000 --- a/client/cli/src/params.rs +++ /dev/null @@ -1,1268 +0,0 @@ -// Copyright 2018-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -use std::{str::FromStr, path::PathBuf}; -use structopt::{StructOpt, clap::arg_enum}; -use sc_service::{ - AbstractService, Configuration, ChainSpecExtension, RuntimeGenesis, ServiceBuilderCommand, - config::DatabaseConfig, -}; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; -use crate::VersionInfo; -use crate::error; -use std::fmt::Debug; -use log::info; -use sc_network::config::build_multiaddr; -use std::io; -use std::fs; -use std::io::{Read, Write, Seek}; -use sp_runtime::generic::BlockId; -use crate::runtime::run_until_exit; -use crate::node_key::node_key_config; -use crate::execution_strategy::*; - -pub use crate::execution_strategy::ExecutionStrategy; - -impl Into for ExecutionStrategy { - fn into(self) -> sc_client_api::ExecutionStrategy { - match self { - ExecutionStrategy::Native => sc_client_api::ExecutionStrategy::NativeWhenPossible, - ExecutionStrategy::Wasm => sc_client_api::ExecutionStrategy::AlwaysWasm, - ExecutionStrategy::Both => sc_client_api::ExecutionStrategy::Both, - ExecutionStrategy::NativeElseWasm => sc_client_api::ExecutionStrategy::NativeElseWasm, - } - } -} - -arg_enum! { - /// How to execute Wasm runtime code - #[allow(missing_docs)] - #[derive(Debug, Clone, Copy)] - pub enum WasmExecutionMethod { - // Uses an interpreter. - Interpreted, - // Uses a compiled runtime. - Compiled, - } -} - -impl WasmExecutionMethod { - /// Returns list of variants that are not disabled by feature flags. - fn enabled_variants() -> Vec<&'static str> { - Self::variants() - .iter() - .cloned() - .filter(|&name| cfg!(feature = "wasmtime") || name != "Compiled") - .collect() - } -} - -impl Into for WasmExecutionMethod { - fn into(self) -> sc_service::config::WasmExecutionMethod { - match self { - WasmExecutionMethod::Interpreted => sc_service::config::WasmExecutionMethod::Interpreted, - #[cfg(feature = "wasmtime")] - WasmExecutionMethod::Compiled => sc_service::config::WasmExecutionMethod::Compiled, - #[cfg(not(feature = "wasmtime"))] - WasmExecutionMethod::Compiled => panic!( - "Substrate must be compiled with \"wasmtime\" feature for compiled Wasm execution" - ), - } - } -} - -arg_enum! { - /// Whether off-chain workers are enabled. - #[allow(missing_docs)] - #[derive(Debug, Clone)] - pub enum OffchainWorkerEnabled { - Always, - Never, - WhenValidating, - } -} - -/// Shared parameters used by all `CoreParams`. -#[derive(Debug, StructOpt, Clone)] -pub struct SharedParams { - /// Specify the chain specification (one of dev, local or staging). - #[structopt(long = "chain", value_name = "CHAIN_SPEC")] - pub chain: Option, - - /// Specify the development chain. - #[structopt(long = "dev")] - pub dev: bool, - - /// Specify custom base path. - #[structopt(long = "base-path", short = "d", value_name = "PATH", parse(from_os_str))] - pub base_path: Option, - - /// Sets a custom logging filter. - #[structopt(short = "l", long = "log", value_name = "LOG_PATTERN")] - pub log: Option, -} - -/// Parameters for block import. -#[derive(Debug, StructOpt, Clone)] -pub struct ImportParams { - /// Specify the state pruning mode, a number of blocks to keep or 'archive'. - /// - /// Default is to keep all block states if the node is running as a - /// validator (i.e. 'archive'), otherwise state is only kept for the last - /// 256 blocks. - #[structopt(long = "pruning", value_name = "PRUNING_MODE")] - pub pruning: Option, - - /// Force start with unsafe pruning settings. - /// - /// When running as a validator it is highly recommended to disable state - /// pruning (i.e. 'archive') which is the default. The node will refuse to - /// start as a validator if pruning is enabled unless this option is set. - #[structopt(long = "unsafe-pruning")] - pub unsafe_pruning: bool, - - /// Method for executing Wasm runtime code. - #[structopt( - long = "wasm-execution", - value_name = "METHOD", - possible_values = &WasmExecutionMethod::enabled_variants(), - case_insensitive = true, - default_value = "Interpreted" - )] - pub wasm_method: WasmExecutionMethod, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub execution_strategies: ExecutionStrategies, - - /// Limit the memory the database cache can use. - #[structopt(long = "db-cache", value_name = "MiB", default_value = "1024")] - pub database_cache_size: u32, - - /// Specify the state cache size. - #[structopt(long = "state-cache-size", value_name = "Bytes", default_value = "67108864")] - pub state_cache_size: usize, - - /// Comma separated list of targets for tracing - #[structopt(long = "tracing-targets", value_name = "TARGETS")] - pub tracing_targets: Option, - - /// Receiver to process tracing messages - #[structopt( - long = "tracing-receiver", - value_name = "RECEIVER", - possible_values = &TracingReceiver::variants(), - case_insensitive = true, - default_value = "Log" - )] - pub tracing_receiver: TracingReceiver, -} - -/// Parameters used to create the network configuration. -#[derive(Debug, StructOpt, Clone)] -pub struct NetworkConfigurationParams { - /// Specify a list of bootnodes. - #[structopt(long = "bootnodes", value_name = "URL")] - pub bootnodes: Vec, - - /// Specify a list of reserved node addresses. - #[structopt(long = "reserved-nodes", value_name = "URL")] - pub reserved_nodes: Vec, - - /// Whether to only allow connections to/from reserved nodes. - /// - /// If you are a validator your node might still connect to other validator - /// nodes regardless of whether they are defined as reserved nodes. - #[structopt(long = "reserved-only")] - pub reserved_only: bool, - - /// Specify a list of sentry node public addresses. - #[structopt( - long = "sentry-nodes", - value_name = "URL", - conflicts_with_all = &[ "sentry" ] - )] - pub sentry_nodes: Vec, - - /// Listen on this multiaddress. - #[structopt(long = "listen-addr", value_name = "LISTEN_ADDR")] - pub listen_addr: Vec, - - /// Specify p2p protocol TCP port. - /// - /// Only used if --listen-addr is not specified. - #[structopt(long = "port", value_name = "PORT")] - pub port: Option, - - /// Forbid connecting to private IPv4 addresses (as specified in - /// [RFC1918](https://tools.ietf.org/html/rfc1918)), unless the address was passed with - /// `--reserved-nodes` or `--bootnodes`. - #[structopt(long = "no-private-ipv4")] - pub no_private_ipv4: bool, - - /// Specify the number of outgoing connections we're trying to maintain. - #[structopt(long = "out-peers", value_name = "COUNT", default_value = "25")] - pub out_peers: u32, - - /// Specify the maximum number of incoming connections we're accepting. - #[structopt(long = "in-peers", value_name = "COUNT", default_value = "25")] - pub in_peers: u32, - - /// Disable mDNS discovery. - /// - /// By default, the network will use mDNS to discover other nodes on the - /// local network. This disables it. Automatically implied when using --dev. - #[structopt(long = "no-mdns")] - pub no_mdns: bool, - - /// Maximum number of peers to ask the same blocks in parallel. - /// - /// This allows downlading announced blocks from multiple peers. Decrease to save - /// traffic and risk increased latency. - #[structopt(long = "max-parallel-downloads", value_name = "COUNT", default_value = "5")] - pub max_parallel_downloads: u32, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub node_key_params: NodeKeyParams, - - /// Experimental feature flag. - #[structopt(long = "use-yamux-flow-control")] - pub use_yamux_flow_control: bool, -} - -arg_enum! { - #[allow(missing_docs)] - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - pub enum NodeKeyType { - Ed25519 - } -} - -/// Parameters used to create the `NodeKeyConfig`, which determines the keypair -/// used for libp2p networking. -#[derive(Debug, StructOpt, Clone)] -pub struct NodeKeyParams { - /// The secret key to use for libp2p networking. - /// - /// The value is a string that is parsed according to the choice of - /// `--node-key-type` as follows: - /// - /// `ed25519`: - /// The value is parsed as a hex-encoded Ed25519 32 bytes secret key, - /// i.e. 64 hex characters. - /// - /// The value of this option takes precedence over `--node-key-file`. - /// - /// WARNING: Secrets provided as command-line arguments are easily exposed. - /// Use of this option should be limited to development and testing. To use - /// an externally managed secret key, use `--node-key-file` instead. - #[structopt(long = "node-key", value_name = "KEY")] - pub node_key: Option, - - /// The type of secret key to use for libp2p networking. - /// - /// The secret key of the node is obtained as follows: - /// - /// * If the `--node-key` option is given, the value is parsed as a secret key - /// according to the type. See the documentation for `--node-key`. - /// - /// * If the `--node-key-file` option is given, the secret key is read from the - /// specified file. See the documentation for `--node-key-file`. - /// - /// * Otherwise, the secret key is read from a file with a predetermined, - /// type-specific name from the chain-specific network config directory - /// inside the base directory specified by `--base-dir`. If this file does - /// not exist, it is created with a newly generated secret key of the - /// chosen type. - /// - /// The node's secret key determines the corresponding public key and hence the - /// node's peer ID in the context of libp2p. - #[structopt( - long = "node-key-type", - value_name = "TYPE", - possible_values = &NodeKeyType::variants(), - case_insensitive = true, - default_value = "Ed25519" - )] - pub node_key_type: NodeKeyType, - - /// The file from which to read the node's secret key to use for libp2p networking. - /// - /// The contents of the file are parsed according to the choice of `--node-key-type` - /// as follows: - /// - /// `ed25519`: - /// The file must contain an unencoded 32 bytes Ed25519 secret key. - /// - /// If the file does not exist, it is created with a newly generated secret key of - /// the chosen type. - #[structopt(long = "node-key-file", value_name = "FILE")] - pub node_key_file: Option, -} - -/// Parameters used to create the pool configuration. -#[derive(Debug, StructOpt, Clone)] -pub struct TransactionPoolParams { - /// Maximum number of transactions in the transaction pool. - #[structopt(long = "pool-limit", value_name = "COUNT", default_value = "8192")] - pub pool_limit: usize, - /// Maximum number of kilobytes of all transactions stored in the pool. - #[structopt(long = "pool-kbytes", value_name = "COUNT", default_value = "20480")] - pub pool_kbytes: usize, -} - -arg_enum! { - #[allow(missing_docs)] - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - pub enum TracingReceiver { - Log, - Telemetry, - Grafana, - } -} - -impl Into for TracingReceiver { - fn into(self) -> sc_tracing::TracingReceiver { - match self { - TracingReceiver::Log => sc_tracing::TracingReceiver::Log, - TracingReceiver::Telemetry => sc_tracing::TracingReceiver::Telemetry, - TracingReceiver::Grafana => sc_tracing::TracingReceiver::Grafana, - } - } -} - -/// Execution strategies parameters. -#[derive(Debug, StructOpt, Clone)] -pub struct ExecutionStrategies { - /// The means of execution used when calling into the runtime while syncing blocks. - #[structopt( - long = "execution-syncing", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - default_value = DEFAULT_EXECUTION_SYNCING.as_str(), - )] - pub execution_syncing: ExecutionStrategy, - - /// The means of execution used when calling into the runtime while importing blocks. - #[structopt( - long = "execution-import-block", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - default_value = DEFAULT_EXECUTION_IMPORT_BLOCK.as_str(), - )] - pub execution_import_block: ExecutionStrategy, - - /// The means of execution used when calling into the runtime while constructing blocks. - #[structopt( - long = "execution-block-construction", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - default_value = DEFAULT_EXECUTION_BLOCK_CONSTRUCTION.as_str(), - )] - pub execution_block_construction: ExecutionStrategy, - - /// The means of execution used when calling into the runtime while using an off-chain worker. - #[structopt( - long = "execution-offchain-worker", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - default_value = DEFAULT_EXECUTION_OFFCHAIN_WORKER.as_str(), - )] - pub execution_offchain_worker: ExecutionStrategy, - - /// The means of execution used when calling into the runtime while not syncing, importing or constructing blocks. - #[structopt( - long = "execution-other", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - default_value = DEFAULT_EXECUTION_OTHER.as_str(), - )] - pub execution_other: ExecutionStrategy, - - /// The execution strategy that should be used by all execution contexts. - #[structopt( - long = "execution", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - conflicts_with_all = &[ - "execution-other", - "execution-offchain-worker", - "execution-block-construction", - "execution-import-block", - "execution-syncing", - ] - )] - pub execution: Option, -} - -/// The `run` command used to run a node. -#[derive(Debug, StructOpt, Clone)] -pub struct RunCmd { - /// Enable validator mode. - /// - /// The node will be started with the authority role and actively - /// participate in any consensus task that it can (e.g. depending on - /// availability of local keys). - #[structopt( - long = "validator", - conflicts_with_all = &[ "sentry" ] - )] - pub validator: bool, - - /// Enable sentry mode. - /// - /// The node will be started with the authority role and participate in - /// consensus tasks as an "observer", it will never actively participate - /// regardless of whether it could (e.g. keys are available locally). This - /// mode is useful as a secure proxy for validators (which would run - /// detached from the network), since we want this node to participate in - /// the full consensus protocols in order to have all needed consensus data - /// available to relay to private nodes. - #[structopt( - long = "sentry", - conflicts_with_all = &[ "validator", "light" ] - )] - pub sentry: bool, - - /// Disable GRANDPA voter when running in validator mode, otherwise disables the GRANDPA observer. - #[structopt(long = "no-grandpa")] - pub no_grandpa: bool, - - /// Experimental: Run in light client mode. - #[structopt(long = "light", conflicts_with = "sentry")] - pub light: bool, - - /// Listen to all RPC interfaces. - /// - /// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use a RPC proxy - /// server to filter out dangerous methods. More details: https://github.com/paritytech/substrate/wiki/Public-RPC. - /// Use `--unsafe-rpc-external` to suppress the warning if you understand the risks. - #[structopt(long = "rpc-external")] - pub rpc_external: bool, - - /// Listen to all RPC interfaces. - /// - /// Same as `--rpc-external`. - #[structopt(long = "unsafe-rpc-external")] - pub unsafe_rpc_external: bool, - - /// Listen to all Websocket interfaces. - /// - /// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use a RPC proxy - /// server to filter out dangerous methods. More details: https://github.com/paritytech/substrate/wiki/Public-RPC. - /// Use `--unsafe-ws-external` to suppress the warning if you understand the risks. - #[structopt(long = "ws-external")] - pub ws_external: bool, - - /// Listen to all Websocket interfaces. - /// - /// Same as `--ws-external`. - #[structopt(long = "unsafe-ws-external")] - pub unsafe_ws_external: bool, - - /// Listen to all Grafana data source interfaces. - /// - /// Default is local. - #[structopt(long = "grafana-external")] - pub grafana_external: bool, - - /// Specify HTTP RPC server TCP port. - #[structopt(long = "rpc-port", value_name = "PORT")] - pub rpc_port: Option, - - /// Specify WebSockets RPC server TCP port. - #[structopt(long = "ws-port", value_name = "PORT")] - pub ws_port: Option, - - /// Maximum number of WS RPC server connections. - #[structopt(long = "ws-max-connections", value_name = "COUNT")] - pub ws_max_connections: Option, - - /// Specify browser Origins allowed to access the HTTP & WS RPC servers. - /// - /// A comma-separated list of origins (protocol://domain or special `null` - /// value). Value of `all` will disable origin validation. Default is to - /// allow localhost, https://polkadot.js.org and - /// https://substrate-ui.parity.io origins. When running in --dev mode the - /// default is to allow all origins. - #[structopt(long = "rpc-cors", value_name = "ORIGINS", parse(try_from_str = parse_cors))] - pub rpc_cors: Option, - - /// Specify Grafana data source server TCP Port. - #[structopt(long = "grafana-port", value_name = "PORT")] - pub grafana_port: Option, - - /// The human-readable name for this node. - /// - /// The node name will be reported to the telemetry server, if enabled. - #[structopt(long = "name", value_name = "NAME")] - pub name: Option, - - /// Disable connecting to the Substrate telemetry server. - /// - /// Telemetry is on by default on global chains. - #[structopt(long = "no-telemetry")] - pub no_telemetry: bool, - - /// The URL of the telemetry server to connect to. - /// - /// This flag can be passed multiple times as a mean to specify multiple - /// telemetry endpoints. Verbosity levels range from 0-9, with 0 denoting - /// the least verbosity. If no verbosity level is specified the default is - /// 0. - #[structopt(long = "telemetry-url", value_name = "URL VERBOSITY", parse(try_from_str = parse_telemetry_endpoints))] - pub telemetry_endpoints: Vec<(String, u8)>, - - /// Should execute offchain workers on every block. - /// - /// By default it's only enabled for nodes that are authoring new blocks. - #[structopt( - long = "offchain-worker", - value_name = "ENABLED", - possible_values = &OffchainWorkerEnabled::variants(), - case_insensitive = true, - default_value = "WhenValidating" - )] - pub offchain_worker: OffchainWorkerEnabled, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub import_params: ImportParams, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub network_config: NetworkConfigurationParams, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub pool_config: TransactionPoolParams, - - /// Shortcut for `--name Alice --validator` with session keys for `Alice` added to keystore. - #[structopt(long, conflicts_with_all = &["bob", "charlie", "dave", "eve", "ferdie", "one", "two"])] - pub alice: bool, - - /// Shortcut for `--name Bob --validator` with session keys for `Bob` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "charlie", "dave", "eve", "ferdie", "one", "two"])] - pub bob: bool, - - /// Shortcut for `--name Charlie --validator` with session keys for `Charlie` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "dave", "eve", "ferdie", "one", "two"])] - pub charlie: bool, - - /// Shortcut for `--name Dave --validator` with session keys for `Dave` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "eve", "ferdie", "one", "two"])] - pub dave: bool, - - /// Shortcut for `--name Eve --validator` with session keys for `Eve` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "ferdie", "one", "two"])] - pub eve: bool, - - /// Shortcut for `--name Ferdie --validator` with session keys for `Ferdie` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "one", "two"])] - pub ferdie: bool, - - /// Shortcut for `--name One --validator` with session keys for `One` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "ferdie", "two"])] - pub one: bool, - - /// Shortcut for `--name Two --validator` with session keys for `Two` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "ferdie", "one"])] - pub two: bool, - - /// Enable authoring even when offline. - #[structopt(long = "force-authoring")] - pub force_authoring: bool, - - /// Specify custom keystore path. - #[structopt(long = "keystore-path", value_name = "PATH", parse(from_os_str))] - pub keystore_path: Option, - - /// Use interactive shell for entering the password used by the keystore. - #[structopt( - long = "password-interactive", - conflicts_with_all = &[ "password", "password-filename" ] - )] - pub password_interactive: bool, - - /// Password used by the keystore. - #[structopt( - long = "password", - conflicts_with_all = &[ "password-interactive", "password-filename" ] - )] - pub password: Option, - - /// File that contains the password used by the keystore. - #[structopt( - long = "password-filename", - value_name = "PATH", - parse(from_os_str), - conflicts_with_all = &[ "password-interactive", "password" ] - )] - pub password_filename: Option -} - -impl RunCmd { - /// Get the `Sr25519Keyring` matching one of the flag - pub fn get_keyring(&self) -> Option { - use sp_keyring::Sr25519Keyring::*; - - if self.alice { Some(Alice) } - else if self.bob { Some(Bob) } - else if self.charlie { Some(Charlie) } - else if self.dave { Some(Dave) } - else if self.eve { Some(Eve) } - else if self.ferdie { Some(Ferdie) } - else if self.one { Some(One) } - else if self.two { Some(Two) } - else { None } - } -} - -/// Default to verbosity level 0, if none is provided. -fn parse_telemetry_endpoints(s: &str) -> Result<(String, u8), Box> { - let pos = s.find(' '); - match pos { - None => { - Ok((s.to_owned(), 0)) - }, - Some(pos_) => { - let verbosity = s[pos_ + 1..].parse()?; - let url = s[..pos_].parse()?; - Ok((url, verbosity)) - } - } -} - -/// CORS setting -/// -/// The type is introduced to overcome `Option>` -/// handling of `structopt`. -#[derive(Clone, Debug)] -pub enum Cors { - /// All hosts allowed - All, - /// Only hosts on the list are allowed. - List(Vec), -} - -impl From for Option> { - fn from(cors: Cors) -> Self { - match cors { - Cors::All => None, - Cors::List(list) => Some(list), - } - } -} - -/// Parse cors origins -fn parse_cors(s: &str) -> Result> { - let mut is_all = false; - let mut origins = Vec::new(); - for part in s.split(',') { - match part { - "all" | "*" => { - is_all = true; - break; - }, - other => origins.push(other.to_owned()), - } - } - - Ok(if is_all { Cors::All } else { Cors::List(origins) }) -} - -/// The `build-spec` command used to build a specification. -#[derive(Debug, StructOpt, Clone)] -pub struct BuildSpecCmd { - /// Force raw genesis storage output. - #[structopt(long = "raw")] - pub raw: bool, - - /// Disable adding the default bootnode to the specification. - /// - /// By default the `/ip4/127.0.0.1/tcp/30333/p2p/NODE_PEER_ID` bootnode is added to the - /// specification when no bootnode exists. - #[structopt(long = "disable-default-bootnode")] - pub disable_default_bootnode: bool, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub node_key_params: NodeKeyParams, -} - -/// Wrapper type of `String` that holds an unsigned integer of arbitrary size, formatted as a decimal. -#[derive(Debug, Clone)] -pub struct BlockNumber(String); - -impl FromStr for BlockNumber { - type Err = String; - - fn from_str(block_number: &str) -> Result { - if block_number.chars().any(|d| !d.is_digit(10)) { - Err(format!( - "Invalid block number: {}, expected decimal formatted unsigned integer", - block_number - )) - } else { - Ok(Self(block_number.to_owned())) - } - } -} - -impl BlockNumber { - /// Wrapper on top of `std::str::parse` but with `Error` as a `String` - /// - /// See `https://doc.rust-lang.org/std/primitive.str.html#method.parse` for more elaborate - /// documentation. - pub fn parse(&self) -> Result - where - N: FromStr, - N::Err: std::fmt::Debug, - { - self.0 - .parse() - .map_err(|e| format!("BlockNumber: {} parsing failed because of {:?}", self.0, e)) - } -} - -/// The `export-blocks` command used to export blocks. -#[derive(Debug, StructOpt, Clone)] -pub struct ExportBlocksCmd { - /// Output file name or stdout if unspecified. - #[structopt(parse(from_os_str))] - pub output: Option, - - /// Specify starting block number. - /// - /// Default is 1. - #[structopt(long = "from", value_name = "BLOCK")] - pub from: Option, - - /// Specify last block number. - /// - /// Default is best block. - #[structopt(long = "to", value_name = "BLOCK")] - pub to: Option, - - /// Use JSON output rather than binary. - #[structopt(long = "json")] - pub json: bool, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, -} - -/// The `import-blocks` command used to import blocks. -#[derive(Debug, StructOpt, Clone)] -pub struct ImportBlocksCmd { - /// Input file or stdin if unspecified. - #[structopt(parse(from_os_str))] - pub input: Option, - - /// The default number of 64KB pages to ever allocate for Wasm execution. - /// - /// Don't alter this unless you know what you're doing. - #[structopt(long = "default-heap-pages", value_name = "COUNT")] - pub default_heap_pages: Option, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub import_params: ImportParams, -} - -/// The `check-block` command used to validate blocks. -#[derive(Debug, StructOpt, Clone)] -pub struct CheckBlockCmd { - /// Block hash or number - #[structopt(value_name = "HASH or NUMBER")] - pub input: String, - - /// The default number of 64KB pages to ever allocate for Wasm execution. - /// - /// Don't alter this unless you know what you're doing. - #[structopt(long = "default-heap-pages", value_name = "COUNT")] - pub default_heap_pages: Option, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub import_params: ImportParams, -} - -/// The `revert` command used revert the chain to a previous state. -#[derive(Debug, StructOpt, Clone)] -pub struct RevertCmd { - /// Number of blocks to revert. - #[structopt(default_value = "256")] - pub num: BlockNumber, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, -} - -/// The `purge-chain` command used to remove the whole chain. -#[derive(Debug, StructOpt, Clone)] -pub struct PurgeChainCmd { - /// Skip interactive prompt by answering yes automatically. - #[structopt(short = "y")] - pub yes: bool, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, -} - -/// The `benchmark` command used to benchmark FRAME Pallets. -#[derive(Debug, StructOpt, Clone)] -pub struct BenchmarkCmd { - /// Select a FRAME Pallet to benchmark. - #[structopt(short, long)] - pub pallet: String, - - /// Select an extrinsic to benchmark. - #[structopt(short, long)] - pub extrinsic: String, - - /// Select how many samples we should take across the variable components. - #[structopt(short, long, default_value = "1")] - pub steps: u32, - - /// Select how many repetitions of this benchmark should run. - #[structopt(short, long, default_value = "1")] - pub repeat: u32, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, - - /// The execution strategy that should be used for benchmarks - #[structopt( - long = "execution", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - )] - pub execution: Option, - - /// Method for executing Wasm runtime code. - #[structopt( - long = "wasm-execution", - value_name = "METHOD", - possible_values = &WasmExecutionMethod::enabled_variants(), - case_insensitive = true, - default_value = "Interpreted" - )] - pub wasm_method: WasmExecutionMethod, -} - -/// All core commands that are provided by default. -/// -/// The core commands are split into multiple subcommands and `Run` is the default subcommand. From -/// the CLI user perspective, it is not visible that `Run` is a subcommand. So, all parameters of -/// `Run` are exported as main executable parameters. -#[derive(Debug, Clone, StructOpt)] -pub enum Subcommand { - /// Build a spec.json file, outputing to stdout. - BuildSpec(BuildSpecCmd), - - /// Export blocks to a file. - ExportBlocks(ExportBlocksCmd), - - /// Import blocks from file. - ImportBlocks(ImportBlocksCmd), - - /// Validate a single block. - CheckBlock(CheckBlockCmd), - - /// Revert chain to the previous state. - Revert(RevertCmd), - - /// Remove the whole chain data. - PurgeChain(PurgeChainCmd), - - /// Run runtime benchmarks. - Benchmark(BenchmarkCmd), -} - -impl Subcommand { - /// Get the shared parameters of a `CoreParams` command - pub fn get_shared_params(&self) -> &SharedParams { - use Subcommand::*; - - match self { - BuildSpec(params) => ¶ms.shared_params, - ExportBlocks(params) => ¶ms.shared_params, - ImportBlocks(params) => ¶ms.shared_params, - CheckBlock(params) => ¶ms.shared_params, - Revert(params) => ¶ms.shared_params, - PurgeChain(params) => ¶ms.shared_params, - Benchmark(params) => ¶ms.shared_params, - } - } - - /// Run any `CoreParams` command - pub fn run( - self, - config: Configuration, - builder: B, - ) -> error::Result<()> - where - B: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - BC: ServiceBuilderCommand + Unpin, - BB: sp_runtime::traits::Block + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, - { - assert!(config.chain_spec.is_some(), "chain_spec must be present before continuing"); - - match self { - Subcommand::BuildSpec(cmd) => cmd.run(config), - Subcommand::ExportBlocks(cmd) => cmd.run(config, builder), - Subcommand::ImportBlocks(cmd) => cmd.run(config, builder), - Subcommand::CheckBlock(cmd) => cmd.run(config, builder), - Subcommand::PurgeChain(cmd) => cmd.run(config), - Subcommand::Benchmark(cmd) => cmd.run(config, builder), - Subcommand::Revert(cmd) => cmd.run(config, builder), - } - } -} - -impl RunCmd { - /// Run the command that runs the node - pub fn run( - self, - mut config: Configuration, - new_light: FNL, - new_full: FNF, - version: &VersionInfo, - ) -> error::Result<()> - where - G: RuntimeGenesis, - E: ChainSpecExtension, - FNL: FnOnce(Configuration) -> Result, - FNF: FnOnce(Configuration) -> Result, - SL: AbstractService + Unpin, - SF: AbstractService + Unpin, - { - assert!(config.chain_spec.is_some(), "chain_spec must be present before continuing"); - - crate::update_config_for_running_node(&mut config, self)?; - - crate::run_node(config, new_light, new_full, &version) - } -} - -impl BuildSpecCmd { - /// Run the build-spec command - pub fn run( - self, - config: Configuration, - ) -> error::Result<()> - where - G: RuntimeGenesis, - E: ChainSpecExtension, - { - assert!(config.chain_spec.is_some(), "chain_spec must be present before continuing"); - - info!("Building chain spec"); - let mut spec = config.expect_chain_spec().clone(); - let raw_output = self.raw; - - if spec.boot_nodes().is_empty() && !self.disable_default_bootnode { - let node_key = node_key_config( - self.node_key_params.clone(), - &Some(config - .in_chain_config_dir(crate::DEFAULT_NETWORK_CONFIG_PATH) - .expect("We provided a base_path")), - )?; - let keys = node_key.into_keypair()?; - let peer_id = keys.public().into_peer_id(); - let addr = build_multiaddr![ - Ip4([127, 0, 0, 1]), - Tcp(30333u16), - P2p(peer_id) - ]; - spec.add_boot_node(addr) - } - - let json = sc_service::chain_ops::build_spec(spec, raw_output)?; - - print!("{}", json); - - Ok(()) - } -} - -impl ExportBlocksCmd { - /// Run the export-blocks command - pub fn run( - self, - mut config: Configuration, - builder: B, - ) -> error::Result<()> - where - B: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - BC: ServiceBuilderCommand + Unpin, - BB: sp_runtime::traits::Block + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, - { - assert!(config.chain_spec.is_some(), "chain_spec must be present before continuing"); - - crate::fill_config_keystore_in_memory(&mut config)?; - - if let DatabaseConfig::Path { ref path, .. } = config.expect_database() { - info!("DB path: {}", path.display()); - } - let from = self.from.as_ref().and_then(|f| f.parse().ok()).unwrap_or(1); - let to = self.to.as_ref().and_then(|t| t.parse().ok()); - - let json = self.json; - - let file: Box = match &self.output { - Some(filename) => Box::new(fs::File::create(filename)?), - None => Box::new(io::stdout()), - }; - - run_until_exit(config, |config| { - Ok(builder(config)?.export_blocks(file, from.into(), to, json)) - }) - } -} - -/// Internal trait used to cast to a dynamic type that implements Read and Seek. -trait ReadPlusSeek: Read + Seek {} - -impl ReadPlusSeek for T {} - -impl ImportBlocksCmd { - /// Run the import-blocks command - pub fn run( - self, - mut config: Configuration, - builder: B, - ) -> error::Result<()> - where - B: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - BC: ServiceBuilderCommand + Unpin, - BB: sp_runtime::traits::Block + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, - { - crate::fill_import_params( - &mut config, - &self.import_params, - sc_service::Roles::FULL, - self.shared_params.dev, - )?; - - let file: Box = match &self.input { - Some(filename) => Box::new(fs::File::open(filename)?), - None => { - let mut buffer = Vec::new(); - io::stdin().read_to_end(&mut buffer)?; - Box::new(io::Cursor::new(buffer)) - }, - }; - - run_until_exit(config, |config| { - Ok(builder(config)?.import_blocks(file, false)) - }) - } -} - -impl CheckBlockCmd { - /// Run the check-block command - pub fn run( - self, - mut config: Configuration, - builder: B, - ) -> error::Result<()> - where - B: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - BC: ServiceBuilderCommand + Unpin, - BB: sp_runtime::traits::Block + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, - { - assert!(config.chain_spec.is_some(), "chain_spec must be present before continuing"); - - crate::fill_import_params( - &mut config, - &self.import_params, - sc_service::Roles::FULL, - self.shared_params.dev, - )?; - crate::fill_config_keystore_in_memory(&mut config)?; - - let input = if self.input.starts_with("0x") { &self.input[2..] } else { &self.input[..] }; - let block_id = match FromStr::from_str(input) { - Ok(hash) => BlockId::hash(hash), - Err(_) => match self.input.parse::() { - Ok(n) => BlockId::number((n as u32).into()), - Err(_) => return Err(error::Error::Input("Invalid hash or number specified".into())), - } - }; - - let start = std::time::Instant::now(); - run_until_exit(config, |config| { - Ok(builder(config)?.check_block(block_id)) - })?; - println!("Completed in {} ms.", start.elapsed().as_millis()); - - Ok(()) - } -} - -impl PurgeChainCmd { - /// Run the purge command - pub fn run( - self, - mut config: Configuration, - ) -> error::Result<()> - where - G: RuntimeGenesis, - E: ChainSpecExtension, - { - assert!(config.chain_spec.is_some(), "chain_spec must be present before continuing"); - - crate::fill_config_keystore_in_memory(&mut config)?; - - let db_path = match config.expect_database() { - DatabaseConfig::Path { path, .. } => path, - _ => { - eprintln!("Cannot purge custom database implementation"); - return Ok(()); - } - }; - - if !self.yes { - print!("Are you sure to remove {:?}? [y/N]: ", &db_path); - io::stdout().flush().expect("failed to flush stdout"); - - let mut input = String::new(); - io::stdin().read_line(&mut input)?; - let input = input.trim(); - - match input.chars().nth(0) { - Some('y') | Some('Y') => {}, - _ => { - println!("Aborted"); - return Ok(()); - }, - } - } - - match fs::remove_dir_all(&db_path) { - Ok(_) => { - println!("{:?} removed.", &db_path); - Ok(()) - }, - Err(ref err) if err.kind() == io::ErrorKind::NotFound => { - eprintln!("{:?} did not exist.", &db_path); - Ok(()) - }, - Err(err) => Result::Err(err.into()) - } - } -} - -impl RevertCmd { - /// Run the revert command - pub fn run( - self, - mut config: Configuration, - builder: B, - ) -> error::Result<()> - where - B: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - BC: ServiceBuilderCommand + Unpin, - BB: sp_runtime::traits::Block + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, - { - assert!(config.chain_spec.is_some(), "chain_spec must be present before continuing"); - - crate::fill_config_keystore_in_memory(&mut config)?; - - let blocks = self.num.parse()?; - builder(config)?.revert_chain(blocks)?; - - Ok(()) - } -} - -impl BenchmarkCmd { - /// Runs the command and benchmarks the chain. - pub fn run( - self, - config: Configuration, - _builder: B, - ) -> error::Result<()> - where - B: FnOnce(Configuration) -> Result, - G: RuntimeGenesis, - E: ChainSpecExtension, - BC: ServiceBuilderCommand + Unpin, - BB: sp_runtime::traits::Block + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, - { - let spec = config.chain_spec.expect("chain_spec is always Some"); - let execution_strategy = self.execution.unwrap_or(ExecutionStrategy::Native).into(); - let wasm_method = self.wasm_method.into(); - let pallet = self.pallet; - let extrinsic = self.extrinsic; - let steps = self.steps; - let repeat = self.repeat; - sc_service::chain_ops::benchmark_runtime::(spec, execution_strategy, wasm_method, pallet, extrinsic, steps, repeat)?; - Ok(()) - } -} diff --git a/client/cli/src/params/import_params.rs b/client/cli/src/params/import_params.rs new file mode 100644 index 0000000000000000000000000000000000000000..95d7623d11da3224284c46a3f78aec7e73fad926 --- /dev/null +++ b/client/cli/src/params/import_params.rs @@ -0,0 +1,194 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use structopt::StructOpt; +use sc_service::{Configuration, RuntimeGenesis, config::DatabaseConfig}; + +use crate::error; +use crate::arg_enums::{ + WasmExecutionMethod, TracingReceiver, ExecutionStrategy, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION, + DEFAULT_EXECUTION_IMPORT_BLOCK, DEFAULT_EXECUTION_OFFCHAIN_WORKER, DEFAULT_EXECUTION_OTHER, + DEFAULT_EXECUTION_SYNCING +}; +use crate::params::PruningParams; + +/// Parameters for block import. +#[derive(Debug, StructOpt, Clone)] +pub struct ImportParams { + #[allow(missing_docs)] + #[structopt(flatten)] + pub pruning_params: PruningParams, + + /// Force start with unsafe pruning settings. + /// + /// When running as a validator it is highly recommended to disable state + /// pruning (i.e. 'archive') which is the default. The node will refuse to + /// start as a validator if pruning is enabled unless this option is set. + #[structopt(long = "unsafe-pruning")] + pub unsafe_pruning: bool, + + /// Method for executing Wasm runtime code. + #[structopt( + long = "wasm-execution", + value_name = "METHOD", + possible_values = &WasmExecutionMethod::enabled_variants(), + case_insensitive = true, + default_value = "Interpreted" + )] + pub wasm_method: WasmExecutionMethod, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub execution_strategies: ExecutionStrategies, + + /// Limit the memory the database cache can use. + #[structopt(long = "db-cache", value_name = "MiB", default_value = "128")] + pub database_cache_size: u32, + + /// Specify the state cache size. + #[structopt(long = "state-cache-size", value_name = "Bytes", default_value = "67108864")] + pub state_cache_size: usize, + + /// Comma separated list of targets for tracing + #[structopt(long = "tracing-targets", value_name = "TARGETS")] + pub tracing_targets: Option, + + /// Receiver to process tracing messages + #[structopt( + long = "tracing-receiver", + value_name = "RECEIVER", + possible_values = &TracingReceiver::variants(), + case_insensitive = true, + default_value = "Log" + )] + pub tracing_receiver: TracingReceiver, +} + +impl ImportParams { + /// Put block import CLI params into `config` object. + pub fn update_config( + &self, + mut config: &mut Configuration, + role: sc_service::Roles, + is_dev: bool, + ) -> error::Result<()> + where + G: RuntimeGenesis, + { + use sc_client_api::execution_extensions::ExecutionStrategies; + + if let Some(DatabaseConfig::Path { ref mut cache_size, .. }) = config.database { + *cache_size = Some(self.database_cache_size); + } + + config.state_cache_size = self.state_cache_size; + + self.pruning_params.update_config(&mut config, role, self.unsafe_pruning)?; + + config.wasm_method = self.wasm_method.into(); + + let exec = &self.execution_strategies; + let exec_all_or = |strat: ExecutionStrategy, default: ExecutionStrategy| { + exec.execution.unwrap_or(if strat == default && is_dev { + ExecutionStrategy::Native + } else { + strat + }).into() + }; + + config.execution_strategies = ExecutionStrategies { + syncing: exec_all_or(exec.execution_syncing, DEFAULT_EXECUTION_SYNCING), + importing: exec_all_or(exec.execution_import_block, DEFAULT_EXECUTION_IMPORT_BLOCK), + block_construction: + exec_all_or(exec.execution_block_construction, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION), + offchain_worker: + exec_all_or(exec.execution_offchain_worker, DEFAULT_EXECUTION_OFFCHAIN_WORKER), + other: exec_all_or(exec.execution_other, DEFAULT_EXECUTION_OTHER), + }; + + Ok(()) + } +} + +/// Execution strategies parameters. +#[derive(Debug, StructOpt, Clone)] +pub struct ExecutionStrategies { + /// The means of execution used when calling into the runtime while syncing blocks. + #[structopt( + long = "execution-syncing", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + default_value = DEFAULT_EXECUTION_SYNCING.as_str(), + )] + pub execution_syncing: ExecutionStrategy, + + /// The means of execution used when calling into the runtime while importing blocks. + #[structopt( + long = "execution-import-block", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + default_value = DEFAULT_EXECUTION_IMPORT_BLOCK.as_str(), + )] + pub execution_import_block: ExecutionStrategy, + + /// The means of execution used when calling into the runtime while constructing blocks. + #[structopt( + long = "execution-block-construction", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + default_value = DEFAULT_EXECUTION_BLOCK_CONSTRUCTION.as_str(), + )] + pub execution_block_construction: ExecutionStrategy, + + /// The means of execution used when calling into the runtime while using an off-chain worker. + #[structopt( + long = "execution-offchain-worker", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + default_value = DEFAULT_EXECUTION_OFFCHAIN_WORKER.as_str(), + )] + pub execution_offchain_worker: ExecutionStrategy, + + /// The means of execution used when calling into the runtime while not syncing, importing or constructing blocks. + #[structopt( + long = "execution-other", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + default_value = DEFAULT_EXECUTION_OTHER.as_str(), + )] + pub execution_other: ExecutionStrategy, + + /// The execution strategy that should be used by all execution contexts. + #[structopt( + long = "execution", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + conflicts_with_all = &[ + "execution-other", + "execution-offchain-worker", + "execution-block-construction", + "execution-import-block", + "execution-syncing", + ] + )] + pub execution: Option, +} diff --git a/client/cli/src/params/mod.rs b/client/cli/src/params/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..f684cab336423159c6a18ff492c3d7741c4cfded --- /dev/null +++ b/client/cli/src/params/mod.rs @@ -0,0 +1,67 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +mod import_params; +mod transaction_pool_params; +mod shared_params; +mod node_key_params; +mod network_configuration_params; +mod pruning_params; + +use std::str::FromStr; +use std::fmt::Debug; + +pub use crate::params::import_params::*; +pub use crate::params::transaction_pool_params::*; +pub use crate::params::shared_params::*; +pub use crate::params::node_key_params::*; +pub use crate::params::network_configuration_params::*; +pub use crate::params::pruning_params::*; + +/// Wrapper type of `String` that holds an unsigned integer of arbitrary size, formatted as a decimal. +#[derive(Debug, Clone)] +pub struct BlockNumber(String); + +impl FromStr for BlockNumber { + type Err = String; + + fn from_str(block_number: &str) -> Result { + if block_number.chars().any(|d| !d.is_digit(10)) { + Err(format!( + "Invalid block number: {}, expected decimal formatted unsigned integer", + block_number, + )) + } else { + Ok(Self(block_number.to_owned())) + } + } +} + +impl BlockNumber { + /// Wrapper on top of `std::str::parse` but with `Error` as a `String` + /// + /// See `https://doc.rust-lang.org/std/primitive.str.html#method.parse` for more elaborate + /// documentation. + pub fn parse(&self) -> Result + where + N: FromStr, + N::Err: std::fmt::Debug, + { + self.0 + .parse() + .map_err(|e| format!("BlockNumber: {} parsing failed because of {:?}", self.0, e)) + } +} diff --git a/client/cli/src/params/network_configuration_params.rs b/client/cli/src/params/network_configuration_params.rs new file mode 100644 index 0000000000000000000000000000000000000000..eef679d6a615fa4acc9c33598c0cc2c1a0c04d03 --- /dev/null +++ b/client/cli/src/params/network_configuration_params.rs @@ -0,0 +1,160 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::path::PathBuf; +use std::iter; +use std::net::Ipv4Addr; +use structopt::StructOpt; +use sc_network::{ + config::{NonReservedPeerMode, TransportConfig}, multiaddr::Protocol, +}; +use sc_service::{Configuration, RuntimeGenesis}; + +use crate::error; +use crate::params::node_key_params::NodeKeyParams; + +/// Parameters used to create the network configuration. +#[derive(Debug, StructOpt, Clone)] +pub struct NetworkConfigurationParams { + /// Specify a list of bootnodes. + #[structopt(long = "bootnodes", value_name = "URL")] + pub bootnodes: Vec, + + /// Specify a list of reserved node addresses. + #[structopt(long = "reserved-nodes", value_name = "URL")] + pub reserved_nodes: Vec, + + /// Whether to only allow connections to/from reserved nodes. + /// + /// If you are a validator your node might still connect to other validator + /// nodes regardless of whether they are defined as reserved nodes. + #[structopt(long = "reserved-only")] + pub reserved_only: bool, + + /// Specify a list of sentry node public addresses. + #[structopt( + long = "sentry-nodes", + value_name = "URL", + conflicts_with_all = &[ "sentry" ] + )] + pub sentry_nodes: Vec, + + /// Listen on this multiaddress. + #[structopt(long = "listen-addr", value_name = "LISTEN_ADDR")] + pub listen_addr: Vec, + + /// Specify p2p protocol TCP port. + /// + /// Only used if --listen-addr is not specified. + #[structopt(long = "port", value_name = "PORT")] + pub port: Option, + + /// Forbid connecting to private IPv4 addresses (as specified in + /// [RFC1918](https://tools.ietf.org/html/rfc1918)), unless the address was passed with + /// `--reserved-nodes` or `--bootnodes`. + #[structopt(long = "no-private-ipv4")] + pub no_private_ipv4: bool, + + /// Specify the number of outgoing connections we're trying to maintain. + #[structopt(long = "out-peers", value_name = "COUNT", default_value = "25")] + pub out_peers: u32, + + /// Specify the maximum number of incoming connections we're accepting. + #[structopt(long = "in-peers", value_name = "COUNT", default_value = "25")] + pub in_peers: u32, + + /// Disable mDNS discovery. + /// + /// By default, the network will use mDNS to discover other nodes on the + /// local network. This disables it. Automatically implied when using --dev. + #[structopt(long = "no-mdns")] + pub no_mdns: bool, + + /// Maximum number of peers to ask the same blocks in parallel. + /// + /// This allows downlading announced blocks from multiple peers. Decrease to save + /// traffic and risk increased latency. + #[structopt(long = "max-parallel-downloads", value_name = "COUNT", default_value = "5")] + pub max_parallel_downloads: u32, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub node_key_params: NodeKeyParams, + + /// Experimental feature flag. + #[structopt(long = "use-yamux-flow-control")] + pub use_yamux_flow_control: bool, +} + +impl NetworkConfigurationParams { + /// Fill the given `NetworkConfiguration` by looking at the cli parameters. + pub fn update_config( + &self, + mut config: &mut Configuration, + config_path: PathBuf, + client_id: String, + is_dev: bool, + ) -> error::Result<()> + where + G: RuntimeGenesis, + { + config.network.boot_nodes.extend(self.bootnodes.clone()); + config.network.config_path = Some(config_path.clone()); + config.network.net_config_path = Some(config_path.clone()); + + config.network.reserved_nodes.extend(self.reserved_nodes.clone()); + if self.reserved_only { + config.network.non_reserved_mode = NonReservedPeerMode::Deny; + } + + config.network.sentry_nodes.extend(self.sentry_nodes.clone()); + + for addr in self.listen_addr.iter() { + let addr = addr.parse().ok().ok_or(error::Error::InvalidListenMultiaddress)?; + config.network.listen_addresses.push(addr); + } + + if config.network.listen_addresses.is_empty() { + let port = match self.port { + Some(port) => port, + None => 30333, + }; + + config.network.listen_addresses = vec![ + iter::once(Protocol::Ip4(Ipv4Addr::new(0, 0, 0, 0))) + .chain(iter::once(Protocol::Tcp(port))) + .collect() + ]; + } + + config.network.client_version = client_id; + self.node_key_params.update_config(&mut config, Some(&config_path))?; + + config.network.in_peers = self.in_peers; + config.network.out_peers = self.out_peers; + + config.network.transport = TransportConfig::Normal { + enable_mdns: !is_dev && !self.no_mdns, + allow_private_ipv4: !self.no_private_ipv4, + wasm_external_transport: None, + use_yamux_flow_control: self.use_yamux_flow_control, + }; + + config.network.max_parallel_downloads = self.max_parallel_downloads; + + Ok(()) + } +} diff --git a/client/cli/src/params/node_key_params.rs b/client/cli/src/params/node_key_params.rs new file mode 100644 index 0000000000000000000000000000000000000000..ddc1d6cc21af7ce91715aa64e4b3cbc5bff06722 --- /dev/null +++ b/client/cli/src/params/node_key_params.rs @@ -0,0 +1,244 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::{path::PathBuf, str::FromStr}; +use structopt::StructOpt; +use sc_service::{Configuration, RuntimeGenesis}; +use sc_network::config::NodeKeyConfig; +use sp_core::H256; + +use crate::error; +use crate::arg_enums::NodeKeyType; + +/// The file name of the node's Ed25519 secret key inside the chain-specific +/// network config directory, if neither `--node-key` nor `--node-key-file` +/// is specified in combination with `--node-key-type=ed25519`. +const NODE_KEY_ED25519_FILE: &str = "secret_ed25519"; + +/// Parameters used to create the `NodeKeyConfig`, which determines the keypair +/// used for libp2p networking. +#[derive(Debug, StructOpt, Clone)] +pub struct NodeKeyParams { + /// The secret key to use for libp2p networking. + /// + /// The value is a string that is parsed according to the choice of + /// `--node-key-type` as follows: + /// + /// `ed25519`: + /// The value is parsed as a hex-encoded Ed25519 32 bytes secret key, + /// i.e. 64 hex characters. + /// + /// The value of this option takes precedence over `--node-key-file`. + /// + /// WARNING: Secrets provided as command-line arguments are easily exposed. + /// Use of this option should be limited to development and testing. To use + /// an externally managed secret key, use `--node-key-file` instead. + #[structopt(long = "node-key", value_name = "KEY")] + pub node_key: Option, + + /// The type of secret key to use for libp2p networking. + /// + /// The secret key of the node is obtained as follows: + /// + /// * If the `--node-key` option is given, the value is parsed as a secret key + /// according to the type. See the documentation for `--node-key`. + /// + /// * If the `--node-key-file` option is given, the secret key is read from the + /// specified file. See the documentation for `--node-key-file`. + /// + /// * Otherwise, the secret key is read from a file with a predetermined, + /// type-specific name from the chain-specific network config directory + /// inside the base directory specified by `--base-dir`. If this file does + /// not exist, it is created with a newly generated secret key of the + /// chosen type. + /// + /// The node's secret key determines the corresponding public key and hence the + /// node's peer ID in the context of libp2p. + #[structopt( + long = "node-key-type", + value_name = "TYPE", + possible_values = &NodeKeyType::variants(), + case_insensitive = true, + default_value = "Ed25519" + )] + pub node_key_type: NodeKeyType, + + /// The file from which to read the node's secret key to use for libp2p networking. + /// + /// The contents of the file are parsed according to the choice of `--node-key-type` + /// as follows: + /// + /// `ed25519`: + /// The file must contain an unencoded 32 bytes Ed25519 secret key. + /// + /// If the file does not exist, it is created with a newly generated secret key of + /// the chosen type. + #[structopt(long = "node-key-file", value_name = "FILE")] + pub node_key_file: Option, +} + +impl NodeKeyParams { + /// Create a `NodeKeyConfig` from the given `NodeKeyParams` in the context + /// of an optional network config storage directory. + pub fn update_config<'a, G, E>( + &self, + mut config: &'a mut Configuration, + net_config_path: Option<&PathBuf>, + ) -> error::Result<&'a NodeKeyConfig> + where + G: RuntimeGenesis, + { + config.network.node_key = match self.node_key_type { + NodeKeyType::Ed25519 => { + let secret = if let Some(node_key) = self.node_key.as_ref() { + parse_ed25519_secret(node_key)? + } else { + let path = self.node_key_file.clone() + .or_else(|| net_config_path.map(|d| d.join(NODE_KEY_ED25519_FILE))); + + if let Some(path) = path { + sc_network::config::Secret::File(path) + } else { + sc_network::config::Secret::New + } + }; + + NodeKeyConfig::Ed25519(secret) + } + }; + + Ok(&config.network.node_key) + } +} + +/// Create an error caused by an invalid node key argument. +fn invalid_node_key(e: impl std::fmt::Display) -> error::Error { + error::Error::Input(format!("Invalid node key: {}", e)) +} + +/// Parse a Ed25519 secret key from a hex string into a `sc_network::Secret`. +fn parse_ed25519_secret(hex: &str) -> error::Result { + H256::from_str(&hex).map_err(invalid_node_key).and_then(|bytes| + sc_network::config::identity::ed25519::SecretKey::from_bytes(bytes) + .map(sc_network::config::Secret::Input) + .map_err(invalid_node_key)) +} + +#[cfg(test)] +mod tests { + use sc_network::config::identity::ed25519; + use super::*; + + #[test] + fn test_node_key_config_input() { + fn secret_input(net_config_dir: Option<&PathBuf>) -> error::Result<()> { + NodeKeyType::variants().iter().try_for_each(|t| { + let mut config = Configuration::<(), ()>::default(); + let node_key_type = NodeKeyType::from_str(t).unwrap(); + let sk = match node_key_type { + NodeKeyType::Ed25519 => ed25519::SecretKey::generate().as_ref().to_vec() + }; + let params = NodeKeyParams { + node_key_type, + node_key: Some(format!("{:x}", H256::from_slice(sk.as_ref()))), + node_key_file: None + }; + params.update_config(&mut config, net_config_dir).and_then(|c| match c { + NodeKeyConfig::Ed25519(sc_network::config::Secret::Input(ref ski)) + if node_key_type == NodeKeyType::Ed25519 && + &sk[..] == ski.as_ref() => Ok(()), + _ => Err(error::Error::Input("Unexpected node key config".into())) + }) + }) + } + + assert!(secret_input(None).is_ok()); + assert!(secret_input(Some(&PathBuf::from_str("x").unwrap())).is_ok()); + } + + #[test] + fn test_node_key_config_file() { + fn secret_file(net_config_dir: Option<&PathBuf>) -> error::Result<()> { + NodeKeyType::variants().iter().try_for_each(|t| { + let mut config = Configuration::<(), ()>::default(); + let node_key_type = NodeKeyType::from_str(t).unwrap(); + let tmp = tempfile::Builder::new().prefix("alice").tempdir()?; + let file = tmp.path().join(format!("{}_mysecret", t)).to_path_buf(); + let params = NodeKeyParams { + node_key_type, + node_key: None, + node_key_file: Some(file.clone()) + }; + params.update_config(&mut config, net_config_dir).and_then(|c| match c { + NodeKeyConfig::Ed25519(sc_network::config::Secret::File(ref f)) + if node_key_type == NodeKeyType::Ed25519 && f == &file => Ok(()), + _ => Err(error::Error::Input("Unexpected node key config".into())) + }) + }) + } + + assert!(secret_file(None).is_ok()); + assert!(secret_file(Some(&PathBuf::from_str("x").unwrap())).is_ok()); + } + + #[test] + fn test_node_key_config_default() { + fn with_def_params(f: F) -> error::Result<()> + where + F: Fn(NodeKeyParams) -> error::Result<()> + { + NodeKeyType::variants().iter().try_for_each(|t| { + let node_key_type = NodeKeyType::from_str(t).unwrap(); + f(NodeKeyParams { + node_key_type, + node_key: None, + node_key_file: None + }) + }) + } + + fn no_config_dir() -> error::Result<()> { + with_def_params(|params| { + let mut config = Configuration::<(), ()>::default(); + let typ = params.node_key_type; + params.update_config(&mut config, None) + .and_then(|c| match c { + NodeKeyConfig::Ed25519(sc_network::config::Secret::New) + if typ == NodeKeyType::Ed25519 => Ok(()), + _ => Err(error::Error::Input("Unexpected node key config".into())) + }) + }) + } + + fn some_config_dir(net_config_dir: &PathBuf) -> error::Result<()> { + with_def_params(|params| { + let mut config = Configuration::<(), ()>::default(); + let dir = PathBuf::from(net_config_dir.clone()); + let typ = params.node_key_type; + params.update_config(&mut config, Some(net_config_dir)) + .and_then(move |c| match c { + NodeKeyConfig::Ed25519(sc_network::config::Secret::File(ref f)) + if typ == NodeKeyType::Ed25519 && + f == &dir.join(NODE_KEY_ED25519_FILE) => Ok(()), + _ => Err(error::Error::Input("Unexpected node key config".into())) + }) + }) + } + + assert!(no_config_dir().is_ok()); + assert!(some_config_dir(&PathBuf::from_str("x").unwrap()).is_ok()); + } +} diff --git a/client/cli/src/params/pruning_params.rs b/client/cli/src/params/pruning_params.rs new file mode 100644 index 0000000000000000000000000000000000000000..ad64d757dcb61acb3fb6928caebfff9c8c778015 --- /dev/null +++ b/client/cli/src/params/pruning_params.rs @@ -0,0 +1,69 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use structopt::StructOpt; +use sc_service::{Configuration, RuntimeGenesis, PruningMode}; + +use crate::error; + +/// Parameters to define the pruning mode +#[derive(Debug, StructOpt, Clone)] +pub struct PruningParams { + /// Specify the state pruning mode, a number of blocks to keep or 'archive'. + /// + /// Default is to keep all block states if the node is running as a + /// validator (i.e. 'archive'), otherwise state is only kept for the last + /// 256 blocks. + #[structopt(long = "pruning", value_name = "PRUNING_MODE")] + pub pruning: Option, +} + +impl PruningParams { + /// Put block pruning CLI params into `config` object. + pub fn update_config( + &self, + mut config: &mut Configuration, + role: sc_service::Roles, + unsafe_pruning: bool, + ) -> error::Result<()> + where + G: RuntimeGenesis, + { + // by default we disable pruning if the node is an authority (i.e. + // `ArchiveAll`), otherwise we keep state for the last 256 blocks. if the + // node is an authority and pruning is enabled explicitly, then we error + // unless `unsafe_pruning` is set. + config.pruning = match &self.pruning { + Some(ref s) if s == "archive" => PruningMode::ArchiveAll, + None if role == sc_service::Roles::AUTHORITY => PruningMode::ArchiveAll, + None => PruningMode::default(), + Some(s) => { + if role == sc_service::Roles::AUTHORITY && !unsafe_pruning { + return Err(error::Error::Input( + "Validators should run with state pruning disabled (i.e. archive). \ + You can ignore this check with `--unsafe-pruning`.".to_string() + )); + } + + PruningMode::keep_blocks(s.parse() + .map_err(|_| error::Error::Input("Invalid pruning mode specified".to_string()))? + ) + }, + }; + + Ok(()) + } +} diff --git a/client/cli/src/params/shared_params.rs b/client/cli/src/params/shared_params.rs new file mode 100644 index 0000000000000000000000000000000000000000..03f44796460a28b138fdf9f3a91754a3b685c354 --- /dev/null +++ b/client/cli/src/params/shared_params.rs @@ -0,0 +1,116 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::path::PathBuf; +use structopt::StructOpt; +use app_dirs::{AppInfo, AppDataType}; +use sc_service::{ + Configuration, ChainSpecExtension, RuntimeGenesis, + config::DatabaseConfig, ChainSpec, +}; + +use crate::VersionInfo; +use crate::error; + +/// default sub directory to store database +const DEFAULT_DB_CONFIG_PATH : &'static str = "db"; + +/// Shared parameters used by all `CoreParams`. +#[derive(Debug, StructOpt, Clone)] +pub struct SharedParams { + /// Specify the chain specification (one of dev, local or staging). + #[structopt(long = "chain", value_name = "CHAIN_SPEC")] + pub chain: Option, + + /// Specify the development chain. + #[structopt(long = "dev")] + pub dev: bool, + + /// Specify custom base path. + #[structopt(long = "base-path", short = "d", value_name = "PATH", parse(from_os_str))] + pub base_path: Option, + + /// Sets a custom logging filter. + #[structopt(short = "l", long = "log", value_name = "LOG_PATTERN")] + pub log: Option, +} + +impl SharedParams { + /// Load spec to `Configuration` from `SharedParams` and spec factory. + pub fn update_config<'a, G, E, F>( + &self, + mut config: &'a mut Configuration, + spec_factory: F, + version: &VersionInfo, + ) -> error::Result<&'a ChainSpec> where + G: RuntimeGenesis, + E: ChainSpecExtension, + F: FnOnce(&str) -> Result>, String>, + { + let chain_key = match self.chain { + Some(ref chain) => chain.clone(), + None => if self.dev { "dev".into() } else { "".into() } + }; + let spec = match spec_factory(&chain_key)? { + Some(spec) => spec, + None => ChainSpec::from_json_file(PathBuf::from(chain_key))? + }; + + config.network.boot_nodes = spec.boot_nodes().to_vec(); + config.telemetry_endpoints = spec.telemetry_endpoints().clone(); + + config.chain_spec = Some(spec); + + if config.config_dir.is_none() { + config.config_dir = Some(base_path(self, version)); + } + + if config.database.is_none() { + config.database = Some(DatabaseConfig::Path { + path: config + .in_chain_config_dir(DEFAULT_DB_CONFIG_PATH) + .expect("We provided a base_path/config_dir."), + cache_size: None, + }); + } + + Ok(config.chain_spec.as_ref().unwrap()) + } + + /// Initialize substrate. This must be done only once. + /// + /// This method: + /// + /// 1. Set the panic handler + /// 2. Raise the FD limit + /// 3. Initialize the logger + pub fn init(&self, version: &VersionInfo) -> error::Result<()> { + crate::init(self.log.as_ref().map(|v| v.as_ref()).unwrap_or(""), version) + } +} + +fn base_path(cli: &SharedParams, version: &VersionInfo) -> PathBuf { + cli.base_path.clone() + .unwrap_or_else(|| + app_dirs::get_app_root( + AppDataType::UserData, + &AppInfo { + name: version.executable_name, + author: version.author + } + ).expect("app directories exist on all supported platforms; qed") + ) +} diff --git a/client/cli/src/params/transaction_pool_params.rs b/client/cli/src/params/transaction_pool_params.rs new file mode 100644 index 0000000000000000000000000000000000000000..80c591d1d2dcf2454d5aa059c0c7098744429b0c --- /dev/null +++ b/client/cli/src/params/transaction_pool_params.rs @@ -0,0 +1,49 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use structopt::StructOpt; +use sc_service::Configuration; +use crate::error; + +/// Parameters used to create the pool configuration. +#[derive(Debug, StructOpt, Clone)] +pub struct TransactionPoolParams { + /// Maximum number of transactions in the transaction pool. + #[structopt(long = "pool-limit", value_name = "COUNT", default_value = "8192")] + pub pool_limit: usize, + /// Maximum number of kilobytes of all transactions stored in the pool. + #[structopt(long = "pool-kbytes", value_name = "COUNT", default_value = "20480")] + pub pool_kbytes: usize, +} + +impl TransactionPoolParams { + /// Fill the given `PoolConfiguration` by looking at the cli parameters. + pub fn update_config( + &self, + config: &mut Configuration, + ) -> error::Result<()> { + // ready queue + config.transaction_pool.ready.count = self.pool_limit; + config.transaction_pool.ready.total_bytes = self.pool_kbytes * 1024; + + // future queue + let factor = 10; + config.transaction_pool.future.count = self.pool_limit / factor; + config.transaction_pool.future.total_bytes = self.pool_kbytes * 1024 / factor; + + Ok(()) + } +} diff --git a/client/consensus/aura/Cargo.toml b/client/consensus/aura/Cargo.toml index e67f1e15a3e76f6d2cd2d9a6820e774583ad947b..be610c16d99b50f57563ea74d3b729770cca4f6d 100644 --- a/client/consensus/aura/Cargo.toml +++ b/client/consensus/aura/Cargo.toml @@ -1,44 +1,44 @@ [package] name = "sc-consensus-aura" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] description = "Aura consensus algorithm for substrate" edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sp-application-crypto = { version = "2.0.0", path = "../../../primitives/application-crypto" } -sp-consensus-aura = { version = "0.8", path = "../../../primitives/consensus/aura" } -sp-block-builder = { version = "2.0.0", path = "../../../primitives/block-builder" } -sc-client = { version = "0.8", path = "../../" } -sc-client-api = { version = "2.0.0", path = "../../api" } -codec = { package = "parity-scale-codec", version = "1.0.0" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } +sp-application-crypto = { version = "2.0.0-alpha.2", path = "../../../primitives/application-crypto" } +sp-consensus-aura = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/aura" } +sp-block-builder = { version = "2.0.0-alpha.2", path = "../../../primitives/block-builder" } +sc-client = { version = "0.8.0-alpha.2", path = "../../" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../../api" } +codec = { package = "parity-scale-codec", version = "1.2.0" } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/common" } derive_more = "0.99.2" futures = "0.3.1" futures-timer = "3.0.1" -sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } -sc-keystore = { version = "2.0.0", path = "../../keystore" } +sp-inherents = { version = "2.0.0-alpha.2", path = "../../../primitives/inherents" } +sc-keystore = { version = "2.0.0-alpha.2", path = "../../keystore" } log = "0.4.8" parking_lot = "0.10.0" -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sp-io = { version = "2.0.0", path = "../../../primitives/io" } -sp-version = { version = "2.0.0", path = "../../../primitives/version" } -sc-consensus-slots = { version = "0.8", path = "../slots" } -sp-api = { version = "2.0.0", path = "../../../primitives/api" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-timestamp = { version = "2.0.0", path = "../../../primitives/timestamp" } -sc-telemetry = { version = "2.0.0", path = "../../telemetry" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../../primitives/blockchain" } +sp-io = { version = "2.0.0-alpha.2", path = "../../../primitives/io" } +sp-version = { version = "2.0.0-alpha.2", path = "../../../primitives/version" } +sc-consensus-slots = { version = "0.8.0-alpha.2", path = "../slots" } +sp-api = { version = "2.0.0-alpha.2", path = "../../../primitives/api" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +sp-timestamp = { version = "2.0.0-alpha.2", path = "../../../primitives/timestamp" } +sc-telemetry = { version = "2.0.0-alpha.2", path = "../../telemetry" } [dev-dependencies] -sp-keyring = { version = "2.0.0", path = "../../../primitives/keyring" } -sc-executor = { version = "0.8", path = "../../executor" } -sc-network = { version = "0.8", path = "../../network" } -sc-network-test = { version = "0.8.0", path = "../../network/test" } -sc-service = { version = "0.8", path = "../../service" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } -tokio = "0.1.22" +sp-keyring = { version = "2.0.0-alpha.2", path = "../../../primitives/keyring" } +sc-executor = { version = "0.8.0-alpha.2", path = "../../executor" } +sc-network = { version = "0.8.0-alpha.2", path = "../../network" } +sc-network-test = { version = "0.8.0-dev", path = "../../network/test" } +sc-service = { version = "0.8.0-alpha.2", path = "../../service" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } env_logger = "0.7.0" tempfile = "3.1.0" -futures01 = { package = "futures", version = "0.1" } diff --git a/client/consensus/aura/src/digest.rs b/client/consensus/aura/src/digests.rs similarity index 100% rename from client/consensus/aura/src/digest.rs rename to client/consensus/aura/src/digests.rs diff --git a/client/consensus/aura/src/lib.rs b/client/consensus/aura/src/lib.rs index db872f28c1b445db8d0c49baa81288477e03a712..5850228628547d061decc0263d7ef9f8ab30bd4a 100644 --- a/client/consensus/aura/src/lib.rs +++ b/client/consensus/aura/src/lib.rs @@ -79,33 +79,23 @@ pub use sp_consensus_aura::{ }, }; pub use sp_consensus::SyncOracle; -pub use digest::CompatibleDigestItem; +pub use digests::CompatibleDigestItem; -mod digest; +mod digests; type AuthorityId

=

::Public; -/// A slot duration. Create with `get_or_compute`. -#[derive(Clone, Copy, Debug, Encode, Decode, Hash, PartialOrd, Ord, PartialEq, Eq)] -pub struct SlotDuration(sc_consensus_slots::SlotDuration); - -impl SlotDuration { - /// Either fetch the slot duration from disk or compute it from the genesis - /// state. - pub fn get_or_compute(client: &C) -> CResult - where - A: Codec, - B: BlockT, - C: AuxStore + ProvideRuntimeApi, - C::Api: AuraApi, - { - sc_consensus_slots::SlotDuration::get_or_compute(client, |a, b| a.slot_duration(b)).map(Self) - } +/// Slot duration type for Aura. +pub type SlotDuration = sc_consensus_slots::SlotDuration; - /// Get the slot duration in milliseconds. - pub fn get(&self) -> u64 { - self.0.get() - } +/// Get type of `SlotDuration` for Aura. +pub fn slot_duration(client: &C) -> CResult where + A: Codec, + B: BlockT, + C: AuxStore + ProvideRuntimeApi, + C::Api: AuraApi, +{ + SlotDuration::get_or_compute(client, |a, b| a.slot_duration(b)) } /// Get slot author for given block along with authorities. @@ -179,10 +169,10 @@ pub fn start_aura( }; register_aura_inherent_data_provider( &inherent_data_providers, - slot_duration.0.slot_duration() + slot_duration.slot_duration() )?; Ok(sc_consensus_slots::start_slot_worker::<_, _, _, _, _, AuraSlotCompatible, _>( - slot_duration.0, + slot_duration, select_chain, worker, sync_oracle, @@ -415,21 +405,17 @@ fn find_pre_digest(header: &B::Header) -> Result( +fn check_header( client: &C, slot_now: u64, mut header: B::Header, hash: B::Hash, authorities: &[AuthorityId

], - _transaction_pool: Option<&T>, ) -> Result)>, Error> where DigestItemFor: CompatibleDigestItem

, P::Signature: Decode, C: sc_client_api::backend::AuxStore, P::Public: Encode + Decode + PartialEq + Clone, - T: Send + Sync + 'static, { let seal = match header.digest_mut().pop() { Some(x) => x, @@ -479,14 +465,13 @@ fn check_header( } /// A verifier for Aura blocks. -pub struct AuraVerifier { +pub struct AuraVerifier { client: Arc, phantom: PhantomData

, inherent_data_providers: sp_inherents::InherentDataProviders, - transaction_pool: Option>, } -impl AuraVerifier +impl AuraVerifier where P: Send + Sync + 'static { fn check_inherents( @@ -541,7 +526,7 @@ impl AuraVerifier } #[forbid(deprecated)] -impl Verifier for AuraVerifier where +impl Verifier for AuraVerifier where C: ProvideRuntimeApi + Send + Sync + @@ -553,7 +538,6 @@ impl Verifier for AuraVerifier where P: Pair + Send + Sync + 'static, P::Public: Send + Sync + Hash + Eq + Clone + Decode + Encode + Debug + 'static, P::Signature: Encode + Decode, - T: Send + Sync + 'static, { fn verify( &mut self, @@ -575,13 +559,12 @@ impl Verifier for AuraVerifier where // we add one to allow for some small drift. // FIXME #1019 in the future, alter this queue to allow deferring of // headers - let checked_header = check_header::( + let checked_header = check_header::( &self.client, slot_now + 1, header, hash, &authorities[..], - self.transaction_pool.as_ref().map(|x| &**x), ).map_err(|e| e.to_string())?; match checked_header { CheckedHeader::Checked(pre_header, (slot_num, seal)) => { @@ -805,14 +788,13 @@ impl BlockImport for AuraBlockImport( +pub fn import_queue( slot_duration: SlotDuration, block_import: I, justification_import: Option>, finality_proof_import: Option>, client: Arc, inherent_data_providers: InherentDataProviders, - transaction_pool: Option>, ) -> Result>, sp_consensus::Error> where B: BlockT, C::Api: BlockBuilderApi + AuraApi> + ApiExt, @@ -822,7 +804,6 @@ pub fn import_queue( P: Pair + Send + Sync + 'static, P::Public: Clone + Eq + Send + Sync + Hash + Debug + Encode + Decode, P::Signature: Encode + Decode, - T: Send + Sync + 'static, { register_aura_inherent_data_provider(&inherent_data_providers, slot_duration.get())?; initialize_authorities_cache(&*client)?; @@ -831,7 +812,6 @@ pub fn import_queue( client: client.clone(), inherent_data_providers, phantom: PhantomData, - transaction_pool, }; Ok(BasicQueue::new( verifier, @@ -849,10 +829,10 @@ mod tests { use sp_runtime::traits::{Block as BlockT, DigestFor}; use sc_network::config::ProtocolConfig; use parking_lot::Mutex; - use tokio::runtime::current_thread; use sp_keyring::sr25519::Keyring; use sc_client::BlockchainEvents; use sp_consensus_aura::sr25519::AuthorityPair; + use std::task::Poll; type Error = sp_blockchain::Error; @@ -906,12 +886,11 @@ mod tests { const SLOT_DURATION: u64 = 1000; pub struct AuraTestNet { - peers: Vec>, + peers: Vec>, } impl TestNetFactory for AuraTestNet { - type Specialization = DummySpecialization; - type Verifier = AuraVerifier; + type Verifier = AuraVerifier; type PeerData = (); /// Create new test network with peers and given config. @@ -926,8 +905,7 @@ mod tests { { match client { PeersClient::Full(client, _) => { - let slot_duration = SlotDuration::get_or_compute(&*client) - .expect("slot duration available"); + let slot_duration = slot_duration(&*client).expect("slot duration available"); let inherent_data_providers = InherentDataProviders::new(); register_aura_inherent_data_provider( &inherent_data_providers, @@ -938,7 +916,6 @@ mod tests { AuraVerifier { client, inherent_data_providers, - transaction_pool: Default::default(), phantom: Default::default(), } }, @@ -946,15 +923,15 @@ mod tests { } } - fn peer(&mut self, i: usize) -> &mut Peer { + fn peer(&mut self, i: usize) -> &mut Peer { &mut self.peers[i] } - fn peers(&self) -> &Vec> { + fn peers(&self) -> &Vec> { &self.peers } - fn mut_peers>)>(&mut self, closure: F) { + fn mut_peers>)>(&mut self, closure: F) { closure(&mut self.peers); } } @@ -973,8 +950,8 @@ mod tests { let net = Arc::new(Mutex::new(net)); let mut import_notifications = Vec::new(); + let mut aura_futures = Vec::new(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut keystore_paths = Vec::new(); for (peer_id, key) in peers { let mut net = net.lock(); @@ -995,15 +972,14 @@ mod tests { .for_each(move |_| future::ready(())) ); - let slot_duration = SlotDuration::get_or_compute(&*client) - .expect("slot duration available"); + let slot_duration = slot_duration(&*client).expect("slot duration available"); let inherent_data_providers = InherentDataProviders::new(); register_aura_inherent_data_provider( &inherent_data_providers, slot_duration.get() ).expect("Registers aura inherent data provider"); - let aura = start_aura::<_, _, _, _, _, AuthorityPair, _, _, _>( + aura_futures.push(start_aura::<_, _, _, _, _, AuthorityPair, _, _, _>( slot_duration, client.clone(), select_chain, @@ -1014,21 +990,19 @@ mod tests { false, keystore, sp_consensus::AlwaysCanAuthor, - ) - .expect("Starts aura") - .unit_error() - .compat(); - - runtime.spawn(aura); + ).expect("Starts aura")); } - runtime.spawn(futures01::future::poll_fn(move || { - net.lock().poll(); - Ok::<_, ()>(futures01::Async::NotReady::<()>) - })); - - runtime.block_on(future::join_all(import_notifications) - .unit_error().compat()).unwrap(); + futures::executor::block_on(future::select( + future::poll_fn(move |cx| { + net.lock().poll(cx); + Poll::<()>::Pending + }), + future::select( + future::join_all(aura_futures), + future::join_all(import_notifications) + ) + )); } #[test] diff --git a/client/consensus/babe/Cargo.toml b/client/consensus/babe/Cargo.toml index 3f5487885fea0e11ba758d421589eeda6a007500..66455adbdf8284a822a98a4bf4a2314964445fe5 100644 --- a/client/consensus/babe/Cargo.toml +++ b/client/consensus/babe/Cargo.toml @@ -1,37 +1,41 @@ [package] name = "sc-consensus-babe" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] description = "BABE consensus algorithm for substrate" edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +documentation = "https://docs.rs/sc-consensus-babe" + [dependencies] -codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } -sp-consensus-babe = { version = "0.8", path = "../../../primitives/consensus/babe" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-application-crypto = { version = "2.0.0", path = "../../../primitives/application-crypto" } +codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +sp-consensus-babe = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/babe" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-application-crypto = { version = "2.0.0-alpha.2", path = "../../../primitives/application-crypto" } num-bigint = "0.2.3" num-rational = "0.2.2" num-traits = "0.2.8" serde = { version = "1.0.104", features = ["derive"] } -sp-version = { version = "2.0.0", path = "../../../primitives/version" } -sp-io = { version = "2.0.0", path = "../../../primitives/io" } -sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } -sp-timestamp = { version = "2.0.0", path = "../../../primitives/timestamp" } -sc-telemetry = { version = "2.0.0", path = "../../telemetry" } -sc-keystore = { version = "2.0.0", path = "../../keystore" } -sc-client-api = { version = "2.0.0", path = "../../api" } -sc-client = { version = "0.8", path = "../../" } -sc-consensus-epochs = { version = "0.8", path = "../epochs" } -sp-api = { version = "2.0.0", path = "../../../primitives/api" } -sp-block-builder = { version = "2.0.0", path = "../../../primitives/block-builder" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } -sc-consensus-uncles = { version = "0.8", path = "../uncles" } -sc-consensus-slots = { version = "0.8", path = "../slots" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -fork-tree = { version = "2.0.0", path = "../../../utils/fork-tree" } +sp-version = { version = "2.0.0-alpha.2", path = "../../../primitives/version" } +sp-io = { version = "2.0.0-alpha.2", path = "../../../primitives/io" } +sp-inherents = { version = "2.0.0-alpha.2", path = "../../../primitives/inherents" } +sp-timestamp = { version = "2.0.0-alpha.2", path = "../../../primitives/timestamp" } +sc-telemetry = { version = "2.0.0-alpha.2", path = "../../telemetry" } +sc-keystore = { version = "2.0.0-alpha.2", path = "../../keystore" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../../api" } +sc-client = { version = "0.8.0-alpha.2", path = "../../" } +sc-consensus-epochs = { version = "0.8.0-alpha.2", path = "../epochs" } +sp-api = { version = "2.0.0-alpha.2", path = "../../../primitives/api" } +sp-block-builder = { version = "2.0.0-alpha.2", path = "../../../primitives/block-builder" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../../primitives/blockchain" } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/common" } +sc-consensus-uncles = { version = "0.8.0-alpha.2", path = "../uncles" } +sc-consensus-slots = { version = "0.8.0-alpha.2", path = "../slots" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +fork-tree = { version = "2.0.0-alpha.2", path = "../../../utils/fork-tree" } futures = "0.3.1" futures-timer = "3.0.1" parking_lot = "0.10.0" @@ -43,17 +47,15 @@ pdqselect = "0.1.0" derive_more = "0.99.2" [dev-dependencies] -sp-keyring = { version = "2.0.0", path = "../../../primitives/keyring" } -sc-executor = { version = "0.8", path = "../../executor" } -sc-network = { version = "0.8", path = "../../network" } -sc-network-test = { version = "0.8.0", path = "../../network/test" } -sc-service = { version = "0.8", path = "../../service" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } -sc-block-builder = { version = "0.8", path = "../../block-builder" } -tokio = "0.1.22" +sp-keyring = { version = "2.0.0-alpha.2", path = "../../../primitives/keyring" } +sc-executor = { version = "0.8.0-alpha.2", path = "../../executor" } +sc-network = { version = "0.8.0-alpha.2", path = "../../network" } +sc-network-test = { version = "0.8.0-dev", path = "../../network/test" } +sc-service = { version = "0.8.0-alpha.2", path = "../../service" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } +sc-block-builder = { version = "0.8.0-alpha.2", path = "../../block-builder" } env_logger = "0.7.0" tempfile = "3.1.0" -futures01 = { package = "futures", version = "0.1" } [features] test-helpers = [] diff --git a/client/consensus/babe/rpc/Cargo.toml b/client/consensus/babe/rpc/Cargo.toml index 3fd0e924af092231d1103471bec8231f40acb872..3f2cc4f0bcbce8913ca9183815edd5cffe39928f 100644 --- a/client/consensus/babe/rpc/Cargo.toml +++ b/client/consensus/babe/rpc/Cargo.toml @@ -1,30 +1,32 @@ [package] name = "sc-consensus-babe-rpc" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] description = "RPC extensions for the BABE consensus algorithm" edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sc-consensus-babe = { version = "0.8.0", path = "../" } +sc-consensus-babe = { version = "0.8.0-alpha.2", path = "../" } jsonrpc-core = "14.0.3" jsonrpc-core-client = "14.0.3" jsonrpc-derive = "14.0.3" -sp-consensus-babe = { version = "0.8", path = "../../../../primitives/consensus/babe" } +sp-consensus-babe = { version = "0.8.0-alpha.2", path = "../../../../primitives/consensus/babe" } serde = { version = "1.0.104", features=["derive"] } -sp-blockchain = { version = "2.0.0", path = "../../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0", path = "../../../../primitives/runtime" } -sc-consensus-epochs = { version = "0.8", path = "../../epochs" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../../primitives/runtime" } +sc-consensus-epochs = { version = "0.8.0-alpha.2", path = "../../epochs" } futures = "0.3.1" derive_more = "0.99.2" -sp-api = { version = "2.0.0", path = "../../../../primitives/api" } -sp-consensus = { version = "0.8", path = "../../../../primitives/consensus/common" } -sp-core = { version = "2.0.0", path = "../../../../primitives/core" } -sc-keystore = { version = "2.0.0", path = "../../../keystore" } +sp-api = { version = "2.0.0-alpha.2", path = "../../../../primitives/api" } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../../../primitives/consensus/common" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../../primitives/core" } +sc-keystore = { version = "2.0.0-alpha.2", path = "../../../keystore" } [dev-dependencies] -substrate-test-runtime-client = { version = "2.0.0", path = "../../../../test-utils/runtime/client" } -sp-application-crypto = { version = "2.0.0", path = "../../../../primitives/application-crypto" } -sp-keyring = { version = "2.0.0", path = "../../../../primitives/keyring" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../../test-utils/runtime/client" } +sp-application-crypto = { version = "2.0.0-alpha.2", path = "../../../../primitives/application-crypto" } +sp-keyring = { version = "2.0.0-alpha.2", path = "../../../../primitives/keyring" } tempfile = "3.1.0" diff --git a/client/consensus/babe/rpc/src/lib.rs b/client/consensus/babe/rpc/src/lib.rs index 033a7d6b98530aa8f55ad4b224931d13188a1cd6..1ea7e423dc7a90c7b975cc0788284647106bfabf 100644 --- a/client/consensus/babe/rpc/src/lib.rs +++ b/client/consensus/babe/rpc/src/lib.rs @@ -230,7 +230,6 @@ mod tests { config.clone(), client.clone(), client.clone(), - client.clone(), ).expect("can initialize block-import"); let epoch_changes = link.epoch_changes().clone(); diff --git a/client/consensus/babe/src/authorship.rs b/client/consensus/babe/src/authorship.rs index 4654f91b89876a9858751e8771bbb3fd0a69b022..a01ea63bbe111ef193d65de6a4264d7eb8a5ede4 100644 --- a/client/consensus/babe/src/authorship.rs +++ b/client/consensus/babe/src/authorship.rs @@ -86,16 +86,15 @@ pub(super) fn secondary_slot_author( Some(&expected_author.0) } -#[allow(deprecated)] pub(super) fn make_transcript( randomness: &[u8], slot_number: u64, epoch: u64, ) -> Transcript { let mut transcript = Transcript::new(&BABE_ENGINE_ID); - transcript.commit_bytes(b"slot number", &slot_number.to_le_bytes()); - transcript.commit_bytes(b"current epoch", &epoch.to_le_bytes()); - transcript.commit_bytes(b"chain randomness", randomness); + transcript.append_u64(b"slot number", slot_number); + transcript.append_u64(b"current epoch", epoch); + transcript.append_message(b"chain randomness", randomness); transcript } diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index b93619b29c885216fbf39ff2cffc694889dc18b3..967a78e7bfc882debba29d27f1b8187338a0fea1 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -93,12 +93,9 @@ use sp_consensus_babe::inherents::BabeInherentData; use sp_timestamp::{TimestampInherentData, InherentType as TimestampInherent}; use sp_consensus::import_queue::{Verifier, BasicQueue, CacheKeyId}; use sc_client_api::{ - backend::{AuxStore, Backend}, - call_executor::CallExecutor, + backend::AuxStore, BlockchainEvents, ProvideUncles, }; -use sc_client::Client; - use sp_block_builder::BlockBuilder as BlockBuilderApi; use futures::prelude::*; @@ -321,7 +318,7 @@ pub struct BabeParams { pub can_author_with: CAW, } -/// Start the babe worker. The returned future should be run in a tokio runtime. +/// Start the babe worker. pub fn start_babe(BabeParams { keystore, client, @@ -655,27 +652,28 @@ impl BabeLink { } /// A verifier for Babe blocks. -pub struct BabeVerifier { - client: Arc>, - api: Arc, +pub struct BabeVerifier { + client: Arc, inherent_data_providers: sp_inherents::InherentDataProviders, config: Config, epoch_changes: SharedEpochChanges, time_source: TimeSource, } -impl BabeVerifier { +impl BabeVerifier + where + Block: BlockT, + Client: HeaderBackend + HeaderMetadata + ProvideRuntimeApi, + Client::Api: BlockBuilderApi, +{ fn check_inherents( &self, block: Block, block_id: BlockId, inherent_data: InherentData, ) -> Result<(), Error> - where - PRA: ProvideRuntimeApi, - PRA::Api: BlockBuilderApi { - let inherent_res = self.api.runtime_api().check_inherents( + let inherent_res = self.client.runtime_api().check_inherents( &block_id, block, inherent_data, @@ -693,57 +691,11 @@ impl BabeVerifier { } } -#[allow(dead_code)] -fn median_algorithm( - median_required_blocks: u64, - slot_duration: u64, - slot_number: u64, - slot_now: u64, - time_source: &mut (Option, Vec<(Instant, u64)>), -) { - let num_timestamps = time_source.1.len(); - if num_timestamps as u64 >= median_required_blocks && median_required_blocks > 0 { - let mut new_list: Vec<_> = time_source.1.iter().map(|&(t, sl)| { - let offset: u128 = u128::from(slot_duration) - .checked_mul(1_000_000u128) // self.config.slot_duration returns milliseconds - .and_then(|x| { - x.checked_mul(u128::from(slot_number).saturating_sub(u128::from(sl))) - }) - .expect("we cannot have timespans long enough for this to overflow; qed"); - - const NANOS_PER_SEC: u32 = 1_000_000_000; - let nanos = (offset % u128::from(NANOS_PER_SEC)) as u32; - let secs = (offset / u128::from(NANOS_PER_SEC)) as u64; - - t + Duration::new(secs, nanos) - }).collect(); - - // Use a partial sort to move the median timestamp to the middle of the list - pdqselect::select(&mut new_list, num_timestamps / 2); - - let &median = new_list - .get(num_timestamps / 2) - .expect("we have at least one timestamp, so this is a valid index; qed"); - - let now = Instant::now(); - if now >= median { - time_source.0.replace(now - median); - } - - time_source.1.clear(); - } else { - time_source.1.push((Instant::now(), slot_now)) - } -} - -impl Verifier for BabeVerifier where +impl Verifier for BabeVerifier where Block: BlockT, - B: Backend + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, - RA: Send + Sync, - PRA: ProvideRuntimeApi + Send + Sync + AuxStore + ProvideCache, - PRA::Api: BlockBuilderApi - + BabeApi, + Client: HeaderMetadata + HeaderBackend + ProvideRuntimeApi + + Send + Sync + AuxStore + ProvideCache, + Client::Api: BlockBuilderApi + BabeApi, { fn verify( &mut self, @@ -812,7 +764,7 @@ impl Verifier for BabeVerifier { +pub struct BabeBlockImport { inner: I, - client: Arc>, - api: Arc, + client: Arc, epoch_changes: SharedEpochChanges, config: Config, } -impl Clone for BabeBlockImport { +impl Clone for BabeBlockImport { fn clone(&self) -> Self { BabeBlockImport { inner: self.inner.clone(), client: self.client.clone(), - api: self.api.clone(), epoch_changes: self.epoch_changes.clone(), config: self.config.clone(), } } } -impl BabeBlockImport { +impl BabeBlockImport { fn new( - client: Arc>, - api: Arc, + client: Arc, epoch_changes: SharedEpochChanges, block_import: I, config: Config, ) -> Self { BabeBlockImport { client, - api, inner: block_import, epoch_changes, config, @@ -938,19 +886,16 @@ impl BabeBlockImport { } } -impl BlockImport for BabeBlockImport where +impl BlockImport for BabeBlockImport where Block: BlockT, - I: BlockImport> + Send + Sync, - I::Error: Into, - B: Backend + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, - Client: AuxStore, - RA: Send + Sync, - PRA: ProvideRuntimeApi + ProvideCache, - PRA::Api: BabeApi + ApiExt, + Inner: BlockImport> + Send + Sync, + Inner::Error: Into, + Client: HeaderBackend + HeaderMetadata + + AuxStore + ProvideRuntimeApi + ProvideCache + Send + Sync, + Client::Api: BabeApi + ApiExt, { type Error = ConsensusError; - type Transaction = sp_api::TransactionFor; + type Transaction = sp_api::TransactionFor; fn import_block( &mut self, @@ -974,7 +919,7 @@ impl BlockImport for BabeBlockImport::ParentUnavailable(parent_hash, hash) @@ -1046,7 +991,7 @@ impl BlockImport for BabeBlockImport BlockImport for BabeBlockImport BlockImport for BabeBlockImport( - client: &Client, +fn prune_finalized( + client: Arc, epoch_changes: &mut EpochChangesFor, ) -> Result<(), ConsensusError> where Block: BlockT, - E: CallExecutor + Send + Sync, - B: Backend, - RA: Send + Sync, + Client: HeaderBackend + HeaderMetadata, { - let info = client.chain_info(); + let info = client.info(); let finalized_slot = { - let finalized_header = client.header(&BlockId::Hash(info.finalized_hash)) + let finalized_header = client.header(BlockId::Hash(info.finalized_hash)) .map_err(|e| ConsensusError::ClientImport(format!("{:?}", e)))? .expect("best finalized hash was given by client; \ finalized headers must exist in db; qed"); @@ -1190,16 +1133,12 @@ fn prune_finalized( /// /// Also returns a link object used to correctly instantiate the import queue /// and background worker. -pub fn block_import( +pub fn block_import( config: Config, wrapped_block_import: I, - client: Arc>, - api: Arc, -) -> ClientResult<(BabeBlockImport, BabeLink)> where - B: Backend, - E: CallExecutor + Send + Sync, - RA: Send + Sync, - Client: AuxStore, + client: Arc, +) -> ClientResult<(BabeBlockImport, BabeLink)> where + Client: AuxStore + HeaderBackend + HeaderMetadata, { let epoch_changes = aux_schema::load_epoch_changes::(&*client)?; let link = BabeLink { @@ -1212,13 +1151,12 @@ pub fn block_import( // epoch tree it is useful as a migration, so that nodes prune long trees on // startup rather than waiting until importing the next epoch change block. prune_finalized( - &client, + client.clone(), &mut epoch_changes.lock(), )?; let import = BabeBlockImport::new( client, - api, epoch_changes, wrapped_block_import, config, @@ -1236,28 +1174,24 @@ pub fn block_import( /// /// The block import object provided must be the `BabeBlockImport` or a wrapper /// of it, otherwise crucial import logic will be omitted. -pub fn import_queue( +pub fn import_queue( babe_link: BabeLink, - block_import: I, + block_import: Inner, justification_import: Option>, finality_proof_import: Option>, - client: Arc>, - api: Arc, + client: Arc, inherent_data_providers: InherentDataProviders, -) -> ClientResult>> where - B: Backend + 'static, - I: BlockImport> +) -> ClientResult>> where + Inner: BlockImport> + Send + Sync + 'static, - E: CallExecutor + Clone + Send + Sync + 'static, - RA: Send + Sync + 'static, - PRA: ProvideRuntimeApi + ProvideCache + Send + Sync + AuxStore + 'static, - PRA::Api: BlockBuilderApi + BabeApi + ApiExt, + Client: ProvideRuntimeApi + ProvideCache + Send + Sync + AuxStore + 'static, + Client: HeaderBackend + HeaderMetadata, + Client::Api: BlockBuilderApi + BabeApi + ApiExt, { register_babe_inherent_data_provider(&inherent_data_providers, babe_link.config.slot_duration)?; let verifier = BabeVerifier { client: client.clone(), - api, inherent_data_providers, config: babe_link.config, epoch_changes: babe_link.epoch_changes, diff --git a/client/consensus/babe/src/tests.rs b/client/consensus/babe/src/tests.rs index 6c1ffa2c3af05109701a361a5f463cf4b3b71d50..a5493918f001de75f828eaa573cbafb9c2fcb8f6 100644 --- a/client/consensus/babe/src/tests.rs +++ b/client/consensus/babe/src/tests.rs @@ -23,7 +23,7 @@ use super::*; use authorship::claim_slot; use sp_consensus_babe::{AuthorityPair, SlotNumber}; -use sc_block_builder::BlockBuilder; +use sc_block_builder::{BlockBuilder, BlockBuilderProvider}; use sp_consensus::{ NoNetwork as DummyOracle, Proposal, RecordProof, import_queue::{BoxBlockImport, BoxJustificationImport, BoxFinalityProofImport}, @@ -32,10 +32,9 @@ use sc_network_test::*; use sc_network_test::{Block as TestBlock, PeersClient}; use sc_network::config::{BoxFinalityProofRequestBuilder, ProtocolConfig}; use sp_runtime::{generic::DigestItem, traits::{Block as BlockT, DigestFor}}; -use tokio::runtime::current_thread; use sc_client_api::{BlockchainEvents, backend::TransactionFor}; use log::debug; -use std::{time::Duration, cell::RefCell}; +use std::{time::Duration, cell::RefCell, task::Poll}; type Item = DigestItem; @@ -199,20 +198,14 @@ impl> BlockImport for PanickingBlockImport< } pub struct BabeTestNet { - peers: Vec, DummySpecialization>>, + peers: Vec>>, } type TestHeader = ::Header; type TestExtrinsic = ::Extrinsic; pub struct TestVerifier { - inner: BabeVerifier< - substrate_test_runtime_client::Backend, - substrate_test_runtime_client::Executor, - TestBlock, - substrate_test_runtime_client::runtime::RuntimeApi, - PeersFullClient, - >, + inner: BabeVerifier, mutator: Mutator, } @@ -242,7 +235,6 @@ pub struct PeerData { } impl TestNetFactory for BabeTestNet { - type Specialization = DummySpecialization; type Verifier = TestVerifier; type PeerData = Option; @@ -271,7 +263,6 @@ impl TestNetFactory for BabeTestNet { config, client.clone(), client.clone(), - client.clone(), ).expect("can initialize block-import"); let block_import = PanickingBlockImport(block_import); @@ -305,7 +296,6 @@ impl TestNetFactory for BabeTestNet { TestVerifier { inner: BabeVerifier { client: client.clone(), - api: client, inherent_data_providers: data.inherent_data_providers.clone(), config: data.link.config.clone(), epoch_changes: data.link.epoch_changes.clone(), @@ -315,17 +305,17 @@ impl TestNetFactory for BabeTestNet { } } - fn peer(&mut self, i: usize) -> &mut Peer { + fn peer(&mut self, i: usize) -> &mut Peer { trace!(target: "babe", "Retrieving a peer"); &mut self.peers[i] } - fn peers(&self) -> &Vec> { + fn peers(&self) -> &Vec> { trace!(target: "babe", "Retrieving peers"); &self.peers } - fn mut_peers>)>( + fn mut_peers>)>( &mut self, closure: F, ) { @@ -363,7 +353,7 @@ fn run_one_test( let net = Arc::new(Mutex::new(net)); let mut import_notifications = Vec::new(); - let mut runtime = current_thread::Runtime::new().unwrap(); + let mut babe_futures = Vec::new(); let mut keystore_paths = Vec::new(); for (peer_id, seed) in peers { @@ -408,7 +398,7 @@ fn run_one_test( ); - runtime.spawn(start_babe(BabeParams { + babe_futures.push(start_babe(BabeParams { block_import: data.block_import.lock().take().expect("import set up during init"), select_chain, client, @@ -419,23 +409,23 @@ fn run_one_test( babe_link: data.link.clone(), keystore, can_author_with: sp_consensus::AlwaysCanAuthor, - }).expect("Starts babe").unit_error().compat()); + }).expect("Starts babe")); } - runtime.spawn(futures01::future::poll_fn(move || { - let mut net = net.lock(); - net.poll(); - for p in net.peers() { - for (h, e) in p.failed_verifications() { - panic!("Verification failed for {:?}: {}", h, e); + futures::executor::block_on(future::select( + futures::future::poll_fn(move |cx| { + let mut net = net.lock(); + net.poll(cx); + for p in net.peers() { + for (h, e) in p.failed_verifications() { + panic!("Verification failed for {:?}: {}", h, e); + } } - } - - Ok::<_, ()>(futures01::Async::NotReady::<()>) - })); - - runtime.block_on(future::join_all(import_notifications) - .unit_error().compat()).unwrap(); + + Poll::<()>::Pending + }), + future::select(future::join_all(import_notifications), future::join_all(babe_futures)) + )); } #[test] diff --git a/client/consensus/epochs/Cargo.toml b/client/consensus/epochs/Cargo.toml index e08553a241d1c41e7c8d175bcaedd3fc03961610..fe2365429892121a4092a1f63021725af6cf5028 100644 --- a/client/consensus/epochs/Cargo.toml +++ b/client/consensus/epochs/Cargo.toml @@ -1,14 +1,17 @@ [package] name = "sc-consensus-epochs" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] description = "Generic epochs-based utilities for consensus" edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } parking_lot = "0.10.0" -fork-tree = { version = "2.0.0", path = "../../../utils/fork-tree" } -sp-runtime = { path = "../../../primitives/runtime" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sc-client-api = { path = "../../api" } +fork-tree = { version = "2.0.0-alpha.2", path = "../../../utils/fork-tree" } +sp-runtime = { path = "../../../primitives/runtime" , version = "2.0.0-alpha.2"} +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../../primitives/blockchain" } +sc-client-api = { path = "../../api" , version = "2.0.0-alpha.2"} diff --git a/client/consensus/manual-seal/Cargo.toml b/client/consensus/manual-seal/Cargo.toml index 71c1f6e5583a27edb96b6d7710a31112ce6f169d..11fee0e3f9e608c73e716f5ac8a84480140e31d1 100644 --- a/client/consensus/manual-seal/Cargo.toml +++ b/client/consensus/manual-seal/Cargo.toml @@ -1,10 +1,12 @@ [package] name = "sc-consensus-manual-seal" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] description = "Manual sealing engine for Substrate" edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] derive_more = "0.99.2" @@ -16,19 +18,19 @@ log = "0.4.8" parking_lot = "0.10.0" serde = { version = "1.0", features=["derive"] } -sc-client = { path = "../../../client" } -sc-client-api = { path = "../../../client/api" } -sc-transaction-pool = { path = "../../transaction-pool" } -sp-blockchain = { path = "../../../primitives/blockchain" } -sp-consensus = { package = "sp-consensus", path = "../../../primitives/consensus/common" } -sp-inherents = { path = "../../../primitives/inherents" } -sp-runtime = { path = "../../../primitives/runtime" } -sp-transaction-pool = { path = "../../../primitives/transaction-pool" } +sc-client = { path = "../../../client" , version = "0.8.0-alpha.2"} +sc-client-api = { path = "../../../client/api" , version = "2.0.0-alpha.2"} +sc-transaction-pool = { path = "../../transaction-pool" , version = "2.0.0-alpha.2"} +sp-blockchain = { path = "../../../primitives/blockchain" , version = "2.0.0-alpha.2"} +sp-consensus = { package = "sp-consensus", path = "../../../primitives/consensus/common" , version = "0.8.0-alpha.2"} +sp-inherents = { path = "../../../primitives/inherents" , version = "2.0.0-alpha.2"} +sp-runtime = { path = "../../../primitives/runtime" , version = "2.0.0-alpha.2"} +sp-transaction-pool = { path = "../../../primitives/transaction-pool" , version = "2.0.0-alpha.2"} [dev-dependencies] -sc-basic-authorship = { path = "../../basic-authorship" } -substrate-test-runtime-client = { path = "../../../test-utils/runtime/client" } -substrate-test-runtime-transaction-pool = { path = "../../../test-utils/runtime/transaction-pool" } +sc-basic-authorship = { path = "../../basic-authorship" , version = "0.8.0-alpha.2"} +substrate-test-runtime-client = { path = "../../../test-utils/runtime/client" , version = "2.0.0-dev"} +substrate-test-runtime-transaction-pool = { path = "../../../test-utils/runtime/transaction-pool" , version = "2.0.0-dev"} tokio = { version = "0.2", features = ["rt-core", "macros"] } env_logger = "0.7.0" tempfile = "3.1.0" diff --git a/client/consensus/manual-seal/src/lib.rs b/client/consensus/manual-seal/src/lib.rs index c4336485a14acbe3ac1c6d7b9493a80bd577e5a8..18dc91ad34d625fd6b7ce661bea7a56aeebdcc38 100644 --- a/client/consensus/manual-seal/src/lib.rs +++ b/client/consensus/manual-seal/src/lib.rs @@ -244,10 +244,10 @@ mod tests { let select_chain = LongestChain::new(backend.clone()); let inherent_data_providers = InherentDataProviders::new(); let pool = Arc::new(BasicPool::new(Options::default(), api()).0); - let env = ProposerFactory { - transaction_pool: pool.clone(), - client: client.clone(), - }; + let env = ProposerFactory::new( + client.clone(), + pool.clone() + ); // this test checks that blocks are created as soon as transactions are imported into the pool. let (sender, receiver) = futures::channel::oneshot::channel(); let mut sender = Arc::new(Some(sender)); @@ -309,10 +309,10 @@ mod tests { let select_chain = LongestChain::new(backend.clone()); let inherent_data_providers = InherentDataProviders::new(); let pool = Arc::new(BasicPool::new(Options::default(), api()).0); - let env = ProposerFactory { - transaction_pool: pool.clone(), - client: client.clone(), - }; + let env = ProposerFactory::new( + client.clone(), + pool.clone() + ); // this test checks that blocks are created as soon as an engine command is sent over the stream. let (mut sink, stream) = futures::channel::mpsc::channel(1024); let future = run_manual_seal( @@ -378,10 +378,10 @@ mod tests { let inherent_data_providers = InherentDataProviders::new(); let pool_api = api(); let pool = Arc::new(BasicPool::new(Options::default(), pool_api.clone()).0); - let env = ProposerFactory { - transaction_pool: pool.clone(), - client: client.clone(), - }; + let env = ProposerFactory::new( + client.clone(), + pool.clone(), + ); // this test checks that blocks are created as soon as an engine command is sent over the stream. let (mut sink, stream) = futures::channel::mpsc::channel(1024); let future = run_manual_seal( diff --git a/client/consensus/pow/Cargo.toml b/client/consensus/pow/Cargo.toml index b31d9406e1e852101016f1af51a9ef9c9cafaaf4..f95d196b62f7a94fec013416252456e50395b02f 100644 --- a/client/consensus/pow/Cargo.toml +++ b/client/consensus/pow/Cargo.toml @@ -1,23 +1,25 @@ [package] name = "sc-consensus-pow" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] description = "PoW consensus algorithm for substrate" edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0", path = "../../../primitives/api" } -sc-client-api = { version = "2.0.0", path = "../../api" } -sp-block-builder = { version = "2.0.0", path = "../../../primitives/block-builder" } -sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } -sp-consensus-pow = { version = "0.8", path = "../../../primitives/consensus/pow" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } +codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-alpha.2", path = "../../../primitives/api" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../../api" } +sp-block-builder = { version = "2.0.0-alpha.2", path = "../../../primitives/block-builder" } +sp-inherents = { version = "2.0.0-alpha.2", path = "../../../primitives/inherents" } +sp-consensus-pow = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/pow" } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/common" } log = "0.4.8" futures = { version = "0.3.1", features = ["compat"] } -sp-timestamp = { version = "2.0.0", path = "../../../primitives/timestamp" } +sp-timestamp = { version = "2.0.0-alpha.2", path = "../../../primitives/timestamp" } derive_more = "0.99.2" diff --git a/client/consensus/slots/Cargo.toml b/client/consensus/slots/Cargo.toml index 67de0a54ed7b657418a7abb9710d0672e042733a..fe7958b257203c9bf24e0582515e249d3a34a74c 100644 --- a/client/consensus/slots/Cargo.toml +++ b/client/consensus/slots/Cargo.toml @@ -1,27 +1,29 @@ [package] name = "sc-consensus-slots" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] description = "Generic slots-based utilities for consensus" edition = "2018" build = "build.rs" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -codec = { package = "parity-scale-codec", version = "1.0.0" } -sc-client-api = { version = "2.0.0", path = "../../api" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-state-machine = { version = "0.8.0", path = "../../../primitives/state-machine" } -sp-api = { version = "2.0.0", path = "../../../primitives/api" } -sc-telemetry = { version = "2.0.0", path = "../../telemetry" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } -sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } +codec = { package = "parity-scale-codec", version = "1.2.0" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../../api" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +sp-state-machine = { version = "0.8.0-alpha.2", path = "../../../primitives/state-machine" } +sp-api = { version = "2.0.0-alpha.2", path = "../../../primitives/api" } +sc-telemetry = { version = "2.0.0-alpha.2", path = "../../telemetry" } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/common" } +sp-inherents = { version = "2.0.0-alpha.2", path = "../../../primitives/inherents" } futures = "0.3.1" futures-timer = "3.0.1" parking_lot = "0.10.0" log = "0.4.8" [dev-dependencies] -substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } diff --git a/client/consensus/slots/src/lib.rs b/client/consensus/slots/src/lib.rs index 892c8b136436b9fa226f2cbd4617c4729870dede..e23b2fb321a306243aa1bb24b7e580289f28045a 100644 --- a/client/consensus/slots/src/lib.rs +++ b/client/consensus/slots/src/lib.rs @@ -37,7 +37,7 @@ use futures_timer::Delay; use sp_inherents::{InherentData, InherentDataProviders}; use log::{debug, error, info, warn}; use sp_runtime::generic::BlockId; -use sp_runtime::traits::{Block as BlockT, Header, HasherFor, NumberFor}; +use sp_runtime::traits::{Block as BlockT, Header, HashFor, NumberFor}; use sp_api::{ProvideRuntimeApi, ApiRef}; use std::{fmt::Debug, ops::Deref, pin::Pin, sync::Arc, time::{Instant, Duration}}; use sc_telemetry::{telemetry, CONSENSUS_DEBUG, CONSENSUS_WARN, CONSENSUS_INFO}; @@ -47,7 +47,7 @@ use parking_lot::Mutex; /// /// See [`sp_state_machine::StorageChanges`] for more information. pub type StorageChanges = - sp_state_machine::StorageChanges, NumberFor>; + sp_state_machine::StorageChanges, NumberFor>; /// A worker that should be invoked at every new slot. pub trait SlotWorker { diff --git a/client/consensus/uncles/Cargo.toml b/client/consensus/uncles/Cargo.toml index f336564c4ec14db9fca99f0e50c2155d81dc35ac..ad325ed79f7a576bd957fd1a06fd5e473eb60394 100644 --- a/client/consensus/uncles/Cargo.toml +++ b/client/consensus/uncles/Cargo.toml @@ -1,16 +1,18 @@ [package] name = "sc-consensus-uncles" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] description = "Generic uncle inclusion utilities for consensus" edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sc-client-api = { version = "2.0.0", path = "../../api" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-authorship = { version = "2.0.0", path = "../../../primitives/authorship" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } -sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../../api" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +sp-authorship = { version = "2.0.0-alpha.2", path = "../../../primitives/authorship" } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/common" } +sp-inherents = { version = "2.0.0-alpha.2", path = "../../../primitives/inherents" } log = "0.4.8" diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml index ef418d43efc183facef019a1e9e57912c3951140..8eaae24e5207cd0cd1fa46b079775fd858073dd7 100644 --- a/client/db/Cargo.toml +++ b/client/db/Cargo.toml @@ -1,9 +1,12 @@ [package] name = "sc-client-db" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Client backend that uses RocksDB database as storage." [dependencies] parking_lot = "0.10.0" @@ -15,22 +18,23 @@ kvdb-memorydb = "0.4.0" linked-hash-map = "0.5.2" hash-db = "0.15.2" parity-util-mem = { version = "0.5.1", default-features = false, features = ["std"] } -codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } -sc-client-api = { version = "2.0.0", path = "../api" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sc-client = { version = "0.8", path = "../" } -sp-state-machine = { version = "0.8", path = "../../primitives/state-machine" } -sc-executor = { version = "0.8", path = "../executor" } -sc-state-db = { version = "0.8", path = "../state-db" } -sp-trie = { version = "2.0.0", path = "../../primitives/trie" } -sp-consensus = { version = "0.8", path = "../../primitives/consensus/common" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../api" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } +sc-client = { version = "0.8.0-alpha.2", path = "../" } +sp-state-machine = { version = "0.8.0-alpha.2", path = "../../primitives/state-machine" } +sc-executor = { version = "0.8.0-alpha.2", path = "../executor" } +sc-state-db = { version = "0.8.0-alpha.2", path = "../state-db" } +sp-trie = { version = "2.0.0-alpha.2", path = "../../primitives/trie" } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../primitives/consensus/common" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../primitives/blockchain" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.8.0-alpha.2", path = "../../utils/prometheus" } [dev-dependencies] -sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } +sp-keyring = { version = "2.0.0-alpha.2", path = "../../primitives/keyring" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } env_logger = "0.7.0" quickcheck = "0.9" kvdb-rocksdb = "0.5" diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 9858a5c148bfaa13f34af57ce4091540e1b39928..b6a6b3f8d499762f8a413a084138e7c6fe600a31 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -24,14 +24,14 @@ use rand::Rng; use hash_db::{Prefix, Hasher}; use sp_trie::{MemoryDB, prefixed_key}; use sp_core::storage::ChildInfo; -use sp_runtime::traits::{Block as BlockT, HasherFor}; +use sp_runtime::traits::{Block as BlockT, HashFor}; use sp_runtime::Storage; use sp_state_machine::{DBValue, backend::Backend as StateBackend}; use kvdb::{KeyValueDB, DBTransaction}; use kvdb_rocksdb::{Database, DatabaseConfig}; type DbState = sp_state_machine::TrieBackend< - Arc>>, HasherFor + Arc>>, HashFor >; struct StorageDb { @@ -39,9 +39,9 @@ struct StorageDb { _block: std::marker::PhantomData, } -impl sp_state_machine::Storage> for StorageDb { +impl sp_state_machine::Storage> for StorageDb { fn get(&self, key: &Block::Hash, prefix: Prefix) -> Result, String> { - let key = prefixed_key::>(key, prefix); + let key = prefixed_key::>(key, prefix); self.db.get(0, &key) .map_err(|e| format!("Database backend error: {:?}", e)) } @@ -53,7 +53,7 @@ pub struct BenchmarkingState { root: Cell, state: RefCell>>, db: Cell>>, - genesis: as StateBackend>>::Transaction, + genesis: as StateBackend>>::Transaction, } impl BenchmarkingState { @@ -64,8 +64,8 @@ impl BenchmarkingState { let path = temp_dir.join(&name); let mut root = B::Hash::default(); - let mut mdb = MemoryDB::>::default(); - sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); + let mut mdb = MemoryDB::>::default(); + sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); std::fs::create_dir(&path).map_err(|_| String::from("Error creating temp dir"))?; let mut state = BenchmarkingState { @@ -108,8 +108,8 @@ impl BenchmarkingState { self.db.set(None); *self.state.borrow_mut() = None; let mut root = B::Hash::default(); - let mut mdb = MemoryDB::>::default(); - sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); + let mut mdb = MemoryDB::>::default(); + sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); self.root.set(root); std::fs::remove_dir_all(&self.path).map_err(|_| "Error removing database dir".into()) @@ -126,10 +126,10 @@ fn state_err() -> String { "State is not open".into() } -impl StateBackend> for BenchmarkingState { - type Error = as StateBackend>>::Error; - type Transaction = as StateBackend>>::Transaction; - type TrieBackendStorage = as StateBackend>>::TrieBackendStorage; +impl StateBackend> for BenchmarkingState { + type Error = as StateBackend>>::Error; + type Transaction = as StateBackend>>::Transaction; + type TrieBackendStorage = as StateBackend>>::TrieBackendStorage; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { self.state.borrow().as_ref().ok_or_else(state_err)?.storage(key) @@ -244,12 +244,12 @@ impl StateBackend> for BenchmarkingState { } fn as_trie_backend(&mut self) - -> Option<&sp_state_machine::TrieBackend>> + -> Option<&sp_state_machine::TrieBackend>> { None } - fn commit(&self, storage_root: as Hasher>::Out, mut transaction: Self::Transaction) + fn commit(&self, storage_root: as Hasher>::Out, mut transaction: Self::Transaction) -> Result<(), Self::Error> { if let Some(db) = self.db.take() { diff --git a/client/db/src/changes_tries_storage.rs b/client/db/src/changes_tries_storage.rs index 99488bbaed0964d4509e38b94c37d68dbe2cfb24..a28cd604fe3633d952dfd10a94ea8df01c4bc975 100644 --- a/client/db/src/changes_tries_storage.rs +++ b/client/db/src/changes_tries_storage.rs @@ -28,7 +28,7 @@ use sc_client_api::backend::PrunableStateChangesTrieStorage; use sp_blockchain::{well_known_cache_keys, Cache as BlockchainCache}; use sp_core::{ChangesTrieConfiguration, ChangesTrieConfigurationRange, convert_hash}; use sp_runtime::traits::{ - Block as BlockT, Header as HeaderT, HasherFor, NumberFor, One, Zero, CheckedSub, + Block as BlockT, Header as HeaderT, HashFor, NumberFor, One, Zero, CheckedSub, }; use sp_runtime::generic::{BlockId, DigestItem, ChangesTrieSignal}; use sp_state_machine::{DBValue, ChangesTrieBuildCache, ChangesTrieCacheAction}; @@ -150,7 +150,7 @@ impl DbChangesTrieStorage { pub fn commit( &self, tx: &mut DBTransaction, - mut changes_trie: MemoryDB>, + mut changes_trie: MemoryDB>, parent_block: ComplexBlockId, block: ComplexBlockId, new_header: &Block::Header, @@ -377,7 +377,7 @@ impl DbChangesTrieStorage { } impl PrunableStateChangesTrieStorage for DbChangesTrieStorage { - fn storage(&self) -> &dyn sp_state_machine::ChangesTrieStorage, NumberFor> { + fn storage(&self) -> &dyn sp_state_machine::ChangesTrieStorage, NumberFor> { self } @@ -396,7 +396,7 @@ impl PrunableStateChangesTrieStorage for DbChangesTrieStor } } -impl sp_state_machine::ChangesTrieRootsStorage, NumberFor> +impl sp_state_machine::ChangesTrieRootsStorage, NumberFor> for DbChangesTrieStorage { fn build_anchor( @@ -469,12 +469,12 @@ impl sp_state_machine::ChangesTrieRootsStorage, } } -impl sp_state_machine::ChangesTrieStorage, NumberFor> +impl sp_state_machine::ChangesTrieStorage, NumberFor> for DbChangesTrieStorage where Block: BlockT, { - fn as_roots_storage(&self) -> &dyn sp_state_machine::ChangesTrieRootsStorage, NumberFor> { + fn as_roots_storage(&self) -> &dyn sp_state_machine::ChangesTrieRootsStorage, NumberFor> { self } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 5173497509cf9ad909813d0baa17df72fe406979..746c73bea2804cfc189632f9a1ec78a7c03154a9 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -46,9 +46,11 @@ use std::path::PathBuf; use std::io; use std::collections::HashMap; -use sc_client_api::{execution_extensions::ExecutionExtensions, ForkBlocks, UsageInfo, MemoryInfo, BadBlocks, IoInfo}; -use sc_client_api::backend::NewBlockState; -use sc_client_api::backend::PrunableStateChangesTrieStorage; +use sc_client_api::{ + ForkBlocks, UsageInfo, MemoryInfo, BadBlocks, IoInfo, MemorySize, + execution_extensions::ExecutionExtensions, + backend::{NewBlockState, PrunableStateChangesTrieStorage}, +}; use sp_blockchain::{ Result as ClientResult, Error as ClientError, well_known_cache_keys, HeaderBackend, @@ -65,7 +67,7 @@ use sp_runtime::{ BuildStorage, }; use sp_runtime::traits::{ - Block as BlockT, Header as HeaderT, NumberFor, Zero, One, SaturatedConversion, HasherFor, + Block as BlockT, Header as HeaderT, NumberFor, Zero, One, SaturatedConversion, HashFor, }; use sc_executor::RuntimeInfo; use sp_state_machine::{ @@ -78,10 +80,11 @@ use crate::changes_tries_storage::{DbChangesTrieStorage, DbChangesTrieStorageTra use sc_client::leaves::{LeafSet, FinalizationDisplaced}; use sc_state_db::StateDb; use sp_blockchain::{CachedHeaderMetadata, HeaderMetadata, HeaderMetadataCache}; -use crate::storage_cache::{CachingState, SharedCache, new_shared_cache}; +use crate::storage_cache::{CachingState, SyncingCachingState, SharedCache, new_shared_cache}; use crate::stats::StateUsageStats; use log::{trace, debug, warn}; pub use sc_state_db::PruningMode; +use prometheus_endpoint::Registry; #[cfg(any(feature = "kvdb-rocksdb", test))] pub use bench::BenchmarkingState; @@ -97,7 +100,7 @@ const DEFAULT_CHILD_RATIO: (usize, usize) = (1, 10); /// DB-backed patricia trie state, transaction type is an overlay of changes to commit. pub type DbState = sp_state_machine::TrieBackend< - Arc>>, HasherFor + Arc>>, HashFor >; /// Re-export the KVDB trait so that one can pass an implementation of it. @@ -137,10 +140,10 @@ impl std::fmt::Debug for RefTrackingState { } } -impl StateBackend> for RefTrackingState { - type Error = as StateBackend>>::Error; - type Transaction = as StateBackend>>::Transaction; - type TrieBackendStorage = as StateBackend>>::TrieBackendStorage; +impl StateBackend> for RefTrackingState { + type Error = as StateBackend>>::Error; + type Transaction = as StateBackend>>::Transaction; + type TrieBackendStorage = as StateBackend>>::TrieBackendStorage; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { self.state.storage(key) @@ -249,7 +252,7 @@ impl StateBackend> for RefTrackingState { } fn as_trie_backend(&mut self) - -> Option<&sp_state_machine::TrieBackend>> + -> Option<&sp_state_machine::TrieBackend>> { self.state.as_trie_backend() } @@ -289,6 +292,7 @@ pub fn new_client( fork_blocks: ForkBlocks, bad_blocks: BadBlocks, execution_extensions: ExecutionExtensions, + prometheus_registry: Option, ) -> Result<( sc_client::Client< Backend, @@ -315,6 +319,7 @@ pub fn new_client( fork_blocks, bad_blocks, execution_extensions, + prometheus_registry, )?, backend, )) @@ -518,11 +523,11 @@ impl HeaderMetadata for BlockchainDb { /// Database transaction pub struct BlockImportOperation { - old_state: CachingState, Block>, - db_updates: PrefixedMemoryDB>, + old_state: SyncingCachingState, Block>, + db_updates: PrefixedMemoryDB>, storage_updates: StorageCollection, child_storage_updates: ChildStorageCollection, - changes_trie_updates: MemoryDB>, + changes_trie_updates: MemoryDB>, changes_trie_build_cache_update: Option>>, changes_trie_config_update: Option>, pending_block: Option>, @@ -544,7 +549,7 @@ impl BlockImportOperation { } impl sc_client_api::backend::BlockImportOperation for BlockImportOperation { - type State = CachingState, Block>; + type State = SyncingCachingState, Block>; fn state(&self) -> ClientResult> { Ok(Some(&self.old_state)) @@ -574,7 +579,7 @@ impl sc_client_api::backend::BlockImportOperation for Bloc // Currently cache isn't implemented on full nodes. } - fn update_db_storage(&mut self, update: PrefixedMemoryDB>) -> ClientResult<()> { + fn update_db_storage(&mut self, update: PrefixedMemoryDB>) -> ClientResult<()> { self.db_updates = update; Ok(()) } @@ -621,7 +626,7 @@ impl sc_client_api::backend::BlockImportOperation for Bloc fn update_changes_trie( &mut self, - update: ChangesTrieTransaction, NumberFor>, + update: ChangesTrieTransaction, NumberFor>, ) -> ClientResult<()> { self.changes_trie_updates = update.0; self.changes_trie_build_cache_update = Some(update.1); @@ -666,9 +671,9 @@ struct StorageDb { pub state_db: StateDb>, } -impl sp_state_machine::Storage> for StorageDb { +impl sp_state_machine::Storage> for StorageDb { fn get(&self, key: &Block::Hash, prefix: Prefix) -> Result, String> { - let key = prefixed_key::>(key, prefix); + let key = prefixed_key::>(key, prefix); self.state_db.get(&key, self) .map_err(|e| format!("Database backend error: {:?}", e)) } @@ -688,13 +693,13 @@ struct DbGenesisStorage(pub Block::Hash); impl DbGenesisStorage { pub fn new() -> Self { let mut root = Block::Hash::default(); - let mut mdb = MemoryDB::>::default(); - sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); + let mut mdb = MemoryDB::>::default(); + sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); DbGenesisStorage(root) } } -impl sp_state_machine::Storage> for DbGenesisStorage { +impl sp_state_machine::Storage> for DbGenesisStorage { fn get(&self, _key: &Block::Hash, _prefix: Prefix) -> Result, String> { Ok(None) } @@ -750,10 +755,10 @@ pub struct Backend { blockchain: BlockchainDb, canonicalization_delay: u64, shared_cache: SharedCache, - import_lock: RwLock<()>, + import_lock: Arc>, is_archive: bool, io_stats: FrozenForDuration<(kvdb::IoStats, StateUsageInfo)>, - state_usage: StateUsageStats, + state_usage: Arc, } impl Backend { @@ -825,7 +830,7 @@ impl Backend { import_lock: Default::default(), is_archive: is_archive_pruning, io_stats: FrozenForDuration::new(std::time::Duration::from_secs(1)), - state_usage: StateUsageStats::new(), + state_usage: Arc::new(StateUsageStats::new()), }) } @@ -1127,8 +1132,14 @@ impl Backend { self.state_usage.tally_writes(ops, bytes); let number_u64 = number.saturated_into::(); - let commit = self.storage.state_db.insert_block(&hash, number_u64, &pending_block.header.parent_hash(), changeset) - .map_err(|e: sc_state_db::Error| sp_blockchain::Error::from(format!("State database error: {:?}", e)))?; + let commit = self.storage.state_db.insert_block( + &hash, + number_u64, + &pending_block.header.parent_hash(), + changeset, + ).map_err(|e: sc_state_db::Error| + sp_blockchain::Error::from(format!("State database error: {:?}", e)) + )?; apply_state_commit(&mut transaction, commit); // Check if need to finalize. Genesis is always finalized instantly. @@ -1156,7 +1167,8 @@ impl Backend { changes_trie_cache_ops, )?); self.state_usage.merge_sm(operation.old_state.usage_info()); - let cache = operation.old_state.release(); // release state reference so that it can be finalized + // release state reference so that it can be finalized + let cache = operation.old_state.into_cache_changes(); if finalized { // TODO: ensure best chain contains this block. @@ -1184,9 +1196,20 @@ impl Backend { displaced_leaf }; - let mut children = children::read_children(&*self.storage.db, columns::META, meta_keys::CHILDREN_PREFIX, parent_hash)?; + let mut children = children::read_children( + &*self.storage.db, + columns::META, + meta_keys::CHILDREN_PREFIX, + parent_hash, + )?; children.push(hash); - children::write_children(&mut transaction, columns::META, meta_keys::CHILDREN_PREFIX, parent_hash, children); + children::write_children( + &mut transaction, + columns::META, + meta_keys::CHILDREN_PREFIX, + parent_hash, + children, + ); meta_updates.push((hash, number, pending_block.leaf_state.is_best(), finalized)); @@ -1196,7 +1219,7 @@ impl Backend { }; let cache_update = if let Some(set_head) = operation.set_head { - if let Some(header) = ::sc_client::blockchain::HeaderBackend::header(&self.blockchain, set_head)? { + if let Some(header) = sc_client::blockchain::HeaderBackend::header(&self.blockchain, set_head)? { let number = header.number(); let hash = header.hash(); @@ -1266,7 +1289,6 @@ impl Backend { Ok(()) } - // write stuff to a transaction after a new block is finalized. // this canonicalizes finalized blocks. Fails if called with a block which // was not a child of the last finalized block. @@ -1354,11 +1376,13 @@ impl sc_client_api::backend::AuxStore for Backend where Block: Blo impl sc_client_api::backend::Backend for Backend { type BlockImportOperation = BlockImportOperation; type Blockchain = BlockchainDb; - type State = CachingState, Block>; + type State = SyncingCachingState, Block>; type OffchainStorage = offchain::LocalStorage; fn begin_operation(&self) -> ClientResult { - let old_state = self.state_at(BlockId::Hash(Default::default()))?; + let mut old_state = self.state_at(BlockId::Hash(Default::default()))?; + old_state.disable_syncing(); + Ok(BlockImportOperation { pending_block: None, old_state, @@ -1381,13 +1405,13 @@ impl sc_client_api::backend::Backend for Backend { block: BlockId, ) -> ClientResult<()> { operation.old_state = self.state_at(block)?; + operation.old_state.disable_syncing(); + operation.commit_state = true; Ok(()) } - fn commit_operation(&self, operation: Self::BlockImportOperation) - -> ClientResult<()> - { + fn commit_operation(&self, operation: Self::BlockImportOperation) -> ClientResult<()> { let usage = operation.old_state.usage_info(); self.state_usage.merge_sm(usage); @@ -1447,7 +1471,6 @@ impl sc_client_api::backend::Backend for Backend { Some(self.offchain_storage.clone()) } - fn usage_info(&self) -> Option { let (io_stats, state_stats) = self.io_stats.take_or_else(|| ( @@ -1455,13 +1478,17 @@ impl sc_client_api::backend::Backend for Backend { self.state_usage.take(), ) ); - let database_cache = parity_util_mem::malloc_size(&*self.storage.db); - let state_cache = (*&self.shared_cache).lock().used_storage_cache_size(); + let database_cache = MemorySize::from_bytes(parity_util_mem::malloc_size(&*self.storage.db)); + let state_cache = MemorySize::from_bytes( + (*&self.shared_cache).lock().used_storage_cache_size(), + ); + let state_db = self.storage.state_db.memory_info(); Some(UsageInfo { memory: MemoryInfo { state_cache, database_cache, + state_db, }, io: IoInfo { transactions: io_stats.transactions, @@ -1568,7 +1595,17 @@ impl sc_client_api::backend::Backend for Backend { let root = genesis_storage.0.clone(); let db_state = DbState::::new(Arc::new(genesis_storage), root); let state = RefTrackingState::new(db_state, self.storage.clone(), None); - return Ok(CachingState::new(state, self.shared_cache.clone(), None)); + let caching_state = CachingState::new( + state, + self.shared_cache.clone(), + None, + ); + return Ok(SyncingCachingState::new( + caching_state, + self.state_usage.clone(), + self.blockchain.meta.clone(), + self.import_lock.clone(), + )); }, _ => {} } @@ -1591,7 +1628,17 @@ impl sc_client_api::backend::Backend for Backend { self.storage.clone(), Some(hash.clone()), ); - Ok(CachingState::new(state, self.shared_cache.clone(), Some(hash))) + let caching_state = CachingState::new( + state, + self.shared_cache.clone(), + Some(hash), + ); + Ok(SyncingCachingState::new( + caching_state, + self.state_usage.clone(), + self.blockchain.meta.clone(), + self.import_lock.clone(), + )) } else { Err( sp_blockchain::Error::UnknownBlock( @@ -1626,17 +1673,8 @@ impl sc_client_api::backend::Backend for Backend { } } - fn destroy_state(&self, state: Self::State) -> ClientResult<()> { - self.state_usage.merge_sm(state.usage_info()); - if let Some(hash) = state.cache.parent_hash.clone() { - let is_best = self.blockchain.meta.read().best_hash == hash; - state.release().sync_cache(&[], &[], vec![], vec![], None, None, is_best); - } - Ok(()) - } - fn get_import_lock(&self) -> &RwLock<()> { - &self.import_lock + &*self.import_lock } } @@ -1647,7 +1685,7 @@ pub(crate) mod tests { use hash_db::{HashDB, EMPTY_PREFIX}; use super::*; use crate::columns; - use sp_core::{Blake2Hasher, H256}; + use sp_core::H256; use sc_client_api::backend::{Backend as BTrait, BlockImportOperation as Op}; use sc_client::blockchain::Backend as BLBTrait; use sp_runtime::testing::{Header, Block as RawBlock, ExtrinsicWrapper}; @@ -1658,11 +1696,11 @@ pub(crate) mod tests { pub(crate) type Block = RawBlock>; - pub fn prepare_changes(changes: Vec<(Vec, Vec)>) -> (H256, MemoryDB) { + pub fn prepare_changes(changes: Vec<(Vec, Vec)>) -> (H256, MemoryDB) { let mut changes_root = H256::default(); - let mut changes_trie_update = MemoryDB::::default(); + let mut changes_trie_update = MemoryDB::::default(); { - let mut trie = TrieDBMut::::new( + let mut trie = TrieDBMut::::new( &mut changes_trie_update, &mut changes_root ); @@ -1835,6 +1873,7 @@ pub(crate) mod tests { op.update_db_storage(overlay).unwrap(); header.state_root = root.into(); + op.update_storage(storage, Vec::new()).unwrap(); op.set_block_data( header, Some(vec![]), @@ -1894,7 +1933,7 @@ pub(crate) mod tests { backend.commit_operation(op).unwrap(); assert_eq!(backend.storage.db.get( columns::STATE, - &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) + &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) ).unwrap().unwrap(), &b"hello"[..]); hash }; @@ -1931,7 +1970,7 @@ pub(crate) mod tests { backend.commit_operation(op).unwrap(); assert_eq!(backend.storage.db.get( columns::STATE, - &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) + &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) ).unwrap().unwrap(), &b"hello"[..]); hash }; @@ -1969,7 +2008,7 @@ pub(crate) mod tests { assert!(backend.storage.db.get( columns::STATE, - &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) + &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) ).unwrap().is_some()); hash }; @@ -2003,7 +2042,7 @@ pub(crate) mod tests { backend.commit_operation(op).unwrap(); assert!(backend.storage.db.get( columns::STATE, - &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) + &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) ).unwrap().is_none()); } @@ -2012,7 +2051,7 @@ pub(crate) mod tests { backend.finalize_block(BlockId::Number(3), None).unwrap(); assert!(backend.storage.db.get( columns::STATE, - &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) + &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) ).unwrap().is_none()); } diff --git a/client/db/src/light.rs b/client/db/src/light.rs index 14ce6ac0f9a05d5511698b169a117d8fbd888d9a..cda1a1195268e74893b6a18440f23d7d1495d971 100644 --- a/client/db/src/light.rs +++ b/client/db/src/light.rs @@ -36,7 +36,7 @@ use sp_blockchain::{ use sc_client::light::blockchain::Storage as LightBlockchainStorage; use codec::{Decode, Encode}; use sp_runtime::generic::{DigestItem, BlockId}; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, One, NumberFor, HasherFor}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, One, NumberFor, HashFor}; use crate::cache::{DbCacheSync, DbCache, ComplexBlockId, EntryType as CacheEntryType}; use crate::utils::{self, meta_keys, DatabaseType, Meta, db_err, read_db, block_id_to_lookup_key, read_meta}; use crate::{DatabaseSettings, FrozenForDuration}; @@ -305,7 +305,7 @@ impl LightStorage { Some(old_current_num) }); - let new_header_cht_root = cht::compute_root::, _>( + let new_header_cht_root = cht::compute_root::, _>( cht::size(), new_cht_number, cht_range.map(|num| self.hash(num)) )?; transaction.put( @@ -317,12 +317,12 @@ impl LightStorage { // if the header includes changes trie root, let's build a changes tries roots CHT if header.digest().log(DigestItem::as_changes_trie_root).is_some() { let mut current_num = new_cht_start; - let cht_range = ::std::iter::from_fn(|| { + let cht_range = std::iter::from_fn(|| { let old_current_num = current_num; current_num = current_num + One::one(); Some(old_current_num) }); - let new_changes_trie_cht_root = cht::compute_root::, _>( + let new_changes_trie_cht_root = cht::compute_root::, _>( cht::size(), new_cht_number, cht_range .map(|num| self.changes_trie_root(BlockId::Number(num))) )?; @@ -572,15 +572,16 @@ impl LightBlockchainStorage for LightStorage #[cfg(not(target_os = "unknown"))] fn usage_info(&self) -> Option { - use sc_client_api::{MemoryInfo, IoInfo}; + use sc_client_api::{MemoryInfo, IoInfo, MemorySize}; - let database_cache = parity_util_mem::malloc_size(&*self.db); + let database_cache = MemorySize::from_bytes(parity_util_mem::malloc_size(&*self.db)); let io_stats = self.io_stats.take_or_else(|| self.db.io_stats(kvdb::IoStatsKind::SincePrevious)); Some(UsageInfo { memory: MemoryInfo { database_cache, - state_cache: 0, + state_cache: Default::default(), + state_db: Default::default(), }, io: IoInfo { transactions: io_stats.transactions, diff --git a/client/db/src/stats.rs b/client/db/src/stats.rs index 805a0f498f7fcb2fe749f09db355cb0b1313d1c1..1d6ed8e7f0493c9adf7d516bfacd0467c231dae7 100644 --- a/client/db/src/stats.rs +++ b/client/db/src/stats.rs @@ -59,8 +59,14 @@ impl StateUsageStats { } /// Tally one child key read. - pub fn tally_child_key_read(&self, key: &(Vec, Vec), val: Option>, cache: bool) -> Option> { - self.tally_read(key.0.len() as u64 + key.1.len() as u64 + val.as_ref().map(|x| x.len() as u64).unwrap_or(0), cache); + pub fn tally_child_key_read( + &self, + key: &(Vec, Vec), + val: Option>, + cache: bool, + ) -> Option> { + let bytes = key.0.len() + key.1.len() + val.as_ref().map(|x| x.len()).unwrap_or(0); + self.tally_read(bytes as u64, cache); val } @@ -80,11 +86,15 @@ impl StateUsageStats { self.bytes_read_cache.fetch_add(info.cache_reads.bytes, AtomicOrdering::Relaxed); } + /// Returns the collected `UsageInfo` and resets the internal state. pub fn take(&self) -> sp_state_machine::UsageInfo { use sp_state_machine::UsageUnit; fn unit(ops: &AtomicU64, bytes: &AtomicU64) -> UsageUnit { - UsageUnit { ops: ops.swap(0, AtomicOrdering::Relaxed), bytes: bytes.swap(0, AtomicOrdering::Relaxed) } + UsageUnit { + ops: ops.swap(0, AtomicOrdering::Relaxed), + bytes: bytes.swap(0, AtomicOrdering::Relaxed), + } } sp_state_machine::UsageInfo { diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index 6ef29f47b8c42c1b5c5702cc9724da611d87d2f2..2ac1ee3dbd5f4af37f6fd21d72a55ee7a5748d1d 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -18,10 +18,11 @@ use std::collections::{VecDeque, HashSet, HashMap}; use std::sync::Arc; +use std::hash::Hash as StdHash; use parking_lot::{Mutex, RwLock, RwLockUpgradableReadGuard}; use linked_hash_map::{LinkedHashMap, Entry}; use hash_db::Hasher; -use sp_runtime::traits::{Block as BlockT, Header, HasherFor, NumberFor}; +use sp_runtime::traits::{Block as BlockT, Header, HashFor, NumberFor}; use sp_core::hexdisplay::HexDisplay; use sp_core::storage::ChildInfo; use sp_state_machine::{ @@ -29,8 +30,7 @@ use sp_state_machine::{ StorageCollection, ChildStorageCollection, }; use log::trace; -use std::hash::Hash as StdHash; -use crate::stats::StateUsageStats; +use crate::{utils::Meta, stats::StateUsageStats}; const STATE_CACHE_BLOCKS: usize = 12; @@ -281,7 +281,7 @@ pub struct CacheChanges { /// Shared canonical state cache. shared_cache: SharedCache, /// Local cache of values for this state. - local_cache: RwLock>>, + local_cache: RwLock>>, /// Hash of the block on top of which this instance was created or /// `None` if cache is disabled pub parent_hash: Option, @@ -296,16 +296,16 @@ pub struct CacheChanges { /// For canonical instances local cache is accumulated and applied /// in `sync_cache` along with the change overlay. /// For non-canonical clones local cache and changes are dropped. -pub struct CachingState>, B: BlockT> { +pub struct CachingState { /// Usage statistics usage: StateUsageStats, /// Backing state. state: S, /// Cache data. - pub cache: CacheChanges, + cache: CacheChanges, } -impl>, B: BlockT> std::fmt::Debug for CachingState { +impl std::fmt::Debug for CachingState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "Block {:?}", self.cache.parent_hash) } @@ -417,12 +417,15 @@ impl CacheChanges { } } } - } -impl>, B: BlockT> CachingState { +impl>, B: BlockT> CachingState { /// Create a new instance wrapping generic State and shared cache. - pub fn new(state: S, shared_cache: SharedCache, parent_hash: Option) -> Self { + pub(crate) fn new( + state: S, + shared_cache: SharedCache, + parent_hash: Option, + ) -> Self { CachingState { usage: StateUsageStats::new(), state, @@ -433,7 +436,7 @@ impl>, B: BlockT> CachingState { hashes: Default::default(), child_storage: Default::default(), }), - parent_hash: parent_hash, + parent_hash, }, } } @@ -445,8 +448,7 @@ impl>, B: BlockT> CachingState { child_key: Option<&ChildStorageKey>, parent_hash: &Option, modifications: &VecDeque> - ) -> bool - { + ) -> bool { let mut parent = match *parent_hash { None => { trace!("Cache lookup skipped for {:?}: no parent hash", key.as_ref().map(HexDisplay::from)); @@ -479,17 +481,15 @@ impl>, B: BlockT> CachingState { } } } - trace!("Cache lookup skipped for {:?}: parent hash is unknown", key.as_ref().map(HexDisplay::from)); + trace!( + "Cache lookup skipped for {:?}: parent hash is unknown", + key.as_ref().map(HexDisplay::from), + ); false } - - /// Dispose state and return cache data. - pub fn release(self) -> CacheChanges { - self.cache - } } -impl>, B: BlockT> StateBackend> for CachingState { +impl>, B: BlockT> StateBackend> for CachingState { type Error = S::Error; type Transaction = S::Transaction; type TrieBackendStorage = S::TrieBackendStorage; @@ -659,7 +659,7 @@ impl>, B: BlockT> StateBackend> for Ca self.state.child_keys(storage_key, child_info, prefix) } - fn as_trie_backend(&mut self) -> Option<&TrieBackend>> { + fn as_trie_backend(&mut self) -> Option<&TrieBackend>> { self.state.as_trie_backend() } @@ -668,12 +668,221 @@ impl>, B: BlockT> StateBackend> for Ca } } +/// Extended [`CachingState`] that will sync the caches on drop. +pub struct SyncingCachingState { + /// The usage statistics of the backend. These will be updated on drop. + state_usage: Arc, + /// Reference to the meta db. + meta: Arc, Block::Hash>>>, + /// Mutex to lock get exlusive access to the backend. + lock: Arc>, + /// The wrapped caching state. + /// + /// This is required to be a `Option`, because sometimes we want to extract + /// the cache changes and Rust does not allow to move fields from types that + /// implement `Drop`. + caching_state: Option>, + /// Disable syncing of the cache. This is by default always `false`. However, + /// we need to disable syncing when this is a state in a + /// [`BlockImportOperation`](crate::BlockImportOperation). The import operation + /// takes care to sync the cache and more importantly we want to prevent a dead + /// lock. + disable_syncing: bool, +} + +impl SyncingCachingState { + /// Create new automatic syncing state. + pub fn new( + caching_state: CachingState, + state_usage: Arc, + meta: Arc, B::Hash>>>, + lock: Arc>, + ) -> Self { + Self { + caching_state: Some(caching_state), + state_usage, + meta, + lock, + disable_syncing: false, + } + } + + /// Returns the reference to the internal [`CachingState`]. + fn caching_state(&self) -> &CachingState { + self.caching_state + .as_ref() + .expect("`caching_state` is always valid for the lifetime of the object; qed") + } + + /// Convert `Self` into the cache changes. + pub fn into_cache_changes(mut self) -> CacheChanges { + self.caching_state + .take() + .expect("`caching_state` is always valid for the lifetime of the object; qed") + .cache + } + + /// Disable syncing the cache on drop. + pub fn disable_syncing(&mut self) { + self.disable_syncing = true; + } +} + +impl std::fmt::Debug for SyncingCachingState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.caching_state().fmt(f) + } +} + +impl>, B: BlockT> StateBackend> for SyncingCachingState { + type Error = S::Error; + type Transaction = S::Transaction; + type TrieBackendStorage = S::TrieBackendStorage; + + fn storage(&self, key: &[u8]) -> Result>, Self::Error> { + self.caching_state().storage(key) + } + + fn storage_hash(&self, key: &[u8]) -> Result, Self::Error> { + self.caching_state().storage_hash(key) + } + + fn child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { + self.caching_state().child_storage(storage_key, child_info, key) + } + + fn exists_storage(&self, key: &[u8]) -> Result { + self.caching_state().exists_storage(key) + } + + fn exists_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result { + self.caching_state().exists_child_storage(storage_key, child_info, key) + } + + fn for_keys_in_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + f: F, + ) { + self.caching_state().for_keys_in_child_storage(storage_key, child_info, f) + } + + fn next_storage_key(&self, key: &[u8]) -> Result>, Self::Error> { + self.caching_state().next_storage_key(key) + } + + fn next_child_storage_key( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { + self.caching_state().next_child_storage_key(storage_key, child_info, key) + } + + fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { + self.caching_state().for_keys_with_prefix(prefix, f) + } + + fn for_key_values_with_prefix(&self, prefix: &[u8], f: F) { + self.caching_state().for_key_values_with_prefix(prefix, f) + } + + fn for_child_keys_with_prefix( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + f: F, + ) { + self.caching_state().for_child_keys_with_prefix(storage_key, child_info, prefix, f) + } + + fn storage_root(&self, delta: I) -> (B::Hash, Self::Transaction) + where + I: IntoIterator, Option>)>, + { + self.caching_state().storage_root(delta) + } + + fn child_storage_root( + &self, + storage_key: &[u8], + child_info: ChildInfo, + delta: I, + ) -> (B::Hash, bool, Self::Transaction) + where + I: IntoIterator, Option>)>, + { + self.caching_state().child_storage_root(storage_key, child_info, delta) + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + self.caching_state().pairs() + } + + fn keys(&self, prefix: &[u8]) -> Vec> { + self.caching_state().keys(prefix) + } + + fn child_keys( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + ) -> Vec> { + self.caching_state().child_keys(storage_key, child_info, prefix) + } + + fn as_trie_backend(&mut self) -> Option<&TrieBackend>> { + self.caching_state + .as_mut() + .expect("`caching_state` is valid for the lifetime of the object; qed") + .as_trie_backend() + } + + fn usage_info(&self) -> sp_state_machine::UsageInfo { + self.caching_state().usage_info() + } +} + +impl Drop for SyncingCachingState { + fn drop(&mut self) { + if self.disable_syncing { + return; + } + + if let Some(mut caching_state) = self.caching_state.take() { + let _lock = self.lock.read(); + + self.state_usage.merge_sm(caching_state.usage.take()); + if let Some(hash) = caching_state.cache.parent_hash.clone() { + let is_best = self.meta.read().best_hash == hash; + caching_state.cache.sync_cache(&[], &[], vec![], vec![], None, None, is_best); + } + } + } +} + #[cfg(test)] mod tests { use super::*; - use sp_runtime::testing::{H256, Block as RawBlock, ExtrinsicWrapper}; + use sp_runtime::{ + traits::BlakeTwo256, + testing::{H256, Block as RawBlock, ExtrinsicWrapper}, + }; use sp_state_machine::InMemoryBackend; - use sp_core::Blake2Hasher; type Block = RawBlock>; @@ -695,7 +904,7 @@ mod tests { // blocks [ 3a(c) 2a(c) 2b 1b 1a(c) 0 ] // state [ 5 5 4 3 2 2 ] let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(root_parent), ); @@ -710,14 +919,14 @@ mod tests { ); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h0), ); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h1a), Some(1), true); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h0), ); @@ -732,7 +941,7 @@ mod tests { ); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h1b), ); @@ -747,7 +956,7 @@ mod tests { ); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h1a), ); @@ -762,35 +971,35 @@ mod tests { ); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h2a), ); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h3a), Some(3), true); let s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h3a), ); assert_eq!(s.storage(&key).unwrap().unwrap(), vec![5]); let s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h1a), ); assert!(s.storage(&key).unwrap().is_none()); let s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h2b), ); assert!(s.storage(&key).unwrap().is_none()); let s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h1b), ); @@ -799,7 +1008,7 @@ mod tests { // reorg to 3b // blocks [ 3b(c) 3a 2a 2b(c) 1b 1a 0 ] let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h2b), ); @@ -813,7 +1022,7 @@ mod tests { true, ); let s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h3a), ); @@ -834,7 +1043,7 @@ mod tests { let shared = new_shared_cache::(256*1024, (0,1)); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(root_parent), ); @@ -849,14 +1058,14 @@ mod tests { ); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h1), ); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h2a), Some(2), true); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h1), ); @@ -871,7 +1080,7 @@ mod tests { ); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h2b), ); @@ -886,7 +1095,7 @@ mod tests { ); let s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h2a), ); @@ -906,21 +1115,21 @@ mod tests { let shared = new_shared_cache::(256*1024, (0,1)); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(root_parent), ); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h1), Some(1), true); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h1), ); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h2a), Some(2), true); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h2a), ); @@ -935,14 +1144,14 @@ mod tests { ); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h1), ); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h2b), Some(2), false); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h2b), ); @@ -957,7 +1166,7 @@ mod tests { ); let s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h3a), ); @@ -971,7 +1180,7 @@ mod tests { let h0 = H256::random(); let mut s = CachingState::new( - InMemoryBackend::::default(), shared.clone(), Some(root_parent.clone()), + InMemoryBackend::::default(), shared.clone(), Some(root_parent.clone()), ); let key = H256::random()[..].to_vec(); @@ -1009,7 +1218,7 @@ mod tests { let h0 = H256::random(); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(root_parent), ); @@ -1053,7 +1262,7 @@ mod tests { let shared = new_shared_cache::(256 * 1024, (0, 1)); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(root_parent.clone()), ); @@ -1068,7 +1277,7 @@ mod tests { ); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h0), ); @@ -1083,7 +1292,7 @@ mod tests { ); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h1), ); @@ -1106,7 +1315,7 @@ mod tests { s.cache.sync_cache(&[], &[], vec![], vec![], None, None, true); let s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), shared.clone(), Some(h1), ); @@ -1121,9 +1330,11 @@ mod qc { use quickcheck::{quickcheck, TestResult, Arbitrary}; use super::*; - use sp_runtime::testing::{H256, Block as RawBlock, ExtrinsicWrapper}; + use sp_runtime::{ + traits::BlakeTwo256, + testing::{H256, Block as RawBlock, ExtrinsicWrapper}, + }; use sp_state_machine::InMemoryBackend; - use sp_core::Blake2Hasher; type Block = RawBlock>; @@ -1250,22 +1461,22 @@ mod qc { } } - fn head_state(&self, hash: H256) -> CachingState, Block> { + fn head_state(&self, hash: H256) -> CachingState, Block> { CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), self.shared.clone(), - Some(hash) + Some(hash), ) } - fn canon_head_state(&self) -> CachingState, Block> { + fn canon_head_state(&self) -> CachingState, Block> { self.head_state(self.canon.last().expect("Expected to be one commit").hash) } fn mutate_static( &mut self, action: Action, - ) -> CachingState, Block> { + ) -> CachingState, Block> { self.mutate(action).expect("Expected to provide only valid actions to the mutate_static") } @@ -1284,7 +1495,7 @@ mod qc { fn mutate( &mut self, action: Action, - ) -> Result, Block>, ()> { + ) -> Result, Block>, ()> { let state = match action { Action::Fork { depth, hash, changes } => { let pos = self.canon.len() as isize - depth as isize; @@ -1321,9 +1532,9 @@ mod qc { }; let mut state = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), self.shared.clone(), - Some(parent) + Some(parent), ); state.cache.sync_cache( @@ -1360,9 +1571,9 @@ mod qc { } let mut state = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), self.shared.clone(), - Some(parent_hash) + Some(parent_hash), ); state.cache.sync_cache( @@ -1407,9 +1618,9 @@ mod qc { self.canon.push(node); let mut state = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::default(), self.shared.clone(), - Some(fork_at) + Some(fork_at), ); let height = pos as u64 + enacted.len() as u64 + 2; diff --git a/client/db/src/utils.rs b/client/db/src/utils.rs index 534cbb2197ebce2c55006b10bea4529227418969..f26714eb5a7dbf9edae0fcac6a850e877478fd0e 100644 --- a/client/db/src/utils.rs +++ b/client/db/src/utils.rs @@ -36,6 +36,7 @@ use crate::{DatabaseSettings, DatabaseSettingsSrc}; /// Number of columns in the db. Must be the same for both full && light dbs. /// Otherwise RocksDb will fail to open database && check its type. +#[cfg(any(feature = "kvdb-rocksdb", feature = "test-helpers", test))] pub const NUM_COLUMNS: u32 = 11; /// Meta column. The set of keys in the column is shared by full && light storages. pub const COLUMN_META: u32 = 0; diff --git a/client/executor/Cargo.toml b/client/executor/Cargo.toml index c8774470febf3ea3ed82e380b8f0775dd7cbd55c..1353bc57307b9c5d664b005f7d1bcbb997ca093f 100644 --- a/client/executor/Cargo.toml +++ b/client/executor/Cargo.toml @@ -1,28 +1,32 @@ [package] name = "sc-executor" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "A crate that provides means of executing/dispatching calls into the runtime." +documentation = "https://docs.rs/sc-executor" [dependencies] derive_more = "0.99.2" -codec = { package = "parity-scale-codec", version = "1.0.0" } -sp-io = { version = "2.0.0", path = "../../primitives/io" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-trie = { version = "2.0.0", path = "../../primitives/trie" } -sp-serializer = { version = "2.0.0", path = "../../primitives/serializer" } -sp-version = { version = "2.0.0", path = "../../primitives/version" } -sp-panic-handler = { version = "2.0.0", path = "../../primitives/panic-handler" } +codec = { package = "parity-scale-codec", version = "1.2.0" } +sp-io = { version = "2.0.0-alpha.2", path = "../../primitives/io" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +sp-trie = { version = "2.0.0-alpha.2", path = "../../primitives/trie" } +sp-serializer = { version = "2.0.0-alpha.2", path = "../../primitives/serializer" } +sp-version = { version = "2.0.0-alpha.2", path = "../../primitives/version" } +sp-panic-handler = { version = "2.0.0-alpha.2", path = "../../primitives/panic-handler" } wasmi = "0.6.2" parity-wasm = "0.41.0" lazy_static = "1.4.0" -sp-wasm-interface = { version = "2.0.0", path = "../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0", path = "../../primitives/runtime-interface" } -sp-externalities = { version = "0.8.0", path = "../../primitives/externalities" } -sc-executor-common = { version = "0.8", path = "common" } -sc-executor-wasmi = { version = "0.8", path = "wasmi" } -sc-executor-wasmtime = { version = "0.8", path = "wasmtime", optional = true } +sp-wasm-interface = { version = "2.0.0-alpha.2", path = "../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-alpha.2", path = "../../primitives/runtime-interface" } +sp-externalities = { version = "0.8.0-alpha.2", path = "../../primitives/externalities" } +sc-executor-common = { version = "0.8.0-alpha.2", path = "common" } +sc-executor-wasmi = { version = "0.8.0-alpha.2", path = "wasmi" } +sc-executor-wasmtime = { version = "0.8.0-alpha.2", path = "wasmtime", optional = true } parking_lot = "0.10.0" log = "0.4.8" libsecp256k1 = "0.3.4" @@ -31,10 +35,11 @@ libsecp256k1 = "0.3.4" assert_matches = "1.3.0" wabt = "0.9.2" hex-literal = "0.2.1" -sc-runtime-test = { version = "2.0.0", path = "runtime-test" } -substrate-test-runtime = { version = "2.0.0", path = "../../test-utils/runtime" } -sp-state-machine = { version = "0.8", path = "../../primitives/state-machine" } +sc-runtime-test = { version = "2.0.0-dev", path = "runtime-test" } +substrate-test-runtime = { version = "2.0.0-dev", path = "../../test-utils/runtime" } +sp-state-machine = { version = "0.8.0-alpha.2", path = "../../primitives/state-machine" } test-case = "0.3.3" +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } [features] default = [ "std" ] diff --git a/client/executor/common/Cargo.toml b/client/executor/common/Cargo.toml index e10afe3448b7e960571c7cb1135f97d72a7cf066..04db56938a4bc1105b140b8f1d74207a2c5d3395 100644 --- a/client/executor/common/Cargo.toml +++ b/client/executor/common/Cargo.toml @@ -1,20 +1,24 @@ [package] name = "sc-executor-common" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "A set of common definitions that are needed for defining execution engines." +documentation = "https://docs.rs/sc-executor-common/" [dependencies] log = "0.4.8" derive_more = "0.99.2" -codec = { package = "parity-scale-codec", version = "1.0.0" } +codec = { package = "parity-scale-codec", version = "1.2.0" } wasmi = "0.6.2" -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-allocator = { version = "2.0.0", path = "../../../primitives/allocator" } -sp-wasm-interface = { version = "2.0.0", path = "../../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0", path = "../../../primitives/runtime-interface" } -sp-serializer = { version = "2.0.0", path = "../../../primitives/serializer" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-allocator = { version = "2.0.0-alpha.2", path = "../../../primitives/allocator" } +sp-wasm-interface = { version = "2.0.0-alpha.2", path = "../../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime-interface" } +sp-serializer = { version = "2.0.0-alpha.2", path = "../../../primitives/serializer" } [features] default = [] diff --git a/client/executor/common/src/wasm_runtime.rs b/client/executor/common/src/wasm_runtime.rs index 7af6c2bd53c0ec1f9de45356d459a100e2cf50e8..b59ca8ba930edb71cdc7d671bbb3e99df9d35853 100644 --- a/client/executor/common/src/wasm_runtime.rs +++ b/client/executor/common/src/wasm_runtime.rs @@ -17,18 +17,25 @@ //! Definitions for a wasm runtime. use crate::error::Error; -use sp_wasm_interface::{Function, Value}; +use sp_wasm_interface::Value; -/// A trait that defines an abstract wasm runtime. +/// A trait that defines an abstract WASM runtime module. /// /// This can be implemented by an execution engine. -pub trait WasmRuntime { - /// Return the host functions that are registered for this Wasm runtime. - fn host_functions(&self) -> &[&'static dyn Function]; +pub trait WasmModule: Sync + Send { + /// Create a new instance. + fn new_instance(&self) -> Result, Error>; +} - /// Call a method in the Substrate runtime by name. Returns the encoded result on success. - fn call(&mut self, method: &str, data: &[u8]) -> Result, Error>; +/// A trait that defines an abstract wasm module instance. +/// +/// This can be implemented by an execution engine. +pub trait WasmInstance: Send { + /// Call a method on this WASM instance and reset it afterwards. + /// Returns the encoded result on success. + fn call(&self, method: &str, data: &[u8]) -> Result, Error>; /// Get the value from a global with the given `name`. - fn get_global_val(&self, name: &str) -> Result, Error>; + /// This method is only suitable for getting immutable globals. + fn get_global_const(&self, name: &str) -> Result, Error>; } diff --git a/client/executor/runtime-test/Cargo.toml b/client/executor/runtime-test/Cargo.toml index 3cb0c03a87c8eea3832ef3993fc6711d58deee86..ad7c44718d243433ae6e2e775d29b213d091ca3a 100644 --- a/client/executor/runtime-test/Cargo.toml +++ b/client/executor/runtime-test/Cargo.toml @@ -1,21 +1,24 @@ [package] name = "sc-runtime-test" -version = "2.0.0" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" license = "GPL-3.0" +publish = false +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sp-std = { version = "2.0.0", default-features = false, path = "../../../primitives/std" } -sp-io = { version = "2.0.0", default-features = false, path = "../../../primitives/io" } -sp-sandbox = { version = "0.8.0", default-features = false, path = "../../../primitives/sandbox" } -sp-core = { version = "2.0.0", default-features = false, path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../../primitives/runtime" } -sp-allocator = { version = "2.0.0", default-features = false, path = "../../../primitives/allocator" } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/std" } +sp-io = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/io" } +sp-sandbox = { version = "0.8.0-alpha.2", default-features = false, path = "../../../primitives/sandbox" } +sp-core = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/runtime" } +sp-allocator = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/allocator" } [build-dependencies] -wasm-builder-runner = { version = "1.0.4", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } +wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } [features] default = [ "std" ] diff --git a/client/executor/src/deprecated_host_interface.rs b/client/executor/src/deprecated_host_interface.rs deleted file mode 100644 index 6ea0b11f5a909ad5c424aa4f7b07b09f53a1cf07..0000000000000000000000000000000000000000 --- a/client/executor/src/deprecated_host_interface.rs +++ /dev/null @@ -1,941 +0,0 @@ -// Copyright 2017-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Definition and implementation of the old and deprecated Substrate runtime interface for the host. - -use codec::Encode; -use std::{convert::TryFrom, str}; -use sp_core::{ - blake2_128, blake2_256, twox_64, twox_128, twox_256, ed25519, sr25519, keccak_256, Blake2Hasher, Pair, - crypto::KeyTypeId, offchain, -}; -use sp_trie::{TrieConfiguration, trie_types::Layout}; -use sp_wasm_interface::{ - Pointer, WordSize, WritePrimitive, ReadPrimitive, FunctionContext, Result as WResult, -}; - -#[cfg(feature="wasm-extern-trace")] -macro_rules! debug_trace { - ( $( $x:tt )* ) => ( trace!( $( $x )* ) ) -} - -#[cfg(not(feature="wasm-extern-trace"))] -macro_rules! debug_trace { - ( $( $x:tt )* ) => () -} - -/// The old and deprecated Substrate externals. These are still required for backwards compatibility -/// reasons. -pub struct SubstrateExternals; - -enum RecoverResult { - Invalid(u32), - Valid(secp256k1::PublicKey), -} - -fn secp256k1_recover( - context: &mut dyn FunctionContext, - msg_data: Pointer, - sig_data: Pointer, -) -> WResult { - let mut sig = [0u8; 65]; - context.read_memory_into(sig_data, &mut sig[..]) - .map_err(|_| "Invalid attempt to get signature in ext_secp256k1_ecdsa_recover")?; - let rs = match secp256k1::Signature::parse_slice(&sig[0..64]) { - Ok(rs) => rs, - _ => return Ok(RecoverResult::Invalid(1)), - }; - - let recovery_id = if sig[64] > 26 { sig[64] - 27 } else { sig[64] } as u8; - let v = match secp256k1::RecoveryId::parse(recovery_id) { - Ok(v) => v, - _ => return Ok(RecoverResult::Invalid(2)), - }; - - let mut msg = [0u8; 32]; - context.read_memory_into(msg_data, &mut msg[..]) - .map_err(|_| "Invalid attempt to get message in ext_secp256k1_ecdsa_recover")?; - - Ok(match secp256k1::recover(&secp256k1::Message::parse(&msg), &rs, &v) { - Ok(pubkey) => RecoverResult::Valid(pubkey), - Err(_) => RecoverResult::Invalid(3), - }) -} - -impl_wasm_host_interface! { - impl SubstrateExternals where context { - ext_malloc(size: WordSize) -> Pointer { - let r = context.allocate_memory(size)?; - debug_trace!(target: "sp-io", "malloc {} bytes at {:?}", size, r); - Ok(r) - } - - ext_free(addr: Pointer) { - context.deallocate_memory(addr)?; - debug_trace!(target: "sp-io", "free {:?}", addr); - Ok(()) - } - - ext_sandbox_instantiate( - dispatch_thunk_idx: u32, - wasm_ptr: Pointer, - wasm_len: WordSize, - imports_ptr: Pointer, - imports_len: WordSize, - state: u32, - ) -> u32 { - let wasm = context.read_memory(wasm_ptr, wasm_len) - .map_err(|_| "OOB while ext_sandbox_instantiate: wasm")?; - let raw_env_def = context.read_memory(imports_ptr, imports_len) - .map_err(|_| "OOB while ext_sandbox_instantiate: imports")?; - - context.sandbox().instance_new(dispatch_thunk_idx, &wasm, &raw_env_def, state) - } - - ext_sandbox_instance_teardown(instance_idx: u32) { - context.sandbox().instance_teardown(instance_idx) - } - - ext_sandbox_invoke( - instance_idx: u32, - export_ptr: Pointer, - export_len: WordSize, - args_ptr: Pointer, - args_len: WordSize, - return_val_ptr: Pointer, - return_val_len: WordSize, - state: u32, - ) -> u32 { - let export = context.read_memory(export_ptr, export_len) - .map_err(|_| "OOB while ext_sandbox_invoke: export") - .and_then(|b| - String::from_utf8(b) - .map_err(|_| "Export name should be a valid utf-8 sequence") - )?; - - // Deserialize arguments and convert them into wasmi types. - let serialized_args = context.read_memory(args_ptr, args_len) - .map_err(|_| "OOB while ext_sandbox_invoke: args")?; - - context.sandbox().invoke( - instance_idx, - &export, - &serialized_args, - return_val_ptr, - return_val_len, - state, - ) - } - - ext_sandbox_memory_new(initial: WordSize, maximum: WordSize) -> u32 { - context.sandbox().memory_new(initial, maximum) - } - - ext_sandbox_memory_get( - memory_idx: u32, - offset: WordSize, - buf_ptr: Pointer, - buf_len: WordSize, - ) -> u32 { - context.sandbox().memory_get(memory_idx, offset, buf_ptr, buf_len) - } - - ext_sandbox_memory_set( - memory_idx: u32, - offset: WordSize, - val_ptr: Pointer, - val_len: WordSize, - ) -> u32 { - context.sandbox().memory_set(memory_idx, offset, val_ptr, val_len) - } - - ext_sandbox_memory_teardown(memory_idx: u32) { - context.sandbox().memory_teardown(memory_idx) - } - - ext_print_utf8(utf8_data: Pointer, utf8_len: WordSize) { - if let Ok(utf8) = context.read_memory(utf8_data, utf8_len) { - sp_io::misc::print_utf8(&utf8); - } - Ok(()) - } - - ext_print_hex(data: Pointer, len: WordSize) { - if let Ok(hex) = context.read_memory(data, len) { - sp_io::misc::print_hex(&hex); - } - Ok(()) - } - - ext_print_num(number: u64) { - sp_io::misc::print_num(number); - Ok(()) - } - - ext_log( - level: u32, - target_data: Pointer, - target_len: WordSize, - message_data: Pointer, - message_len: WordSize, - ) { - let target = context.read_memory(target_data, target_len) - .map_err(|_| "Invalid attempt to determine target in ext_log")?; - let message = context.read_memory(message_data, message_len) - .map_err(|_| "Invalid attempt to determine message in ext_log")?; - - let target_str = std::str::from_utf8(&target) - .map_err(|_| "Target invalid utf8 in ext_log")?; - - sp_io::logging::log(level.into(), &target_str, &message); - Ok(()) - } - - ext_set_storage( - key_data: Pointer, - key_len: WordSize, - value_data: Pointer, - value_len: WordSize, - ) { - let key = context.read_memory(key_data, key_len) - .map_err(|_| "Invalid attempt to determine key in ext_set_storage")?; - let value = context.read_memory(value_data, value_len) - .map_err(|_| "Invalid attempt to determine value in ext_set_storage")?; - Ok(sp_io::storage::set(&key, &value)) - } - - ext_clear_storage(key_data: Pointer, key_len: WordSize) { - let key = context.read_memory(key_data, key_len) - .map_err(|_| "Invalid attempt to determine key in ext_clear_storage")?; - Ok(sp_io::storage::clear(&key)) - } - - ext_exists_storage(key_data: Pointer, key_len: WordSize) -> u32 { - let key = context.read_memory(key_data, key_len) - .map_err(|_| "Invalid attempt to determine key in ext_exists_storage")?; - Ok(if sp_io::storage::exists(&key) { 1 } else { 0 }) - } - - ext_clear_prefix(prefix_data: Pointer, prefix_len: WordSize) { - let prefix = context.read_memory(prefix_data, prefix_len) - .map_err(|_| "Invalid attempt to determine prefix in ext_clear_prefix")?; - Ok(sp_io::storage::clear_prefix(&prefix)) - } - - ext_get_allocated_storage( - key_data: Pointer, - key_len: WordSize, - written_out: Pointer, - ) -> Pointer { - let key = context.read_memory(key_data, key_len) - .map_err(|_| "Invalid attempt to determine key in ext_get_allocated_storage")?; - - if let Some(value) = sp_io::storage::get(&key) { - let offset = context.allocate_memory(value.len() as u32)?; - context.write_memory(offset, &value) - .map_err(|_| "Invalid attempt to set memory in ext_get_allocated_storage")?; - context.write_primitive(written_out, value.len() as u32) - .map_err(|_| "Invalid attempt to write written_out in ext_get_allocated_storage")?; - Ok(offset) - } else { - context.write_primitive(written_out, u32::max_value()) - .map_err(|_| "Invalid attempt to write failed written_out in ext_get_allocated_storage")?; - Ok(Pointer::null()) - } - } - - ext_get_storage_into( - key_data: Pointer, - key_len: WordSize, - value_data: Pointer, - value_len: WordSize, - value_offset: WordSize, - ) -> WordSize { - let key = context.read_memory(key_data, key_len) - .map_err(|_| "Invalid attempt to get key in ext_get_storage_into")?; - - if let Some(value) = sp_io::storage::get(&key) { - let data = &value[value.len().min(value_offset as usize)..]; - let written = std::cmp::min(value_len as usize, data.len()); - context.write_memory(value_data, &data[..written]) - .map_err(|_| "Invalid attempt to set value in ext_get_storage_into")?; - Ok(value.len() as u32) - } else { - Ok(u32::max_value()) - } - } - - ext_storage_root(result: Pointer) { - context.write_memory(result, sp_io::storage::root().as_ref()) - .map_err(|_| "Invalid attempt to set memory in ext_storage_root".into()) - } - - ext_storage_changes_root( - parent_hash_data: Pointer, - _len: WordSize, - result: Pointer, - ) -> u32 { - let mut parent_hash = [0u8; 32]; - context.read_memory_into(parent_hash_data, &mut parent_hash[..]) - .map_err(|_| "Invalid attempt to get parent_hash in ext_storage_changes_root")?; - - if let Some(r) = sp_io::storage::changes_root(&parent_hash) { - context.write_memory(result, &r[..]) - .map_err(|_| "Invalid attempt to set memory in ext_storage_changes_root")?; - Ok(1) - } else { - Ok(0) - } - } - - ext_blake2_256_enumerated_trie_root( - values_data: Pointer, - lens_data: Pointer, - lens_len: WordSize, - result: Pointer, - ) { - let values = (0..lens_len) - .map(|i| context.read_primitive(lens_data.offset(i).ok_or("Pointer overflow")?)) - .collect::, _>>()? - .into_iter() - .scan(0u32, |acc, v| { let o = *acc; *acc += v; Some((o, v)) }) - .map(|(offset, len)| - context.read_memory(values_data.offset(offset).ok_or("Pointer overflow")?, len) - .map_err(|_| - "Invalid attempt to get memory in ext_blake2_256_enumerated_trie_root" - ) - ) - .collect::, _>>()?; - let r = Layout::::ordered_trie_root(values.into_iter()); - context.write_memory(result, &r[..]) - .map_err(|_| "Invalid attempt to set memory in ext_blake2_256_enumerated_trie_root")?; - Ok(()) - } - - ext_chain_id() -> u64 { - Ok(sp_io::misc::chain_id()) - } - - ext_twox_64(data: Pointer, len: WordSize, out: Pointer) { - let result: [u8; 8] = if len == 0 { - let hashed = twox_64(&[0u8; 0]); - hashed - } else { - let key = context.read_memory(data, len) - .map_err(|_| "Invalid attempt to get key in ext_twox_64")?; - let hashed_key = twox_64(&key); - hashed_key - }; - - context.write_memory(out, &result) - .map_err(|_| "Invalid attempt to set result in ext_twox_64")?; - Ok(()) - } - - ext_twox_128(data: Pointer, len: WordSize, out: Pointer) { - let result: [u8; 16] = if len == 0 { - let hashed = twox_128(&[0u8; 0]); - hashed - } else { - let key = context.read_memory(data, len) - .map_err(|_| "Invalid attempt to get key in ext_twox_128")?; - let hashed_key = twox_128(&key); - hashed_key - }; - - context.write_memory(out, &result) - .map_err(|_| "Invalid attempt to set result in ext_twox_128")?; - Ok(()) - } - - ext_twox_256(data: Pointer, len: WordSize, out: Pointer) { - let result: [u8; 32] = if len == 0 { - twox_256(&[0u8; 0]) - } else { - let mem = context.read_memory(data, len) - .map_err(|_| "Invalid attempt to get data in ext_twox_256")?; - twox_256(&mem) - }; - context.write_memory(out, &result) - .map_err(|_| "Invalid attempt to set result in ext_twox_256")?; - Ok(()) - } - - ext_blake2_128(data: Pointer, len: WordSize, out: Pointer) { - let result: [u8; 16] = if len == 0 { - let hashed = blake2_128(&[0u8; 0]); - hashed - } else { - let key = context.read_memory(data, len) - .map_err(|_| "Invalid attempt to get key in ext_blake2_128")?; - let hashed_key = blake2_128(&key); - hashed_key - }; - - context.write_memory(out, &result) - .map_err(|_| "Invalid attempt to set result in ext_blake2_128")?; - Ok(()) - } - - ext_blake2_256(data: Pointer, len: WordSize, out: Pointer) { - let result: [u8; 32] = if len == 0 { - blake2_256(&[0u8; 0]) - } else { - let mem = context.read_memory(data, len) - .map_err(|_| "Invalid attempt to get data in ext_blake2_256")?; - blake2_256(&mem) - }; - context.write_memory(out, &result) - .map_err(|_| "Invalid attempt to set result in ext_blake2_256")?; - Ok(()) - } - - ext_keccak_256(data: Pointer, len: WordSize, out: Pointer) { - let result: [u8; 32] = if len == 0 { - keccak_256(&[0u8; 0]) - } else { - let mem = context.read_memory(data, len) - .map_err(|_| "Invalid attempt to get data in ext_keccak_256")?; - keccak_256(&mem) - }; - context.write_memory(out, &result) - .map_err(|_| "Invalid attempt to set result in ext_keccak_256")?; - Ok(()) - } - - ext_ed25519_public_keys(id_data: Pointer, result_len: Pointer) -> Pointer { - let mut id = [0u8; 4]; - context.read_memory_into(id_data, &mut id[..]) - .map_err(|_| "Invalid attempt to get id in ext_ed25519_public_keys")?; - let key_type = KeyTypeId(id); - - let keys = sp_io::crypto::ed25519_public_keys(key_type).encode(); - - let len = keys.len() as u32; - let offset = context.allocate_memory(len)?; - - context.write_memory(offset, keys.as_ref()) - .map_err(|_| "Invalid attempt to set memory in ext_ed25519_public_keys")?; - context.write_primitive(result_len, len) - .map_err(|_| "Invalid attempt to write result_len in ext_ed25519_public_keys")?; - - Ok(offset) - } - - ext_ed25519_verify( - msg_data: Pointer, - msg_len: WordSize, - sig_data: Pointer, - pubkey_data: Pointer, - ) -> u32 { - let mut sig = [0u8; 64]; - context.read_memory_into(sig_data, &mut sig[..]) - .map_err(|_| "Invalid attempt to get signature in ext_ed25519_verify")?; - let mut pubkey = [0u8; 32]; - context.read_memory_into(pubkey_data, &mut pubkey[..]) - .map_err(|_| "Invalid attempt to get pubkey in ext_ed25519_verify")?; - let msg = context.read_memory(msg_data, msg_len) - .map_err(|_| "Invalid attempt to get message in ext_ed25519_verify")?; - - Ok(if ed25519::Pair::verify_weak(&sig, &msg, &pubkey) { - 0 - } else { - 1 - }) - } - - ext_ed25519_generate( - id_data: Pointer, - seed: Pointer, - seed_len: WordSize, - out: Pointer, - ) { - let mut id = [0u8; 4]; - context.read_memory_into(id_data, &mut id[..]) - .map_err(|_| "Invalid attempt to get id in ext_ed25519_generate")?; - let key_type = KeyTypeId(id); - - let seed = if seed_len == 0 { - None - } else { - Some( - context.read_memory(seed, seed_len) - .map_err(|_| "Invalid attempt to get seed in ext_ed25519_generate")? - ) - }; - - let pubkey = sp_io::crypto::ed25519_generate(key_type, seed); - - context.write_memory(out, pubkey.as_ref()) - .map_err(|_| "Invalid attempt to set out in ext_ed25519_generate".into()) - } - - ext_ed25519_sign( - id_data: Pointer, - pubkey_data: Pointer, - msg_data: Pointer, - msg_len: WordSize, - out: Pointer, - ) -> u32 { - let mut id = [0u8; 4]; - context.read_memory_into(id_data, &mut id[..]) - .map_err(|_| "Invalid attempt to get id in ext_ed25519_sign")?; - let key_type = KeyTypeId(id); - - let mut pubkey = [0u8; 32]; - context.read_memory_into(pubkey_data, &mut pubkey[..]) - .map_err(|_| "Invalid attempt to get pubkey in ext_ed25519_sign")?; - - let msg = context.read_memory(msg_data, msg_len) - .map_err(|_| "Invalid attempt to get message in ext_ed25519_sign")?; - - let pub_key = ed25519::Public::try_from(pubkey.as_ref()) - .map_err(|_| "Invalid `ed25519` public key")?; - - let signature = sp_io::crypto::ed25519_sign(key_type, &pub_key, &msg); - - match signature { - Some(signature) => { - context.write_memory(out, signature.as_ref()) - .map_err(|_| "Invalid attempt to set out in ext_ed25519_sign")?; - Ok(0) - }, - None => Ok(1), - } - } - - ext_sr25519_public_keys(id_data: Pointer, result_len: Pointer) -> Pointer { - let mut id = [0u8; 4]; - context.read_memory_into(id_data, &mut id[..]) - .map_err(|_| "Invalid attempt to get id in ext_sr25519_public_keys")?; - let key_type = KeyTypeId(id); - - let keys = sp_io::crypto::sr25519_public_keys(key_type).encode(); - - let len = keys.len() as u32; - let offset = context.allocate_memory(len)?; - - context.write_memory(offset, keys.as_ref()) - .map_err(|_| "Invalid attempt to set memory in ext_sr25519_public_keys")?; - context.write_primitive(result_len, len) - .map_err(|_| "Invalid attempt to write result_len in ext_sr25519_public_keys")?; - - Ok(offset) - } - - ext_sr25519_verify( - msg_data: Pointer, - msg_len: WordSize, - sig_data: Pointer, - pubkey_data: Pointer, - ) -> u32 { - let mut sig = [0u8; 64]; - context.read_memory_into(sig_data, &mut sig[..]) - .map_err(|_| "Invalid attempt to get signature in ext_sr25519_verify")?; - let mut pubkey = [0u8; 32]; - context.read_memory_into(pubkey_data, &mut pubkey[..]) - .map_err(|_| "Invalid attempt to get pubkey in ext_sr25519_verify")?; - let msg = context.read_memory(msg_data, msg_len) - .map_err(|_| "Invalid attempt to get message in ext_sr25519_verify")?; - - Ok(if sr25519::Pair::verify_weak(&sig, &msg, &pubkey) { - 0 - } else { - 1 - }) - } - - ext_sr25519_generate( - id_data: Pointer, - seed: Pointer, - seed_len: WordSize, - out: Pointer, - ) { - let mut id = [0u8; 4]; - context.read_memory_into(id_data, &mut id[..]) - .map_err(|_| "Invalid attempt to get id in ext_sr25519_generate")?; - let key_type = KeyTypeId(id); - let seed = if seed_len == 0 { - None - } else { - Some( - context.read_memory(seed, seed_len) - .map_err(|_| "Invalid attempt to get seed in ext_sr25519_generate")? - ) - }; - - let pubkey = sp_io::crypto::sr25519_generate(key_type, seed); - - context.write_memory(out, pubkey.as_ref()) - .map_err(|_| "Invalid attempt to set out in ext_sr25519_generate".into()) - } - - ext_sr25519_sign( - id_data: Pointer, - pubkey_data: Pointer, - msg_data: Pointer, - msg_len: WordSize, - out: Pointer, - ) -> u32 { - let mut id = [0u8; 4]; - context.read_memory_into(id_data, &mut id[..]) - .map_err(|_| "Invalid attempt to get id in ext_sr25519_sign")?; - let key_type = KeyTypeId(id); - - let mut pubkey = [0u8; 32]; - context.read_memory_into(pubkey_data, &mut pubkey[..]) - .map_err(|_| "Invalid attempt to get pubkey in ext_sr25519_sign")?; - - let msg = context.read_memory(msg_data, msg_len) - .map_err(|_| "Invalid attempt to get message in ext_sr25519_sign")?; - - let pub_key = sr25519::Public::try_from(pubkey.as_ref()) - .map_err(|_| "Invalid `sr25519` public key")?; - - let signature = sp_io::crypto::sr25519_sign(key_type, &pub_key, &msg); - - match signature { - Some(signature) => { - context.write_memory(out, signature.as_ref()) - .map_err(|_| "Invalid attempt to set out in ext_sr25519_sign")?; - Ok(0) - }, - None => Ok(1), - } - } - - ext_secp256k1_ecdsa_recover( - msg_data: Pointer, - sig_data: Pointer, - pubkey_data: Pointer, - ) -> u32 { - match secp256k1_recover(context, msg_data, sig_data)? { - RecoverResult::Invalid(c) => Ok(c), - RecoverResult::Valid(pubkey) => { - context.write_memory(pubkey_data, &pubkey.serialize()[1..65]) - .map_err(|_| "Invalid attempt to set pubkey in ext_secp256k1_ecdsa_recover")?; - Ok(0) - } - } - } - - ext_secp256k1_ecdsa_recover_compressed( - msg_data: Pointer, - sig_data: Pointer, - pubkey_data: Pointer, - ) -> u32 { - match secp256k1_recover(context, msg_data, sig_data)? { - RecoverResult::Invalid(c) => Ok(c), - RecoverResult::Valid(pubkey) => { - context.write_memory(pubkey_data, &pubkey.serialize_compressed()[..]) - .map_err(|_| "Invalid attempt to set pubkey in ext_secp256k1_ecdsa_recover")?; - Ok(0) - } - } - } - - ext_is_validator() -> u32 { - if sp_io::offchain::is_validator() { Ok(1) } else { Ok(0) } - } - - ext_submit_transaction(msg_data: Pointer, len: WordSize) -> u32 { - let extrinsic = context.read_memory(msg_data, len) - .map_err(|_| "OOB while ext_submit_transaction: wasm")?; - - let res = sp_io::offchain::submit_transaction(extrinsic); - - Ok(if res.is_ok() { 0 } else { 1 }) - } - - ext_network_state(written_out: Pointer) -> Pointer { - let res = sp_io::offchain::network_state(); - - let encoded = res.encode(); - let len = encoded.len() as u32; - let offset = context.allocate_memory(len)?; - context.write_memory(offset, &encoded) - .map_err(|_| "Invalid attempt to set memory in ext_network_state")?; - - context.write_primitive(written_out, len) - .map_err(|_| "Invalid attempt to write written_out in ext_network_state")?; - - Ok(offset) - } - - ext_timestamp() -> u64 { - Ok(sp_io::offchain::timestamp().unix_millis()) - } - - ext_sleep_until(deadline: u64) { - sp_io::offchain::sleep_until(offchain::Timestamp::from_unix_millis(deadline)); - Ok(()) - } - - ext_random_seed(seed_data: Pointer) { - // NOTE the runtime as assumptions about seed size. - let seed = sp_io::offchain::random_seed(); - - context.write_memory(seed_data, &seed) - .map_err(|_| "Invalid attempt to set value in ext_random_seed")?; - Ok(()) - } - - ext_local_storage_set( - kind: u32, - key: Pointer, - key_len: WordSize, - value: Pointer, - value_len: WordSize, - ) { - let kind = offchain::StorageKind::try_from(kind) - .map_err(|_| "storage kind OOB while ext_local_storage_set: wasm")?; - let key = context.read_memory(key, key_len) - .map_err(|_| "OOB while ext_local_storage_set: wasm")?; - let value = context.read_memory(value, value_len) - .map_err(|_| "OOB while ext_local_storage_set: wasm")?; - - sp_io::offchain::local_storage_set(kind, &key, &value); - - Ok(()) - } - - ext_local_storage_get( - kind: u32, - key: Pointer, - key_len: WordSize, - value_len: Pointer, - ) -> Pointer { - let kind = offchain::StorageKind::try_from(kind) - .map_err(|_| "storage kind OOB while ext_local_storage_get: wasm")?; - let key = context.read_memory(key, key_len) - .map_err(|_| "OOB while ext_local_storage_get: wasm")?; - - let maybe_value = sp_io::offchain::local_storage_get(kind, &key); - - let (offset, len) = if let Some(value) = maybe_value { - let offset = context.allocate_memory(value.len() as u32)?; - context.write_memory(offset, &value) - .map_err(|_| "Invalid attempt to set memory in ext_local_storage_get")?; - (offset, value.len() as u32) - } else { - (Pointer::null(), u32::max_value()) - }; - - context.write_primitive(value_len, len) - .map_err(|_| "Invalid attempt to write value_len in ext_local_storage_get")?; - - Ok(offset) - } - - ext_local_storage_compare_and_set( - kind: u32, - key: Pointer, - key_len: WordSize, - old_value: Pointer, - old_value_len: WordSize, - new_value: Pointer, - new_value_len: WordSize, - ) -> u32 { - let kind = offchain::StorageKind::try_from(kind) - .map_err(|_| "storage kind OOB while ext_local_storage_compare_and_set: wasm")?; - let key = context.read_memory(key, key_len) - .map_err(|_| "OOB while ext_local_storage_compare_and_set: wasm")?; - let new_value = context.read_memory(new_value, new_value_len) - .map_err(|_| "OOB while ext_local_storage_compare_and_set: wasm")?; - - let old_value = if old_value_len == u32::max_value() { - None - } else { - Some( - context.read_memory(old_value, old_value_len) - .map_err(|_| "OOB while ext_local_storage_compare_and_set: wasm")? - ) - }; - - let res = sp_io::offchain::local_storage_compare_and_set( - kind, - &key, - old_value, - &new_value, - ); - - Ok(if res { 0 } else { 1 }) - } - - ext_http_request_start( - method: Pointer, - method_len: WordSize, - url: Pointer, - url_len: WordSize, - meta: Pointer, - meta_len: WordSize, - ) -> u32 { - let method = context.read_memory(method, method_len) - .map_err(|_| "OOB while ext_http_request_start: wasm")?; - let url = context.read_memory(url, url_len) - .map_err(|_| "OOB while ext_http_request_start: wasm")?; - let meta = context.read_memory(meta, meta_len) - .map_err(|_| "OOB while ext_http_request_start: wasm")?; - - let method_str = str::from_utf8(&method) - .map_err(|_| "invalid str while ext_http_request_start: wasm")?; - let url_str = str::from_utf8(&url) - .map_err(|_| "invalid str while ext_http_request_start: wasm")?; - - let id = sp_io::offchain::http_request_start(method_str, url_str, &meta); - - if let Ok(id) = id { - Ok(id.into()) - } else { - Ok(u32::max_value()) - } - } - - ext_http_request_add_header( - request_id: u32, - name: Pointer, - name_len: WordSize, - value: Pointer, - value_len: WordSize, - ) -> u32 { - let name = context.read_memory(name, name_len) - .map_err(|_| "OOB while ext_http_request_add_header: wasm")?; - let value = context.read_memory(value, value_len) - .map_err(|_| "OOB while ext_http_request_add_header: wasm")?; - - let name_str = str::from_utf8(&name) - .map_err(|_| "Invalid str while ext_http_request_add_header: wasm")?; - let value_str = str::from_utf8(&value) - .map_err(|_| "Invalid str while ext_http_request_add_header: wasm")?; - - let res = sp_io::offchain::http_request_add_header( - offchain::HttpRequestId(request_id as u16), - name_str, - value_str, - ); - - Ok(if res.is_ok() { 0 } else { 1 }) - } - - ext_http_request_write_body( - request_id: u32, - chunk: Pointer, - chunk_len: WordSize, - deadline: u64, - ) -> u32 { - let chunk = context.read_memory(chunk, chunk_len) - .map_err(|_| "OOB while ext_http_request_write_body: wasm")?; - - let res = sp_io::offchain::http_request_write_body( - offchain::HttpRequestId(request_id as u16), - &chunk, - deadline_to_timestamp(deadline), - ); - - Ok(match res { - Ok(()) => 0, - Err(e) => e.into(), - }) - } - - ext_http_response_wait( - ids: Pointer, - ids_len: WordSize, - statuses: Pointer, - deadline: u64, - ) { - let ids = (0..ids_len) - .map(|i| - context.read_primitive(ids.offset(i).ok_or("Point overflow")?) - .map(|id: u32| offchain::HttpRequestId(id as u16)) - .map_err(|_| "OOB while ext_http_response_wait: wasm") - ) - .collect::, _>>()?; - - let res = sp_io::offchain::http_response_wait(&ids, deadline_to_timestamp(deadline)) - .into_iter() - .map(|status| u32::from(status)) - .enumerate() - // make sure to take up to `ids_len` to avoid exceeding the mem. - .take(ids_len as usize); - - for (i, status) in res { - context.write_primitive(statuses.offset(i as u32).ok_or("Point overflow")?, status) - .map_err(|_| "Invalid attempt to set memory in ext_http_response_wait")?; - } - - Ok(()) - } - - ext_http_response_headers( - request_id: u32, - written_out: Pointer, - ) -> Pointer { - use codec::Encode; - - let headers = sp_io::offchain::http_response_headers( - offchain::HttpRequestId(request_id as u16), - ); - - let encoded = headers.encode(); - let len = encoded.len() as u32; - let offset = context.allocate_memory(len)?; - - context.write_memory(offset, &encoded) - .map_err(|_| "Invalid attempt to set memory in ext_http_response_headers")?; - context.write_primitive(written_out, len) - .map_err(|_| "Invalid attempt to write written_out in ext_http_response_headers")?; - - Ok(offset) - } - - ext_http_response_read_body( - request_id: u32, - buffer: Pointer, - buffer_len: WordSize, - deadline: u64, - ) -> WordSize { - let mut internal_buffer = Vec::with_capacity(buffer_len as usize); - internal_buffer.resize(buffer_len as usize, 0); - - let res = sp_io::offchain::http_response_read_body( - offchain::HttpRequestId(request_id as u16), - &mut internal_buffer, - deadline_to_timestamp(deadline), - ); - - Ok(match res { - Ok(read) => { - context.write_memory(buffer, &internal_buffer[..read as usize]) - .map_err(|_| "Invalid attempt to set memory in ext_http_response_read_body")?; - - read as u32 - }, - Err(err) => { - u32::max_value() - u32::from(err) + 1 - } - }) - } - } -} - -fn deadline_to_timestamp(deadline: u64) -> Option { - if deadline == 0 { - None - } else { - Some(offchain::Timestamp::from_unix_millis(deadline)) - } -} diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index c0516d3ac7dfaa8b3447faf234834791efb56842..e787b229ec85c40c7c5f0318d9ba3ed6be7b813f 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -19,19 +19,20 @@ mod sandbox; use codec::{Encode, Decode}; use hex_literal::hex; use sp_core::{ - Blake2Hasher, blake2_128, blake2_256, ed25519, sr25519, map, Pair, + blake2_128, blake2_256, ed25519, sr25519, map, Pair, offchain::{OffchainExt, testing}, - traits::Externalities, + traits::{Externalities, CallInWasm}, }; use sc_runtime_test::WASM_BINARY; use sp_state_machine::TestExternalities as CoreTestExternalities; use test_case::test_case; use sp_trie::{TrieConfiguration, trie_types::Layout}; use sp_wasm_interface::HostFunctions as _; +use sp_runtime::traits::BlakeTwo256; use crate::WasmExecutionMethod; -pub type TestExternalities = CoreTestExternalities; +pub type TestExternalities = CoreTestExternalities; type HostFunctions = sp_io::SubstrateHostFunctions; fn call_in_wasm( @@ -39,15 +40,18 @@ fn call_in_wasm( call_data: &[u8], execution_method: WasmExecutionMethod, ext: &mut E, -) -> crate::error::Result> { - crate::call_in_wasm::( +) -> Result, String> { + let executor = crate::WasmExecutor::new( + execution_method, + Some(1024), + HostFunctions::host_functions(), + true, + ); + executor.call_in_wasm( + &WASM_BINARY[..], function, call_data, - execution_method, ext, - &WASM_BINARY[..], - 1024, - true, ) } @@ -83,12 +87,12 @@ fn call_not_existing_function(wasm_method: WasmExecutionMethod) { match wasm_method { WasmExecutionMethod::Interpreted => assert_eq!( &format!("{:?}", e), - "Wasmi(Trap(Trap { kind: Host(Other(\"Function `missing_external` is only a stub. Calling a stub is not allowed.\")) }))" + "\"Trap: Trap { kind: Host(Other(\\\"Function `missing_external` is only a stub. Calling a stub is not allowed.\\\")) }\"" ), #[cfg(feature = "wasmtime")] WasmExecutionMethod::Compiled => assert_eq!( &format!("{:?}", e), - "Other(\"Wasm execution trapped: call to a missing function env:missing_external\")" + "\"Wasm execution trapped: call to a missing function env:missing_external\"" ), } } @@ -112,12 +116,12 @@ fn call_yet_another_not_existing_function(wasm_method: WasmExecutionMethod) { match wasm_method { WasmExecutionMethod::Interpreted => assert_eq!( &format!("{:?}", e), - "Wasmi(Trap(Trap { kind: Host(Other(\"Function `yet_another_missing_external` is only a stub. Calling a stub is not allowed.\")) }))" + "\"Trap: Trap { kind: Host(Other(\\\"Function `yet_another_missing_external` is only a stub. Calling a stub is not allowed.\\\")) }\"" ), #[cfg(feature = "wasmtime")] WasmExecutionMethod::Compiled => assert_eq!( &format!("{:?}", e), - "Other(\"Wasm execution trapped: call to a missing function env:yet_another_missing_external\")" + "\"Wasm execution trapped: call to a missing function env:yet_another_missing_external\"" ), } } @@ -440,7 +444,7 @@ fn ordered_trie_root_should_work(wasm_method: WasmExecutionMethod) { wasm_method, &mut ext.ext(), ).unwrap(), - Layout::::ordered_trie_root(trie_input.iter()).as_bytes().encode(), + Layout::::ordered_trie_root(trie_input.iter()).as_bytes().encode(), ); } @@ -501,29 +505,32 @@ fn offchain_http_should_work(wasm_method: WasmExecutionMethod) { fn should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); - crate::call_in_wasm::( + let executor = crate::WasmExecutor::new( + wasm_method, + Some(17), // `17` is the initial number of pages compiled into the binary. + HostFunctions::host_functions(), + true, + ); + executor.call_in_wasm( + &WASM_BINARY[..], "test_exhaust_heap", &[0], - wasm_method, &mut ext.ext(), - &WASM_BINARY[..], - // `17` is the initial number of pages compiled into the binary. - 17, - true, ).unwrap(); } #[test_case(WasmExecutionMethod::Interpreted)] #[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] fn returns_mutable_static(wasm_method: WasmExecutionMethod) { - let mut instance = crate::wasm_runtime::create_wasm_runtime_with_code( + let runtime = crate::wasm_runtime::create_wasm_runtime_with_code( wasm_method, 1024, &WASM_BINARY[..], HostFunctions::host_functions(), true, - ).expect("Creates instance"); + ).expect("Creates runtime"); + let instance = runtime.new_instance().unwrap(); let res = instance.call("returns_mutable_static", &[0]).unwrap(); assert_eq!(33, u64::decode(&mut &res[..]).unwrap()); @@ -549,13 +556,14 @@ fn restoration_of_globals(wasm_method: WasmExecutionMethod) { // to our allocator algorithm there are inefficiencies. const REQUIRED_MEMORY_PAGES: u64 = 32; - let mut instance = crate::wasm_runtime::create_wasm_runtime_with_code( + let runtime = crate::wasm_runtime::create_wasm_runtime_with_code( wasm_method, REQUIRED_MEMORY_PAGES, &WASM_BINARY[..], HostFunctions::host_functions(), true, - ).expect("Creates instance"); + ).expect("Creates runtime"); + let instance = runtime.new_instance().unwrap(); // On the first invocation we allocate approx. 768KB (75%) of stack and then trap. let res = instance.call("allocates_huge_stack_array", &true.encode()); @@ -568,15 +576,16 @@ fn restoration_of_globals(wasm_method: WasmExecutionMethod) { #[test_case(WasmExecutionMethod::Interpreted)] fn heap_is_reset_between_calls(wasm_method: WasmExecutionMethod) { - let mut instance = crate::wasm_runtime::create_wasm_runtime_with_code( + let runtime = crate::wasm_runtime::create_wasm_runtime_with_code( wasm_method, 1024, &WASM_BINARY[..], HostFunctions::host_functions(), true, - ).expect("Creates instance"); + ).expect("Creates runtime"); + let instance = runtime.new_instance().unwrap(); - let heap_base = instance.get_global_val("__heap_base") + let heap_base = instance.get_global_const("__heap_base") .expect("`__heap_base` is valid") .expect("`__heap_base` exists") .as_i32() diff --git a/client/executor/src/lib.rs b/client/executor/src/lib.rs index 152e3a498485dfb310efcc25abd8a0b2eb163e02..1c21026fd8d35c5afb10c0db139399555cd7a4d0 100644 --- a/client/executor/src/lib.rs +++ b/client/executor/src/lib.rs @@ -29,92 +29,31 @@ #![warn(missing_docs)] #![recursion_limit="128"] -#[macro_use] -mod wasm_utils; #[macro_use] mod native_executor; -pub mod deprecated_host_interface; mod wasm_runtime; #[cfg(test)] mod integration_tests; pub use wasmi; -pub use native_executor::{with_externalities_safe, NativeExecutor, NativeExecutionDispatch}; +pub use native_executor::{with_externalities_safe, NativeExecutor, WasmExecutor, NativeExecutionDispatch}; pub use sp_version::{RuntimeVersion, NativeVersion}; pub use codec::Codec; #[doc(hidden)] -pub use sp_core::traits::Externalities; +pub use sp_core::traits::{Externalities, CallInWasm}; #[doc(hidden)] pub use sp_wasm_interface; pub use wasm_runtime::WasmExecutionMethod; pub use sc_executor_common::{error, sandbox}; -/// Call the given `function` in the given wasm `code`. -/// -/// The signature of `function` needs to follow the default Substrate function signature. -/// -/// - `call_data`: Will be given as input parameters to `function` -/// - `execution_method`: The execution method to use. -/// - `ext`: The externalities that should be set while executing the wasm function. -/// If `None` is given, no externalities will be set. -/// - `heap_pages`: The number of heap pages to allocate. -/// -/// Returns the `Vec` that contains the return value of the function. -pub fn call_in_wasm( - function: &str, - call_data: &[u8], - execution_method: WasmExecutionMethod, - ext: &mut dyn Externalities, - code: &[u8], - heap_pages: u64, - allow_missing_func_imports: bool, -) -> error::Result> { - call_in_wasm_with_host_functions( - function, - call_data, - execution_method, - ext, - code, - heap_pages, - HF::host_functions(), - allow_missing_func_imports, - ) -} - -/// Non-generic version of [`call_in_wasm`] that takes the `host_functions` as parameter. -/// For more information please see [`call_in_wasm`]. -pub fn call_in_wasm_with_host_functions( - function: &str, - call_data: &[u8], - execution_method: WasmExecutionMethod, - ext: &mut dyn Externalities, - code: &[u8], - heap_pages: u64, - host_functions: Vec<&'static dyn sp_wasm_interface::Function>, - allow_missing_func_imports: bool, -) -> error::Result> { - let instance = wasm_runtime::create_wasm_runtime_with_code( - execution_method, - heap_pages, - code, - host_functions, - allow_missing_func_imports, - )?; - - // It is safe, as we delete the instance afterwards. - let mut instance = std::panic::AssertUnwindSafe(instance); - - with_externalities_safe(ext, move || instance.call(function, call_data)).and_then(|r| r) -} - /// Provides runtime information. pub trait RuntimeInfo { /// Native runtime information. fn native_version(&self) -> &NativeVersion; /// Extract RuntimeVersion of given :code block - fn runtime_version (&self, ext: &mut E) -> error::Result; + fn runtime_version(&self, ext: &mut dyn Externalities) -> error::Result; } #[cfg(test)] @@ -122,19 +61,25 @@ mod tests { use super::*; use sc_runtime_test::WASM_BINARY; use sp_io::TestExternalities; + use sp_wasm_interface::HostFunctions; + use sp_core::traits::CallInWasm; #[test] fn call_in_interpreted_wasm_works() { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let res = call_in_wasm::( + + let executor = WasmExecutor::new( + WasmExecutionMethod::Interpreted, + Some(8), + sp_io::SubstrateHostFunctions::host_functions(), + true, + ); + let res = executor.call_in_wasm( + &WASM_BINARY[..], "test_empty_return", &[], - WasmExecutionMethod::Interpreted, &mut ext, - &WASM_BINARY, - 8, - true, ).unwrap(); assert_eq!(res, vec![0u8; 0]); } diff --git a/client/executor/src/native_executor.rs b/client/executor/src/native_executor.rs index 4fe7a205f53f9eec793c33cea0fe7ca452ff5f9d..dfc88d2ede746d73e339481419316d077a9bdae3 100644 --- a/client/executor/src/native_executor.rs +++ b/client/executor/src/native_executor.rs @@ -16,19 +16,15 @@ use crate::{ RuntimeInfo, error::{Error, Result}, - wasm_runtime::{RuntimesCache, WasmExecutionMethod}, + wasm_runtime::{RuntimeCache, WasmExecutionMethod, CodeSource}, }; use sp_version::{NativeVersion, RuntimeVersion}; use codec::{Decode, Encode}; use sp_core::{NativeOrEncoded, traits::{CodeExecutor, Externalities}}; use log::trace; -use std::{result, cell::RefCell, panic::{UnwindSafe, AssertUnwindSafe}, sync::Arc}; +use std::{result, panic::{UnwindSafe, AssertUnwindSafe}, sync::Arc}; use sp_wasm_interface::{HostFunctions, Function}; -use sc_executor_common::wasm_runtime::WasmRuntime; - -thread_local! { - static RUNTIMES_CACHE: RefCell = RefCell::new(RuntimesCache::new()); -} +use sc_executor_common::wasm_runtime::WasmInstance; /// Default num of pages for the heap const DEFAULT_HEAP_PAGES: u64 = 1024; @@ -75,46 +71,43 @@ pub trait NativeExecutionDispatch: Send + Sync { fn native_version() -> NativeVersion; } -/// A generic `CodeExecutor` implementation that uses a delegate to determine wasm code equivalence -/// and dispatch to native code when possible, falling back on `WasmExecutor` when not. -pub struct NativeExecutor { - /// Dummy field to avoid the compiler complaining about us not using `D`. - _dummy: std::marker::PhantomData, +/// An abstraction over Wasm code executor. Supports selecting execution backend and +/// manages runtime cache. +#[derive(Clone)] +pub struct WasmExecutor { /// Method used to execute fallback Wasm code. - fallback_method: WasmExecutionMethod, - /// Native runtime version info. - native_version: NativeVersion, + method: WasmExecutionMethod, /// The number of 64KB pages to allocate for Wasm execution. default_heap_pages: u64, /// The host functions registered with this instance. host_functions: Arc>, + /// WASM runtime cache. + cache: Arc, + /// Allow missing function imports. + allow_missing_func_imports: bool, } -impl NativeExecutor { +impl WasmExecutor { /// Create new instance. /// /// # Parameters /// - /// `fallback_method` - Method used to execute fallback Wasm code. + /// `method` - Method used to execute Wasm code. /// /// `default_heap_pages` - Number of 64KB pages to allocate for Wasm execution. /// Defaults to `DEFAULT_HEAP_PAGES` if `None` is provided. - pub fn new(fallback_method: WasmExecutionMethod, default_heap_pages: Option) -> Self { - let mut host_functions = sp_io::SubstrateHostFunctions::host_functions(); - // Add the old and deprecated host functions as well, so that we support old wasm runtimes. - host_functions.extend( - crate::deprecated_host_interface::SubstrateExternals::host_functions(), - ); - - // Add the custom host functions provided by the user. - host_functions.extend(D::ExtendHostFunctions::host_functions()); - - NativeExecutor { - _dummy: Default::default(), - fallback_method, - native_version: D::native_version(), + pub fn new( + method: WasmExecutionMethod, + default_heap_pages: Option, + host_functions: Vec<&'static dyn Function>, + allow_missing_func_imports: bool, + ) -> Self { + WasmExecutor { + method, default_heap_pages: default_heap_pages.unwrap_or(DEFAULT_HEAP_PAGES), host_functions: Arc::new(host_functions), + cache: Arc::new(RuntimeCache::new()), + allow_missing_func_imports, } } @@ -131,46 +124,90 @@ impl NativeExecutor { /// runtime is invalidated on any `panic!` to prevent a poisoned state. `ext` is already /// implicitly handled as unwind safe, as we store it in a global variable while executing the /// native runtime. - fn with_runtime( + fn with_instance<'c, R, F>( &self, - ext: &mut E, - f: impl for<'a> FnOnce( - AssertUnwindSafe<&'a mut (dyn WasmRuntime + 'static)>, - &'a RuntimeVersion, - AssertUnwindSafe<&'a mut E>, + code: CodeSource<'c>, + ext: &mut dyn Externalities, + f: F, + ) -> Result + where F: FnOnce( + AssertUnwindSafe<&dyn WasmInstance>, + Option<&RuntimeVersion>, + AssertUnwindSafe<&mut dyn Externalities>, ) -> Result>, - ) -> Result where E: Externalities { - RUNTIMES_CACHE.with(|cache| { - let mut cache = cache.borrow_mut(); - let (runtime, version, code_hash) = cache.fetch_runtime( - ext, - self.fallback_method, - self.default_heap_pages, - &*self.host_functions, - )?; - - let runtime = AssertUnwindSafe(runtime); - let ext = AssertUnwindSafe(ext); - - match f(runtime, version, ext) { - Ok(res) => res, - Err(e) => { - cache.invalidate_runtime(self.fallback_method, code_hash); - Err(e) - } + { + match self.cache.with_instance( + code, + ext, + self.method, + self.default_heap_pages, + &*self.host_functions, + self.allow_missing_func_imports, + |instance, version, ext| { + let instance = AssertUnwindSafe(instance); + let ext = AssertUnwindSafe(ext); + f(instance, version, ext) } - }) + )? { + Ok(r) => r, + Err(e) => Err(e), + } } } -impl Clone for NativeExecutor { - fn clone(&self) -> Self { +impl sp_core::traits::CallInWasm for WasmExecutor { + fn call_in_wasm( + &self, + wasm_blob: &[u8], + method: &str, + call_data: &[u8], + ext: &mut dyn Externalities, + ) -> std::result::Result, String> { + self.with_instance(CodeSource::Custom(wasm_blob), ext, |instance, _, mut ext| { + with_externalities_safe( + &mut **ext, + move || instance.call(method, call_data), + ) + }).map_err(|e| e.to_string()) + } +} + +/// A generic `CodeExecutor` implementation that uses a delegate to determine wasm code equivalence +/// and dispatch to native code when possible, falling back on `WasmExecutor` when not. +pub struct NativeExecutor { + /// Dummy field to avoid the compiler complaining about us not using `D`. + _dummy: std::marker::PhantomData, + /// Native runtime version info. + native_version: NativeVersion, + /// Fallback wasm executor. + wasm: WasmExecutor, +} + +impl NativeExecutor { + /// Create new instance. + /// + /// # Parameters + /// + /// `fallback_method` - Method used to execute fallback Wasm code. + /// + /// `default_heap_pages` - Number of 64KB pages to allocate for Wasm execution. + /// Defaults to `DEFAULT_HEAP_PAGES` if `None` is provided. + pub fn new(fallback_method: WasmExecutionMethod, default_heap_pages: Option) -> Self { + let mut host_functions = sp_io::SubstrateHostFunctions::host_functions(); + + // Add the custom host functions provided by the user. + host_functions.extend(D::ExtendHostFunctions::host_functions()); + let wasm_executor = WasmExecutor::new( + fallback_method, + default_heap_pages, + host_functions, + false, + ); + NativeExecutor { _dummy: Default::default(), - fallback_method: self.fallback_method, native_version: D::native_version(), - default_heap_pages: self.default_heap_pages, - host_functions: self.host_functions.clone(), + wasm: wasm_executor, } } } @@ -180,90 +217,109 @@ impl RuntimeInfo for NativeExecutor { &self.native_version } - fn runtime_version( + fn runtime_version( &self, - ext: &mut E, + ext: &mut dyn Externalities, ) -> Result { - self.with_runtime(ext, |_runtime, version, _ext| Ok(Ok(version.clone()))) + self.wasm.with_instance(CodeSource::Externalities, ext, + |_instance, version, _ext| + Ok(version.cloned().ok_or_else(|| Error::ApiError("Unknown version".into()))) + ) } } impl CodeExecutor for NativeExecutor { type Error = Error; - fn call - < - E: Externalities, + fn call< R: Decode + Encode + PartialEq, NC: FnOnce() -> result::Result + UnwindSafe, >( &self, - ext: &mut E, + ext: &mut dyn Externalities, method: &str, data: &[u8], use_native: bool, native_call: Option, - ) -> (Result>, bool){ + ) -> (Result>, bool) { let mut used_native = false; - let result = self.with_runtime(ext, |mut runtime, onchain_version, mut ext| { - match ( - use_native, - onchain_version.can_call_with(&self.native_version.runtime_version), - native_call, - ) { - (_, false, _) => { - trace!( - target: "executor", - "Request for native execution failed (native: {}, chain: {})", - self.native_version.runtime_version, - onchain_version, - ); - - with_externalities_safe( - &mut **ext, - move || runtime.call(method, data).map(NativeOrEncoded::Encoded) - ) - } - (false, _, _) => { - with_externalities_safe( - &mut **ext, - move || runtime.call(method, data).map(NativeOrEncoded::Encoded) - ) - }, - (true, true, Some(call)) => { - trace!( - target: "executor", - "Request for native execution with native call succeeded (native: {}, chain: {}).", - self.native_version.runtime_version, - onchain_version, - ); - - used_native = true; - let res = with_externalities_safe(&mut **ext, move || (call)()) - .and_then(|r| r - .map(NativeOrEncoded::Native) - .map_err(|s| Error::ApiError(s.to_string())) + let result = self.wasm.with_instance( + CodeSource::Externalities, + ext, + |instance, onchain_version, mut ext| { + let onchain_version = onchain_version.ok_or_else( + || Error::ApiError("Unknown version".into()) + )?; + match ( + use_native, + onchain_version.can_call_with(&self.native_version.runtime_version), + native_call, + ) { + (_, false, _) => { + trace!( + target: "executor", + "Request for native execution failed (native: {}, chain: {})", + self.native_version.runtime_version, + onchain_version, ); - Ok(res) - } - _ => { - trace!( - target: "executor", - "Request for native execution succeeded (native: {}, chain: {})", - self.native_version.runtime_version, - onchain_version - ); - - used_native = true; - Ok(D::dispatch(&mut **ext, method, data).map(NativeOrEncoded::Encoded)) + with_externalities_safe( + &mut **ext, + move || instance.call(method, data).map(NativeOrEncoded::Encoded) + ) + } + (false, _, _) => { + with_externalities_safe( + &mut **ext, + move || instance.call(method, data).map(NativeOrEncoded::Encoded) + ) + }, + (true, true, Some(call)) => { + trace!( + target: "executor", + "Request for native execution with native call succeeded \ + (native: {}, chain: {}).", + self.native_version.runtime_version, + onchain_version, + ); + + used_native = true; + let res = with_externalities_safe(&mut **ext, move || (call)()) + .and_then(|r| r + .map(NativeOrEncoded::Native) + .map_err(|s| Error::ApiError(s.to_string())) + ); + + Ok(res) + } + _ => { + trace!( + target: "executor", + "Request for native execution succeeded (native: {}, chain: {})", + self.native_version.runtime_version, + onchain_version + ); + + used_native = true; + Ok(D::dispatch(&mut **ext, method, data).map(NativeOrEncoded::Encoded)) + } } } - }); + ); (result, used_native) } } +impl Clone for NativeExecutor { + fn clone(&self) -> Self { + NativeExecutor { + _dummy: Default::default(), + native_version: D::native_version(), + wasm: self.wasm.clone(), + } + } +} + impl sp_core::traits::CallInWasm for NativeExecutor { fn call_in_wasm( &self, @@ -272,16 +328,7 @@ impl sp_core::traits::CallInWasm for NativeExecutor< call_data: &[u8], ext: &mut dyn Externalities, ) -> std::result::Result, String> { - crate::call_in_wasm_with_host_functions( - method, - call_data, - self.fallback_method, - ext, - wasm_blob, - self.default_heap_pages, - (*self.host_functions).clone(), - false, - ).map_err(|e| e.to_string()) + sp_core::traits::CallInWasm::call_in_wasm(&self.wasm, wasm_blob, method, call_data, ext) } } @@ -383,7 +430,7 @@ mod tests { let executor = NativeExecutor::::new(WasmExecutionMethod::Interpreted, None); my_interface::HostFunctions::host_functions().iter().for_each(|function| { assert_eq!( - executor.host_functions.iter().filter(|f| f == &function).count(), + executor.wasm.host_functions.iter().filter(|f| f == &function).count(), 2, ); }); diff --git a/client/executor/src/wasm_runtime.rs b/client/executor/src/wasm_runtime.rs index 9d54246ee07638562fd7ae95ab2cbd0e9efb808a..180baf0a2f8089cfd0d093ca4cf4a2bb835e0ea0 100644 --- a/client/executor/src/wasm_runtime.rs +++ b/client/executor/src/wasm_runtime.rs @@ -19,13 +19,15 @@ //! The primary means of accessing the runtimes is through a cache which saves the reusable //! components of the runtime that are expensive to initialize. +use std::sync::Arc; +use std::borrow::Cow; use crate::error::{Error, WasmError}; -use log::{trace, warn}; +use parking_lot::{Mutex, RwLock}; use codec::Decode; use sp_core::{storage::well_known_keys, traits::Externalities}; use sp_version::RuntimeVersion; -use std::{collections::hash_map::{Entry, HashMap}, panic::AssertUnwindSafe}; -use sc_executor_common::wasm_runtime::WasmRuntime; +use std::panic::AssertUnwindSafe; +use sc_executor_common::wasm_runtime::{WasmModule, WasmInstance}; use sp_wasm_interface::Function; @@ -39,15 +41,33 @@ pub enum WasmExecutionMethod { Compiled, } +/// Executoed code origin. +pub enum CodeSource<'a> { + /// Take code from storage, + Externalities, + /// Use provided code, + Custom(&'a [u8]), +} + /// A Wasm runtime object along with its cached runtime version. struct VersionedRuntime { - runtime: Box, + /// Runtime code hash. + code_hash: Vec, + /// Wasm runtime type. + wasm_method: WasmExecutionMethod, + /// Shared runtime that can spawn instances. + module: Box, /// The number of WebAssembly heap pages this instance was created with. heap_pages: u64, - /// Runtime version according to `Core_version`. - version: RuntimeVersion, + /// Runtime version according to `Core_version` if any. + version: Option, + /// Cached instance pool. + instances: RwLock<[Option>>>; MAX_INSTANCES]>, } +const MAX_RUNTIMES: usize = 2; +const MAX_INSTANCES: usize = 8; + /// Cache for the runtimes. /// /// When an instance is requested for the first time it is added to this cache. Metadata is kept @@ -60,130 +80,184 @@ struct VersionedRuntime { /// /// For now the cache grows indefinitely, but that should be fine for now since runtimes can only be /// upgraded rarely and there are no other ways to make the node to execute some other runtime. -pub struct RuntimesCache { - /// A cache of runtime instances along with metadata, ready to be reused. +pub struct RuntimeCache { + /// A cache of runtimes along with metadata. /// - /// Instances are keyed by the Wasm execution method and the hash of their code. - instances: HashMap<(WasmExecutionMethod, Vec), Result>, + /// Runtimes sorted by recent usage. The most recently used is at the front. + runtimes: Mutex<[Option>; MAX_RUNTIMES]>, } -impl RuntimesCache { +impl RuntimeCache { /// Creates a new instance of a runtimes cache. - pub fn new() -> RuntimesCache { - RuntimesCache { - instances: HashMap::new(), + pub fn new() -> RuntimeCache { + RuntimeCache { + runtimes: Default::default(), } } - /// Fetches an instance of the runtime. - /// - /// On first use we create a new runtime instance, save it to the cache - /// and persist its initial memory. - /// - /// Each subsequent request will return this instance, with its memory restored - /// to the persisted initial memory. Thus, we reuse one single runtime instance - /// for every `fetch_runtime` invocation. + /// Prepares a WASM module instance and executes given function for it. /// + /// This uses internal cache to find avaiable instance or create a new one. /// # Parameters /// + /// `code` - Provides external code or tells the executor to fetch it from storage. + /// /// `ext` - Externalities to use for the runtime. This is used for setting /// up an initial runtime instance. /// /// `default_heap_pages` - Number of 64KB pages to allocate for Wasm execution. /// + /// `wasm_method` - Type of WASM backend to use. + /// /// `host_functions` - The host functions that should be registered for the Wasm runtime. /// - /// # Return value + /// `allow_missing_func_imports` - Ignore missing function imports. /// - /// If no error occurred a tuple `(&mut WasmRuntime, H256)` is - /// returned. `H256` is the hash of the runtime code. + /// `f` - Function to execute. /// + /// # Returns result of `f` wrapped in an additonal result. /// In case of failure one of two errors can be returned: /// /// `Err::InvalidCode` is returned for runtime code issues. /// /// `Error::InvalidMemoryReference` is returned if no memory export with the /// identifier `memory` can be found in the runtime. - pub fn fetch_runtime( - &mut self, - ext: &mut E, + pub fn with_instance<'c, R, F>( + &self, + code: CodeSource<'c>, + ext: &mut dyn Externalities, wasm_method: WasmExecutionMethod, default_heap_pages: u64, host_functions: &[&'static dyn Function], - ) -> Result<(&mut (dyn WasmRuntime + 'static), &RuntimeVersion, Vec), Error> { - let code_hash = ext - .original_storage_hash(well_known_keys::CODE) - .ok_or(Error::InvalidCode("`CODE` not found in storage.".into()))?; - - let heap_pages = ext - .storage(well_known_keys::HEAP_PAGES) - .and_then(|pages| u64::decode(&mut &pages[..]).ok()) - .unwrap_or(default_heap_pages); + allow_missing_func_imports: bool, + f: F, + ) -> Result, Error> + where F: FnOnce( + &dyn WasmInstance, + Option<&RuntimeVersion>, + &mut dyn Externalities) + -> Result, + { + let (code_hash, heap_pages) = match &code { + CodeSource::Externalities => { + ( + ext + .original_storage_hash(well_known_keys::CODE) + .ok_or(Error::InvalidCode("`CODE` not found in storage.".into()))?, + ext + .storage(well_known_keys::HEAP_PAGES) + .and_then(|pages| u64::decode(&mut &pages[..]).ok()) + .unwrap_or(default_heap_pages), + ) + }, + CodeSource::Custom(code) => { + (sp_core::blake2_256(code).to_vec(), default_heap_pages) + } + }; - let result = match self.instances.entry((wasm_method, code_hash.clone())) { - Entry::Occupied(o) => { - let result = o.into_mut(); - if let Ok(ref mut cached_runtime) = result { - let heap_pages_changed = cached_runtime.heap_pages != heap_pages; - let host_functions_changed = cached_runtime.runtime.host_functions() - != host_functions; - if heap_pages_changed || host_functions_changed { - let changed = if heap_pages_changed { - "heap_pages" - } else { - "host functions" - }; + let mut runtimes = self.runtimes.lock(); // this must be released prior to calling f + let pos = runtimes.iter().position(|r| r.as_ref().map_or( + false, + |r| r.wasm_method == wasm_method && + r.code_hash == code_hash && + r.heap_pages == heap_pages + )); - trace!( - target: "runtimes_cache", - "{} were changed. Reinstantiating the instance", - changed, - ); - *result = create_versioned_wasm_runtime( - ext, - wasm_method, - heap_pages, - host_functions.into(), - ); - if let Err(ref err) = result { - warn!(target: "runtimes_cache", "cannot create a runtime: {:?}", err); - } + let runtime = match pos { + Some(n) => runtimes[n] + .clone() + .expect("`position` only returns `Some` for entries that are `Some`"), + None => { + let code = match code { + CodeSource::Externalities => { + Cow::Owned(ext.original_storage(well_known_keys::CODE) + .ok_or(WasmError::CodeNotFound)?) } - } - result - }, - Entry::Vacant(v) => { - trace!(target: "runtimes_cache", "no instance found in cache, creating now."); + CodeSource::Custom(code) => { + Cow::Borrowed(code) + } + }; + let result = create_versioned_wasm_runtime( + &code, + code_hash, ext, wasm_method, heap_pages, host_functions.into(), + allow_missing_func_imports, ); if let Err(ref err) = result { - warn!(target: "runtimes_cache", "cannot create a runtime: {:?}", err); + log::warn!(target: "wasm-runtime", "Cannot create a runtime: {:?}", err); } - v.insert(result) + Arc::new(result?) } }; - result.as_mut() - .map(|entry| (entry.runtime.as_mut(), &entry.version, code_hash)) - .map_err(|ref e| Error::InvalidCode(format!("{:?}", e))) - } + // Rearrange runtimes by last recently used. + match pos { + Some(0) => {}, + Some(n) => { + for i in (1 .. n + 1).rev() { + runtimes.swap(i, i - 1); + } + } + None => { + runtimes[MAX_RUNTIMES-1] = Some(runtime.clone()); + for i in (1 .. MAX_RUNTIMES).rev() { + runtimes.swap(i, i - 1); + } + } + } + drop(runtimes); - /// Invalidate the runtime for the given `wasm_method` and `code_hash`. - /// - /// Invalidation of a runtime is useful when there was a `panic!` in native while executing it. - /// The `panic!` maybe have brought the runtime into a poisoned state and so, it is better to - /// invalidate this runtime instance. - pub fn invalidate_runtime( - &mut self, - wasm_method: WasmExecutionMethod, - code_hash: Vec, - ) { - // Just remove the instance, it will be re-created the next time it is requested. - self.instances.remove(&(wasm_method, code_hash)); + let result = { + // Find a free instance + let instance_pool = runtime.instances.read().clone(); + let instance = instance_pool + .iter() + .find_map(|i| i.as_ref().and_then(|i| i.try_lock())); + if let Some(mut locked) = instance { + let result = f(&**locked, runtime.version.as_ref(), ext); + if let Err(e) = &result { + log::warn!(target: "wasm-runtime", "Evicting failed runtime instance: {:?}", e); + *locked = runtime.module.new_instance()?; + } + result + } else { + // Allocate a new instance + let instance = runtime.module.new_instance()?; + + let result = f(&*instance, runtime.version.as_ref(), ext); + match &result { + Ok(_) => { + let mut instance_pool = runtime.instances.write(); + if let Some(ref mut slot) = instance_pool.iter_mut().find(|s| s.is_none()) { + **slot = Some(Arc::new(Mutex::new(instance))); + log::debug!( + target: "wasm-runtime", + "Allocated WASM instance {}/{}", + instance_pool.len(), + MAX_INSTANCES, + ); + } else { + log::warn!(target: "wasm-runtime", "Ran out of free WASM instances"); + } + } + Err(e) => { + log::warn!( + target: + "wasm-runtime", + "Fresh runtime instance failed with {:?}", + e, + ); + } + } + result + } + }; + + Ok(result) } } @@ -194,28 +268,43 @@ pub fn create_wasm_runtime_with_code( code: &[u8], host_functions: Vec<&'static dyn Function>, allow_missing_func_imports: bool, -) -> Result, WasmError> { +) -> Result, WasmError> { match wasm_method { WasmExecutionMethod::Interpreted => - sc_executor_wasmi::create_instance(code, heap_pages, host_functions, allow_missing_func_imports) - .map(|runtime| -> Box { Box::new(runtime) }), + sc_executor_wasmi::create_runtime( + code, + heap_pages, + host_functions, + allow_missing_func_imports + ).map(|runtime| -> Box { Box::new(runtime) }), #[cfg(feature = "wasmtime")] WasmExecutionMethod::Compiled => - sc_executor_wasmtime::create_instance(code, heap_pages, host_functions, allow_missing_func_imports) - .map(|runtime| -> Box { Box::new(runtime) }), + sc_executor_wasmtime::create_runtime( + code, + heap_pages, + host_functions, + allow_missing_func_imports + ).map(|runtime| -> Box { Box::new(runtime) }), } } -fn create_versioned_wasm_runtime( - ext: &mut E, +fn create_versioned_wasm_runtime( + code: &[u8], + code_hash: Vec, + ext: &mut dyn Externalities, wasm_method: WasmExecutionMethod, heap_pages: u64, host_functions: Vec<&'static dyn Function>, + allow_missing_func_imports: bool, ) -> Result { - let code = ext - .original_storage(well_known_keys::CODE) - .ok_or(WasmError::CodeNotFound)?; - let mut runtime = create_wasm_runtime_with_code(wasm_method, heap_pages, &code, host_functions, false)?; + let time = std::time::Instant::now(); + let mut runtime = create_wasm_runtime_with_code( + wasm_method, + heap_pages, + &code, + host_functions, + allow_missing_func_imports, + )?; // Call to determine runtime version. let version_result = { @@ -224,21 +313,33 @@ fn create_versioned_wasm_runtime( // The following unwind safety assertion is OK because if the method call panics, the // runtime will be dropped. - let mut runtime = AssertUnwindSafe(runtime.as_mut()); + let runtime = AssertUnwindSafe(runtime.as_mut()); crate::native_executor::with_externalities_safe( &mut **ext, - move || runtime.call("Core_version", &[]) + move || runtime.new_instance()?.call("Core_version", &[]) ).map_err(|_| WasmError::Instantiation("panic in call to get runtime version".into()))? }; - let encoded_version = version_result - .map_err(|e| WasmError::Instantiation(format!("failed to call \"Core_version\": {}", e)))?; - let version = RuntimeVersion::decode(&mut encoded_version.as_slice()) - .map_err(|_| WasmError::Instantiation("failed to decode \"Core_version\" result".into()))?; + let version = match version_result { + Ok(version) => Some(RuntimeVersion::decode(&mut version.as_slice()) + .map_err(|_| + WasmError::Instantiation("failed to decode \"Core_version\" result".into()) + )?), + Err(_) => None, + }; + log::debug!( + target: "wasm-runtime", + "Prepared new runtime version {:?} in {} ms.", + version, + time.elapsed().as_millis(), + ); Ok(VersionedRuntime { - runtime, + code_hash, + module: runtime, version, heap_pages, + wasm_method, + instances: Default::default(), }) } diff --git a/client/executor/src/wasm_utils.rs b/client/executor/src/wasm_utils.rs deleted file mode 100644 index 539e210a9467e0825d146637aaef0b4608722392..0000000000000000000000000000000000000000 --- a/client/executor/src/wasm_utils.rs +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright 2017-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Utilities for defining the wasm host environment. - -/// Converts arguments into respective WASM types. -#[macro_export] -macro_rules! convert_args { - () => ([]); - ( $( $t:ty ),* ) => ( [ $( <$t as $crate::sp_wasm_interface::IntoValue>::VALUE_TYPE, )* ] ); -} - -/// Generates a WASM signature for given list of parameters. -#[macro_export] -macro_rules! gen_signature { - ( ( $( $params: ty ),* ) ) => ( - $crate::sp_wasm_interface::Signature { - args: std::borrow::Cow::Borrowed(&convert_args!( $( $params ),* )[..]), - return_value: None, - } - ); - ( ( $( $params: ty ),* ) -> $returns:ty ) => ( - $crate::sp_wasm_interface::Signature { - args: std::borrow::Cow::Borrowed(&convert_args!( $( $params ),* )[..]), - return_value: Some(<$returns as $crate::sp_wasm_interface::IntoValue>::VALUE_TYPE), - } - ); -} - -macro_rules! gen_functions { - (@INTERNAL - { $( $generated:tt )* } - $context:ident, - ) => ( - vec![ $( $generated )* ] - ); - (@INTERNAL - { $( $generated:tt )* } - $context:ident, - $name:ident ( $( $names:ident: $params:ty ),* ) $( -> $returns:ty )? { $( $body:tt )* } - $( $tail:tt )* - ) => ( - gen_functions! { - @INTERNAL - { - $( $generated )* - { - struct $name; - - #[allow(unused)] - impl $crate::sp_wasm_interface::Function for $name { - fn name(&self) -> &str { - stringify!($name) - } - fn signature(&self) -> $crate::sp_wasm_interface::Signature { - gen_signature!( ( $( $params ),* ) $( -> $returns )? ) - } - fn execute( - &self, - context: &mut dyn $crate::sp_wasm_interface::FunctionContext, - args: &mut dyn Iterator, - ) -> ::std::result::Result, String> { - let mut $context = context; - marshall! { - args, - ( $( $names : $params ),* ) $( -> $returns )? => { $( $body )* } - } - } - } - - &$name as &dyn $crate::sp_wasm_interface::Function - }, - } - $context, - $( $tail )* - } - ); - - ( $context:ident, $( $tail:tt )* ) => ( - gen_functions!(@INTERNAL {} $context, $($tail)*); - ); -} - -/// Converts the list of arguments coming from WASM into their native types. -#[macro_export] -macro_rules! unmarshall_args { - ( $body:tt, $args_iter:ident, $( $names:ident : $params:ty ),*) => ({ - $( - let $names : $params = - $args_iter.next() - .and_then(|val| <$params as $crate::sp_wasm_interface::TryFromValue>::try_from_value(val)) - .expect( - "`$args_iter` comes from an argument of Externals::execute_function; - args to an external call always matches the signature of the external; - external signatures are built with count and types and in order defined by `$params`; - here, we iterating on `$params`; - qed; - " - ); - )* - $body - }) -} - -/// Since we can't specify the type of closure directly at binding site: -/// -/// ```nocompile -/// let f: FnOnce() -> Result<::NativeType, _> = || { /* ... */ }; -/// ``` -/// -/// we use this function to constrain the type of the closure. -#[inline(always)] -pub fn constrain_closure(f: F) -> F -where - F: FnOnce() -> Result -{ - f -} - -/// Pass the list of parameters by converting them to respective WASM types. -#[macro_export] -macro_rules! marshall { - ( $args_iter:ident, ( $( $names:ident : $params:ty ),* ) -> $returns:ty => $body:tt ) => ({ - let body = $crate::wasm_utils::constrain_closure::<$returns, _>(|| { - unmarshall_args!($body, $args_iter, $( $names : $params ),*) - }); - let r = body()?; - return Ok(Some($crate::sp_wasm_interface::IntoValue::into_value(r))) - }); - ( $args_iter:ident, ( $( $names:ident : $params:ty ),* ) => $body:tt ) => ({ - let body = $crate::wasm_utils::constrain_closure::<(), _>(|| { - unmarshall_args!($body, $args_iter, $( $names : $params ),*) - }); - body()?; - return Ok(None) - }) -} - -/// Implements the wasm host interface for the given type. -#[macro_export] -macro_rules! impl_wasm_host_interface { - ( - impl $interface_name:ident where $context:ident { - $( - $name:ident($( $names:ident : $params:ty ),* $(,)? ) $( -> $returns:ty )? - { $( $body:tt )* } - )* - } - ) => ( - impl $crate::sp_wasm_interface::HostFunctions for $interface_name { - #[allow(non_camel_case_types)] - fn host_functions() -> Vec<&'static dyn $crate::sp_wasm_interface::Function> { - gen_functions!( - $context, - $( $name( $( $names: $params ),* ) $( -> $returns )? { $( $body )* } )* - ) - } - } - ); -} diff --git a/client/executor/wasmi/Cargo.toml b/client/executor/wasmi/Cargo.toml index 9e968fdc685f0896abb3b7593fc425e2e5af6c17..aa259e29f8dfb3dfb564b3b8060af8f238f537e6 100644 --- a/client/executor/wasmi/Cargo.toml +++ b/client/executor/wasmi/Cargo.toml @@ -1,17 +1,21 @@ [package] name = "sc-executor-wasmi" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "This crate provides an implementation of `WasmRuntime` that is baked by wasmi." +documentation = "https://docs.rs/sc-execturo-wasmi" [dependencies] log = "0.4.8" wasmi = "0.6.2" parity-wasm = "0.41.0" -codec = { package = "parity-scale-codec", version = "1.0.0" } -sc-executor-common = { version = "0.8", path = "../common" } -sp-wasm-interface = { version = "2.0.0", path = "../../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0", path = "../../../primitives/runtime-interface" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-allocator = { version = "2.0.0", path = "../../../primitives/allocator" } +codec = { package = "parity-scale-codec", version = "1.2.0" } +sc-executor-common = { version = "0.8.0-alpha.2", path = "../common" } +sp-wasm-interface = { version = "2.0.0-alpha.2", path = "../../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime-interface" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-allocator = { version = "2.0.0-alpha.2", path = "../../../primitives/allocator" } diff --git a/client/executor/wasmi/src/lib.rs b/client/executor/wasmi/src/lib.rs index a0e11dfcf8b93f4799624688e84c8e109f98ca9d..6348c2413357f03f9de161d4ff537cf8a77e818d 100644 --- a/client/executor/wasmi/src/lib.rs +++ b/client/executor/wasmi/src/lib.rs @@ -14,10 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! This crate provides an implementation of `WasmRuntime` that is baked by wasmi. +//! This crate provides an implementation of `WasmModule` that is baked by wasmi. use sc_executor_common::{error::{Error, WasmError}, sandbox}; -use std::{str, mem, cell::RefCell}; +use std::{str, mem, cell::RefCell, sync::Arc}; use wasmi::{ Module, ModuleInstance, MemoryInstance, MemoryRef, TableRef, ImportsBuilder, ModuleRef, memory_units::Pages, RuntimeValue::{I32, I64, self}, @@ -30,7 +30,7 @@ use sp_wasm_interface::{ FunctionContext, Pointer, WordSize, Sandbox, MemoryId, Result as WResult, Function, }; use sp_runtime_interface::unpack_ptr_and_len; -use sc_executor_common::wasm_runtime::WasmRuntime; +use sc_executor_common::wasm_runtime::{WasmModule, WasmInstance}; struct FunctionExecutor<'a> { sandbox_store: sandbox::Store, @@ -623,9 +623,77 @@ impl StateSnapshot { } } -/// A runtime along with its initial state snapshot. -#[derive(Clone)] +/// A runtime along with initial copy of data segments. pub struct WasmiRuntime { + /// A wasm module. + module: Module, + /// The host functions registered for this instance. + host_functions: Arc>, + /// Enable stub generation for functions that are not available in `host_functions`. + /// These stubs will error when the wasm blob tries to call them. + allow_missing_func_imports: bool, + /// Numer of heap pages this runtime uses. + heap_pages: u64, + /// Data segments created for each new instance. + data_segments: Vec, +} + +impl WasmModule for WasmiRuntime { + fn new_instance(&self) -> Result, Error> { + // Instantiate this module. + let (instance, missing_functions, memory) = instantiate_module( + self.heap_pages as usize, + &self.module, + &self.host_functions, + self.allow_missing_func_imports, + ).map_err(|e| WasmError::Instantiation(e.to_string()))?; + + // Take state snapshot before executing anything. + let state_snapshot = StateSnapshot::take(&instance, self.data_segments.clone()) + .expect( + "`take` returns `Err` if the module is not valid; + we already loaded module above, thus the `Module` is proven to be valid at this point; + qed + ", + ); + + Ok(Box::new(WasmiInstance { + instance, + memory, + state_snapshot, + host_functions: self.host_functions.clone(), + allow_missing_func_imports: self.allow_missing_func_imports, + missing_functions, + })) + } +} + +/// Create a new `WasmiRuntime` given the code. This function loads the module and +/// stores it in the instance. +pub fn create_runtime( + code: &[u8], + heap_pages: u64, + host_functions: Vec<&'static dyn Function>, + allow_missing_func_imports: bool, +) -> Result { + let module = Module::from_buffer(&code).map_err(|_| WasmError::InvalidModule)?; + + // Extract the data segments from the wasm code. + // + // A return of this error actually indicates that there is a problem in logic, since + // we just loaded and validated the `module` above. + let data_segments = extract_data_segments(&code)?; + Ok(WasmiRuntime { + module, + data_segments, + host_functions: Arc::new(host_functions), + allow_missing_func_imports, + heap_pages, + }) +} + +/// Wasmi instance wrapper along with the state snapshot. +pub struct WasmiInstance { /// A wasm module instance. instance: ModuleRef, /// The memory instance of used by the wasm module. @@ -633,7 +701,7 @@ pub struct WasmiRuntime { /// The snapshot of the instance's state taken just after the instantiation. state_snapshot: StateSnapshot, /// The host functions registered for this instance. - host_functions: Vec<&'static dyn Function>, + host_functions: Arc>, /// Enable stub generation for functions that are not available in `host_functions`. /// These stubs will error when the wasm blob tries to call them. allow_missing_func_imports: bool, @@ -641,13 +709,12 @@ pub struct WasmiRuntime { missing_functions: Vec, } -impl WasmRuntime for WasmiRuntime { - fn host_functions(&self) -> &[&'static dyn Function] { - &self.host_functions - } +// This is safe because `WasmiInstance` does not leak any references to `self.memory` and `self.instance` +unsafe impl Send for WasmiInstance {} +impl WasmInstance for WasmiInstance { fn call( - &mut self, + &self, method: &str, data: &[u8], ) -> Result, Error> { @@ -664,67 +731,26 @@ impl WasmRuntime for WasmiRuntime { &self.memory, method, data, - &self.host_functions, + self.host_functions.as_ref(), self.allow_missing_func_imports, - &self.missing_functions, + self.missing_functions.as_ref(), ) } - fn get_global_val(&self, name: &str) -> Result, Error> { + fn get_global_const(&self, name: &str) -> Result, Error> { match self.instance.export_by_name(name) { Some(global) => Ok(Some( global - .as_global() - .ok_or_else(|| format!("`{}` is not a global", name))? - .get() - .into() + .as_global() + .ok_or_else(|| format!("`{}` is not a global", name))? + .get() + .into() )), None => Ok(None), } } } -pub fn create_instance( - code: &[u8], - heap_pages: u64, - host_functions: Vec<&'static dyn Function>, - allow_missing_func_imports: bool, -) -> Result { - let module = Module::from_buffer(&code).map_err(|_| WasmError::InvalidModule)?; - - // Extract the data segments from the wasm code. - // - // A return of this error actually indicates that there is a problem in logic, since - // we just loaded and validated the `module` above. - let data_segments = extract_data_segments(&code)?; - - // Instantiate this module. - let (instance, missing_functions, memory) = instantiate_module( - heap_pages as usize, - &module, - &host_functions, - allow_missing_func_imports, - ).map_err(|e| WasmError::Instantiation(e.to_string()))?; - - // Take state snapshot before executing anything. - let state_snapshot = StateSnapshot::take(&instance, data_segments) - .expect( - "`take` returns `Err` if the module is not valid; - we already loaded module above, thus the `Module` is proven to be valid at this point; - qed - ", - ); - - Ok(WasmiRuntime { - instance, - memory, - state_snapshot, - host_functions, - allow_missing_func_imports, - missing_functions, - }) -} - /// Extract the data segments from the given wasm code. /// /// Returns `Err` if the given wasm code cannot be deserialized. diff --git a/client/executor/wasmtime/Cargo.toml b/client/executor/wasmtime/Cargo.toml index eb41adb2714e03b8b99291c70a6dbabb668020ee..9f8784cc981b7d1a79438846255d17afcf27d9d3 100644 --- a/client/executor/wasmtime/Cargo.toml +++ b/client/executor/wasmtime/Cargo.toml @@ -1,22 +1,24 @@ [package] name = "sc-executor-wasmtime" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Defines a `WasmRuntime` that uses the Wasmtime JIT to execute." [dependencies] log = "0.4.8" -wasmi = "0.6.2" +scoped-tls = "1.0" parity-wasm = "0.41.0" -codec = { package = "parity-scale-codec", version = "1.0.0" } -sc-executor-common = { version = "0.8", path = "../common" } -sp-wasm-interface = { version = "2.0.0", path = "../../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0", path = "../../../primitives/runtime-interface" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-allocator = { version = "2.0.0", path = "../../../primitives/allocator" } - -wasmtime = "0.11" +codec = { package = "parity-scale-codec", version = "1.2.0" } +sc-executor-common = { version = "0.8.0-alpha.2", path = "../common" } +sp-wasm-interface = { version = "2.0.0-alpha.2", path = "../../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime-interface" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-allocator = { version = "2.0.0-alpha.2", path = "../../../primitives/allocator" } +wasmtime = { git = "https://github.com/paritytech/wasmtime", branch = "a-thread-safe-api" } [dev-dependencies] assert_matches = "1.3.0" diff --git a/client/executor/wasmtime/src/host.rs b/client/executor/wasmtime/src/host.rs index e0cc6ecc9ae7a1dd738352a0d287ff9d5ad7ff5b..29187ac66338e87e8694bb89a4dc0fa19c9e534d 100644 --- a/client/executor/wasmtime/src/host.rs +++ b/client/executor/wasmtime/src/host.rs @@ -19,7 +19,7 @@ use crate::instance_wrapper::InstanceWrapper; use crate::util; -use std::cell::RefCell; +use std::{cell::RefCell, rc::Rc}; use log::trace; use codec::{Encode, Decode}; use sp_allocator::FreeingBumpHeapAllocator; @@ -51,12 +51,12 @@ pub struct HostState { // borrow after performing necessary queries/changes. sandbox_store: RefCell>, allocator: RefCell, - instance: InstanceWrapper, + instance: Rc, } impl HostState { /// Constructs a new `HostState`. - pub fn new(allocator: FreeingBumpHeapAllocator, instance: InstanceWrapper) -> Self { + pub fn new(allocator: FreeingBumpHeapAllocator, instance: Rc) -> Self { HostState { sandbox_store: RefCell::new(sandbox::Store::new()), allocator: RefCell::new(allocator), @@ -64,11 +64,6 @@ impl HostState { } } - /// Destruct the host state and extract the `InstanceWrapper` passed at the creation. - pub fn into_instance(self) -> InstanceWrapper { - self.instance - } - /// Materialize `HostContext` that can be used to invoke a substrate host `dyn Function`. pub fn materialize<'a>(&'a self) -> HostContext<'a> { HostContext(self) diff --git a/client/executor/wasmtime/src/imports.rs b/client/executor/wasmtime/src/imports.rs index 349f84a0d74d27031ea3d3cba613cb3f36f02fbf..48299ffd62de1caef2e03016d9b8060188548365 100644 --- a/client/executor/wasmtime/src/imports.rs +++ b/client/executor/wasmtime/src/imports.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use crate::state_holder::StateHolder; +use crate::state_holder; use sc_executor_common::error::WasmError; use sp_wasm_interface::{Function, Value, ValueType}; use std::any::Any; @@ -34,7 +34,6 @@ pub struct Imports { /// Goes over all imports of a module and prepares a vector of `Extern`s that can be used for /// instantiation of the module. Returns an error if there are imports that cannot be satisfied. pub fn resolve_imports( - state_holder: &StateHolder, module: &Module, host_functions: &[&'static dyn Function], heap_pages: u32, @@ -58,7 +57,6 @@ pub fn resolve_imports( } _ => resolve_func_import( module, - state_holder, import_ty, host_functions, allow_missing_func_imports, @@ -112,7 +110,6 @@ fn resolve_memory_import( fn resolve_func_import( module: &Module, - state_holder: &StateHolder, import_ty: &ImportType, host_functions: &[&'static dyn Function], allow_missing_func_imports: bool, @@ -152,7 +149,7 @@ fn resolve_func_import( ))); } - Ok(HostFuncHandler::new(&state_holder, *host_func).into_extern(module)) + Ok(HostFuncHandler::new(*host_func).into_extern(module)) } /// Returns `true` if `lhs` and `rhs` represent the same signature. @@ -163,14 +160,12 @@ fn signature_matches(lhs: &wasmtime::FuncType, rhs: &wasmtime::FuncType) -> bool /// This structure implements `Callable` and acts as a bridge between wasmtime and /// substrate host functions. struct HostFuncHandler { - state_holder: StateHolder, host_func: &'static dyn Function, } impl HostFuncHandler { - fn new(state_holder: &StateHolder, host_func: &'static dyn Function) -> Self { + fn new(host_func: &'static dyn Function) -> Self { Self { - state_holder: state_holder.clone(), host_func, } } @@ -188,7 +183,7 @@ impl Callable for HostFuncHandler { wasmtime_params: &[Val], wasmtime_results: &mut [Val], ) -> Result<(), wasmtime::Trap> { - let unwind_result = self.state_holder.with_context(|host_ctx| { + let unwind_result = state_holder::with_context(|host_ctx| { let mut host_ctx = host_ctx.expect( "host functions can be called only from wasm instance; wasm instance is always called initializing context; diff --git a/client/executor/wasmtime/src/lib.rs b/client/executor/wasmtime/src/lib.rs index 8f4801e6da1d0de171ff50d2e8b04c0a08d1f00d..66e4e085235ac3adc2e4026679ec849d002ed94a 100644 --- a/client/executor/wasmtime/src/lib.rs +++ b/client/executor/wasmtime/src/lib.rs @@ -23,4 +23,4 @@ mod imports; mod instance_wrapper; mod util; -pub use runtime::create_instance; +pub use runtime::create_runtime; diff --git a/client/executor/wasmtime/src/runtime.rs b/client/executor/wasmtime/src/runtime.rs index b99d3347872059d778f50c3c00581252e224a167..02acd33e69a628705675105e8644bad177bfc15a 100644 --- a/client/executor/wasmtime/src/runtime.rs +++ b/client/executor/wasmtime/src/runtime.rs @@ -15,57 +15,85 @@ // along with Substrate. If not, see . //! Defines the compiled Wasm runtime that uses Wasmtime internally. +use std::rc::Rc; +use std::sync::Arc; use crate::host::HostState; -use crate::imports::{resolve_imports, Imports}; +use crate::imports::{Imports, resolve_imports}; use crate::instance_wrapper::InstanceWrapper; -use crate::state_holder::StateHolder; +use crate::state_holder; use sc_executor_common::{ error::{Error, Result, WasmError}, - wasm_runtime::WasmRuntime, + wasm_runtime::{WasmModule, WasmInstance}, }; use sp_allocator::FreeingBumpHeapAllocator; use sp_runtime_interface::unpack_ptr_and_len; use sp_wasm_interface::{Function, Pointer, WordSize, Value}; use wasmtime::{Config, Engine, Module, Store}; -/// A `WasmRuntime` implementation using wasmtime to compile the runtime module to machine code +/// A `WasmModule` implementation using wasmtime to compile the runtime module to machine code /// and execute the compiled code. pub struct WasmtimeRuntime { - module: Module, - imports: Imports, - state_holder: StateHolder, + module: Arc, heap_pages: u32, + allow_missing_func_imports: bool, host_functions: Vec<&'static dyn Function>, } -impl WasmRuntime for WasmtimeRuntime { - fn host_functions(&self) -> &[&'static dyn Function] { - &self.host_functions +impl WasmModule for WasmtimeRuntime { + fn new_instance(&self) -> Result> { + // Scan all imports, find the matching host functions, and create stubs that adapt arguments + // and results. + let imports = resolve_imports( + &self.module, + &self.host_functions, + self.heap_pages, + self.allow_missing_func_imports, + )?; + + Ok(Box::new(WasmtimeInstance { + module: self.module.clone(), + imports, + heap_pages: self.heap_pages, + })) } +} + +/// A `WasmInstance` implementation that reuses compiled module and spawns instances +/// to execute the compiled code. +pub struct WasmtimeInstance { + module: Arc, + imports: Imports, + heap_pages: u32, +} + +// This is safe because `WasmtimeInstance` does not leak reference to `self.imports` +// and all imports don't reference any anything, other than host functions and memory +unsafe impl Send for WasmtimeInstance {} - fn call(&mut self, method: &str, data: &[u8]) -> Result> { +impl WasmInstance for WasmtimeInstance { + fn call(&self, method: &str, data: &[u8]) -> Result> { + // TODO: reuse the instance and reset globals after call + // https://github.com/paritytech/substrate/issues/5141 + let instance = Rc::new(InstanceWrapper::new(&self.module, &self.imports, self.heap_pages)?); call_method( - &self.module, - &mut self.imports, - &self.state_holder, + instance, method, data, - self.heap_pages, ) } - fn get_global_val(&self, name: &str) -> Result> { - // Yeah, there is no better way currently :( - InstanceWrapper::new(&self.module, &self.imports, self.heap_pages)? - .get_global_val(name) + fn get_global_const(&self, name: &str) -> Result> { + let instance = InstanceWrapper::new(&self.module, &self.imports, self.heap_pages)?; + instance.get_global_val(name) } } + /// Create a new `WasmtimeRuntime` given the code. This function performs translation from Wasm to /// machine code, which can be computationally heavy. -pub fn create_instance( +pub fn create_runtime( code: &[u8], heap_pages: u64, host_functions: Vec<&'static dyn Function>, @@ -80,55 +108,37 @@ pub fn create_instance( let module = Module::new(&store, code) .map_err(|e| WasmError::Other(format!("cannot create module: {}", e)))?; - let state_holder = StateHolder::empty(); - - // Scan all imports, find the matching host functions, and create stubs that adapt arguments - // and results. - let imports = resolve_imports( - &state_holder, - &module, - &host_functions, - heap_pages as u32, - allow_missing_func_imports, - )?; - Ok(WasmtimeRuntime { - module, - imports, - state_holder, + module: Arc::new(module), heap_pages: heap_pages as u32, + allow_missing_func_imports, host_functions, }) } /// Call a function inside a precompiled Wasm module. fn call_method( - module: &Module, - imports: &mut Imports, - state_holder: &StateHolder, + instance_wrapper: Rc, method: &str, data: &[u8], - heap_pages: u32, ) -> Result> { - let instance_wrapper = InstanceWrapper::new(module, imports, heap_pages)?; let entrypoint = instance_wrapper.resolve_entrypoint(method)?; let heap_base = instance_wrapper.extract_heap_base()?; let allocator = FreeingBumpHeapAllocator::new(heap_base); - perform_call(data, state_holder, instance_wrapper, entrypoint, allocator) + perform_call(data, instance_wrapper, entrypoint, allocator) } fn perform_call( data: &[u8], - state_holder: &StateHolder, - instance_wrapper: InstanceWrapper, + instance_wrapper: Rc, entrypoint: wasmtime::Func, mut allocator: FreeingBumpHeapAllocator, ) -> Result> { let (data_ptr, data_len) = inject_input_data(&instance_wrapper, &mut allocator, data)?; - let host_state = HostState::new(allocator, instance_wrapper); - let (ret, host_state) = state_holder.with_initialized_state(host_state, || { + let host_state = HostState::new(allocator, instance_wrapper.clone()); + let ret = state_holder::with_initialized_state(&host_state, || { match entrypoint.call(&[ wasmtime::Val::I32(u32::from(data_ptr) as i32), wasmtime::Val::I32(u32::from(data_len) as i32), @@ -146,9 +156,7 @@ fn perform_call( } }); let (output_ptr, output_len) = ret?; - - let instance = host_state.into_instance(); - let output = extract_output_data(&instance, output_ptr, output_len)?; + let output = extract_output_data(&instance_wrapper, output_ptr, output_len)?; Ok(output) } diff --git a/client/executor/wasmtime/src/state_holder.rs b/client/executor/wasmtime/src/state_holder.rs index 57564ed3ec414506e1960c41654e42be96e44576..42cb79e7a356807176033266dd0a4f22eacb5103 100644 --- a/client/executor/wasmtime/src/state_holder.rs +++ b/client/executor/wasmtime/src/state_holder.rs @@ -15,63 +15,29 @@ // along with Substrate. If not, see . use crate::host::{HostContext, HostState}; -use std::cell::RefCell; -use std::rc::Rc; -/// A common place to store a reference to the `HostState`. -/// -/// This structure is passed into each host function handler and retained in the implementation of -/// `WasmRuntime`. Whenever a call into a runtime method is initiated, the host state is populated -/// with the state for that runtime method call. +scoped_tls::scoped_thread_local!(static HOST_STATE: HostState); + +/// Provide `HostState` for the runtime method call and execute the given function `f`. /// -/// During the execution of the runtime method call, wasm can call imported host functions. When -/// that happens the host function handler gets a `HostContext` (obtainable through having a -/// `HostState` reference). -#[derive(Clone)] -pub struct StateHolder { - // This is `Some` only during a call. - state: Rc>>, +/// During the execution of the provided function `with_context` will be callable. +pub fn with_initialized_state(s: &HostState, f: F) -> R +where + F: FnOnce() -> R, +{ + HOST_STATE.set(s, f) } -impl StateHolder { - /// Create a placeholder `StateHolder`. - pub fn empty() -> StateHolder { - StateHolder { - state: Rc::new(RefCell::new(None)), - } - } - - /// Provide `HostState` for the runtime method call and execute the given function `f`. - /// - /// During the execution of the provided function `with_context` will be callable. - pub fn with_initialized_state(&self, state: HostState, f: F) -> (R, HostState) - where - F: FnOnce() -> R, - { - *self.state.borrow_mut() = Some(state); - - let ret = f(); - let state = self - .state - .borrow_mut() - .take() - .expect("cannot be None since was just assigned; qed"); - - (ret, state) - } - - /// Create a `HostContext` from the contained `HostState` and execute the given function `f`. - /// - /// This function is only callable within closure passed to `init_state`. Otherwise, the passed - /// context will be `None`. - pub fn with_context(&self, f: F) -> R - where - F: FnOnce(Option) -> R, - { - let state = self.state.borrow(); - match *state { - Some(ref state) => f(Some(state.materialize())), - None => f(None), - } +/// Create a `HostContext` from the contained `HostState` and execute the given function `f`. +/// +/// This function is only callable within closure passed to `init_state`. Otherwise, the passed +/// context will be `None`. +pub fn with_context(f: F) -> R +where + F: FnOnce(Option) -> R, +{ + if !HOST_STATE.is_set() { + return f(None) } + HOST_STATE.with(|state| f(Some(state.materialize()))) } diff --git a/client/finality-grandpa/Cargo.toml b/client/finality-grandpa/Cargo.toml index e96792258a279cd507f2ecba583925e5742af9b3..6cee0cd8526f6dcbd2ed9019b172dc30b009ca05 100644 --- a/client/finality-grandpa/Cargo.toml +++ b/client/finality-grandpa/Cargo.toml @@ -1,47 +1,54 @@ [package] name = "sc-finality-grandpa" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Integration of the GRANDPA finality gadget into substrate." +documentation = "https://docs.rs/sc-finality-grandpa" + [dependencies] -fork-tree = { version = "2.0.0", path = "../../utils/fork-tree" } +fork-tree = { version = "2.0.0-alpha.2", path = "../../utils/fork-tree" } futures = "0.3.1" futures-timer = "3.0.1" log = "0.4.8" parking_lot = "0.10.0" rand = "0.7.2" assert_matches = "1.3.0" -parity-scale-codec = { version = "1.0.0", features = ["derive"] } -sp-arithmetic = { version = "2.0.0", path = "../../primitives/arithmetic" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sp-consensus = { version = "0.8", path = "../../primitives/consensus/common" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sc-telemetry = { version = "2.0.0", path = "../telemetry" } -sc-keystore = { version = "2.0.0", path = "../keystore" } +parity-scale-codec = { version = "1.2.0", features = ["derive"] } +sp-arithmetic = { version = "2.0.0-alpha.2", path = "../../primitives/arithmetic" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } +sp-consensus = { version = "0.8.0-alpha.1", path = "../../primitives/consensus/common" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +sp-api = { version = "2.0.0-alpha.2", path = "../../primitives/api" } +sc-telemetry = { version = "2.0.0-alpha.2", path = "../telemetry" } +sc-keystore = { version = "2.0.0-alpha.2", path = "../keystore" } serde_json = "1.0.41" -sc-client-api = { version = "2.0.0", path = "../api" } -sc-client = { version = "0.8", path = "../" } -sp-inherents = { version = "2.0.0", path = "../../primitives/inherents" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } -sc-network = { version = "0.8", path = "../network" } -sc-network-gossip = { version = "0.8", path = "../network-gossip" } -sp-finality-tracker = { version = "2.0.0", path = "../../primitives/finality-tracker" } -sp-finality-grandpa = { version = "2.0.0", path = "../../primitives/finality-grandpa" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../api" } +sc-client = { version = "0.8.0-alpha.2", path = "../" } +sp-inherents = { version = "2.0.0-alpha.2", path = "../../primitives/inherents" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../primitives/blockchain" } +sc-network = { version = "0.8.0-alpha.2", path = "../network" } +sc-network-gossip = { version = "0.8.0-alpha.2", path = "../network-gossip" } +sp-finality-tracker = { version = "2.0.0-alpha.2", path = "../../primitives/finality-tracker" } +sp-finality-grandpa = { version = "2.0.0-alpha.2", path = "../../primitives/finality-grandpa" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-alpha.2" } +sc-block-builder = { version = "0.8.0-alpha.2", path = "../block-builder" } finality-grandpa = { version = "0.11.1", features = ["derive-codec"] } pin-project = "0.4.6" [dev-dependencies] finality-grandpa = { version = "0.11.1", features = ["derive-codec", "test-helpers"] } -sc-network = { version = "0.8", path = "../network" } -sc-network-test = { version = "0.8.0", path = "../network/test" } -sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } -sp-consensus-babe = { version = "0.8", path = "../../primitives/consensus/babe" } -sp-state-machine = { version = "0.8", path = "../../primitives/state-machine" } +sc-network = { version = "0.8.0-alpha.2", path = "../network" } +sc-network-test = { version = "0.8.0-dev", path = "../network/test" } +sp-keyring = { version = "2.0.0-alpha.2", path = "../../primitives/keyring" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } +sp-consensus-babe = { version = "0.8.0-alpha.2", path = "../../primitives/consensus/babe" } +sp-state-machine = { version = "0.8.0-alpha.2", path = "../../primitives/state-machine" } env_logger = "0.7.0" -tokio = "0.1.22" +tokio = { version = "0.2", features = ["rt-core"] } tempfile = "3.1.0" -sp-api = { version = "2.0.0", path = "../../primitives/api" } -futures01 = { package = "futures", version = "0.1.29" } +sp-api = { version = "2.0.0-alpha.2", path = "../../primitives/api" } diff --git a/client/finality-grandpa/src/communication/mod.rs b/client/finality-grandpa/src/communication/mod.rs index 050a3c8642fde81056af71f6076fd5534f9c8a6d..1f4e22359712fa2f398bd846e1fbe1620a47efa4 100644 --- a/client/finality-grandpa/src/communication/mod.rs +++ b/client/finality-grandpa/src/communication/mod.rs @@ -65,6 +65,7 @@ mod periodic; pub(crate) mod tests; pub use sp_finality_grandpa::GRANDPA_ENGINE_ID; +pub const GRANDPA_PROTOCOL_NAME: &[u8] = b"/paritytech/grandpa/1"; // cost scalars for reporting peers. mod cost { @@ -119,9 +120,8 @@ pub trait Network: GossipNetwork + Clone + Send + 'static fn set_sync_fork_request(&self, peers: Vec, hash: Block::Hash, number: NumberFor); } -impl Network for Arc> where +impl Network for Arc> where B: BlockT, - S: sc_network::specialization::NetworkSpecialization, H: sc_network::ExHashT, { fn set_sync_fork_request(&self, peers: Vec, hash: B::Hash, number: NumberFor) { @@ -185,7 +185,12 @@ impl> NetworkBridge { ); let validator = Arc::new(validator); - let gossip_engine = GossipEngine::new(service.clone(), GRANDPA_ENGINE_ID, validator.clone()); + let gossip_engine = GossipEngine::new( + service.clone(), + GRANDPA_ENGINE_ID, + GRANDPA_PROTOCOL_NAME, + validator.clone() + ); { // register all previous votes with the gossip service so that they're diff --git a/client/finality-grandpa/src/communication/tests.rs b/client/finality-grandpa/src/communication/tests.rs index 040ee4c7bbd2257e6b188a07ee32b557ddd821e4..96761a2f3c07c567aaefdc3f0540dee3c0cabb1c 100644 --- a/client/finality-grandpa/src/communication/tests.rs +++ b/client/finality-grandpa/src/communication/tests.rs @@ -25,7 +25,7 @@ use std::sync::Arc; use sp_keyring::Ed25519Keyring; use parity_scale_codec::Encode; use sp_runtime::{ConsensusEngineId, traits::NumberFor}; -use std::{pin::Pin, task::{Context, Poll}}; +use std::{borrow::Cow, pin::Pin, task::{Context, Poll}}; use crate::environment::SharedVoterSetState; use sp_finality_grandpa::{AuthorityList, GRANDPA_ENGINE_ID}; use super::gossip::{self, GossipValidator}; @@ -44,20 +44,12 @@ pub(crate) struct TestNetwork { sender: mpsc::UnboundedSender, } -impl TestNetwork { - fn event_stream_03(&self) -> Pin + Send>> { +impl sc_network_gossip::Network for TestNetwork { + fn event_stream(&self) -> Pin + Send>> { let (tx, rx) = mpsc::unbounded(); let _ = self.sender.unbounded_send(Event::EventStream(tx)); Box::pin(rx) } -} - -impl sc_network_gossip::Network for TestNetwork { - fn event_stream(&self) -> Box + Send> { - Box::new( - self.event_stream_03().map(Ok::<_, ()>).compat() - ) - } fn report_peer(&self, who: sc_network::PeerId, cost_benefit: sc_network::ReputationChange) { let _ = self.sender.unbounded_send(Event::Report(who, cost_benefit)); @@ -69,7 +61,7 @@ impl sc_network_gossip::Network for TestNetwork { let _ = self.sender.unbounded_send(Event::WriteNotification(who, message)); } - fn register_notifications_protocol(&self, _: ConsensusEngineId) {} + fn register_notifications_protocol(&self, _: ConsensusEngineId, _: Cow<'static, [u8]>) {} fn announce(&self, block: Hash, _associated_data: Vec) { let _ = self.sender.unbounded_send(Event::Announce(block)); diff --git a/client/finality-grandpa/src/environment.rs b/client/finality-grandpa/src/environment.rs index fd88113776c9c02c9feca3292e096e3958d50693..a0f37f20cb390926cb6c43662e1370772863313b 100644 --- a/client/finality-grandpa/src/environment.rs +++ b/client/finality-grandpa/src/environment.rs @@ -25,18 +25,11 @@ use parity_scale_codec::{Decode, Encode}; use futures::prelude::*; use futures_timer::Delay; use parking_lot::RwLock; -use sp_blockchain::{HeaderBackend, Error as ClientError}; - -use sc_client_api::{ - BlockchainEvents, - backend::{AuxStore, Backend}, - Finalizer, - call_executor::CallExecutor, - utils::is_descendent_of, -}; -use sc_client::{ - apply_aux, Client, -}; +use sp_blockchain::{HeaderBackend, Error as ClientError, HeaderMetadata}; +use std::marker::PhantomData; + +use sc_client_api::{backend::Backend, utils::is_descendent_of}; +use sc_client::apply_aux; use finality_grandpa::{ BlockNumberOps, Equivocation, Error as GrandpaError, round::State as RoundState, voter, voter_set::VoterSet, @@ -62,6 +55,7 @@ use crate::justification::GrandpaJustification; use crate::until_imported::UntilVoteTargetImported; use crate::voting_rule::VotingRule; use sp_finality_grandpa::{AuthorityId, AuthoritySignature, SetId, RoundNumber}; +use prometheus_endpoint::{Gauge, U64, register, PrometheusError}; type HistoricalVotes = finality_grandpa::HistoricalVotes< ::Hash, @@ -376,9 +370,27 @@ impl SharedVoterSetState { } } +/// Prometheus metrics for GRANDPA. +#[derive(Clone)] +pub(crate) struct Metrics { + finality_grandpa_round: Gauge, +} + +impl Metrics { + pub(crate) fn register(registry: &prometheus_endpoint::Registry) -> Result { + Ok(Self { + finality_grandpa_round: register( + Gauge::new("finality_grandpa_round", "Highest completed GRANDPA round.")?, + registry + )?, + }) + } +} + + /// The environment we run GRANDPA in. -pub(crate) struct Environment, RA, SC, VR> { - pub(crate) client: Arc>, +pub(crate) struct Environment, SC, VR> { + pub(crate) client: Arc, pub(crate) select_chain: SC, pub(crate) voters: Arc>, pub(crate) config: Config, @@ -388,9 +400,11 @@ pub(crate) struct Environment, RA, SC, V pub(crate) set_id: SetId, pub(crate) voter_set_state: SharedVoterSetState, pub(crate) voting_rule: VR, + pub(crate) metrics: Option, + pub(crate) _phantom: PhantomData, } -impl, RA, SC, VR> Environment { +impl, SC, VR> Environment { /// Updates the voter set state using the given closure. The write lock is /// held during evaluation of the closure and the environment's voter set /// state is set to its result if successful. @@ -400,23 +414,33 @@ impl, RA, SC, VR> Environment +impl finality_grandpa::Chain> -for Environment +for Environment where Block: 'static, - B: Backend + 'static, - E: CallExecutor + Send + Sync, + BE: Backend, + C: crate::ClientForGrandpa, N: NetworkT + 'static + Send, SC: SelectChain + 'static, - VR: VotingRule>, - RA: Send + Sync, + VR: VotingRule, NumberFor: BlockNumberOps, { fn ancestry(&self, base: Block::Hash, block: Block::Hash) -> Result, GrandpaError> { @@ -432,7 +456,7 @@ where return None; } - let base_header = match self.client.header(&BlockId::Hash(block)).ok()? { + let base_header = match self.client.header(BlockId::Hash(block)).ok()? { Some(h) => h, None => { debug!(target: "afg", "Encountered error finding best chain containing {:?}: couldn't find base block", block); @@ -450,7 +474,7 @@ where match self.select_chain.finality_target(block, None) { Ok(Some(best_hash)) => { - let best_header = self.client.header(&BlockId::Hash(best_hash)).ok()? + let best_header = self.client.header(BlockId::Hash(best_hash)).ok()? .expect("Header known to exist after `finality_target` call; qed"); // check if our vote is currently being limited due to a pending change @@ -474,7 +498,7 @@ where break; } - target_header = self.client.header(&BlockId::Hash(*target_header.parent_hash())).ok()? + target_header = self.client.header(BlockId::Hash(*target_header.parent_hash())).ok()? .expect("Header known to exist after `finality_target` call; qed"); } @@ -519,17 +543,16 @@ where } -pub(crate) fn ancestry( - client: &Client, +pub(crate) fn ancestry( + client: &Arc, base: Block::Hash, block: Block::Hash, ) -> Result, GrandpaError> where - B: Backend, - E: CallExecutor, + Client: HeaderMetadata, { if base == block { return Err(GrandpaError::NotDescendent) } - let tree_route_res = sp_blockchain::tree_route(client, block, base); + let tree_route_res = sp_blockchain::tree_route(&**client, block, base); let tree_route = match tree_route_res { Ok(tree_route) => tree_route, @@ -550,19 +573,17 @@ pub(crate) fn ancestry( Ok(tree_route.retracted().iter().skip(1).map(|e| e.hash).collect()) } -impl +impl voter::Environment> -for Environment +for Environment where Block: 'static, - B: Backend + 'static, - E: CallExecutor + 'static + Send + Sync, + B: Backend, + C: crate::ClientForGrandpa + 'static, N: NetworkT + 'static + Send, - RA: 'static + Send + Sync, SC: SelectChain + 'static, - VR: VotingRule>, + VR: VotingRule, NumberFor: BlockNumberOps, - Client: AuxStore, { type Timer = Pin> + Send>>; type Id = AuthorityId; @@ -882,7 +903,7 @@ where commit: Commit, ) -> Result<(), Self::Error> { finalize_block( - &*self.client, + self.client.clone(), &self.authority_set, &self.consensus_changes, Some(self.config.justification_period.into()), @@ -940,8 +961,8 @@ impl From> for JustificationOrCommit< /// authority set change is enacted then a justification is created (if not /// given) and stored with the block when finalizing it. /// This method assumes that the block being finalized has already been imported. -pub(crate) fn finalize_block( - client: &Client, +pub(crate) fn finalize_block( + client: Arc, authority_set: &SharedAuthoritySet>, consensus_changes: &SharedConsensusChanges>, justification_period: Option>, @@ -949,16 +970,16 @@ pub(crate) fn finalize_block( number: NumberFor, justification_or_commit: JustificationOrCommit, ) -> Result<(), CommandOrError>> where - B: Backend, - E: CallExecutor + Send + Sync, - RA: Send + Sync, + Block: BlockT, + BE: Backend, + Client: crate::ClientForGrandpa, { // NOTE: lock must be held through writing to DB to avoid race. this lock // also implicitly synchronizes the check for last finalized number // below. let mut authority_set = authority_set.inner().write(); - let status = client.chain_info(); + let status = client.info(); if number <= status.finalized_number && client.hash(number)? == Some(hash) { // This can happen after a forced change (triggered by the finality tracker when finality is stalled), since // the voter will be restarted at the median last finalized block, which can be lower than the local best @@ -981,14 +1002,14 @@ pub(crate) fn finalize_block( let mut consensus_changes = consensus_changes.lock(); let canon_at_height = |canon_number| { // "true" because the block is finalized - canonical_at_height(client, (hash, number), true, canon_number) + canonical_at_height(&*client, (hash, number), true, canon_number) }; let update_res: Result<_, Error> = client.lock_import_and_run(|import_op| { let status = authority_set.apply_standard_changes( hash, number, - &is_descendent_of::(client, None), + &is_descendent_of::(&*client, None), ).map_err(|e| Error::Safety(e.to_string()))?; // check if this is this is the first finalization of some consensus changes @@ -1031,7 +1052,7 @@ pub(crate) fn finalize_block( // finalization to remote nodes if !justification_required { if let Some(justification_period) = justification_period { - let last_finalized_number = client.chain_info().finalized_number; + let last_finalized_number = client.info().finalized_number; justification_required = (!last_finalized_number.is_zero() || number - last_finalized_number == justification_period) && (last_finalized_number / justification_period != number / justification_period); @@ -1040,7 +1061,7 @@ pub(crate) fn finalize_block( if justification_required { let justification = GrandpaJustification::from_commit( - client, + &client, round_number, commit, )?; diff --git a/client/finality-grandpa/src/finality_proof.rs b/client/finality-grandpa/src/finality_proof.rs index 9da99ab531ae59044c71b10794e0ced3b1546632..2c85839b5e3640ca19398767dffb42ea4a3afa05 100644 --- a/client/finality-grandpa/src/finality_proof.rs +++ b/client/finality-grandpa/src/finality_proof.rs @@ -34,16 +34,15 @@ //! finality proof (that finalizes some block C that is ancestor of the B and descendant //! of the U) could be returned. -use std::iter; use std::sync::Arc; use log::{trace, warn}; use sp_blockchain::{Backend as BlockchainBackend, Error as ClientError, Result as ClientResult}; use sc_client_api::{ - backend::Backend, CallExecutor, StorageProof, + backend::Backend, StorageProof, light::{FetchChecker, RemoteReadRequest}, + StorageProvider, ProofProvider, }; -use sc_client::Client; use parity_scale_codec::{Encode, Decode}; use finality_grandpa::BlockNumberOps; use sp_runtime::{ @@ -67,12 +66,25 @@ pub trait AuthoritySetForFinalityProver: Send + Sync { fn prove_authorities(&self, block: &BlockId) -> ClientResult; } -/// Client-based implementation of AuthoritySetForFinalityProver. -impl AuthoritySetForFinalityProver for Client +/// Trait that combines `StorageProvider` and `ProofProvider` +pub trait StorageAndProofProvider: StorageProvider + ProofProvider + Send + Sync where - B: Backend + Send + Sync + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, - RA: Send + Sync, + Block: BlockT, + BE: Backend + Send + Sync, +{} + +/// Blanket implementation. +impl StorageAndProofProvider for P + where + Block: BlockT, + BE: Backend + Send + Sync, + P: StorageProvider + ProofProvider + Send + Sync, +{} + +/// Implementation of AuthoritySetForFinalityProver. +impl AuthoritySetForFinalityProver for Arc> + where + BE: Backend + Send + Sync + 'static, { fn authorities(&self, block: &BlockId) -> ClientResult { let storage_key = StorageKey(GRANDPA_AUTHORITIES_KEY.to_vec()); @@ -83,7 +95,7 @@ impl AuthoritySetForFinalityProver for Client) -> ClientResult { - self.read_proof(block, iter::once(GRANDPA_AUTHORITIES_KEY)) + self.read_proof(block, &mut std::iter::once(GRANDPA_AUTHORITIES_KEY)) } } @@ -146,15 +158,17 @@ impl FinalityProofProvider /// /// - backend for accessing blockchain data; /// - authority_provider for calling and proving runtime methods. - pub fn new( + pub fn new

( backend: Arc, - authority_provider: Arc>, - ) -> Self { - FinalityProofProvider { backend, authority_provider } + authority_provider: P, + ) -> Self + where P: AuthoritySetForFinalityProver + 'static, + { + FinalityProofProvider { backend, authority_provider: Arc::new(authority_provider) } } } -impl sc_network::FinalityProofProvider for FinalityProofProvider +impl sc_network::config::FinalityProofProvider for FinalityProofProvider where Block: BlockT, NumberFor: BlockNumberOps, diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index 2eb1b4a7c889c3929c57ac3149e373a6c86c736f..ea1deccdafbf5d4a428947917b797779ff0957fe 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -21,9 +21,10 @@ use parity_scale_codec::Encode; use futures::channel::mpsc; use parking_lot::RwLockWriteGuard; -use sp_blockchain::{HeaderBackend, BlockStatus, well_known_cache_keys}; -use sc_client_api::{backend::{TransactionFor, Backend}, CallExecutor, utils::is_descendent_of}; -use sc_client::Client; +use sp_blockchain::{BlockStatus, well_known_cache_keys}; +use sc_client_api::{backend::Backend, utils::is_descendent_of}; +use sp_api::{TransactionFor}; + use sp_consensus::{ BlockImport, Error as ConsensusError, BlockCheckParams, BlockImportParams, ImportResult, JustificationImport, @@ -41,6 +42,7 @@ use crate::authorities::{AuthoritySet, SharedAuthoritySet, DelayKind, PendingCha use crate::consensus_changes::SharedConsensusChanges; use crate::environment::finalize_block; use crate::justification::GrandpaJustification; +use std::marker::PhantomData; /// A block-import handler for GRANDPA. /// @@ -51,16 +53,17 @@ use crate::justification::GrandpaJustification; /// /// When using GRANDPA, the block import worker should be using this block import /// object. -pub struct GrandpaBlockImport { - inner: Arc>, +pub struct GrandpaBlockImport { + inner: Arc, select_chain: SC, authority_set: SharedAuthoritySet>, send_voter_commands: mpsc::UnboundedSender>>, consensus_changes: SharedConsensusChanges>, + _phantom: PhantomData, } -impl Clone for - GrandpaBlockImport +impl Clone for + GrandpaBlockImport { fn clone(&self) -> Self { GrandpaBlockImport { @@ -69,24 +72,24 @@ impl Clone for authority_set: self.authority_set.clone(), send_voter_commands: self.send_voter_commands.clone(), consensus_changes: self.consensus_changes.clone(), + _phantom: PhantomData, } } } -impl JustificationImport - for GrandpaBlockImport where +impl JustificationImport + for GrandpaBlockImport where NumberFor: finality_grandpa::BlockNumberOps, - B: Backend + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, DigestFor: Encode, - RA: Send + Sync, + BE: Backend, + Client: crate::ClientForGrandpa, SC: SelectChain, { type Error = ConsensusError; fn on_start(&mut self) -> Vec<(Block::Hash, NumberFor)> { let mut out = Vec::new(); - let chain_info = self.inner.chain_info(); + let chain_info = self.inner.info(); // request justifications for all pending changes for which change blocks have already been imported let authorities = self.authority_set.inner().read(); @@ -105,7 +108,7 @@ impl JustificationImport }; if let Ok(Some(hash)) = effective_block_hash { - if let Ok(Some(header)) = self.inner.header(&BlockId::Hash(hash)) { + if let Ok(Some(header)) = self.inner.header(BlockId::Hash(hash)) { if *header.number() == pending_change.effective_number() { out.push((header.hash(), *header.number())); } @@ -123,7 +126,7 @@ impl JustificationImport number: NumberFor, justification: Justification, ) -> Result<(), Self::Error> { - self.import_justification(hash, number, justification, false) + GrandpaBlockImport::import_justification(self, hash, number, justification, false) } } @@ -200,14 +203,13 @@ fn find_forced_change(header: &B::Header) header.digest().convert_first(|l| l.try_to(id).and_then(filter_log)) } -impl - GrandpaBlockImport +impl + GrandpaBlockImport where NumberFor: finality_grandpa::BlockNumberOps, - B: Backend + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, DigestFor: Encode, - RA: Send + Sync, + BE: Backend, + Client: crate::ClientForGrandpa, { // check for a new authority set change. fn check_new_change(&self, header: &Block::Header, hash: Block::Hash) @@ -235,11 +237,11 @@ where }) } - fn make_authorities_changes<'a>( - &'a self, - block: &mut BlockImportParams>, + fn make_authorities_changes( + &self, + block: &mut BlockImportParams>, hash: Block::Hash, - ) -> Result, ConsensusError> { + ) -> Result, ConsensusError> { // when we update the authorities, we need to hold the lock // until the block is written to prevent a race if we need to restore // the old authority set on error or panic. @@ -325,10 +327,10 @@ where // for the canon block the new authority set should start // with. we use the minimum between the median and the local // best finalized block. - let best_finalized_number = self.inner.chain_info().finalized_number; + let best_finalized_number = self.inner.info().finalized_number; let canon_number = best_finalized_number.min(median_last_finalized_number); let canon_hash = - self.inner.header(&BlockId::Number(canon_number)) + self.inner.header(BlockId::Number(canon_number)) .map_err(|e| ConsensusError::ClientImport(e.to_string()))? .expect("the given block number is less or equal than the current best finalized number; \ current best finalized number must exist in chain; qed.") @@ -380,18 +382,17 @@ where } } -impl BlockImport - for GrandpaBlockImport where +impl BlockImport + for GrandpaBlockImport where NumberFor: finality_grandpa::BlockNumberOps, - B: Backend + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, DigestFor: Encode, - RA: Send + Sync, - for<'a> &'a Client: - BlockImport>, + BE: Backend, + Client: crate::ClientForGrandpa, + for<'a> &'a Client: + BlockImport>, { type Error = ConsensusError; - type Transaction = TransactionFor; + type Transaction = TransactionFor; fn import_block( &mut self, @@ -521,31 +522,30 @@ impl BlockImport } } -impl GrandpaBlockImport { +impl GrandpaBlockImport { pub(crate) fn new( - inner: Arc>, + inner: Arc, select_chain: SC, authority_set: SharedAuthoritySet>, send_voter_commands: mpsc::UnboundedSender>>, consensus_changes: SharedConsensusChanges>, - ) -> GrandpaBlockImport { + ) -> GrandpaBlockImport { GrandpaBlockImport { inner, select_chain, authority_set, send_voter_commands, consensus_changes, + _phantom: PhantomData, } } } -impl - GrandpaBlockImport +impl GrandpaBlockImport where + BE: Backend, + Client: crate::ClientForGrandpa, NumberFor: finality_grandpa::BlockNumberOps, - B: Backend + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, - RA: Send + Sync, { /// Import a block justification and finalize the block. @@ -572,7 +572,7 @@ where }; let result = finalize_block( - &*self.inner, + self.inner.clone(), &self.authority_set, &self.consensus_changes, None, diff --git a/client/finality-grandpa/src/justification.rs b/client/finality-grandpa/src/justification.rs index ad96956454fd727a57b67969756e9d7d012d94e5..084c0042ab19a08fe5f17e31aeffd1820958fcb0 100644 --- a/client/finality-grandpa/src/justification.rs +++ b/client/finality-grandpa/src/justification.rs @@ -15,10 +15,9 @@ // along with Substrate. If not, see . use std::collections::{HashMap, HashSet}; +use std::sync::Arc; -use sc_client::Client; -use sc_client_api::{CallExecutor, backend::Backend}; -use sp_blockchain::Error as ClientError; +use sp_blockchain::{Error as ClientError, HeaderBackend}; use parity_scale_codec::{Encode, Decode}; use finality_grandpa::voter_set::VoterSet; use finality_grandpa::{Error as GrandpaError}; @@ -47,14 +46,12 @@ pub struct GrandpaJustification { impl GrandpaJustification { /// Create a GRANDPA justification from the given commit. This method /// assumes the commit is valid and well-formed. - pub(crate) fn from_commit( - client: &Client, + pub(crate) fn from_commit( + client: &Arc, round: u64, commit: Commit, ) -> Result, Error> where - B: Backend, - E: CallExecutor + Send + Sync, - RA: Send + Sync, + C: HeaderBackend, { let mut votes_ancestries_hashes = HashSet::new(); let mut votes_ancestries = Vec::new(); @@ -69,7 +66,7 @@ impl GrandpaJustification { loop { if current_hash == commit.target_hash { break; } - match client.header(&BlockId::Hash(current_hash))? { + match client.header(BlockId::Hash(current_hash))? { Some(current_header) => { if *current_header.number() <= commit.target_number { return error(); diff --git a/client/finality-grandpa/src/lib.rs b/client/finality-grandpa/src/lib.rs index e931271df91001b79276f12fd5acb1d36642ff33..afee2bec53576a56c3aeba69716ad4d144bbd851 100644 --- a/client/finality-grandpa/src/lib.rs +++ b/client/finality-grandpa/src/lib.rs @@ -56,15 +56,18 @@ use futures::prelude::*; use futures::StreamExt; use log::{debug, info}; use futures::channel::mpsc; -use sc_client_api::{BlockchainEvents, CallExecutor, backend::{AuxStore, Backend}, ExecutionStrategy}; -use sp_blockchain::{HeaderBackend, Error as ClientError}; -use sc_client::Client; +use sc_client_api::{ + backend::{AuxStore, Backend}, + LockImportRun, BlockchainEvents, CallExecutor, + ExecutionStrategy, Finalizer, TransactionFor, ExecutorProvider, +}; +use sp_blockchain::{HeaderBackend, Error as ClientError, HeaderMetadata}; use parity_scale_codec::{Decode, Encode}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{NumberFor, Block as BlockT, DigestFor, Zero}; use sc_keystore::KeyStorePtr; use sp_inherents::InherentDataProviders; -use sp_consensus::SelectChain; +use sp_consensus::{SelectChain, BlockImport}; use sp_core::Pair; use sc_telemetry::{telemetry, CONSENSUS_INFO, CONSENSUS_DEBUG}; use serde_json; @@ -93,16 +96,15 @@ mod observer; mod until_imported; mod voting_rule; -pub use finality_proof::FinalityProofProvider; +pub use finality_proof::{FinalityProofProvider, StorageAndProofProvider}; pub use justification::GrandpaJustification; pub use light_import::light_block_import; -pub use observer::run_grandpa_observer; pub use voting_rule::{ BeforeBestBlockBy, ThreeQuartersOfTheUnfinalizedChain, VotingRule, VotingRulesBuilder }; use aux_schema::PersistentData; -use environment::{Environment, VoterSetState}; +use environment::{Environment, VoterSetState, Metrics}; use import::GrandpaBlockImport; use until_imported::UntilGlobalMessageBlocksImported; use communication::{NetworkBridge, Network as NetworkT}; @@ -110,6 +112,8 @@ use sp_finality_grandpa::{AuthorityList, AuthorityPair, AuthoritySignature, SetI // Re-export these two because it's just so damn convenient. pub use sp_finality_grandpa::{AuthorityId, ScheduledChange}; +use sp_api::ProvideRuntimeApi; +use std::marker::PhantomData; #[cfg(test)] mod tests; @@ -246,10 +250,8 @@ pub(crate) trait BlockStatus { fn block_number(&self, hash: Block::Hash) -> Result>, Error>; } -impl BlockStatus for Arc> where - B: Backend, - E: CallExecutor + Send + Sync, - RA: Send + Sync, +impl BlockStatus for Arc where + Client: HeaderBackend, NumberFor: BlockNumberOps, { fn block_number(&self, hash: Block::Hash) -> Result>, Error> { @@ -258,6 +260,29 @@ impl BlockStatus for Arc } } +/// A trait that includes all the client functionalities grandpa requires. +/// Ideally this would be a trait alias, we're not there yet. +/// tracking issue https://github.com/rust-lang/rust/issues/41517 +pub trait ClientForGrandpa: + LockImportRun + Finalizer + AuxStore + + HeaderMetadata + HeaderBackend + + BlockchainEvents + ProvideRuntimeApi + ExecutorProvider + + BlockImport, Error = sp_consensus::Error> + where + BE: Backend, + Block: BlockT, +{} + +impl ClientForGrandpa for T + where + BE: Backend, + Block: BlockT, + T: LockImportRun + Finalizer + AuxStore + + HeaderMetadata + HeaderBackend + + BlockchainEvents + ProvideRuntimeApi + ExecutorProvider + + BlockImport, Error = sp_consensus::Error>, +{} + /// Something that one can ask to do a block sync request. pub(crate) trait BlockSyncRequester { /// Notifies the sync service to try and sync the given block from the given @@ -349,8 +374,8 @@ impl fmt::Display for CommandOrError { } } -pub struct LinkHalf { - client: Arc>, +pub struct LinkHalf { + client: Arc, select_chain: SC, persistent_data: PersistentData, voter_commands_rx: mpsc::UnboundedReceiver>>, @@ -362,11 +387,8 @@ pub trait GenesisAuthoritySetProvider { fn get(&self) -> Result; } -impl GenesisAuthoritySetProvider for Client - where - B: Backend + Send + Sync + 'static, - E: CallExecutor + Send + Sync, - RA: Send + Sync, +impl GenesisAuthoritySetProvider for Arc> + where E: CallExecutor, { fn get(&self) -> Result { // This implementation uses the Grandpa runtime API instead of reading directly from the @@ -391,22 +413,20 @@ impl GenesisAuthoritySetProvider for Client( - client: Arc>, +pub fn block_import( + client: Arc, genesis_authorities_provider: &dyn GenesisAuthoritySetProvider, select_chain: SC, ) -> Result<( - GrandpaBlockImport, - LinkHalf + GrandpaBlockImport, + LinkHalf, ), ClientError> where - B: Backend + 'static, - E: CallExecutor + Send + Sync, - RA: Send + Sync, SC: SelectChain, - Client: AuxStore, + BE: Backend + 'static, + Client: ClientForGrandpa + 'static, { - let chain_info = client.chain_info(); + let chain_info = client.info(); let genesis_hash = chain_info.genesis_hash; let persistent_data = aux_schema::load_persistent( @@ -441,10 +461,10 @@ where )) } -fn global_communication( +fn global_communication( set_id: SetId, voters: &Arc>, - client: &Arc>, + client: Arc, network: &NetworkBridge, keystore: &Option, ) -> ( @@ -456,10 +476,9 @@ fn global_communication( Error = CommandOrError>, > + Unpin, ) where - B: Backend, - E: CallExecutor + Send + Sync, + BE: Backend + 'static, + C: ClientForGrandpa + 'static, N: NetworkT, - RA: Send + Sync, NumberFor: BlockNumberOps, { let is_voter = is_voter(voters, keystore).is_some(); @@ -488,20 +507,18 @@ fn global_communication( /// Register the finality tracker inherent data provider (which is used by /// GRANDPA), if not registered already. -fn register_finality_tracker_inherent_data_provider( - client: Arc>, +fn register_finality_tracker_inherent_data_provider( + client: Arc, inherent_data_providers: &InherentDataProviders, ) -> Result<(), sp_consensus::Error> where - B: Backend + 'static, - E: CallExecutor + Send + Sync + 'static, - RA: Send + Sync + 'static, + Client: HeaderBackend + 'static, { if !inherent_data_providers.has_provider(&sp_finality_tracker::INHERENT_IDENTIFIER) { inherent_data_providers .register_provider(sp_finality_tracker::InherentDataProvider::new(move || { #[allow(deprecated)] { - let info = client.chain_info(); + let info = client.info(); telemetry!(CONSENSUS_INFO; "afg.finalized"; "finalized_number" => ?info.finalized_number, "finalized_hash" => ?info.finalized_hash, @@ -516,50 +533,53 @@ fn register_finality_tracker_inherent_data_provider( } /// Parameters used to run Grandpa. -pub struct GrandpaParams { +pub struct GrandpaParams { /// Configuration for the GRANDPA service. pub config: Config, /// A link to the block import worker. - pub link: LinkHalf, + pub link: LinkHalf, /// The Network instance. pub network: N, /// The inherent data providers. pub inherent_data_providers: InherentDataProviders, - /// Handle to a future that will resolve on exit. - pub on_exit: X, /// If supplied, can be used to hook on telemetry connection established events. pub telemetry_on_connect: Option>, /// A voting rule used to potentially restrict target votes. pub voting_rule: VR, + /// The prometheus metrics registry. + pub prometheus_registry: Option, } /// Run a GRANDPA voter as a task. Provide configuration and a link to a /// block import worker that has already been instantiated with `block_import`. -pub fn run_grandpa_voter( - grandpa_params: GrandpaParams, +pub fn run_grandpa_voter( + grandpa_params: GrandpaParams, ) -> sp_blockchain::Result + Unpin + Send + 'static> where Block::Hash: Ord, - B: Backend + 'static, - E: CallExecutor + Send + Sync + 'static, + BE: Backend + 'static, N: NetworkT + Send + Sync + Clone + 'static, SC: SelectChain + 'static, - VR: VotingRule> + Clone + 'static, + VR: VotingRule + Clone + 'static, NumberFor: BlockNumberOps, DigestFor: Encode, - RA: Send + Sync + 'static, - X: futures::Future + Clone + Send + Unpin + 'static, - Client: AuxStore, + C: ClientForGrandpa + 'static, { let GrandpaParams { - config, + mut config, link, network, inherent_data_providers, - on_exit, telemetry_on_connect, voting_rule, + prometheus_registry, } = grandpa_params; + // NOTE: we have recently removed `run_grandpa_observer` from the public + // API, I felt it is easier to just ignore this field rather than removing + // it from the config temporarily. This should be removed after #5013 is + // fixed and we re-add the observer to the public API. + config.observer_enabled = false; + let LinkHalf { client, select_chain, @@ -610,6 +630,7 @@ pub fn run_grandpa_voter( voting_rule, persistent_data, voter_commands_rx, + prometheus_registry, ); let voter_work = voter_work @@ -619,38 +640,37 @@ pub fn run_grandpa_voter( let telemetry_task = telemetry_task .then(|_| future::pending::<()>()); - Ok(future::select(future::select(voter_work, on_exit), telemetry_task).map(drop)) + Ok(future::select(voter_work, telemetry_task).map(drop)) } /// Future that powers the voter. #[must_use] -struct VoterWork, RA, SC, VR> { +struct VoterWork, SC, VR> { voter: Pin>>> + Send>>, - env: Arc>, + env: Arc>, voter_commands_rx: mpsc::UnboundedReceiver>>, network: NetworkBridge, } -impl VoterWork +impl VoterWork where Block: BlockT, + B: Backend + 'static, + C: ClientForGrandpa + 'static, N: NetworkT + Sync, NumberFor: BlockNumberOps, - RA: 'static + Send + Sync, - E: CallExecutor + Send + Sync + 'static, - B: Backend + 'static, SC: SelectChain + 'static, - VR: VotingRule> + Clone + 'static, - Client: AuxStore, + VR: VotingRule + Clone + 'static, { fn new( - client: Arc>, + client: Arc, config: Config, network: NetworkBridge, select_chain: SC, voting_rule: VR, persistent_data: PersistentData, voter_commands_rx: mpsc::UnboundedReceiver>>, + prometheus_registry: Option, ) -> Self { let voters = persistent_data.authority_set.current_authorities(); @@ -665,6 +685,11 @@ where authority_set: persistent_data.authority_set.clone(), consensus_changes: persistent_data.consensus_changes.clone(), voter_set_state: persistent_data.set_state.clone(), + metrics: prometheus_registry.map(|registry| { + Metrics::register(®istry) + .expect("Other metrics would have failed to register before these; qed") + }), + _phantom: PhantomData, }); let mut work = VoterWork { @@ -695,7 +720,7 @@ where "authority_id" => authority_id.to_string(), ); - let chain_info = self.env.client.chain_info(); + let chain_info = self.env.client.info(); telemetry!(CONSENSUS_INFO; "afg.authority_set"; "number" => ?chain_info.finalized_number, "hash" => ?chain_info.finalized_hash, @@ -719,7 +744,7 @@ where let global_comms = global_communication( self.env.set_id, &self.env.voters, - &self.env.client, + self.env.client.clone(), &self.env.network, &self.env.config.keystore, ); @@ -784,6 +809,8 @@ where consensus_changes: self.env.consensus_changes.clone(), network: self.env.network.clone(), voting_rule: self.env.voting_rule.clone(), + metrics: self.env.metrics.clone(), + _phantom: PhantomData, }); self.rebuild_voter(); @@ -808,17 +835,15 @@ where } } -impl Future for VoterWork +impl Future for VoterWork where Block: BlockT, + B: Backend + 'static, N: NetworkT + Sync, NumberFor: BlockNumberOps, - RA: 'static + Send + Sync, - E: CallExecutor + Send + Sync + 'static, - B: Backend + 'static, SC: SelectChain + 'static, - VR: VotingRule> + Clone + 'static, - Client: AuxStore, + C: ClientForGrandpa + 'static, + VR: VotingRule + Clone + 'static, { type Output = Result<(), Error>; @@ -863,15 +888,13 @@ where /// discards all GRANDPA messages (otherwise, we end up banning nodes that send /// us a `Neighbor` message, since there is no registered gossip validator for /// the engine id defined in the message.) -pub fn setup_disabled_grandpa( - client: Arc>, +pub fn setup_disabled_grandpa( + client: Arc, inherent_data_providers: &InherentDataProviders, network: N, ) -> Result<(), sp_consensus::Error> where - B: Backend + 'static, - E: CallExecutor + Send + Sync + 'static, - RA: Send + Sync + 'static, N: NetworkT + Send + Clone + 'static, + Client: HeaderBackend + 'static, { register_finality_tracker_inherent_data_provider( client, @@ -881,7 +904,10 @@ pub fn setup_disabled_grandpa( // We register the GRANDPA protocol so that we don't consider it an anomaly // to receive GRANDPA messages on the network. We don't process the // messages. - network.register_notifications_protocol(communication::GRANDPA_ENGINE_ID); + network.register_notifications_protocol( + communication::GRANDPA_ENGINE_ID, + From::from(communication::GRANDPA_PROTOCOL_NAME), + ); Ok(()) } diff --git a/client/finality-grandpa/src/light_import.rs b/client/finality-grandpa/src/light_import.rs index 0181a93f1088cb35c46715a9db4f32b937950f21..276f5d0f28d7ab1c9c3d376bc23ce4f55f03b1a7 100644 --- a/client/finality-grandpa/src/light_import.rs +++ b/client/finality-grandpa/src/light_import.rs @@ -18,8 +18,9 @@ use std::collections::HashMap; use std::sync::Arc; use log::{info, trace, warn}; use parking_lot::RwLock; -use sc_client::Client; -use sc_client_api::{CallExecutor, backend::{AuxStore, Backend, Finalizer, TransactionFor}}; +use sc_client_api::{ + backend::{AuxStore, Backend, Finalizer, TransactionFor}, +}; use sp_blockchain::{HeaderBackend, Error as ClientError, well_known_cache_keys}; use parity_scale_codec::{Encode, Decode}; use sp_consensus::{ @@ -48,17 +49,15 @@ const LIGHT_AUTHORITY_SET_KEY: &[u8] = b"grandpa_voters"; const LIGHT_CONSENSUS_CHANGES_KEY: &[u8] = b"grandpa_consensus_changes"; /// Create light block importer. -pub fn light_block_import( - client: Arc>, - backend: Arc, +pub fn light_block_import( + client: Arc, + backend: Arc, genesis_authorities_provider: &dyn GenesisAuthoritySetProvider, authority_set_provider: Arc>, -) -> Result, ClientError> +) -> Result, ClientError> where - B: Backend + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, - RA: Send + Sync, - Client: AuxStore, + BE: Backend, + Client: crate::ClientForGrandpa, { let info = client.info(); let import_data = load_aux_import_data( @@ -79,14 +78,14 @@ pub fn light_block_import( /// It is responsible for: /// - checking GRANDPA justifications; /// - fetching finality proofs for blocks that are enacting consensus changes. -pub struct GrandpaLightBlockImport { - client: Arc>, - backend: Arc, +pub struct GrandpaLightBlockImport { + client: Arc, + backend: Arc, authority_set_provider: Arc>, data: Arc>>, } -impl Clone for GrandpaLightBlockImport { +impl Clone for GrandpaLightBlockImport { fn clone(&self) -> Self { GrandpaLightBlockImport { client: self.client.clone(), @@ -111,27 +110,26 @@ struct LightAuthoritySet { authorities: AuthorityList, } -impl GrandpaLightBlockImport { +impl GrandpaLightBlockImport { /// Create finality proof request builder. pub fn create_finality_proof_request_builder(&self) -> BoxFinalityProofRequestBuilder { Box::new(GrandpaFinalityProofRequestBuilder(self.data.clone())) as _ } } -impl BlockImport - for GrandpaLightBlockImport where +impl BlockImport + for GrandpaLightBlockImport where NumberFor: finality_grandpa::BlockNumberOps, - B: Backend + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, DigestFor: Encode, - RA: Send + Sync, - for<'a> &'a Client: - BlockImport> - + Finalizer + BE: Backend + 'static, + for<'a> &'a Client: + HeaderBackend + + BlockImport> + + Finalizer + AuxStore, { type Error = ConsensusError; - type Transaction = TransactionFor; + type Transaction = TransactionFor; fn import_block( &mut self, @@ -151,23 +149,22 @@ impl BlockImport } } -impl FinalityProofImport - for GrandpaLightBlockImport where +impl FinalityProofImport + for GrandpaLightBlockImport where NumberFor: finality_grandpa::BlockNumberOps, - B: Backend + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, DigestFor: Encode, - RA: Send + Sync, - for<'a> &'a Client: - BlockImport> - + Finalizer + BE: Backend + 'static, + for<'a> &'a Client: + HeaderBackend + + BlockImport> + + Finalizer + AuxStore, { type Error = ConsensusError; fn on_start(&mut self) -> Vec<(Block::Hash, NumberFor)> { let mut out = Vec::new(); - let chain_info = self.client.chain_info(); + let chain_info = (&*self.client).info(); let data = self.data.read(); for (pending_number, pending_hash) in data.consensus_changes.pending_changes() { @@ -567,7 +564,7 @@ fn on_post_finalization_error(error: ClientError, value_type: &str) -> Consensus #[cfg(test)] pub mod tests { use super::*; - use sp_consensus::ForkChoiceStrategy; + use sp_consensus::{ForkChoiceStrategy, BlockImport}; use sp_finality_grandpa::AuthorityId; use sp_core::{H256, crypto::Public}; use substrate_test_runtime_client::sc_client::in_mem::Blockchain as InMemoryAuxStore; @@ -575,37 +572,36 @@ pub mod tests { use crate::tests::TestApi; use crate::finality_proof::tests::TestJustification; - pub struct NoJustificationsImport( - pub GrandpaLightBlockImport + pub struct NoJustificationsImport( + pub GrandpaLightBlockImport ); - impl Clone - for NoJustificationsImport where + impl Clone + for NoJustificationsImport where NumberFor: finality_grandpa::BlockNumberOps, - B: Backend + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, DigestFor: Encode, - RA: Send + Sync, + BE: Backend + 'static, { fn clone(&self) -> Self { NoJustificationsImport(self.0.clone()) } } - impl BlockImport - for NoJustificationsImport where + impl BlockImport + for NoJustificationsImport where NumberFor: finality_grandpa::BlockNumberOps, - B: Backend + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, DigestFor: Encode, - RA: Send + Sync, - for<'a> &'a Client: - BlockImport> - + Finalizer + BE: Backend + 'static, + for <'a > &'a Client: + HeaderBackend + + BlockImport> + + Finalizer + AuxStore, + GrandpaLightBlockImport: + BlockImport, Error = ConsensusError> { type Error = ConsensusError; - type Transaction = TransactionFor; + type Transaction = TransactionFor; fn import_block( &mut self, @@ -624,16 +620,15 @@ pub mod tests { } } - impl FinalityProofImport - for NoJustificationsImport where + impl FinalityProofImport + for NoJustificationsImport where NumberFor: finality_grandpa::BlockNumberOps, - B: Backend + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, + BE: Backend + 'static, DigestFor: Encode, - RA: Send + Sync, - for<'a> &'a Client: - BlockImport> - + Finalizer + for <'a > &'a Client: + HeaderBackend + + BlockImport> + + Finalizer + AuxStore, { type Error = ConsensusError; @@ -654,19 +649,15 @@ pub mod tests { } /// Creates light block import that ignores justifications that came outside of finality proofs. - pub fn light_block_import_without_justifications( - client: Arc>, - backend: Arc, + pub fn light_block_import_without_justifications( + client: Arc, + backend: Arc, genesis_authorities_provider: &dyn GenesisAuthoritySetProvider, authority_set_provider: Arc>, - ) -> Result, ClientError> + ) -> Result, ClientError> where - B: Backend + 'static, - E: CallExecutor + 'static + Clone + Send + Sync, - RA: Send + Sync, - Client: BlockImport - + Finalizer - + AuxStore, + BE: Backend + 'static, + Client: crate::ClientForGrandpa, { light_block_import(client, backend, genesis_authorities_provider, authority_set_provider) .map(NoJustificationsImport) @@ -677,6 +668,7 @@ pub mod tests { justification: Option, ) -> ImportResult { let (client, _backend) = substrate_test_runtime_client::new_light(); + let client = Arc::new(client); let mut import_data = LightImportData { last_finalized: Default::default(), authority_set: LightAuthoritySet::genesis(vec![(AuthorityId::from_slice(&[1; 32]), 1)]), @@ -696,7 +688,7 @@ pub mod tests { block.fork_choice = Some(ForkChoiceStrategy::LongestChain); do_import_block::<_, _, _, TestJustification>( - &client, + &*client, &mut import_data, block, new_cache, diff --git a/client/finality-grandpa/src/observer.rs b/client/finality-grandpa/src/observer.rs index 77227909dc856b0ec0a99d04fefeacd20ad31ce3..921e5a3dd5b1453846a606b633f9e228e4017232 100644 --- a/client/finality-grandpa/src/observer.rs +++ b/client/finality-grandpa/src/observer.rs @@ -26,10 +26,9 @@ use finality_grandpa::{ use log::{debug, info, warn}; use sp_consensus::SelectChain; -use sc_client_api::{CallExecutor, backend::{Backend, AuxStore}}; -use sc_client::Client; +use sc_client_api::backend::Backend; use sp_runtime::traits::{NumberFor, Block as BlockT}; - +use sp_blockchain::HeaderMetadata; use crate::{ global_communication, CommandOrError, CommunicationIn, Config, environment, LinkHalf, Error, aux_schema::PersistentData, VoterCommand, VoterSetState, @@ -38,17 +37,21 @@ use crate::authorities::SharedAuthoritySet; use crate::communication::{Network as NetworkT, NetworkBridge}; use crate::consensus_changes::SharedConsensusChanges; use sp_finality_grandpa::AuthorityId; +use std::marker::{PhantomData, Unpin}; -struct ObserverChain<'a, Block: BlockT, B, E, RA>(&'a Client); +struct ObserverChain<'a, Block: BlockT, Client> { + client: &'a Arc, + _phantom: PhantomData, +} -impl<'a, Block: BlockT, B, E, RA> finality_grandpa::Chain> - for ObserverChain<'a, Block, B, E, RA> where - B: Backend, - E: CallExecutor, +impl<'a, Block, Client> finality_grandpa::Chain> + for ObserverChain<'a, Block, Client> where + Block: BlockT, + Client: HeaderMetadata, NumberFor: BlockNumberOps, { fn ancestry(&self, base: Block::Hash, block: Block::Hash) -> Result, GrandpaError> { - environment::ancestry(&self.0, base, block) + environment::ancestry(&self.client, base, block) } fn best_chain_containing(&self, _block: Block::Hash) -> Option<(Block::Hash, NumberFor)> { @@ -57,8 +60,8 @@ impl<'a, Block: BlockT, B, E, RA> finality_grandpa::Chain( - client: &Arc>, +fn grandpa_observer( + client: &Arc, authority_set: &SharedAuthoritySet>, consensus_changes: &SharedConsensusChanges>, voters: &Arc>, @@ -67,13 +70,12 @@ fn grandpa_observer( note_round: F, ) -> impl Future>>> where NumberFor: BlockNumberOps, - B: Backend, - E: CallExecutor + Send + Sync + 'static, - RA: Send + Sync, S: Stream< Item = Result, CommandOrError>>, >, F: Fn(u64), + BE: Backend, + Client: crate::ClientForGrandpa, { let authority_set = authority_set.clone(); let consensus_changes = consensus_changes.clone(); @@ -101,7 +103,7 @@ fn grandpa_observer( let validation_result = match finality_grandpa::validate_commit( &commit, &voters, - &ObserverChain(&*client), + &ObserverChain { client: &client, _phantom: PhantomData }, ) { Ok(r) => r, Err(e) => return future::err(e.into()), @@ -113,7 +115,7 @@ fn grandpa_observer( // commit is valid, finalize the block it targets match environment::finalize_block( - &client, + client.clone(), &authority_set, &consensus_changes, None, @@ -150,19 +152,20 @@ fn grandpa_observer( /// listening for and validating GRANDPA commits instead of following the full /// protocol. Provide configuration and a link to a block import worker that has /// already been instantiated with `block_import`. -pub fn run_grandpa_observer( +/// NOTE: this is currently not part of the crate's public API since we don't consider +/// it stable enough to use on a live network. +#[allow(unused)] +pub fn run_grandpa_observer( config: Config, - link: LinkHalf, + link: LinkHalf, network: N, - on_exit: impl futures::Future + Clone + Send + Unpin + 'static, -) -> sp_blockchain::Result + Unpin + Send + 'static> where - B: Backend + 'static, - E: CallExecutor + Send + Sync + 'static, +) -> sp_blockchain::Result + Unpin + Send + 'static> +where + BE: Backend + Unpin + 'static, N: NetworkT + Send + Clone + 'static, SC: SelectChain + 'static, NumberFor: BlockNumberOps, - RA: Send + Sync + 'static, - Client: AuxStore, + Client: crate::ClientForGrandpa + 'static, { let LinkHalf { client, @@ -191,33 +194,32 @@ pub fn run_grandpa_observer( warn!("GRANDPA Observer failed: {:?}", e); }); - Ok(future::select(observer_work, on_exit).map(drop)) + Ok(observer_work.map(drop)) } /// Future that powers the observer. #[must_use] -struct ObserverWork, E, Backend, RA> { +struct ObserverWork> { observer: Pin>>> + Send>>, - client: Arc>, + client: Arc, network: NetworkBridge, persistent_data: PersistentData, keystore: Option, voter_commands_rx: mpsc::UnboundedReceiver>>, + _phantom: PhantomData, } -impl ObserverWork +impl ObserverWork where B: BlockT, - N: NetworkT, + BE: Backend + 'static, + Client: crate::ClientForGrandpa + 'static, + Network: NetworkT, NumberFor: BlockNumberOps, - RA: 'static + Send + Sync, - E: CallExecutor + Send + Sync + 'static, - Bk: Backend + 'static, - Client: AuxStore, { fn new( - client: Arc>, - network: NetworkBridge, + client: Arc, + network: NetworkBridge, persistent_data: PersistentData, keystore: Option, voter_commands_rx: mpsc::UnboundedReceiver>>, @@ -232,6 +234,7 @@ where persistent_data, keystore, voter_commands_rx, + _phantom: PhantomData, }; work.rebuild_observer(); work @@ -248,12 +251,12 @@ where let (global_in, _) = global_communication( set_id, &voters, - &self.client, + self.client.clone(), &self.network, &self.keystore, ); - let last_finalized_number = self.client.chain_info().finalized_number; + let last_finalized_number = self.client.info().finalized_number; // NOTE: since we are not using `round_communication` we have to // manually note the round with the gossip validator, otherwise we won't @@ -321,15 +324,13 @@ where } } -impl Future for ObserverWork +impl Future for ObserverWork where B: BlockT, + BE: Backend + Unpin + 'static, + C: crate::ClientForGrandpa + 'static, N: NetworkT, NumberFor: BlockNumberOps, - RA: 'static + Send + Sync, - E: CallExecutor + Send + Sync + 'static, - Bk: Backend + 'static, - Client: AuxStore, { type Output = Result<(), Error>; @@ -376,6 +377,7 @@ mod tests { use crate::{aux_schema, communication::tests::{Event, make_test_network}}; use substrate_test_runtime_client::{TestClientBuilder, TestClientBuilderExt}; use sc_network::PeerId; + use sp_blockchain::HeaderBackend as _; use futures::executor; @@ -403,7 +405,7 @@ mod tests { let persistent_data = aux_schema::load_persistent( &*backend, - client.chain_info().genesis_hash, + client.info().genesis_hash, 0, || Ok(vec![]), ).unwrap(); diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index 9b9063f2c17e973246d58a968dc286120a665224..0774194d7eb5200d21222e754b71b4aa78b52dc2 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -19,13 +19,13 @@ use super::*; use environment::HasVoted; use sc_network_test::{ - Block, DummySpecialization, Hash, TestNetFactory, BlockImportAdapter, Peer, - PeersClient, PassThroughVerifier, + Block, Hash, TestNetFactory, BlockImportAdapter, Peer, + PeersClient, PassThroughVerifier, PeersFullClient, }; use sc_network::config::{ProtocolConfig, Roles, BoxFinalityProofRequestBuilder}; use parking_lot::Mutex; use futures_timer::Delay; -use tokio::runtime::current_thread; +use tokio::runtime::{Runtime, Handle}; use sp_keyring::Ed25519Keyring; use sc_client::LongestChain; use sc_client_api::backend::TransactionFor; @@ -39,36 +39,33 @@ use sp_consensus::{ use std::{ collections::{HashMap, HashSet}, result, - pin::Pin, task, + pin::Pin, }; use parity_scale_codec::Decode; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT, HasherFor}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT, HashFor}; use sp_runtime::generic::{BlockId, DigestItem}; use sp_core::{H256, NativeOrEncoded, ExecutionContext, crypto::Public}; use sp_finality_grandpa::{GRANDPA_ENGINE_ID, AuthorityList, GrandpaApi}; use sp_state_machine::{InMemoryBackend, prove_read, read_proof_check}; -use futures01::Async; -use futures::compat::Future01CompatExt; use authorities::AuthoritySet; use finality_proof::{ FinalityProofProvider, AuthoritySetForFinalityProver, AuthoritySetForFinalityChecker, }; use consensus_changes::ConsensusChanges; +use sc_block_builder::BlockBuilderProvider; type PeerData = Mutex< Option< LinkHalf< - substrate_test_runtime_client::Backend, - substrate_test_runtime_client::Executor, Block, - substrate_test_runtime_client::runtime::RuntimeApi, + PeersFullClient, LongestChain > > >; -type GrandpaPeer = Peer; +type GrandpaPeer = Peer; struct GrandpaTestNet { peers: Vec, @@ -90,7 +87,6 @@ impl GrandpaTestNet { } impl TestNetFactory for GrandpaTestNet { - type Specialization = DummySpecialization; type Verifier = PassThroughVerifier; type PeerData = PeerData; @@ -171,11 +167,10 @@ impl TestNetFactory for GrandpaTestNet { fn make_finality_proof_provider( &self, client: PeersClient - ) -> Option>> { + ) -> Option>> { match client { PeersClient::Full(_, ref backend) => { - let authorities_provider = Arc::new(self.test_config.clone()); - Some(Arc::new(FinalityProofProvider::new(backend.clone(), authorities_provider))) + Some(Arc::new(FinalityProofProvider::new(backend.clone(), self.test_config.clone()))) }, PeersClient::Light(_, _) => None, } @@ -194,17 +189,6 @@ impl TestNetFactory for GrandpaTestNet { } } -#[derive(Clone)] -struct Exit; - -impl futures::Future for Exit { - type Output = (); - - fn poll(self: Pin<&mut Self>, _: &mut task::Context) -> task::Poll<()> { - task::Poll::Pending - } -} - #[derive(Default, Clone)] pub(crate) struct TestApi { genesis_authorities: AuthorityList, @@ -293,7 +277,7 @@ impl ApiExt for RuntimeApi { fn into_storage_changes( &self, _: &Self::StateBackend, - _: Option<&sp_api::ChangesTrieState, sp_api::NumberFor>>, + _: Option<&sp_api::ChangesTrieState, sp_api::NumberFor>>, _: ::Hash, ) -> std::result::Result, String> where Self: Sized @@ -327,7 +311,7 @@ impl AuthoritySetForFinalityProver for TestApi { fn prove_authorities(&self, block: &BlockId) -> Result { let authorities = self.authorities(block)?; - let backend = >>::from(vec![ + let backend = >>::from(vec![ (None, vec![(b"authorities".to_vec(), Some(authorities.encode()))]) ]); let proof = prove_read(backend, vec![b"authorities"]) @@ -343,7 +327,7 @@ impl AuthoritySetForFinalityChecker for TestApi { header: ::Header, proof: StorageProof, ) -> Result { - let results = read_proof_check::, _>( + let results = read_proof_check::, _>( *header.state_root(), proof, vec![b"authorities"] ) .expect("failure checking read proof for authorities"); @@ -372,27 +356,25 @@ fn create_keystore(authority: Ed25519Keyring) -> (KeyStorePtr, tempfile::TempDir (keystore, keystore_path) } -fn block_until_complete(future: impl Future + Unpin, net: &Arc>, runtime: &mut current_thread::Runtime) { - let drive_to_completion = futures01::future::poll_fn(|| { - net.lock().poll(); Ok::, ()>(Async::NotReady) +fn block_until_complete(future: impl Future + Unpin, net: &Arc>, runtime: &mut Runtime) { + let drive_to_completion = futures::future::poll_fn(|cx| { + net.lock().poll(cx); Poll::<()>::Pending }); runtime.block_on( - future::select(future, drive_to_completion.compat()) - .map(|_| Ok::<(), ()>(())) - .compat() - ).unwrap(); + future::select(future, drive_to_completion) + ); } // run the voters to completion. provide a closure to be invoked after // the voters are spawned but before blocking on them. fn run_to_completion_with( - runtime: &mut current_thread::Runtime, + runtime: &mut Runtime, blocks: u64, net: Arc>, peers: &[Ed25519Keyring], with: F, ) -> u64 where - F: FnOnce(current_thread::Handle) -> Option>>> + F: FnOnce(Handle) -> Option>>> { use parking_lot::RwLock; @@ -400,7 +382,7 @@ fn run_to_completion_with( let highest_finalized = Arc::new(RwLock::new(0)); - if let Some(f) = (with)(runtime.handle()) { + if let Some(f) = (with)(runtime.handle().clone()) { wait_for.push(f); }; @@ -450,15 +432,15 @@ fn run_to_completion_with( link: link, network: net_service, inherent_data_providers: InherentDataProviders::new(), - on_exit: Exit, telemetry_on_connect: None, voting_rule: (), + prometheus_registry: None, }; let voter = run_grandpa_voter(grandpa_params).expect("all in order with client and network"); assert_send(&voter); - runtime.spawn(voter.unit_error().compat()); + runtime.spawn(voter); } // wait for all finalized on each. @@ -470,7 +452,7 @@ fn run_to_completion_with( } fn run_to_completion( - runtime: &mut current_thread::Runtime, + runtime: &mut Runtime, blocks: u64, net: Arc>, peers: &[Ed25519Keyring] @@ -499,13 +481,13 @@ fn add_forced_change( #[test] fn finalize_3_voters_no_observers() { let _ = env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); + let mut runtime = Runtime::new().unwrap(); let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let voters = make_ids(peers); let mut net = GrandpaTestNet::new(TestApi::new(voters), 3); net.peer(0).push_blocks(20, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); for i in 0..3 { assert_eq!(net.peer(i).client().info().best_number, 20, @@ -524,14 +506,14 @@ fn finalize_3_voters_no_observers() { #[test] fn finalize_3_voters_1_full_observer() { - let mut runtime = current_thread::Runtime::new().unwrap(); + let mut runtime = Runtime::new().unwrap(); let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let voters = make_ids(peers); let mut net = GrandpaTestNet::new(TestApi::new(voters), 4); net.peer(0).push_blocks(20, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); let net = Arc::new(Mutex::new(net)); let mut finality_notifications = Vec::new(); @@ -581,16 +563,16 @@ fn finalize_3_voters_1_full_observer() { link: link, network: net_service, inherent_data_providers: InherentDataProviders::new(), - on_exit: Exit, telemetry_on_connect: None, voting_rule: (), + prometheus_registry: None, }; voters.push(run_grandpa_voter(grandpa_params).expect("all in order with client and network")); } for voter in voters { - runtime.spawn(voter.unit_error().compat()); + runtime.spawn(voter); } // wait for all finalized on each. @@ -628,10 +610,10 @@ fn transition_3_voters_twice_1_full_observer() { let api = TestApi::new(genesis_voters); let net = Arc::new(Mutex::new(GrandpaTestNet::new(api, 8))); - let mut runtime = current_thread::Runtime::new().unwrap(); + let mut runtime = Runtime::new().unwrap(); net.lock().peer(0).push_blocks(1, false); - net.lock().block_until_sync(&mut runtime); + net.lock().block_until_sync(); for (i, peer) in net.lock().peers().iter().enumerate() { let full_client = peer.client().as_full().expect("only full clients are used in test"); @@ -691,7 +673,7 @@ fn transition_3_voters_twice_1_full_observer() { future::ready(()) }); - runtime.spawn(block_production.unit_error().compat()); + runtime.spawn(block_production); } let mut finality_notifications = Vec::new(); @@ -744,13 +726,13 @@ fn transition_3_voters_twice_1_full_observer() { link: link, network: net_service, inherent_data_providers: InherentDataProviders::new(), - on_exit: Exit, telemetry_on_connect: None, voting_rule: (), + prometheus_registry: None, }; let voter = run_grandpa_voter(grandpa_params).expect("all in order with client and network"); - runtime.spawn(voter.unit_error().compat()); + runtime.spawn(voter); } // wait for all finalized on each. @@ -761,14 +743,14 @@ fn transition_3_voters_twice_1_full_observer() { #[test] fn justification_is_emitted_when_consensus_data_changes() { - let mut runtime = current_thread::Runtime::new().unwrap(); + let mut runtime = Runtime::new().unwrap(); let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let mut net = GrandpaTestNet::new(TestApi::new(make_ids(peers)), 3); // import block#1 WITH consensus data change let new_authorities = vec![sp_consensus_babe::AuthorityId::from_slice(&[42; 32])]; net.peer(0).push_authorities_change_block(new_authorities); - net.block_until_sync(&mut runtime); + net.block_until_sync(); let net = Arc::new(Mutex::new(net)); run_to_completion(&mut runtime, 1, net.clone(), peers); @@ -779,13 +761,13 @@ fn justification_is_emitted_when_consensus_data_changes() { #[test] fn justification_is_generated_periodically() { - let mut runtime = current_thread::Runtime::new().unwrap(); + let mut runtime = Runtime::new().unwrap(); let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let voters = make_ids(peers); let mut net = GrandpaTestNet::new(TestApi::new(voters), 3); net.peer(0).push_blocks(32, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); let net = Arc::new(Mutex::new(net)); run_to_completion(&mut runtime, 32, net.clone(), peers); @@ -818,7 +800,7 @@ fn consensus_changes_works() { #[test] fn sync_justifications_on_change_blocks() { - let mut runtime = current_thread::Runtime::new().unwrap(); + let mut runtime = Runtime::new().unwrap(); let peers_a = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let peers_b = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob]; let voters = make_ids(peers_b); @@ -842,7 +824,7 @@ fn sync_justifications_on_change_blocks() { // add more blocks on top of it (until we have 25) net.peer(0).push_blocks(4, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); for i in 0..4 { assert_eq!(net.peer(i).client().info().best_number, 25, @@ -859,9 +841,9 @@ fn sync_justifications_on_change_blocks() { } // the last peer should get the justification by syncing from other peers - futures::executor::block_on(futures::future::poll_fn(move |_| { + futures::executor::block_on(futures::future::poll_fn(move |cx| { if net.lock().peer(3).client().justification(&BlockId::Number(21)).unwrap().is_none() { - net.lock().poll(); + net.lock().poll(cx); Poll::Pending } else { Poll::Ready(()) @@ -872,7 +854,7 @@ fn sync_justifications_on_change_blocks() { #[test] fn finalizes_multiple_pending_changes_in_order() { let _ = env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); + let mut runtime = Runtime::new().unwrap(); let peers_a = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let peers_b = &[Ed25519Keyring::Dave, Ed25519Keyring::Eve, Ed25519Keyring::Ferdie]; @@ -917,7 +899,7 @@ fn finalizes_multiple_pending_changes_in_order() { // add more blocks on top of it (until we have 30) net.peer(0).push_blocks(4, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); // all peers imported both change blocks for i in 0..6 { @@ -932,7 +914,7 @@ fn finalizes_multiple_pending_changes_in_order() { #[test] fn force_change_to_new_set() { let _ = env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); + let mut runtime = Runtime::new().unwrap(); // two of these guys are offline. let genesis_authorities = &[ Ed25519Keyring::Alice, @@ -967,7 +949,7 @@ fn force_change_to_new_set() { }); net.lock().peer(0).push_blocks(25, false); - net.lock().block_until_sync(&mut runtime); + net.lock().block_until_sync(); for (i, peer) in net.lock().peers().iter().enumerate() { assert_eq!(peer.client().info().best_number, 26, @@ -1095,7 +1077,7 @@ fn voter_persists_its_votes() { use futures::channel::mpsc; let _ = env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); + let mut runtime = Runtime::new().unwrap(); // we have two authorities but we'll only be running the voter for alice // we are going to be listening for the prevotes it casts @@ -1105,7 +1087,7 @@ fn voter_persists_its_votes() { // alice has a chain with 20 blocks let mut net = GrandpaTestNet::new(TestApi::new(voters.clone()), 2); net.peer(0).push_blocks(20, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); assert_eq!(net.peer(0).client().info().best_number, 20, "Peer #{} failed to sync", 0); @@ -1169,9 +1151,9 @@ fn voter_persists_its_votes() { link, network: this.net.lock().peers[0].network_service().clone(), inherent_data_providers: InherentDataProviders::new(), - on_exit: Exit, telemetry_on_connect: None, voting_rule: VotingRulesBuilder::default().build(), + prometheus_registry: None, }; let voter = run_grandpa_voter(grandpa_params) @@ -1203,7 +1185,7 @@ fn voter_persists_its_votes() { net: net.clone(), client: client.clone(), keystore, - }.unit_error().compat()); + }); } let (exit_tx, exit_rx) = futures::channel::oneshot::channel::<()>(); @@ -1248,10 +1230,7 @@ fn voter_persists_its_votes() { HasVoted::No, ); - runtime.spawn( - network.map_err(|e| panic!("network bridge should not error: {:?}", e)) - .compat(), - ); + runtime.spawn(network); let round_tx = Arc::new(Mutex::new(round_tx)); let exit_tx = Arc::new(Mutex::new(Some(exit_tx))); @@ -1343,7 +1322,7 @@ fn voter_persists_its_votes() { panic!() } } - }).map(Ok).boxed().compat()); + })); } block_until_complete(exit_rx.into_future(), &net, &mut runtime); @@ -1352,13 +1331,13 @@ fn voter_persists_its_votes() { #[test] fn finalize_3_voters_1_light_observer() { let _ = env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); + let mut runtime = Runtime::new().unwrap(); let authorities = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let voters = make_ids(authorities); let mut net = GrandpaTestNet::new(TestApi::new(voters), 4); net.peer(0).push_blocks(20, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); for i in 0..4 { assert_eq!(net.peer(i).client().info().best_number, 20, @@ -1376,7 +1355,7 @@ fn finalize_3_voters_1_light_observer() { run_to_completion_with(&mut runtime, 20, net.clone(), authorities, |executor| { executor.spawn( - run_grandpa_observer( + observer::run_grandpa_observer( Config { gossip_duration: TEST_GOSSIP_DURATION, justification_period: 32, @@ -1387,9 +1366,8 @@ fn finalize_3_voters_1_light_observer() { }, link, net.lock().peers[3].network_service().clone(), - Exit, - ).unwrap().unit_error().compat() - ).unwrap(); + ).unwrap() + ); Some(Box::pin(finality_notifications.map(|_| ()))) }); @@ -1398,7 +1376,7 @@ fn finalize_3_voters_1_light_observer() { #[test] fn finality_proof_is_fetched_by_light_client_when_consensus_data_changes() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); + let mut runtime = Runtime::new().unwrap(); let peers = &[Ed25519Keyring::Alice]; let mut net = GrandpaTestNet::new(TestApi::new(make_ids(peers)), 1); @@ -1409,18 +1387,17 @@ fn finality_proof_is_fetched_by_light_client_when_consensus_data_changes() { net.peer(0).push_authorities_change_block(vec![sp_consensus_babe::AuthorityId::from_slice(&[42; 32])]); let net = Arc::new(Mutex::new(net)); run_to_completion(&mut runtime, 1, net.clone(), peers); - net.lock().block_until_sync(&mut runtime); + net.lock().block_until_sync(); // check that the block#1 is finalized on light client - let mut runtime = current_thread::Runtime::new().unwrap(); - let _ = runtime.block_on(futures::future::poll_fn(move |_| { + runtime.block_on(futures::future::poll_fn(move |cx| { if net.lock().peer(1).client().info().finalized_number == 1 { Poll::Ready(()) } else { - net.lock().poll(); + net.lock().poll(cx); Poll::Pending } - }).unit_error().compat()); + })); } #[test] @@ -1429,7 +1406,7 @@ fn empty_finality_proof_is_returned_to_light_client_when_authority_set_is_differ const FORCE_CHANGE: bool = true; let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); + let mut runtime = Runtime::new().unwrap(); // two of these guys are offline. let genesis_authorities = if FORCE_CHANGE { @@ -1474,14 +1451,14 @@ fn empty_finality_proof_is_returned_to_light_client_when_authority_set_is_differ vec![sp_consensus_babe::AuthorityId::from_slice(&[42; 32])] ); // #10 net.lock().peer(0).push_blocks(1, false); // best is #11 - net.lock().block_until_sync(&mut runtime); + net.lock().block_until_sync(); // finalize block #11 on full clients run_to_completion(&mut runtime, 11, net.clone(), peers_a); // request finalization by light client net.lock().add_light_peer(&GrandpaTestNet::default_config()); - net.lock().block_until_sync(&mut runtime); + net.lock().block_until_sync(); // check block, finalized on light client assert_eq!( @@ -1493,14 +1470,14 @@ fn empty_finality_proof_is_returned_to_light_client_when_authority_set_is_differ #[test] fn voter_catches_up_to_latest_round_when_behind() { let _ = env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); + let mut runtime = Runtime::new().unwrap(); let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob]; let voters = make_ids(peers); let mut net = GrandpaTestNet::new(TestApi::new(voters), 3); net.peer(0).push_blocks(50, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); let net = Arc::new(Mutex::new(net)); let mut finality_notifications = Vec::new(); @@ -1518,9 +1495,9 @@ fn voter_catches_up_to_latest_round_when_behind() { link, network: net.lock().peer(peer_id).network_service().clone(), inherent_data_providers: InherentDataProviders::new(), - on_exit: Exit, telemetry_on_connect: None, voting_rule: (), + prometheus_registry: None, }; Box::pin(run_grandpa_voter(grandpa_params).expect("all in order with client and network")) @@ -1550,19 +1527,18 @@ fn voter_catches_up_to_latest_round_when_behind() { let voter = voter(Some(keystore), peer_id, link, net.clone()); - runtime.spawn(voter.unit_error().compat()); + runtime.spawn(voter); } // wait for them to finalize block 50. since they'll vote on 3/4 of the // unfinalized chain it will take at least 4 rounds to do it. - let wait_for_finality = ::futures::future::join_all(finality_notifications) - .map(|_| ()); + let wait_for_finality = ::futures::future::join_all(finality_notifications); // spawn a new voter, it should be behind by at least 4 rounds and should be // able to catch up to the latest round let test = { let net = net.clone(); - let runtime = runtime.handle(); + let runtime = runtime.handle().clone(); wait_for_finality.then(move |_| { let peer_id = 2; @@ -1576,7 +1552,7 @@ fn voter_catches_up_to_latest_round_when_behind() { let voter = voter(None, peer_id, link, net); - runtime.spawn(voter.unit_error().compat()).unwrap(); + runtime.spawn(voter); let start_time = std::time::Instant::now(); let timeout = Duration::from_secs(5 * 60); @@ -1597,14 +1573,12 @@ fn voter_catches_up_to_latest_round_when_behind() { }) }; - let drive_to_completion = futures01::future::poll_fn(|| { - net.lock().poll(); Ok::, ()>(Async::NotReady) + let drive_to_completion = futures::future::poll_fn(|cx| { + net.lock().poll(cx); Poll::<()>::Pending }); runtime.block_on( - future::select(test, drive_to_completion.compat()) - .map(|_| Ok::<(), ()>(())) - .compat() - ).unwrap(); + future::select(test, drive_to_completion) + ); } #[test] @@ -1655,6 +1629,8 @@ fn grandpa_environment_respects_voting_rules() { voters: Arc::new(authority_set.current_authorities()), network, voting_rule, + metrics: None, + _phantom: PhantomData, } }; diff --git a/client/informant/Cargo.toml b/client/informant/Cargo.toml index 197f320889c0d556bc475d1fb3f410ce31e6c9b4..a29d793c3601b46b9e902fdde25cab18dfb8fc4b 100644 --- a/client/informant/Cargo.toml +++ b/client/informant/Cargo.toml @@ -1,10 +1,12 @@ [package] name = "sc-informant" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] description = "Substrate informant." edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] ansi_term = "0.12.1" @@ -12,8 +14,8 @@ futures = "0.3.1" log = "0.4.8" parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } wasm-timer = "0.2" -sc-client-api = { version = "2.0.0", path = "../api" } -sc-network = { version = "0.8", path = "../network" } -sc-service = { version = "0.8", default-features = false, path = "../service" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../api" } +sc-network = { version = "0.8.0-alpha.2", path = "../network" } +sc-service = { version = "0.8.0-alpha.2", default-features = false, path = "../service" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } diff --git a/client/informant/src/lib.rs b/client/informant/src/lib.rs index 699dcfdd7425e2b7b9e0f82eba481df2e81bf951..d104a64a2dbd121a00d27dde8de80f61fee0c12d 100644 --- a/client/informant/src/lib.rs +++ b/client/informant/src/lib.rs @@ -46,7 +46,10 @@ pub fn build(service: &impl AbstractService, format: OutputFormat) -> impl futur if let Some(ref usage) = info.usage { trace!(target: "usage", "Usage statistics: {}", usage); } else { - trace!(target: "usage", "Usage statistics not displayed as backend does not provide it") + trace!( + target: "usage", + "Usage statistics not displayed as backend does not provide it", + ) } #[cfg(not(target_os = "unknown"))] trace!( diff --git a/client/keystore/Cargo.toml b/client/keystore/Cargo.toml index 89ab787cb4bdc9e9c4fd68c12f79dd0aef63f997..247376bc46e17952cf6c1551becb4324cba90ba7 100644 --- a/client/keystore/Cargo.toml +++ b/client/keystore/Cargo.toml @@ -1,14 +1,19 @@ [package] name = "sc-keystore" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Keystore (and session key management) for ed25519 based chains like Polkadot." +documentation = "https://docs.rs/sc-keystore" + [dependencies] derive_more = "0.99.2" -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-application-crypto = { version = "2.0.0", path = "../../primitives/application-crypto" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +sp-application-crypto = { version = "2.0.0-alpha.2", path = "../../primitives/application-crypto" } hex = "0.4.0" rand = "0.7.2" serde_json = "1.0.41" diff --git a/client/network-gossip/Cargo.toml b/client/network-gossip/Cargo.toml index 98b2bd0590af94aeea4a8f87b0c14b47a7bca38b..08c304c6e0614edad4bfa0cd8902176a5a4c1c10 100644 --- a/client/network-gossip/Cargo.toml +++ b/client/network-gossip/Cargo.toml @@ -1,19 +1,22 @@ [package] description = "Gossiping for the Substrate network protocol" name = "sc-network-gossip" -version = "0.8.0" +version = "0.8.0-alpha.3" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +documentation = "https://docs.rs/sc-network-gossip" + [dependencies] -log = "0.4.8" -futures = { version = "0.3.1", features = ["compat"] } -wasm-timer = "0.2" +futures = "0.3.1" futures-timer = "3.0.1" -futures01 = { package = "futures", version = "0.1.29" } -libp2p = { version = "0.16.0", default-features = false, features = ["libp2p-websocket"] } -lru = "0.1.2" +libp2p = { version = "0.16.2", default-features = false, features = ["libp2p-websocket"] } +log = "0.4.8" +lru = "0.4.3" parking_lot = "0.10.0" -sc-network = { version = "0.8", path = "../network" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } +sc-network = { version = "0.8.0-alpha.2", path = "../network" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } +wasm-timer = "0.2" diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index 87958cbc14563dab3c41e3d9e84a6941c8468447..c911766aba40a4f41b0588ae2c33a9a51d5daab4 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -20,12 +20,11 @@ use crate::state_machine::{ConsensusGossip, TopicNotification, PERIODIC_MAINTENA use sc_network::message::generic::ConsensusMessage; use sc_network::{Event, ReputationChange}; -use futures::{prelude::*, channel::mpsc, compat::Compat01As03}; -use futures01::stream::Stream as Stream01; +use futures::{prelude::*, channel::mpsc}; use libp2p::PeerId; use parking_lot::Mutex; use sp_runtime::{traits::Block as BlockT, ConsensusEngineId}; -use std::{pin::Pin, sync::Arc, task::{Context, Poll}}; +use std::{borrow::Cow, pin::Pin, sync::Arc, task::{Context, Poll}}; /// Wraps around an implementation of the `Network` crate and provides gossiping capabilities on /// top of it. @@ -38,7 +37,7 @@ struct GossipEngineInner { state_machine: ConsensusGossip, network: Box + Send>, periodic_maintenance_interval: futures_timer::Delay, - network_event_stream: Compat01As03 + Send>>, + network_event_stream: Pin + Send>>, engine_id: ConsensusEngineId, } @@ -49,6 +48,7 @@ impl GossipEngine { pub fn new + Send + Clone + 'static>( mut network: N, engine_id: ConsensusEngineId, + protocol_name: impl Into>, validator: Arc>, ) -> Self where B: 'static { let mut state_machine = ConsensusGossip::new(); @@ -57,14 +57,14 @@ impl GossipEngine { // might miss events. let network_event_stream = network.event_stream(); - network.register_notifications_protocol(engine_id); + network.register_notifications_protocol(engine_id, protocol_name.into()); state_machine.register_validator(&mut network, engine_id, validator); let inner = Arc::new(Mutex::new(GossipEngineInner { state_machine, network: Box::new(network), periodic_maintenance_interval: futures_timer::Delay::new(PERIODIC_MAINTENANCE_INTERVAL), - network_event_stream: Compat01As03::new(network_event_stream), + network_event_stream, engine_id, })); @@ -178,7 +178,7 @@ impl Future for GossipEngineInner { fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { let this = &mut *self; - while let Poll::Ready(Some(Ok(event))) = this.network_event_stream.poll_next_unpin(cx) { + while let Poll::Ready(Some(event)) = this.network_event_stream.poll_next_unpin(cx) { match event { Event::NotificationStreamOpened { remote, engine_id: msg_engine_id, roles } => { if msg_engine_id != this.engine_id { diff --git a/client/network-gossip/src/lib.rs b/client/network-gossip/src/lib.rs index 705a27210ac53de386aedd4c9d681467d9acda55..4e4d32366f29d4c907561fc844e107edc60b8627 100644 --- a/client/network-gossip/src/lib.rs +++ b/client/network-gossip/src/lib.rs @@ -59,9 +59,9 @@ pub use self::state_machine::TopicNotification; pub use self::validator::{DiscardAll, MessageIntent, Validator, ValidatorContext, ValidationResult}; use futures::prelude::*; -use sc_network::{specialization::NetworkSpecialization, Event, ExHashT, NetworkService, PeerId, ReputationChange}; +use sc_network::{Event, ExHashT, NetworkService, PeerId, ReputationChange}; use sp_runtime::{traits::Block as BlockT, ConsensusEngineId}; -use std::sync::Arc; +use std::{borrow::Cow, pin::Pin, sync::Arc}; mod bridge; mod state_machine; @@ -70,7 +70,7 @@ mod validator; /// Abstraction over a network. pub trait Network { /// Returns a stream of events representing what happens on the network. - fn event_stream(&self) -> Box + Send>; + fn event_stream(&self) -> Pin + Send>>; /// Adjust the reputation of a node. fn report_peer(&self, peer_id: PeerId, reputation: ReputationChange); @@ -86,7 +86,8 @@ pub trait Network { /// See the documentation of [`NetworkService:register_notifications_protocol`] for more information. fn register_notifications_protocol( &self, - engine_id: ConsensusEngineId + engine_id: ConsensusEngineId, + protocol_name: Cow<'static, [u8]>, ); /// Notify everyone we're connected to that we have the given block. @@ -96,9 +97,9 @@ pub trait Network { fn announce(&self, block: B::Hash, associated_data: Vec); } -impl, H: ExHashT> Network for Arc> { - fn event_stream(&self) -> Box + Send> { - Box::new(NetworkService::event_stream(self).map(|v| Ok::<_, ()>(v)).compat()) +impl Network for Arc> { + fn event_stream(&self) -> Pin + Send>> { + Box::pin(NetworkService::event_stream(self)) } fn report_peer(&self, peer_id: PeerId, reputation: ReputationChange) { @@ -116,8 +117,9 @@ impl, H: ExHashT> Network for Arc, ) { - NetworkService::register_notifications_protocol(self, engine_id) + NetworkService::register_notifications_protocol(self, engine_id, protocol_name) } fn announce(&self, block: B::Hash, associated_data: Vec) { diff --git a/client/network-gossip/src/state_machine.rs b/client/network-gossip/src/state_machine.rs index 26433e63ec3ea7e09a6185628f4a5a3ec894e484..db5ea3603dcacea771f7bb1d458eac7b3c9bd9cc 100644 --- a/client/network-gossip/src/state_machine.rs +++ b/client/network-gossip/src/state_machine.rs @@ -258,6 +258,7 @@ impl ConsensusGossip { let mut context = NetworkContext { gossip: self, network, engine_id: engine_id.clone() }; v.peer_disconnected(&mut context, &who); } + self.peers.remove(&who); } /// Perform periodic maintenance @@ -644,4 +645,52 @@ mod tests { let _ = consensus.live_message_sinks.remove(&([0, 0, 0, 0], topic)); assert_eq!(stream.next(), None); } + + #[test] + fn peer_is_removed_on_disconnect() { + struct TestNetwork; + impl Network for TestNetwork { + fn event_stream( + &self, + ) -> std::pin::Pin + Send>> { + unimplemented!("Not required in tests") + } + + fn report_peer(&self, _: PeerId, _: crate::ReputationChange) { + unimplemented!("Not required in tests") + } + + fn disconnect_peer(&self, _: PeerId) { + unimplemented!("Not required in tests") + } + + fn write_notification(&self, _: PeerId, _: crate::ConsensusEngineId, _: Vec) { + unimplemented!("Not required in tests") + } + + fn register_notifications_protocol( + &self, + _: ConsensusEngineId, + _: std::borrow::Cow<'static, [u8]>, + ) { + unimplemented!("Not required in tests") + } + + fn announce(&self, _: H256, _: Vec) { + unimplemented!("Not required in tests") + } + } + + let mut consensus = ConsensusGossip::::new(); + consensus.register_validator_internal([0, 0, 0, 0], Arc::new(AllowAll)); + + let mut network = TestNetwork; + + let peer_id = PeerId::random(); + consensus.new_peer(&mut network, peer_id.clone(), Roles::FULL); + assert!(consensus.peers.contains_key(&peer_id)); + + consensus.peer_disconnected(&mut network, peer_id.clone()); + assert!(!consensus.peers.contains_key(&peer_id)); + } } diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index e53ba13c8d2e5861a0be48d3a6d76f472134b406..d99dce2fd2cdd6c78411e2cf412e4227a71aeafa 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -1,10 +1,14 @@ [package] description = "Substrate network protocol" name = "sc-network" -version = "0.8.0" +version = "0.8.0-alpha.3" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +documentation = "https://docs.rs/sc-network" + [build-dependencies] prost-build = "0.6.1" @@ -12,17 +16,17 @@ prost-build = "0.6.1" [dependencies] bitflags = "1.2.0" bytes = "0.5.0" -codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } derive_more = "0.99.2" either = "1.5.3" erased-serde = "0.3.9" fnv = "1.0.6" -fork-tree = { version = "2.0.0", path = "../../utils/fork-tree" } +fork-tree = { version = "2.0.0-alpha.2", path = "../../utils/fork-tree" } futures = "0.3.1" futures_codec = "0.3.3" futures-timer = "3.0.1" wasm-timer = "0.2" -libp2p = { version = "0.16.0", default-features = false, features = ["libp2p-websocket"] } +libp2p = { version = "0.16.2", default-features = false, features = ["libp2p-websocket"] } linked-hash-map = "0.5.2" linked_hash_set = "0.1.3" log = "0.4.8" @@ -32,26 +36,25 @@ parking_lot = "0.10.0" prost = "0.6.1" rand = "0.7.2" rustc-hex = "2.0.1" -sc-block-builder = { version = "0.8", path = "../block-builder" } -sc-client = { version = "0.8", path = "../" } -sc-client-api = { version = "2.0.0", path = "../api" } -sc-peerset = { version = "2.0.0", path = "../peerset" } +sc-block-builder = { version = "0.8.0-alpha.2", path = "../block-builder" } +sc-client = { version = "0.8.0-alpha.2", path = "../" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../api" } +sc-peerset = { version = "2.0.0-alpha.2", path = "../peerset" } +pin-project = "0.4.6" serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" slog = { version = "2.5.2", features = ["nested-values"] } slog_derive = "0.2.0" smallvec = "0.6.10" -sp-arithmetic = { version = "2.0.0", path = "../../primitives/arithmetic" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } -sp-consensus = { version = "0.8", path = "../../primitives/consensus/common" } -sp-consensus-babe = { version = "0.8", path = "../../primitives/consensus/babe" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-keyring = { version = "2.0.0", optional = true, path = "../../primitives/keyring" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -substrate-test-client = { version = "2.0.0", optional = true, path = "../../test-utils/client" } -substrate-test-runtime-client = { version = "2.0.0", optional = true, path = "../../test-utils/runtime/client" } +sp-arithmetic = { version = "2.0.0-alpha.2", path = "../../primitives/arithmetic" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../primitives/blockchain" } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../primitives/consensus/common" } +sp-consensus-babe = { version = "0.8.0-alpha.2", path = "../../primitives/consensus/babe" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.8.0-alpha.2", path = "../../utils/prometheus" } thiserror = "1" -unsigned-varint = { version = "0.3.0", features = ["futures-codec"] } +unsigned-varint = { version = "0.3.1", features = ["futures", "futures-codec"] } void = "1.0.2" zeroize = "1.0.0" @@ -61,12 +64,12 @@ assert_matches = "1.3" env_logger = "0.7.0" quickcheck = "0.9.0" rand = "0.7.2" -sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } -sp-test-primitives = { version = "2.0.0", path = "../../primitives/test-primitives" } -substrate-test-runtime = { version = "2.0.0", path = "../../test-utils/runtime" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } +sp-keyring = { version = "2.0.0-alpha.2", path = "../../primitives/keyring" } +sp-test-primitives = { version = "2.0.0-dev", path = "../../primitives/test-primitives" } +substrate-test-runtime = { version = "2.0.0-dev", path = "../../test-utils/runtime" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } tempfile = "3.1.0" [features] default = [] -test-helpers = ["sp-keyring", "substrate-test-runtime-client"] + diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index c8c5e59fe62cc3bc34d00aa441194b5993f6cd69..e7aca1975cd0d03f450e23528137db6511b1490e 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -16,9 +16,8 @@ use crate::{ debug_info, discovery::DiscoveryBehaviour, discovery::DiscoveryOut, DiscoveryNetBehaviour, - Event, protocol::event::DhtEvent + Event, protocol::event::DhtEvent, ExHashT, }; -use crate::{ExHashT, specialization::NetworkSpecialization}; use crate::protocol::{self, light_client_handler, CustomMessageOutcome, Protocol}; use libp2p::NetworkBehaviour; use libp2p::core::{Multiaddr, PeerId, PublicKey}; @@ -33,9 +32,9 @@ use void; /// General behaviour of the network. Combines all protocols together. #[derive(NetworkBehaviour)] #[behaviour(out_event = "BehaviourOut", poll_method = "poll")] -pub struct Behaviour, H: ExHashT> { +pub struct Behaviour { /// All the substrate-specific protocols. - substrate: Protocol, + substrate: Protocol, /// Periodically pings and identifies the nodes we are connected to, and store information in a /// cache. debug_info: debug_info::DebugInfoBehaviour, @@ -55,13 +54,15 @@ pub enum BehaviourOut { BlockImport(BlockOrigin, Vec>), JustificationImport(Origin, B::Hash, NumberFor, Justification), FinalityProofImport(Origin, B::Hash, NumberFor, Vec), + /// Started a random Kademlia discovery query. + RandomKademliaStarted, Event(Event), } -impl, H: ExHashT> Behaviour { +impl Behaviour { /// Builds a new `Behaviour`. pub async fn new( - substrate: Protocol, + substrate: Protocol, user_agent: String, local_public_key: PublicKey, known_addresses: Vec<(PeerId, Multiaddr)>, @@ -97,6 +98,11 @@ impl, H: ExHashT> Behaviour { self.discovery.add_known_address(peer_id, addr) } + /// Returns the number of nodes that are in the Kademlia k-buckets. + pub fn num_kbuckets_entries(&mut self) -> usize { + self.discovery.num_kbuckets_entries() + } + /// Borrows `self` and returns a struct giving access to the information about a node. /// /// Returns `None` if we don't know anything about this node. Always returns `Some` for nodes @@ -107,12 +113,12 @@ impl, H: ExHashT> Behaviour { } /// Returns a shared reference to the user protocol. - pub fn user_protocol(&self) -> &Protocol { + pub fn user_protocol(&self) -> &Protocol { &self.substrate } /// Returns a mutable reference to the user protocol. - pub fn user_protocol_mut(&mut self) -> &mut Protocol { + pub fn user_protocol_mut(&mut self) -> &mut Protocol { &mut self.substrate } @@ -133,15 +139,15 @@ impl, H: ExHashT> Behaviour { } } -impl, H: ExHashT> NetworkBehaviourEventProcess for -Behaviour { +impl NetworkBehaviourEventProcess for +Behaviour { fn inject_event(&mut self, event: void::Void) { void::unreachable(event) } } -impl, H: ExHashT> NetworkBehaviourEventProcess> for -Behaviour { +impl NetworkBehaviourEventProcess> for +Behaviour { fn inject_event(&mut self, event: CustomMessageOutcome) { match event { CustomMessageOutcome::BlockImport(origin, blocks) => @@ -174,8 +180,8 @@ Behaviour { } } -impl, H: ExHashT> NetworkBehaviourEventProcess - for Behaviour { +impl NetworkBehaviourEventProcess + for Behaviour { fn inject_event(&mut self, event: debug_info::DebugInfoEvent) { let debug_info::DebugInfoEvent::Identified { peer_id, mut info } = event; if info.listen_addrs.len() > 30 { @@ -192,8 +198,8 @@ impl, H: ExHashT> NetworkBehaviourEventPr } } -impl, H: ExHashT> NetworkBehaviourEventProcess - for Behaviour { +impl NetworkBehaviourEventProcess + for Behaviour { fn inject_event(&mut self, out: DiscoveryOut) { match out { DiscoveryOut::UnroutablePeer(_peer_id) => { @@ -217,11 +223,14 @@ impl, H: ExHashT> NetworkBehaviourEventPr DiscoveryOut::ValuePutFailed(key) => { self.events.push(BehaviourOut::Event(Event::Dht(DhtEvent::ValuePutFailed(key)))); } + DiscoveryOut::RandomKademliaStarted => { + self.events.push(BehaviourOut::RandomKademliaStarted); + } } } } -impl, H: ExHashT> Behaviour { +impl Behaviour { fn poll(&mut self, _: &mut Context, _: &mut impl PollParameters) -> Poll>> { if !self.events.is_empty() { return Poll::Ready(NetworkBehaviourAction::GenerateEvent(self.events.remove(0))) diff --git a/client/network/src/chain.rs b/client/network/src/chain.rs index b991a0e65208c2b8bee5c89162ddaec4755ef3f6..3c075ec881ca19395b719b58eaa20b68894c6106 100644 --- a/client/network/src/chain.rs +++ b/client/network/src/chain.rs @@ -18,7 +18,7 @@ use sc_client::Client as SubstrateClient; use sp_blockchain::{Error, Info as BlockchainInfo}; -use sc_client_api::{ChangesProof, StorageProof, CallExecutor}; +use sc_client_api::{ChangesProof, StorageProof, CallExecutor, ProofProvider}; use sp_consensus::{BlockImport, BlockStatus, Error as ConsensusError}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; use sp_runtime::generic::{BlockId}; @@ -50,7 +50,11 @@ pub trait Client: Send + Sync { -> Result<(Block::Header, StorageProof), Error>; /// Get storage read execution proof. - fn read_proof(&self, block: &Block::Hash, keys: &[Vec]) -> Result; + fn read_proof( + &self, + block: &Block::Hash, + keys: &mut dyn Iterator, + ) -> Result; /// Get child storage read execution proof. fn read_child_proof( @@ -58,7 +62,7 @@ pub trait Client: Send + Sync { block: &Block::Hash, storage_key: &[u8], child_info: ChildInfo, - keys: &[Vec], + keys: &mut dyn Iterator, ) -> Result; /// Get method execution proof. @@ -125,14 +129,19 @@ impl Client for SubstrateClient where (self as &SubstrateClient).justification(id) } - fn header_proof(&self, block_number: ::Number) - -> Result<(Block::Header, StorageProof), Error> - { - (self as &SubstrateClient).header_proof(&BlockId::Number(block_number)) + fn header_proof( + &self, + block_number: ::Number, + )-> Result<(Block::Header, StorageProof), Error> { + ProofProvider::::header_proof(self, &BlockId::Number(block_number)) } - fn read_proof(&self, block: &Block::Hash, keys: &[Vec]) -> Result { - (self as &SubstrateClient).read_proof(&BlockId::Hash(block.clone()), keys) + fn read_proof( + &self, + block: &Block::Hash, + keys: &mut dyn Iterator, + ) -> Result { + ProofProvider::::read_proof(self, &BlockId::Hash(block.clone()), keys) } fn read_child_proof( @@ -140,10 +149,9 @@ impl Client for SubstrateClient where block: &Block::Hash, storage_key: &[u8], child_info: ChildInfo, - keys: &[Vec], + keys: &mut dyn Iterator, ) -> Result { - (self as &SubstrateClient) - .read_child_proof(&BlockId::Hash(block.clone()), storage_key, child_info, keys) + ProofProvider::::read_child_proof(self, &BlockId::Hash(block.clone()), storage_key, child_info, keys) } fn execution_proof( @@ -152,7 +160,8 @@ impl Client for SubstrateClient where method: &str, data: &[u8], ) -> Result<(Vec, StorageProof), Error> { - (self as &SubstrateClient).execution_proof( + ProofProvider::::execution_proof( + self, &BlockId::Hash(block.clone()), method, data, @@ -168,7 +177,7 @@ impl Client for SubstrateClient where storage_key: Option<&StorageKey>, key: &StorageKey, ) -> Result, Error> { - (self as &SubstrateClient).key_changes_proof(first, last, min, max, storage_key, key) + ProofProvider::::key_changes_proof(self, first, last, min, max, storage_key, key) } fn is_descendent_of(&self, base: &Block::Hash, block: &Block::Hash) -> Result { diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 2f920c6663955aad80c8525543c9fca490887b79..76c0c9b5440f75a9eadca73539186a051afa9cca 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -19,12 +19,18 @@ //! The [`Params`] struct is the struct that must be passed in order to initialize the networking. //! See the documentation of [`Params`]. -pub use crate::protocol::ProtocolConfig; +pub use crate::chain::{Client, FinalityProofProvider}; +pub use crate::on_demand_layer::OnDemand; +pub use crate::service::TransactionPool; pub use libp2p::{identity, core::PublicKey, wasm_ext::ExtTransport, build_multiaddr}; -use crate::chain::{Client, FinalityProofProvider}; -use crate::on_demand_layer::OnDemand; -use crate::service::{ExHashT, TransactionPool}; +// Note: this re-export shouldn't be part of the public API of the crate and will be removed in +// the future. +#[doc(hidden)] +pub use crate::protocol::ProtocolConfig; + +use crate::service::ExHashT; + use bitflags::bitflags; use sp_consensus::{block_validation::BlockAnnounceValidator, import_queue::ImportQueue}; use sp_runtime::traits::{Block as BlockT}; @@ -35,9 +41,10 @@ use core::{fmt, iter}; use std::{future::Future, pin::Pin}; use std::{error::Error, fs, io::{self, Write}, net::Ipv4Addr, path::{Path, PathBuf}, sync::Arc}; use zeroize::Zeroize; +use prometheus_endpoint::Registry; /// Network initialization parameters. -pub struct Params { +pub struct Params { /// Assigned roles for our node (full, light, ...). pub roles: Roles, @@ -82,11 +89,11 @@ pub struct Params { /// valid. pub import_queue: Box>, - /// Customization of the network. Use this to plug additional networking capabilities. - pub specialization: S, - /// Type to check incoming block announcements. pub block_announce_validator: Box + Send>, + + /// Registry for recording prometheus metrics to. + pub metrics_registry: Option, } bitflags! { @@ -120,6 +127,12 @@ impl Roles { } } +impl fmt::Display for Roles { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + impl codec::Encode for Roles { fn encode_to(&self, dest: &mut T) { dest.push_byte(self.bits()) @@ -241,9 +254,9 @@ impl From for ParseErr { #[derive(Clone, Debug)] pub struct NetworkConfiguration { /// Directory path to store general network configuration. None means nothing will be saved. - pub config_path: Option, + pub config_path: Option, /// Directory path to store network-specific configuration. None means nothing will be saved. - pub net_config_path: Option, + pub net_config_path: Option, /// Multiaddresses to listen for incoming connections. pub listen_addresses: Vec, /// Multiaddresses to advertise. Detected automatically if empty. diff --git a/client/network/src/discovery.rs b/client/network/src/discovery.rs index a236ceb1a8b6d8ba4e3981f83e7577fb06224d4e..ecce7d81e30cd62ed2856f03aed35eaa480ce215 100644 --- a/client/network/src/discovery.rs +++ b/client/network/src/discovery.rs @@ -176,6 +176,11 @@ impl DiscoveryBehaviour { pub fn put_value(&mut self, key: record::Key, value: Vec) { self.kademlia.put_record(Record::new(key, value), Quorum::All); } + + /// Returns the number of nodes that are in the Kademlia k-buckets. + pub fn num_kbuckets_entries(&mut self) -> usize { + self.kademlia.kbuckets_entries().count() + } } /// Event generated by the `DiscoveryBehaviour`. @@ -203,6 +208,9 @@ pub enum DiscoveryOut { /// Inserting a value into the DHT failed. ValuePutFailed(record::Key), + + /// Started a random Kademlia query. + RandomKademliaStarted, } impl NetworkBehaviour for DiscoveryBehaviour { @@ -330,25 +338,33 @@ impl NetworkBehaviour for DiscoveryBehaviour { // Poll the stream that fires when we need to start a random Kademlia query. while let Poll::Ready(_) = self.next_kad_random_query.poll_unpin(cx) { - if self.num_connections < self.discovery_only_if_under_num { + let actually_started = if self.num_connections < self.discovery_only_if_under_num { let random_peer_id = PeerId::random(); debug!(target: "sub-libp2p", "Libp2p <= Starting random Kademlia request for \ {:?}", random_peer_id); self.kademlia.get_closest_peers(random_peer_id); + true + } else { debug!( target: "sub-libp2p", "Kademlia paused due to high number of connections ({})", self.num_connections ); - } + false + }; // Schedule the next random query with exponentially increasing delay, // capped at 60 seconds. self.next_kad_random_query = Delay::new(self.duration_to_next_kad); self.duration_to_next_kad = cmp::min(self.duration_to_next_kad * 2, Duration::from_secs(60)); + + if actually_started { + let ev = DiscoveryOut::RandomKademliaStarted; + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); + } } // Poll Kademlia. @@ -391,7 +407,14 @@ impl NetworkBehaviour for DiscoveryBehaviour { DiscoveryOut::ValueFound(results) } + Err(e @ libp2p::kad::GetRecordError::NotFound { .. }) => { + trace!(target: "sub-libp2p", + "Libp2p => Failed to get record: {:?}", e); + DiscoveryOut::ValueNotFound(e.into_key()) + } Err(e) => { + warn!(target: "sub-libp2p", + "Libp2p => Failed to get record: {:?}", e); DiscoveryOut::ValueNotFound(e.into_key()) } }; @@ -401,6 +424,8 @@ impl NetworkBehaviour for DiscoveryBehaviour { let ev = match res { Ok(ok) => DiscoveryOut::ValuePut(ok.key), Err(e) => { + warn!(target: "sub-libp2p", + "Libp2p => Failed to put record: {:?}", e); DiscoveryOut::ValuePutFailed(e.into_key()) } }; diff --git a/client/network/src/error.rs b/client/network/src/error.rs index ba5d5c2d0d2b5d3dde2329281e3e02d77edd961e..158e75fcf1d721551a3f8cfb51f4a3d8b93b73f4 100644 --- a/client/network/src/error.rs +++ b/client/network/src/error.rs @@ -45,6 +45,8 @@ pub enum Error { /// The second peer id that was found for the bootnode. second_id: PeerId, }, + /// Prometheus metrics error. + Prometheus(prometheus_endpoint::PrometheusError) } // Make `Debug` use the `Display` implementation. @@ -60,6 +62,7 @@ impl std::error::Error for Error { Error::Io(ref err) => Some(err), Error::Client(ref err) => Some(err), Error::DuplicateBootnode { .. } => None, + Error::Prometheus(ref err) => Some(err), } } } diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index dd156360330acfa73f8115e027fc31910034ed81..a5397a4e3e6263f2c6c33263b7002549cedcacae 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -78,12 +78,8 @@ //! - DNS for addresses of the form `/dns4/example.com/tcp/5` or `/dns4/example.com/tcp/5/ws`. A //! node's address can contain a domain name. //! -//! The following encryption protocols are supported: -//! -//! - [Secio](https://github.com/libp2p/specs/tree/master/secio). A TLS-1.2-like protocol but -//! without certificates. Support for secio will likely be deprecated in the far future. -//! - [Noise](https://noiseprotocol.org/). Support for noise is very experimental. The details are -//! very blurry and may change at any moment. +//! On top of the base-layer protocol, the [Noise](https://noiseprotocol.org/) protocol is +//! negotiated and applied. The exact handshake protocol is experimental and is subject to change. //! //! The following multiplexing protocols are supported: //! @@ -140,10 +136,6 @@ //! - Light-client requests. When a light client requires information, a random node we have a //! substream open with is chosen, and the information is requested from it. //! - Gossiping. Used for example by grandpa. -//! - Network specialization. The network protocol can be specialized through a template parameter -//! of the network service. This specialization is free to send and receive messages with the -//! remote. This is meant to be used by the chain that is being built on top of Substrate -//! (eg. Polkadot). //! //! It is intended that in the future each of these components gets more isolated, so that they //! are free to open and close their own substreams, and so that syncing and light client requests @@ -160,7 +152,8 @@ //! //! After the `NetworkWorker` has been created, the important things to do are: //! -//! - Calling `NetworkWorker::poll` in order to advance the network. +//! - Calling `NetworkWorker::poll` in order to advance the network. This can be done by +//! dispatching a background task with the [`NetworkWorker`]. //! - Calling `on_block_import` whenever a block is added to the client. //! - Calling `on_block_finalized` whenever a block is finalized. //! - Calling `trigger_repropagate` when a transaction is added to the pool. @@ -180,34 +173,27 @@ mod utils; pub mod config; pub mod error; +pub mod network_state; -pub use chain::{Client as ClientHandle, FinalityProofProvider}; -pub use service::{ - NetworkService, NetworkWorker, TransactionPool, ExHashT, ReportHandle, - NetworkStateInfo, -}; -pub use protocol::{PeerInfo, Context, ProtocolConfig, message, specialization}; +pub use service::{NetworkService, NetworkStateInfo, NetworkWorker, ExHashT, ReportHandle}; +pub use protocol::PeerInfo; pub use protocol::event::{Event, DhtEvent}; pub use protocol::sync::SyncState; pub use libp2p::{Multiaddr, PeerId}; #[doc(inline)] pub use libp2p::multiaddr; -pub use message::{generic as generic_message, RequestId, Status as StatusMessage}; -pub use on_demand_layer::{OnDemand, RemoteResponse}; -pub use sc_peerset::ReputationChange; - -// Used by the `construct_simple_protocol!` macro. +// Note: these re-exports shouldn't be part of the public API of the crate and will be removed in +// the future. +#[doc(hidden)] +pub use protocol::message; #[doc(hidden)] -pub use sp_runtime::traits::Block as BlockT; +pub use protocol::message::Status as StatusMessage; -use libp2p::core::ConnectedPoint; -use serde::{Deserialize, Serialize}; -use slog_derive::SerdeValue; -use std::{collections::{HashMap, HashSet}, time::Duration}; +pub use sc_peerset::ReputationChange; /// Extension trait for `NetworkBehaviour` that also accepts discovering nodes. -pub trait DiscoveryNetBehaviour { +trait DiscoveryNetBehaviour { /// Notify the protocol that we have learned about the existence of nodes. /// /// Can (or most likely will) be called multiple times with the same `PeerId`s. @@ -216,90 +202,3 @@ pub trait DiscoveryNetBehaviour { /// system, or remove nodes that will fail to reach. fn add_discovered_nodes(&mut self, nodes: impl Iterator); } - -/// Returns general information about the networking. -/// -/// Meant for general diagnostic purposes. -/// -/// **Warning**: This API is not stable. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, SerdeValue)] -#[serde(rename_all = "camelCase")] -pub struct NetworkState { - /// PeerId of the local node. - pub peer_id: String, - /// List of addresses the node is currently listening on. - pub listened_addresses: HashSet, - /// List of addresses the node knows it can be reached as. - pub external_addresses: HashSet, - /// List of node we're connected to. - pub connected_peers: HashMap, - /// List of node that we know of but that we're not connected to. - pub not_connected_peers: HashMap, - /// Downloaded bytes per second averaged over the past few seconds. - pub average_download_per_sec: u64, - /// Uploaded bytes per second averaged over the past few seconds. - pub average_upload_per_sec: u64, - /// State of the peerset manager. - pub peerset: serde_json::Value, -} - -/// Part of the `NetworkState` struct. Unstable. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct NetworkStatePeer { - /// How we are connected to the node. - pub endpoint: NetworkStatePeerEndpoint, - /// Node information, as provided by the node itself. Can be empty if not known yet. - pub version_string: Option, - /// Latest ping duration with this node. - pub latest_ping_time: Option, - /// If true, the peer is "enabled", which means that we try to open Substrate-related protocols - /// with this peer. If false, we stick to Kademlia and/or other network-only protocols. - pub enabled: bool, - /// If true, the peer is "open", which means that we have a Substrate-related protocol - /// with this peer. - pub open: bool, - /// List of addresses known for this node. - pub known_addresses: HashSet, -} - -/// Part of the `NetworkState` struct. Unstable. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct NetworkStateNotConnectedPeer { - /// List of addresses known for this node. - pub known_addresses: HashSet, - /// Node information, as provided by the node itself, if we were ever connected to this node. - pub version_string: Option, - /// Latest ping duration with this node, if we were ever connected to this node. - pub latest_ping_time: Option, -} - -/// Part of the `NetworkState` struct. Unstable. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum NetworkStatePeerEndpoint { - /// We are dialing the given address. - Dialing(Multiaddr), - /// We are listening. - Listening { - /// Local address of the connection. - local_addr: Multiaddr, - /// Address data is sent back to. - send_back_addr: Multiaddr, - }, -} - -impl From for NetworkStatePeerEndpoint { - fn from(endpoint: ConnectedPoint) -> Self { - match endpoint { - ConnectedPoint::Dialer { address } => - NetworkStatePeerEndpoint::Dialing(address), - ConnectedPoint::Listener { local_addr, send_back_addr } => - NetworkStatePeerEndpoint::Listening { - local_addr, - send_back_addr - } - } - } -} diff --git a/client/network/src/network_state.rs b/client/network/src/network_state.rs new file mode 100644 index 0000000000000000000000000000000000000000..00d53976ae8fc8960e9045b1dbab7d1621429f26 --- /dev/null +++ b/client/network/src/network_state.rs @@ -0,0 +1,111 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Information about the networking, for diagnostic purposes. +//! +//! **Warning**: These APIs are not stable. + +use libp2p::{core::ConnectedPoint, Multiaddr}; +use serde::{Deserialize, Serialize}; +use slog_derive::SerdeValue; +use std::{collections::{HashMap, HashSet}, time::Duration}; + +/// Returns general information about the networking. +/// +/// Meant for general diagnostic purposes. +/// +/// **Warning**: This API is not stable. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, SerdeValue)] +#[serde(rename_all = "camelCase")] +pub struct NetworkState { + /// PeerId of the local node. + pub peer_id: String, + /// List of addresses the node is currently listening on. + pub listened_addresses: HashSet, + /// List of addresses the node knows it can be reached as. + pub external_addresses: HashSet, + /// List of node we're connected to. + pub connected_peers: HashMap, + /// List of node that we know of but that we're not connected to. + pub not_connected_peers: HashMap, + /// Downloaded bytes per second averaged over the past few seconds. + pub average_download_per_sec: u64, + /// Uploaded bytes per second averaged over the past few seconds. + pub average_upload_per_sec: u64, + /// State of the peerset manager. + pub peerset: serde_json::Value, +} + +/// Part of the `NetworkState` struct. Unstable. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Peer { + /// How we are connected to the node. + pub endpoint: PeerEndpoint, + /// Node information, as provided by the node itself. Can be empty if not known yet. + pub version_string: Option, + /// Latest ping duration with this node. + pub latest_ping_time: Option, + /// If true, the peer is "enabled", which means that we try to open Substrate-related protocols + /// with this peer. If false, we stick to Kademlia and/or other network-only protocols. + pub enabled: bool, + /// If true, the peer is "open", which means that we have a Substrate-related protocol + /// with this peer. + pub open: bool, + /// List of addresses known for this node. + pub known_addresses: HashSet, +} + +/// Part of the `NetworkState` struct. Unstable. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NotConnectedPeer { + /// List of addresses known for this node. + pub known_addresses: HashSet, + /// Node information, as provided by the node itself, if we were ever connected to this node. + pub version_string: Option, + /// Latest ping duration with this node, if we were ever connected to this node. + pub latest_ping_time: Option, +} + +/// Part of the `NetworkState` struct. Unstable. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum PeerEndpoint { + /// We are dialing the given address. + Dialing(Multiaddr), + /// We are listening. + Listening { + /// Local address of the connection. + local_addr: Multiaddr, + /// Address data is sent back to. + send_back_addr: Multiaddr, + }, +} + +impl From for PeerEndpoint { + fn from(endpoint: ConnectedPoint) -> Self { + match endpoint { + ConnectedPoint::Dialer { address } => + PeerEndpoint::Dialing(address), + ConnectedPoint::Listener { local_addr, send_back_addr } => + PeerEndpoint::Listening { + local_addr, + send_back_addr + } + } + } +} diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index d5e7ae6252ca147e59da50be6c22b8acb7d772f0..17208aab50fe06d171fab48fadf75c3c487ec711 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -15,10 +15,10 @@ // along with Substrate. If not, see . use crate::{DiscoveryNetBehaviour, config::ProtocolId}; -use legacy_proto::{LegacyProto, LegacyProtoOut}; use crate::utils::interval; use bytes::{Bytes, BytesMut}; use futures::prelude::*; +use generic_proto::{GenericProto, GenericProtoOut}; use libp2p::{Multiaddr, PeerId}; use libp2p::core::{ConnectedPoint, nodes::listeners::ListenerId}; use libp2p::swarm::{ProtocolsHandler, IntoProtocolsHandler}; @@ -36,13 +36,14 @@ use sp_runtime::traits::{ }; use sp_arithmetic::traits::SaturatedConversion; use message::{BlockAnnounce, BlockAttributes, Direction, FromBlock, Message, RequestId}; -use message::generic::{Message as GenericMessage, ConsensusMessage}; +use message::generic::Message as GenericMessage; use light_dispatch::{LightDispatch, LightDispatchNetwork, RequestData}; -use specialization::NetworkSpecialization; +use prometheus_endpoint::{Registry, Gauge, register, PrometheusError, U64}; use sync::{ChainSync, SyncState}; use crate::service::{TransactionPool, ExHashT}; use crate::config::{BoxFinalityProofRequestBuilder, Roles}; use rustc_hex::ToHex; +use std::borrow::Cow; use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; use std::fmt::Write; @@ -64,7 +65,7 @@ pub mod api { } } -mod legacy_proto; +mod generic_proto; mod util; pub mod block_requests; @@ -72,7 +73,6 @@ pub mod message; pub mod event; pub mod light_client_handler; pub mod light_dispatch; -pub mod specialization; pub mod sync; pub use block_requests::BlockRequests; @@ -132,10 +132,111 @@ mod rep { pub const BAD_PROTOCOL: Rep = Rep::new_fatal("Unsupported protocol"); /// Peer role does not match (e.g. light peer connecting to another light peer). pub const BAD_ROLE: Rep = Rep::new_fatal("Unsupported role"); + /// Peer response data does not have requested bits. + pub const BAD_RESPONSE: Rep = Rep::new(-(1 << 12), "Incomplete response"); +} + +struct Metrics { + handshaking_peers: Gauge, + obsolete_requests: Gauge, + peers: Gauge, + queued_blocks: Gauge, + fork_targets: Gauge, + finality_proofs_pending: Gauge, + finality_proofs_active: Gauge, + finality_proofs_failed: Gauge, + finality_proofs_importing: Gauge, + justifications_pending: Gauge, + justifications_active: Gauge, + justifications_failed: Gauge, + justifications_importing: Gauge +} + +impl Metrics { + fn register(r: &Registry) -> Result { + Ok(Metrics { + handshaking_peers: { + let g = Gauge::new("sync_handshaking_peers", "number of newly connected peers")?; + register(g, r)? + }, + obsolete_requests: { + let g = Gauge::new("sync_obsolete_requests", "total number of obsolete requests")?; + register(g, r)? + }, + peers: { + let g = Gauge::new("sync_peers", "number of peers we sync with")?; + register(g, r)? + }, + queued_blocks: { + let g = Gauge::new("sync_queued_blocks", "number of blocks in import queue")?; + register(g, r)? + }, + fork_targets: { + let g = Gauge::new("sync_fork_targets", "fork sync targets")?; + register(g, r)? + }, + justifications_pending: { + let g = Gauge::new( + "sync_extra_justifications_pending", + "number of pending extra justifications requests" + )?; + register(g, r)? + }, + justifications_active: { + let g = Gauge::new( + "sync_extra_justifications_active", + "number of active extra justifications requests" + )?; + register(g, r)? + }, + justifications_failed: { + let g = Gauge::new( + "sync_extra_justifications_failed", + "number of failed extra justifications requests" + )?; + register(g, r)? + }, + justifications_importing: { + let g = Gauge::new( + "sync_extra_justifications_importing", + "number of importing extra justifications requests" + )?; + register(g, r)? + }, + finality_proofs_pending: { + let g = Gauge::new( + "sync_extra_finality_proofs_pending", + "number of pending extra finality proof requests" + )?; + register(g, r)? + }, + finality_proofs_active: { + let g = Gauge::new( + "sync_extra_finality_proofs_active", + "number of active extra finality proof requests" + )?; + register(g, r)? + }, + finality_proofs_failed: { + let g = Gauge::new( + "sync_extra_finality_proofs_failed", + "number of failed extra finality proof requests" + )?; + register(g, r)? + }, + finality_proofs_importing: { + let g = Gauge::new( + "sync_extra_finality_proofs_importing", + "number of importing extra finality proof requests" + )?; + register(g, r)? + }, + }) + } } // Lock must always be taken in order declared here. -pub struct Protocol, H: ExHashT> { +pub struct Protocol { /// Interval at which we call `tick`. tick_timeout: Pin + Send>>, /// Interval at which we call `propagate_extrinsics`. @@ -145,7 +246,6 @@ pub struct Protocol, H: ExHashT> { light_dispatch: LightDispatch, genesis_hash: B::Hash, sync: ChainSync, - specialization: S, context_data: ContextData, /// List of nodes for which we perform additional logging because they are important for the /// user. @@ -158,9 +258,13 @@ pub struct Protocol, H: ExHashT> { /// When asked for a proof of finality, we use this struct to build one. finality_proof_provider: Option>>, /// Handles opening the unique substream and sending and receiving raw messages. - behaviour: LegacyProto, - /// List of notification protocols that have been registered. - registered_notif_protocols: HashSet, + behaviour: GenericProto, + /// For each legacy gossiping engine ID, the corresponding new protocol name. + protocol_name_by_engine: HashMap>, + /// For each protocol name, the legacy gossiping engine ID. + protocol_engine_by_name: HashMap, ConsensusEngineId>, + /// Prometheus metrics. + metrics: Option, } #[derive(Default)] @@ -207,7 +311,7 @@ pub struct PeerInfo { } struct LightDispatchIn<'a> { - behaviour: &'a mut LegacyProto, + behaviour: &'a mut GenericProto, peerset: sc_peerset::PeersetHandle, } @@ -332,55 +436,6 @@ impl<'a, B: BlockT> LightDispatchNetwork for LightDispatchIn<'a> { } } -/// Context for a network-specific handler. -pub trait Context { - /// Adjusts the reputation of the peer. Use this to point out that a peer has been malign or - /// irresponsible or appeared lazy. - fn report_peer(&mut self, who: PeerId, reputation: sc_peerset::ReputationChange); - - /// Force disconnecting from a peer. Use this when a peer misbehaved. - fn disconnect_peer(&mut self, who: PeerId); - - /// Send a chain-specific message to a peer. - fn send_chain_specific(&mut self, who: PeerId, message: Vec); -} - -/// Protocol context. -struct ProtocolContext<'a, B: 'a + BlockT, H: 'a + ExHashT> { - behaviour: &'a mut LegacyProto, - context_data: &'a mut ContextData, - peerset_handle: &'a sc_peerset::PeersetHandle, -} - -impl<'a, B: BlockT + 'a, H: 'a + ExHashT> ProtocolContext<'a, B, H> { - fn new( - context_data: &'a mut ContextData, - behaviour: &'a mut LegacyProto, - peerset_handle: &'a sc_peerset::PeersetHandle, - ) -> Self { - ProtocolContext { context_data, peerset_handle, behaviour } - } -} - -impl<'a, B: BlockT + 'a, H: ExHashT + 'a> Context for ProtocolContext<'a, B, H> { - fn report_peer(&mut self, who: PeerId, reputation: sc_peerset::ReputationChange) { - self.peerset_handle.report_peer(who, reputation) - } - - fn disconnect_peer(&mut self, who: PeerId) { - self.behaviour.disconnect_peer(&who) - } - - fn send_chain_specific(&mut self, who: PeerId, message: Vec) { - send_message:: ( - self.behaviour, - &mut self.context_data.stats, - &who, - GenericMessage::ChainSpecific(message) - ) - } -} - /// Data necessary to create a context. struct ContextData { // All connected peers @@ -407,20 +462,20 @@ impl Default for ProtocolConfig { } } -impl, H: ExHashT> Protocol { +impl Protocol { /// Create a new instance. pub fn new( config: ProtocolConfig, chain: Arc>, checker: Arc>, - specialization: S, transaction_pool: Arc>, finality_proof_provider: Option>>, finality_proof_request_builder: Option>, protocol_id: ProtocolId, peerset_config: sc_peerset::PeersetConfig, - block_announce_validator: Box + Send> - ) -> error::Result<(Protocol, sc_peerset::PeersetHandle)> { + block_announce_validator: Box + Send>, + metrics_registry: Option<&Registry> + ) -> error::Result<(Protocol, sc_peerset::PeersetHandle)> { let info = chain.info(); let sync = ChainSync::new( config.roles, @@ -442,7 +497,7 @@ impl, H: ExHashT> Protocol { let (peerset, peerset_handle) = sc_peerset::Peerset::from_config(peerset_config); let versions = &((MIN_VERSION as u8)..=(CURRENT_VERSION as u8)).collect::>(); - let behaviour = LegacyProto::new(protocol_id, versions, peerset); + let behaviour = GenericProto::new(protocol_id, versions, peerset); let protocol = Protocol { tick_timeout: Box::pin(interval(TICK_TIMEOUT)), @@ -456,14 +511,19 @@ impl, H: ExHashT> Protocol { light_dispatch: LightDispatch::new(checker), genesis_hash: info.genesis_hash, sync, - specialization, handshaking_peers: HashMap::new(), important_peers, transaction_pool, finality_proof_provider, peerset_handle: peerset_handle.clone(), behaviour, - registered_notif_protocols: HashSet::new(), + protocol_name_by_engine: HashMap::new(), + protocol_engine_by_name: HashMap::new(), + metrics: if let Some(r) = metrics_registry { + Some(Metrics::register(r)?) + } else { + None + } }; Ok((protocol, peerset_handle)) @@ -479,6 +539,16 @@ impl, H: ExHashT> Protocol { self.behaviour.is_open(peer_id) } + /// Returns the list of all the peers that the peerset currently requests us to be connected to. + pub fn requested_peers(&self) -> impl Iterator { + self.behaviour.requested_peers() + } + + /// Returns the number of discovered nodes that we keep in memory. + pub fn num_discovered_peers(&self) -> usize { + self.behaviour.num_discovered_peers() + } + /// Disconnects the given peer if we are connected to it. pub fn disconnect_peer(&mut self, peer_id: &PeerId) { self.behaviour.disconnect_peer(peer_id) @@ -646,7 +716,7 @@ impl, H: ExHashT> Protocol { GenericMessage::RemoteReadChildRequest(request) => self.on_remote_read_child_request(who, request), GenericMessage::Consensus(msg) => - return if self.registered_notif_protocols.contains(&msg.engine_id) { + return if self.protocol_name_by_engine.contains_key(&msg.engine_id) { CustomMessageOutcome::NotificationsReceived { remote: who.clone(), messages: vec![(msg.engine_id, From::from(msg.data))], @@ -659,7 +729,7 @@ impl, H: ExHashT> Protocol { let messages = messages .into_iter() .filter_map(|msg| { - if self.registered_notif_protocols.contains(&msg.engine_id) { + if self.protocol_name_by_engine.contains_key(&msg.engine_id) { Some((msg.engine_id, From::from(msg.data))) } else { warn!(target: "sync", "Received message on non-registered protocol: {:?}", msg.engine_id); @@ -677,11 +747,6 @@ impl, H: ExHashT> Protocol { CustomMessageOutcome::None }; }, - GenericMessage::ChainSpecific(msg) => self.specialization.on_message( - &mut ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle), - who, - msg, - ), } CustomMessageOutcome::None @@ -706,14 +771,6 @@ impl, H: ExHashT> Protocol { ); } - /// Locks `self` and returns a context plus the network specialization. - pub fn specialization_lock<'a>( - &'a mut self, - ) -> (impl Context + 'a, &'a mut S) { - let context = ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle); - (context, &mut self.specialization) - } - /// Called when a new peer is connected pub fn on_peer_connected(&mut self, who: PeerId) { trace!(target: "sync", "Connecting {}", who); @@ -735,9 +792,7 @@ impl, H: ExHashT> Protocol { self.context_data.peers.remove(&peer) }; if let Some(_peer_data) = removed { - let mut context = ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle); self.sync.peer_disconnected(peer.clone()); - self.specialization.on_disconnect(&mut context, peer.clone()); self.light_dispatch.on_disconnect(LightDispatchIn { behaviour: &mut self.behaviour, peerset: self.peerset_handle.clone(), @@ -766,12 +821,14 @@ impl, H: ExHashT> Protocol { peer: PeerId, request: message::BlockRequest ) { - trace!(target: "sync", "BlockRequest {} from {}: from {:?} to {:?} max {:?}", + trace!(target: "sync", "BlockRequest {} from {}: from {:?} to {:?} max {:?} for {:?}", request.id, peer, request.from, request.to, - request.max); + request.max, + request.fields, + ); // sending block requests to the node that is unable to serve it is considered a bad behavior if !self.config.roles.is_full() { @@ -819,6 +876,11 @@ impl, H: ExHashT> Protocol { message_queue: None, justification, }; + // Stop if we don't have requested block body + if get_body && block_data.body.is_none() { + trace!(target: "sync", "Missing data for block request."); + break; + } blocks.push(block_data); match request.direction { message::Direction::Ascending => id = BlockId::Number(number + One::one()), @@ -849,7 +911,7 @@ impl, H: ExHashT> Protocol { request: message::BlockRequest, response: message::BlockResponse, ) -> CustomMessageOutcome { - let blocks_range = match ( + let blocks_range = || match ( response.blocks.first().and_then(|b| b.header.as_ref().map(|h| h.number())), response.blocks.last().and_then(|b| b.header.as_ref().map(|h| h.number())), ) { @@ -861,7 +923,7 @@ impl, H: ExHashT> Protocol { response.id, peer, response.blocks.len(), - blocks_range + blocks_range(), ); if request.fields == message::BlockAttributes::JUSTIFICATION { @@ -876,6 +938,20 @@ impl, H: ExHashT> Protocol { } } } else { + // Validate fields against the request. + if request.fields.contains(message::BlockAttributes::HEADER) && response.blocks.iter().any(|b| b.header.is_none()) { + self.behaviour.disconnect_peer(&peer); + self.peerset_handle.report_peer(peer, rep::BAD_RESPONSE); + trace!(target: "sync", "Missing header for a block"); + return CustomMessageOutcome::None + } + if request.fields.contains(message::BlockAttributes::BODY) && response.blocks.iter().any(|b| b.body.is_none()) { + self.behaviour.disconnect_peer(&peer); + self.peerset_handle.report_peer(peer, rep::BAD_RESPONSE); + trace!(target: "sync", "Missing body for a block"); + return CustomMessageOutcome::None + } + match self.sync.on_block_data(peer, Some(request), response) { Ok(sync::OnBlockData::Import(origin, blocks)) => CustomMessageOutcome::BlockImport(origin, blocks), @@ -901,6 +977,7 @@ impl, H: ExHashT> Protocol { behaviour: &mut self.behaviour, peerset: self.peerset_handle.clone(), }); + self.report_metrics() } fn maintain_peers(&mut self) { @@ -936,9 +1013,6 @@ impl, H: ExHashT> Protocol { } } - self.specialization.maintain_peers( - &mut ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle) - ); for p in aborting { self.behaviour.disconnect_peer(&p); self.peerset_handle.report_peer(p, rep::TIMEOUT); @@ -1054,13 +1128,10 @@ impl, H: ExHashT> Protocol { } } - let mut context = ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle); - self.specialization.on_connect(&mut context, who.clone(), status); - // Notify all the notification protocols as open. CustomMessageOutcome::NotificationStreamOpened { remote: who, - protocols: self.registered_notif_protocols.iter().cloned().collect(), + protocols: self.protocol_name_by_engine.keys().cloned().collect(), roles: info.roles, } } @@ -1075,18 +1146,15 @@ impl, H: ExHashT> Protocol { engine_id: ConsensusEngineId, message: impl Into> ) { - if !self.registered_notif_protocols.contains(&engine_id) { + if let Some(protocol_name) = self.protocol_name_by_engine.get(&engine_id) { + self.behaviour.write_notification(&target, engine_id, protocol_name.clone(), message); + } else { error!( target: "sub-libp2p", "Sending a notification with a protocol that wasn't registered: {:?}", engine_id ); } - - self.send_message(&target, GenericMessage::Consensus(ConsensusMessage { - engine_id, - data: message.into(), - })); } /// Registers a new notifications protocol. @@ -1096,9 +1164,14 @@ impl, H: ExHashT> Protocol { pub fn register_notifications_protocol( &mut self, engine_id: ConsensusEngineId, + protocol_name: impl Into>, ) -> Vec { - if !self.registered_notif_protocols.insert(engine_id) { - error!(target: "sub-libp2p", "Notifications protocol already registered: {:?}", engine_id); + let protocol_name = protocol_name.into(); + if self.protocol_name_by_engine.insert(engine_id, protocol_name.clone()).is_some() { + error!(target: "sub-libp2p", "Notifications protocol already registered: {:?}", protocol_name); + } else { + self.behaviour.register_notif_protocol(protocol_name.clone(), engine_id, Vec::new()); + self.protocol_engine_by_name.insert(protocol_name, engine_id); } // Registering a protocol while we already have open connections isn't great, but for now @@ -1286,7 +1359,7 @@ impl, H: ExHashT> Protocol { roles: self.config.roles.into(), best_number: info.best_number, best_hash: info.best_hash, - chain_status: self.specialization.status(), + chain_status: Vec::new(), // TODO: find a way to make this backwards-compatible }; self.send_message(&who, GenericMessage::Status(status)) @@ -1355,15 +1428,10 @@ impl, H: ExHashT> Protocol { /// Call this when a block has been imported in the import queue and we should announce it on /// the network. - pub fn on_block_imported(&mut self, hash: B::Hash, header: &B::Header, data: Vec, is_best: bool) { + pub fn on_block_imported(&mut self, header: &B::Header, data: Vec, is_best: bool) { if is_best { self.sync.update_chain_info(header); } - self.specialization.on_block_imported( - &mut ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle), - hash.clone(), - header, - ); // blocks are not announced by light clients if self.config.roles.is_light() { @@ -1524,7 +1592,10 @@ impl, H: ExHashT> Protocol { trace!(target: "sync", "Remote read request {} from {} ({} at {})", request.id, who, keys_str(), request.block); - let proof = match self.context_data.chain.read_proof(&request.block, &request.keys) { + let proof = match self.context_data.chain.read_proof( + &request.block, + &mut request.keys.iter().map(AsRef::as_ref) + ) { Ok(proof) => proof, Err(error) => { trace!(target: "sync", "Remote read request {} from {} ({} at {}) failed with: {}", @@ -1574,7 +1645,7 @@ impl, H: ExHashT> Protocol { &request.block, &request.storage_key, child_info, - &request.keys, + &mut request.keys.iter().map(AsRef::as_ref), ) { Ok(proof) => proof, Err(error) => { @@ -1815,6 +1886,40 @@ impl, H: ExHashT> Protocol { } out } + + fn report_metrics(&self) { + use std::convert::TryInto; + + if let Some(metrics) = &self.metrics { + let mut obsolete_requests: u64 = 0; + for peer in self.context_data.peers.values() { + let n = peer.obsolete_requests.len().try_into().unwrap_or(std::u64::MAX); + obsolete_requests = obsolete_requests.saturating_add(n); + } + metrics.obsolete_requests.set(obsolete_requests); + + let n = self.handshaking_peers.len().try_into().unwrap_or(std::u64::MAX); + metrics.handshaking_peers.set(n); + + let n = self.context_data.peers.len().try_into().unwrap_or(std::u64::MAX); + metrics.peers.set(n); + + let m = self.sync.metrics(); + + metrics.fork_targets.set(m.fork_targets.into()); + metrics.queued_blocks.set(m.queued_blocks.into()); + + metrics.justifications_pending.set(m.justifications.pending_requests.into()); + metrics.justifications_active.set(m.justifications.active_requests.into()); + metrics.justifications_failed.set(m.justifications.failed_requests.into()); + metrics.justifications_importing.set(m.justifications.importing_requests.into()); + + metrics.finality_proofs_pending.set(m.finality_proofs.pending_requests.into()); + metrics.finality_proofs_active.set(m.finality_proofs.active_requests.into()); + metrics.finality_proofs_failed.set(m.finality_proofs.failed_requests.into()); + metrics.finality_proofs_importing.set(m.finality_proofs.importing_requests.into()); + } + } } /// Outcome of an incoming custom message. @@ -1833,7 +1938,7 @@ pub enum CustomMessageOutcome { } fn send_request( - behaviour: &mut LegacyProto, + behaviour: &mut GenericProto, stats: &mut HashMap<&'static str, PacketStats>, peers: &mut HashMap>, who: &PeerId, @@ -1854,7 +1959,7 @@ fn send_request( } fn send_message( - behaviour: &mut LegacyProto, + behaviour: &mut GenericProto, stats: &mut HashMap<&'static str, PacketStats>, who: &PeerId, message: Message, @@ -1866,9 +1971,8 @@ fn send_message( behaviour.send_packet(who, encoded); } -impl, H: ExHashT> NetworkBehaviour for -Protocol { - type ProtocolsHandler = ::ProtocolsHandler; +impl NetworkBehaviour for Protocol { + type ProtocolsHandler = ::ProtocolsHandler; type OutEvent = CustomMessageOutcome; fn new_handler(&mut self) -> Self::ProtocolsHandler { @@ -1954,25 +2058,21 @@ Protocol { }; let outcome = match event { - LegacyProtoOut::CustomProtocolOpen { peer_id, version, .. } => { - debug_assert!( - version <= CURRENT_VERSION as u8 - && version >= MIN_VERSION as u8 - ); + GenericProtoOut::CustomProtocolOpen { peer_id, .. } => { self.on_peer_connected(peer_id.clone()); CustomMessageOutcome::None } - LegacyProtoOut::CustomProtocolClosed { peer_id, .. } => { + GenericProtoOut::CustomProtocolClosed { peer_id, .. } => { self.on_peer_disconnected(peer_id.clone()); // Notify all the notification protocols as closed. CustomMessageOutcome::NotificationStreamClosed { remote: peer_id, - protocols: self.registered_notif_protocols.iter().cloned().collect(), + protocols: self.protocol_name_by_engine.keys().cloned().collect(), } }, - LegacyProtoOut::CustomMessage { peer_id, message } => + GenericProtoOut::CustomMessage { peer_id, message } => self.on_custom_message(peer_id, message), - LegacyProtoOut::Clogged { peer_id, messages } => { + GenericProtoOut::Clogged { peer_id, messages } => { debug!(target: "sync", "{} clogging messages:", messages.len()); for msg in messages.into_iter().take(5) { let message: Option> = Decode::decode(&mut &msg[..]).ok(); @@ -2028,13 +2128,13 @@ Protocol { } } -impl, H: ExHashT> DiscoveryNetBehaviour for Protocol { +impl DiscoveryNetBehaviour for Protocol { fn add_discovered_nodes(&mut self, peer_ids: impl Iterator) { self.behaviour.add_discovered_nodes(peer_ids) } } -impl, H: ExHashT> Drop for Protocol { +impl Drop for Protocol { fn drop(&mut self) { debug!(target: "sync", "Network stats:\n{}", self.format_stats()); } diff --git a/client/network/src/protocol/block_requests.rs b/client/network/src/protocol/block_requests.rs index ef970657c5ff9477bc939b6dd98963b4e41387e9..20a99378b16c992436ea2a4ee686fa0fc684d7f3 100644 --- a/client/network/src/protocol/block_requests.rs +++ b/client/network/src/protocol/block_requests.rs @@ -111,7 +111,7 @@ impl Config { let mut v = Vec::new(); v.extend_from_slice(b"/"); v.extend_from_slice(id.as_bytes()); - v.extend_from_slice(b"/sync/1"); + v.extend_from_slice(b"/sync/2"); self.protocol = v.into(); self } @@ -146,8 +146,7 @@ where , request: &api::v1::BlockRequest ) -> Result { - log::trace!("block request {} from peer {}: from block {:?} to block {:?}, max blocks {:?}", - request.id, + log::trace!("block request from peer {}: from block {:?} to block {:?}, max blocks {:?}", peer, request.from_block, request.to_block, @@ -242,7 +241,7 @@ where } } - Ok(api::v1::BlockResponse { id: request.id, blocks }) + Ok(api::v1::BlockResponse { blocks }) } } @@ -274,10 +273,10 @@ where fn inject_node_event(&mut self, peer: PeerId, Request(request, mut stream): Request) { match self.on_block_request(&peer, &request) { Ok(res) => { - log::trace!("enqueueing block response {} for peer {} with {} blocks", res.id, peer, res.blocks.len()); + log::trace!("enqueueing block response for peer {} with {} blocks", peer, res.blocks.len()); let mut data = Vec::with_capacity(res.encoded_len()); if let Err(e) = res.encode(&mut data) { - log::debug!("error encoding block response {} for peer {}: {}", res.id, peer, e) + log::debug!("error encoding block response for peer {}: {}", peer, e) } else { let future = async move { if let Err(e) = write_one(&mut stream, data).await { @@ -287,7 +286,7 @@ where self.outgoing.push(future.boxed()) } } - Err(e) => log::debug!("error handling block request {} from peer {}: {}", request.id, peer, e) + Err(e) => log::debug!("error handling block request from peer {}: {}", peer, e) } } diff --git a/client/network/src/protocol/legacy_proto.rs b/client/network/src/protocol/generic_proto.rs similarity index 86% rename from client/network/src/protocol/legacy_proto.rs rename to client/network/src/protocol/generic_proto.rs index 434782f7d5065de6372fa651f1ed1ac29b0209aa..f703287f386fdcf9aebcfc8d3fa3d0c5971c7eff 100644 --- a/client/network/src/protocol/legacy_proto.rs +++ b/client/network/src/protocol/generic_proto.rs @@ -17,10 +17,10 @@ //! Implementation of libp2p's `NetworkBehaviour` trait that opens a single substream with the //! remote and then allows any communication with them. //! -//! The `Protocol` struct uses `LegacyProto` in order to open substreams with the rest of the +//! The `Protocol` struct uses `GenericProto` in order to open substreams with the rest of the //! network, then performs the Substrate protocol handling on top. -pub use self::behaviour::{LegacyProto, LegacyProtoOut}; +pub use self::behaviour::{GenericProto, GenericProtoOut}; mod behaviour; mod handler; diff --git a/client/network/src/protocol/legacy_proto/behaviour.rs b/client/network/src/protocol/generic_proto/behaviour.rs similarity index 84% rename from client/network/src/protocol/legacy_proto/behaviour.rs rename to client/network/src/protocol/generic_proto/behaviour.rs index 69c89be9a36957f808773a76d54e8b43ea195251..727415baaf5bcd9a4b06ca847d9aeb6968f78ab1 100644 --- a/client/network/src/protocol/legacy_proto/behaviour.rs +++ b/client/network/src/protocol/generic_proto/behaviour.rs @@ -15,9 +15,12 @@ // along with Substrate. If not, see . use crate::{DiscoveryNetBehaviour, config::ProtocolId}; -use crate::protocol::legacy_proto::handler::{CustomProtoHandlerProto, CustomProtoHandlerOut, CustomProtoHandlerIn}; -use crate::protocol::legacy_proto::upgrade::RegisteredProtocol; +use crate::protocol::message::generic::{Message as GenericMessage, ConsensusMessage}; +use crate::protocol::generic_proto::handler::{NotifsHandlerProto, NotifsHandlerOut, NotifsHandlerIn}; +use crate::protocol::generic_proto::upgrade::RegisteredProtocol; + use bytes::BytesMut; +use codec::Encode as _; use fnv::FnvHashMap; use futures::prelude::*; use libp2p::core::{ConnectedPoint, Multiaddr, PeerId}; @@ -25,16 +28,32 @@ use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}; use log::{debug, error, trace, warn}; use rand::distributions::{Distribution as _, Uniform}; use smallvec::SmallVec; -use std::{borrow::Cow, collections::hash_map::Entry, cmp, error, mem, pin::Pin}; -use std::time::Duration; -use wasm_timer::Instant; +use sp_runtime::ConsensusEngineId; +use std::{borrow::Cow, collections::hash_map::Entry, cmp}; +use std::{error, mem, pin::Pin, str, time::Duration}; use std::task::{Context, Poll}; +use wasm_timer::Instant; /// Network behaviour that handles opening substreams for custom protocols with other nodes. /// +/// ## Legacy vs new protocol +/// +/// The `GenericProto` behaves as following: +/// +/// - Whenever a connection is established, we open a single substream (called "legay protocol" in +/// the source code). This substream name depends on the `protocol_id` and `versions` passed at +/// initialization. If the remote refuses this substream, we close the connection. +/// +/// - For each registered protocol, we also open an additional substream for this protocol. If the +/// remote refuses this substream, then it's fine. +/// +/// - Whenever we want to send a message, we can call either `send_packet` to force the legacy +/// substream, or `write_notification` to indicate a registered protocol. If the registered +/// protocol was refused or isn't supported by the remote, we always use the legacy instead. +/// /// ## How it works /// -/// The role of the `LegacyProto` is to synchronize the following components: +/// The role of the `GenericProto` is to synchronize the following components: /// /// - The libp2p swarm that opens new connections and reports disconnects. /// - The connection handler (see `handler.rs`) that handles individual connections. @@ -60,9 +79,12 @@ use std::task::{Context, Poll}; /// Note that this "banning" system is not an actual ban. If a "banned" node tries to connect to /// us, we accept the connection. The "banning" system is only about delaying dialing attempts. /// -pub struct LegacyProto { - /// List of protocols to open with peers. Never modified. - protocol: RegisteredProtocol, +pub struct GenericProto { + /// Legacy protocol to open with peers. Never modified. + legacy_protocol: RegisteredProtocol, + + /// Notification protocols. Entries are only ever added and not removed. + notif_protocols: Vec<(Cow<'static, [u8]>, ConsensusEngineId, Vec)>, /// Receiver for instructions about who to connect to or disconnect from. peerset: sc_peerset::Peerset, @@ -79,7 +101,7 @@ pub struct LegacyProto { next_incoming_index: sc_peerset::IncomingIndex, /// Events to produce from `poll()`. - events: SmallVec<[NetworkBehaviourAction; 4]>, + events: SmallVec<[NetworkBehaviourAction; 4]>, } /// State of a peer we're connected to. @@ -169,6 +191,20 @@ impl PeerState { PeerState::Incoming { .. } => false, } } + + /// True if that node has been requested by the PSM. + fn is_requested(&self) -> bool { + match self { + PeerState::Poisoned => false, + PeerState::Banned { .. } => false, + PeerState::PendingRequest { .. } => true, + PeerState::Requested => true, + PeerState::Disabled { .. } => false, + PeerState::DisabledPendingEnable { .. } => true, + PeerState::Enabled { .. } => true, + PeerState::Incoming { .. } => false, + } + } } /// State of an "incoming" message sent to the peer set manager. @@ -183,13 +219,11 @@ struct IncomingPeer { incoming_id: sc_peerset::IncomingIndex, } -/// Event that can be emitted by the `LegacyProto`. +/// Event that can be emitted by the `GenericProto`. #[derive(Debug)] -pub enum LegacyProtoOut { +pub enum GenericProtoOut { /// Opened a custom protocol with the remote. CustomProtocolOpen { - /// Version of the protocol that has been opened. - version: u8, /// Id of the node we have opened a connection with. peer_id: PeerId, /// Endpoint used for this custom protocol. @@ -205,6 +239,8 @@ pub enum LegacyProtoOut { }, /// Receives a message on a custom protocol substream. + /// + /// Also concerns received notifications for the notifications API. CustomMessage { /// Id of the peer the message came from. peer_id: PeerId, @@ -222,17 +258,18 @@ pub enum LegacyProtoOut { }, } -impl LegacyProto { +impl GenericProto { /// Creates a `CustomProtos`. pub fn new( protocol: impl Into, versions: &[u8], peerset: sc_peerset::Peerset, ) -> Self { - let protocol = RegisteredProtocol::new(protocol, versions); + let legacy_protocol = RegisteredProtocol::new(protocol, versions); - LegacyProto { - protocol, + GenericProto { + legacy_protocol, + notif_protocols: Vec::new(), peerset, peers: FnvHashMap::default(), incoming: SmallVec::new(), @@ -241,6 +278,24 @@ impl LegacyProto { } } + /// Registers a new notifications protocol. + /// + /// You are very strongly encouraged to call this method very early on. Any open connection + /// will retain the protocols that were registered then, and not any new one. + pub fn register_notif_protocol( + &mut self, + protocol_name: impl Into>, + engine_id: ConsensusEngineId, + handshake_msg: impl Into> + ) { + self.notif_protocols.push((protocol_name.into(), engine_id, handshake_msg.into())); + } + + /// Returns the number of discovered nodes that we keep in memory. + pub fn num_discovered_peers(&self) -> usize { + self.peerset.num_discovered_peers() + } + /// Returns the list of all the peers we have an open channel to. pub fn open_peers<'a>(&'a self) -> impl Iterator + 'a { self.peers.iter().filter(|(_, state)| state.is_open()).map(|(id, _)| id) @@ -292,7 +347,7 @@ impl LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", peer_id); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: peer_id.clone(), - event: CustomProtoHandlerIn::Disable, + event: NotifsHandlerIn::Disable, }); let banned_until = ban.map(|dur| Instant::now() + dur); *entry.into_mut() = PeerState::Disabled { open, connected_point, banned_until } @@ -313,7 +368,7 @@ impl LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", peer_id); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: peer_id.clone(), - event: CustomProtoHandlerIn::Disable, + event: NotifsHandlerIn::Disable, }); let banned_until = ban.map(|dur| Instant::now() + dur); *entry.into_mut() = PeerState::Disabled { open: false, connected_point, banned_until } @@ -324,6 +379,11 @@ impl LegacyProto { } } + /// Returns the list of all the peers that the peerset currently requests us to be connected to. + pub fn requested_peers<'a>(&'a self) -> impl Iterator + 'a { + self.peers.iter().filter(|(_, state)| state.is_requested()).map(|(id, _)| id) + } + /// Returns true if we try to open protocols with the given peer. pub fn is_enabled(&self, peer_id: &PeerId) -> bool { match self.peers.get(peer_id) { @@ -339,6 +399,44 @@ impl LegacyProto { } } + /// Sends a notification to a peer. + /// + /// Has no effect if the custom protocol is not open with the given peer. + /// + /// Also note that even if we have a valid open substream, it may in fact be already closed + /// without us knowing, in which case the packet will not be received. + /// + /// > **Note**: Ideally the `engine_id` parameter wouldn't be necessary. See the documentation + /// > of [`NotifsHandlerIn`] for more information. + pub fn write_notification( + &mut self, + target: &PeerId, + engine_id: ConsensusEngineId, + protocol_name: Cow<'static, [u8]>, + message: impl Into>, + ) { + if !self.is_open(target) { + return; + } + + trace!( + target: "sub-libp2p", + "External API => Notification for {:?} with protocol {:?}", + target, + str::from_utf8(&protocol_name) + ); + trace!(target: "sub-libp2p", "Handler({:?}) <= Packet", target); + + self.events.push(NetworkBehaviourAction::SendEvent { + peer_id: target.clone(), + event: NotifsHandlerIn::SendNotification { + message: message.into(), + engine_id, + protocol_name, + }, + }); + } + /// Sends a message to a peer. /// /// Has no effect if the custom protocol is not open with the given peer. @@ -354,7 +452,7 @@ impl LegacyProto { trace!(target: "sub-libp2p", "Handler({:?}) <= Packet", target); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: target.clone(), - event: CustomProtoHandlerIn::SendCustomMessage { + event: NotifsHandlerIn::SendLegacy { message, } }); @@ -416,7 +514,7 @@ impl LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", occ_entry.key()); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: occ_entry.key().clone(), - event: CustomProtoHandlerIn::Enable, + event: NotifsHandlerIn::Enable, }); *occ_entry.into_mut() = PeerState::Enabled { connected_point, open }; }, @@ -434,7 +532,7 @@ impl LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", occ_entry.key()); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: occ_entry.key().clone(), - event: CustomProtoHandlerIn::Enable, + event: NotifsHandlerIn::Enable, }); *occ_entry.into_mut() = PeerState::Enabled { connected_point, open: false }; }, @@ -491,7 +589,7 @@ impl LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", entry.key()); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: entry.key().clone(), - event: CustomProtoHandlerIn::Disable, + event: NotifsHandlerIn::Disable, }); *entry.into_mut() = PeerState::Disabled { open, connected_point, banned_until: None } }, @@ -555,7 +653,7 @@ impl LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", incoming.peer_id); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: incoming.peer_id, - event: CustomProtoHandlerIn::Enable, + event: NotifsHandlerIn::Enable, }); *state = PeerState::Enabled { open: false, connected_point }; @@ -597,13 +695,13 @@ impl LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", incoming.peer_id); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: incoming.peer_id, - event: CustomProtoHandlerIn::Disable, + event: NotifsHandlerIn::Disable, }); *state = PeerState::Disabled { open: false, connected_point, banned_until: None }; } } -impl DiscoveryNetBehaviour for LegacyProto { +impl DiscoveryNetBehaviour for GenericProto { fn add_discovered_nodes(&mut self, peer_ids: impl Iterator) { self.peerset.discovered(peer_ids.into_iter().map(|peer_id| { debug!(target: "sub-libp2p", "PSM <= Discovered({:?})", peer_id); @@ -612,12 +710,12 @@ impl DiscoveryNetBehaviour for LegacyProto { } } -impl NetworkBehaviour for LegacyProto { - type ProtocolsHandler = CustomProtoHandlerProto; - type OutEvent = LegacyProtoOut; +impl NetworkBehaviour for GenericProto { + type ProtocolsHandler = NotifsHandlerProto; + type OutEvent = GenericProtoOut; fn new_handler(&mut self) -> Self::ProtocolsHandler { - CustomProtoHandlerProto::new(self.protocol.clone()) + NotifsHandlerProto::new(self.legacy_protocol.clone(), self.notif_protocols.clone()) } fn addresses_of_peer(&mut self, _: &PeerId) -> Vec { @@ -634,7 +732,7 @@ impl NetworkBehaviour for LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", peer_id); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: peer_id.clone(), - event: CustomProtoHandlerIn::Enable, + event: NotifsHandlerIn::Enable, }); *st = PeerState::Enabled { open: false, connected_point }; } @@ -677,7 +775,7 @@ impl NetworkBehaviour for LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", peer_id); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: peer_id.clone(), - event: CustomProtoHandlerIn::Disable, + event: NotifsHandlerIn::Disable, }); *st = PeerState::Disabled { open: false, connected_point, banned_until }; } @@ -707,7 +805,7 @@ impl NetworkBehaviour for LegacyProto { } if open { debug!(target: "sub-libp2p", "External API <= Closed({:?})", peer_id); - let event = LegacyProtoOut::CustomProtocolClosed { + let event = GenericProtoOut::CustomProtocolClosed { peer_id: peer_id.clone(), reason: "Disconnected by libp2p".into(), }; @@ -724,7 +822,7 @@ impl NetworkBehaviour for LegacyProto { self.peers.insert(peer_id.clone(), PeerState::Banned { until: timer_deadline }); if open { debug!(target: "sub-libp2p", "External API <= Closed({:?})", peer_id); - let event = LegacyProtoOut::CustomProtocolClosed { + let event = GenericProtoOut::CustomProtocolClosed { peer_id: peer_id.clone(), reason: "Disconnected by libp2p".into(), }; @@ -746,7 +844,7 @@ impl NetworkBehaviour for LegacyProto { if open { debug!(target: "sub-libp2p", "External API <= Closed({:?})", peer_id); - let event = LegacyProtoOut::CustomProtocolClosed { + let event = GenericProtoOut::CustomProtocolClosed { peer_id: peer_id.clone(), reason: "Disconnected by libp2p".into(), }; @@ -817,10 +915,10 @@ impl NetworkBehaviour for LegacyProto { fn inject_node_event( &mut self, source: PeerId, - event: CustomProtoHandlerOut, + event: NotifsHandlerOut, ) { match event { - CustomProtoHandlerOut::CustomProtocolClosed { reason } => { + NotifsHandlerOut::Closed { reason } => { debug!(target: "sub-libp2p", "Handler({:?}) => Closed: {}", source, reason); let mut entry = if let Entry::Occupied(entry) = self.peers.entry(source.clone()) { @@ -831,7 +929,7 @@ impl NetworkBehaviour for LegacyProto { }; debug!(target: "sub-libp2p", "External API <= Closed({:?})", source); - let event = LegacyProtoOut::CustomProtocolClosed { + let event = GenericProtoOut::CustomProtocolClosed { reason, peer_id: source.clone(), }; @@ -847,7 +945,7 @@ impl NetworkBehaviour for LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", source); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: source.clone(), - event: CustomProtoHandlerIn::Disable, + event: NotifsHandlerIn::Disable, }); *entry.into_mut() = PeerState::Disabled { @@ -873,8 +971,8 @@ impl NetworkBehaviour for LegacyProto { } } - CustomProtoHandlerOut::CustomProtocolOpen { version } => { - debug!(target: "sub-libp2p", "Handler({:?}) => Open: version {:?}", source, version); + NotifsHandlerOut::Open => { + debug!(target: "sub-libp2p", "Handler({:?}) => Open", source); let endpoint = match self.peers.get_mut(&source) { Some(PeerState::Enabled { ref mut open, ref connected_point }) | Some(PeerState::DisabledPendingEnable { ref mut open, ref connected_point, .. }) | @@ -889,8 +987,7 @@ impl NetworkBehaviour for LegacyProto { }; debug!(target: "sub-libp2p", "External API <= Open({:?})", source); - let event = LegacyProtoOut::CustomProtocolOpen { - version, + let event = GenericProtoOut::CustomProtocolOpen { peer_id: source, endpoint, }; @@ -898,11 +995,11 @@ impl NetworkBehaviour for LegacyProto { self.events.push(NetworkBehaviourAction::GenerateEvent(event)); } - CustomProtoHandlerOut::CustomMessage { message } => { + NotifsHandlerOut::CustomMessage { message } => { debug_assert!(self.is_open(&source)); trace!(target: "sub-libp2p", "Handler({:?}) => Message", source); trace!(target: "sub-libp2p", "External API <= Message({:?})", source); - let event = LegacyProtoOut::CustomMessage { + let event = GenericProtoOut::CustomMessage { peer_id: source, message, }; @@ -910,25 +1007,50 @@ impl NetworkBehaviour for LegacyProto { self.events.push(NetworkBehaviourAction::GenerateEvent(event)); } - CustomProtoHandlerOut::Clogged { messages } => { + NotifsHandlerOut::Notification { protocol_name, engine_id, message } => { + debug_assert!(self.is_open(&source)); + trace!( + target: "sub-libp2p", + "Handler({:?}) => Notification({:?})", + source, + str::from_utf8(&protocol_name) + ); + trace!(target: "sub-libp2p", "External API <= Message({:?})", source); + let event = GenericProtoOut::CustomMessage { + peer_id: source, + message: { + let message = GenericMessage::<(), (), (), ()>::Consensus(ConsensusMessage { + engine_id, + data: message.to_vec(), + }); + + // Note that we clone `message` here. + From::from(&message.encode()[..]) + }, + }; + + self.events.push(NetworkBehaviourAction::GenerateEvent(event)); + } + + NotifsHandlerOut::Clogged { messages } => { debug_assert!(self.is_open(&source)); trace!(target: "sub-libp2p", "Handler({:?}) => Clogged", source); trace!(target: "sub-libp2p", "External API <= Clogged({:?})", source); warn!(target: "sub-libp2p", "Queue of packets to send to {:?} is \ pretty large", source); - self.events.push(NetworkBehaviourAction::GenerateEvent(LegacyProtoOut::Clogged { + self.events.push(NetworkBehaviourAction::GenerateEvent(GenericProtoOut::Clogged { peer_id: source, messages, })); } // Don't do anything for non-severe errors except report them. - CustomProtoHandlerOut::ProtocolError { is_severe, ref error } if !is_severe => { + NotifsHandlerOut::ProtocolError { is_severe, ref error } if !is_severe => { debug!(target: "sub-libp2p", "Handler({:?}) => Benign protocol error: {:?}", source, error) } - CustomProtoHandlerOut::ProtocolError { error, .. } => { + NotifsHandlerOut::ProtocolError { error, .. } => { debug!(target: "sub-libp2p", "Handler({:?}) => Severe protocol error: {:?}", source, error); // A severe protocol error happens when we detect a "bad" node, such as a node on @@ -950,7 +1072,7 @@ impl NetworkBehaviour for LegacyProto { _params: &mut impl PollParameters, ) -> Poll< NetworkBehaviourAction< - CustomProtoHandlerIn, + NotifsHandlerIn, Self::OutEvent, >, > { @@ -1005,7 +1127,7 @@ impl NetworkBehaviour for LegacyProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Enable now that ban has expired", peer_id); self.events.push(NetworkBehaviourAction::SendEvent { peer_id: peer_id.clone(), - event: CustomProtoHandlerIn::Enable, + event: NotifsHandlerIn::Enable, }); *peer_state = PeerState::Enabled { connected_point, open }; } diff --git a/client/cli/src/traits.rs b/client/network/src/protocol/generic_proto/handler.rs similarity index 73% rename from client/cli/src/traits.rs rename to client/network/src/protocol/generic_proto/handler.rs index 96216a172b9700059f7ca4622cc4be9d582c93fa..e97176cfbbfbb98cfa6f1f1d2f25dc9f5f777990 100644 --- a/client/cli/src/traits.rs +++ b/client/network/src/protocol/generic_proto/handler.rs @@ -1,4 +1,4 @@ -// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// Copyright 2020 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Substrate is free software: you can redistribute it and/or modify @@ -14,10 +14,9 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use crate::params::SharedParams; +pub use self::group::{NotifsHandlerProto, NotifsHandler, NotifsHandlerIn, NotifsHandlerOut}; -/// Supports getting common params. -pub trait GetSharedParams { - /// Returns shared params if any. - fn shared_params(&self) -> Option<&SharedParams>; -} +mod group; +mod legacy; +mod notif_in; +mod notif_out; diff --git a/client/network/src/protocol/generic_proto/handler/group.rs b/client/network/src/protocol/generic_proto/handler/group.rs new file mode 100644 index 0000000000000000000000000000000000000000..d6d9919d3e14df0c2261731cddb0811aa291f920 --- /dev/null +++ b/client/network/src/protocol/generic_proto/handler/group.rs @@ -0,0 +1,523 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Implementations of the `IntoProtocolsHandler` and `ProtocolsHandler` traits for both incoming +//! and outgoing substreams for all gossiping protocols together. +//! +//! This is the main implementation of `ProtocolsHandler` in this crate, that handles all the +//! protocols that are Substrate-related and outside of the scope of libp2p. +//! +//! # Usage +//! +//! The handler can be in one of the following states: `Initial`, `Enabled`, `Disabled`. +//! +//! The `Initial` state is the state that the handler initially is in. It is a temporary state +//! during which the user must either enable or disable the handler. After that, the handler stays +//! either enabled or disabled. +//! +//! On the wire, we try to open the following substreams: +//! +//! - One substream for each notification protocol passed as parameter to the +//! `NotifsHandlerProto::new` function. +//! - One "legacy" substream used for anything non-related to gossiping, and used as a fallback +//! in case the notification protocol can't be opened. +//! +//! When the handler is in the `Enabled` state, we immediately open and try to maintain all the +//! aforementioned substreams. When the handler is in the `Disabled` state, we immediately close +//! (or abort opening) all these substreams. It is intended that in the future we allow states in +//! which some protocols are open and not others. Symmetrically, we allow incoming +//! Substrate-related substreams if and only if we are in the `Enabled` state. +//! +//! The user has the choice between sending a message with `SendNotification`, to send a +//! notification, and `SendLegacy`, to send any other kind of message. +//! + +use crate::protocol::generic_proto::{ + handler::legacy::{LegacyProtoHandler, LegacyProtoHandlerProto, LegacyProtoHandlerIn, LegacyProtoHandlerOut}, + handler::notif_in::{NotifsInHandlerProto, NotifsInHandler, NotifsInHandlerIn, NotifsInHandlerOut}, + handler::notif_out::{NotifsOutHandlerProto, NotifsOutHandler, NotifsOutHandlerIn, NotifsOutHandlerOut}, + upgrade::{NotificationsIn, NotificationsOut, NotificationsHandshakeError, RegisteredProtocol, UpgradeCollec}, +}; +use crate::protocol::message::generic::{Message as GenericMessage, ConsensusMessage}; + +use bytes::BytesMut; +use codec::Encode as _; +use libp2p::core::{either::{EitherError, EitherOutput}, ConnectedPoint, PeerId}; +use libp2p::core::upgrade::{EitherUpgrade, UpgradeError, SelectUpgrade, InboundUpgrade, OutboundUpgrade}; +use libp2p::swarm::{ + ProtocolsHandler, ProtocolsHandlerEvent, + IntoProtocolsHandler, + KeepAlive, + ProtocolsHandlerUpgrErr, + SubstreamProtocol, + NegotiatedSubstream, +}; +use log::error; +use sp_runtime::ConsensusEngineId; +use std::{borrow::Cow, error, io, task::{Context, Poll}}; + +/// Implements the `IntoProtocolsHandler` trait of libp2p. +/// +/// Every time a connection with a remote starts, an instance of this struct is created and +/// sent to a background task dedicated to this connection. Once the connection is established, +/// it is turned into a [`NotifsHandler`]. +/// +/// See the documentation at the module level for more information. +pub struct NotifsHandlerProto { + /// Prototypes for handlers for inbound substreams. + in_handlers: Vec<(NotifsInHandlerProto, ConsensusEngineId)>, + + /// Prototypes for handlers for outbound substreams. + out_handlers: Vec<(NotifsOutHandlerProto, ConsensusEngineId)>, + + /// Prototype for handler for backwards-compatibility. + legacy: LegacyProtoHandlerProto, +} + +/// The actual handler once the connection has been established. +/// +/// See the documentation at the module level for more information. +pub struct NotifsHandler { + /// Handlers for inbound substreams. + in_handlers: Vec<(NotifsInHandler, ConsensusEngineId)>, + + /// Handlers for outbound substreams. + out_handlers: Vec<(NotifsOutHandler, ConsensusEngineId)>, + + /// Handler for backwards-compatibility. + legacy: LegacyProtoHandler, + + /// State of this handler. + enabled: EnabledState, + + /// If we receive inbound substream requests while in initialization mode, + /// we push the corresponding index here and process them when the handler + /// gets enabled/disabled. + pending_in: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum EnabledState { + Initial, + Enabled, + Disabled, +} + +impl IntoProtocolsHandler for NotifsHandlerProto { + type Handler = NotifsHandler; + + fn inbound_protocol(&self) -> SelectUpgrade, RegisteredProtocol> { + let in_handlers = self.in_handlers.iter() + .map(|(h, _)| h.inbound_protocol()) + .collect::>(); + + SelectUpgrade::new(in_handlers, self.legacy.inbound_protocol()) + } + + fn into_handler(self, remote_peer_id: &PeerId, connected_point: &ConnectedPoint) -> Self::Handler { + NotifsHandler { + in_handlers: self.in_handlers + .into_iter() + .map(|(p, e)| (p.into_handler(remote_peer_id, connected_point), e)) + .collect(), + out_handlers: self.out_handlers + .into_iter() + .map(|(p, e)| (p.into_handler(remote_peer_id, connected_point), e)) + .collect(), + legacy: self.legacy.into_handler(remote_peer_id, connected_point), + enabled: EnabledState::Initial, + pending_in: Vec::new(), + } + } +} + +/// Event that can be received by a `NotifsHandler`. +#[derive(Debug)] +pub enum NotifsHandlerIn { + /// The node should start using custom protocols. + Enable, + + /// The node should stop using custom protocols. + Disable, + + /// Sends a message through the custom protocol substream. + /// + /// > **Note**: This must **not** be an encoded `ConsensusMessage` message. + SendLegacy { + /// The message to send. + message: Vec, + }, + + /// Sends a notifications message. + SendNotification { + /// Name of the protocol for the message. + /// + /// Must match one of the registered protocols. For backwards-compatibility reasons, if + /// the remote doesn't support this protocol, we use the legacy substream to send a + /// `ConsensusMessage` message. + protocol_name: Cow<'static, [u8]>, + + /// The engine ID to use, in case we need to send this message over the legacy substream. + /// + /// > **Note**: Ideally this field wouldn't be necessary, and we would deduce the engine + /// > ID from the existing handlers. However, it is possible (especially in test + /// > situations) that we open connections before all the notification protocols + /// > have been registered, in which case we always rely on the legacy substream. + engine_id: ConsensusEngineId, + + /// The message to send. + message: Vec, + }, +} + +/// Event that can be emitted by a `NotifsHandler`. +#[derive(Debug)] +pub enum NotifsHandlerOut { + /// Opened the substreams with the remote. + Open, + + /// Closed the substreams with the remote. + Closed { + /// Reason why the substream closed, for diagnostic purposes. + reason: Cow<'static, str>, + }, + + /// Received a non-gossiping message on the legacy substream. + CustomMessage { + /// Message that has been received. + /// + /// Keep in mind that this can be a `ConsensusMessage` message, which then contains a + /// notification. + message: BytesMut, + }, + + /// Received a message on a custom protocol substream. + Notification { + /// Engine corresponding to the message. + protocol_name: Cow<'static, [u8]>, + + /// For legacy reasons, the name to use if we had received the message from the legacy + /// substream. + engine_id: ConsensusEngineId, + + /// Message that has been received. + /// + /// If `protocol_name` is `None`, this decodes to a `Message`. If `protocol_name` is `Some`, + /// this is directly a gossiping message. + message: BytesMut, + }, + + /// A substream to the remote is clogged. The send buffer is very large, and we should print + /// a diagnostic message and/or avoid sending more data. + Clogged { + /// Copy of the messages that are within the buffer, for further diagnostic. + messages: Vec>, + }, + + /// An error has happened on the protocol level with this node. + ProtocolError { + /// If true the error is severe, such as a protocol violation. + is_severe: bool, + /// The error that happened. + error: Box, + }, +} + +impl NotifsHandlerProto { + /// Builds a new handler. + pub fn new(legacy: RegisteredProtocol, list: impl Into, ConsensusEngineId, Vec)>>) -> Self { + let list = list.into(); + + NotifsHandlerProto { + in_handlers: list.clone().into_iter().map(|(p, e, _)| (NotifsInHandlerProto::new(p), e)).collect(), + out_handlers: list.clone().into_iter().map(|(p, e, _)| (NotifsOutHandlerProto::new(p), e)).collect(), + legacy: LegacyProtoHandlerProto::new(legacy), + } + } +} + +impl ProtocolsHandler for NotifsHandler { + type InEvent = NotifsHandlerIn; + type OutEvent = NotifsHandlerOut; + type Error = EitherError< + EitherError< + ::Error, + ::Error, + >, + ::Error, + >; + type InboundProtocol = SelectUpgrade, RegisteredProtocol>; + type OutboundProtocol = EitherUpgrade; + // Index within the `out_handlers`; None for legacy + type OutboundOpenInfo = Option; + + fn listen_protocol(&self) -> SubstreamProtocol { + let in_handlers = self.in_handlers.iter() + .map(|h| h.0.listen_protocol().into_upgrade().1) + .collect::>(); + + let proto = SelectUpgrade::new(in_handlers, self.legacy.listen_protocol().into_upgrade().1); + SubstreamProtocol::new(proto) + } + + fn inject_fully_negotiated_inbound( + &mut self, + out: >::Output + ) { + match out { + EitherOutput::First((out, num)) => + self.in_handlers[num].0.inject_fully_negotiated_inbound(out), + EitherOutput::Second(out) => + self.legacy.inject_fully_negotiated_inbound(out), + } + } + + fn inject_fully_negotiated_outbound( + &mut self, + out: >::Output, + num: Self::OutboundOpenInfo + ) { + match (out, num) { + (EitherOutput::First(out), Some(num)) => + self.out_handlers[num].0.inject_fully_negotiated_outbound(out, ()), + (EitherOutput::Second(out), None) => + self.legacy.inject_fully_negotiated_outbound(out, ()), + _ => error!("inject_fully_negotiated_outbound called with wrong parameters"), + } + } + + fn inject_event(&mut self, message: NotifsHandlerIn) { + match message { + NotifsHandlerIn::Enable => { + self.enabled = EnabledState::Enabled; + self.legacy.inject_event(LegacyProtoHandlerIn::Enable); + for (handler, _) in &mut self.out_handlers { + handler.inject_event(NotifsOutHandlerIn::Enable { + initial_message: vec![] + }); + } + for num in self.pending_in.drain(..) { + self.in_handlers[num].0.inject_event(NotifsInHandlerIn::Accept(vec![])); + } + }, + NotifsHandlerIn::Disable => { + self.legacy.inject_event(LegacyProtoHandlerIn::Disable); + // The notifications protocols start in the disabled state. If we were in the + // "Initial" state, then we shouldn't disable the notifications protocols again. + if self.enabled != EnabledState::Initial { + for (handler, _) in &mut self.out_handlers { + handler.inject_event(NotifsOutHandlerIn::Disable); + } + } + self.enabled = EnabledState::Disabled; + for num in self.pending_in.drain(..) { + self.in_handlers[num].0.inject_event(NotifsInHandlerIn::Refuse); + } + }, + NotifsHandlerIn::SendLegacy { message } => + self.legacy.inject_event(LegacyProtoHandlerIn::SendCustomMessage { message }), + NotifsHandlerIn::SendNotification { message, engine_id, protocol_name } => { + for (handler, ngn_id) in &mut self.out_handlers { + if handler.protocol_name() != &protocol_name[..] { + break; + } + + if handler.is_open() { + handler.inject_event(NotifsOutHandlerIn::Send(message)); + return; + } else { + debug_assert_eq!(engine_id, *ngn_id); + } + } + + let message = GenericMessage::<(), (), (), ()>::Consensus(ConsensusMessage { + engine_id, + data: message, + }); + + self.legacy.inject_event(LegacyProtoHandlerIn::SendCustomMessage { + message: message.encode() + }); + }, + } + } + + fn inject_dial_upgrade_error( + &mut self, + num: Option, + err: ProtocolsHandlerUpgrErr> + ) { + match (err, num) { + (ProtocolsHandlerUpgrErr::Timeout, Some(num)) => + self.out_handlers[num].0.inject_dial_upgrade_error( + (), + ProtocolsHandlerUpgrErr::Timeout + ), + (ProtocolsHandlerUpgrErr::Timeout, None) => + self.legacy.inject_dial_upgrade_error((), ProtocolsHandlerUpgrErr::Timeout), + (ProtocolsHandlerUpgrErr::Timer, Some(num)) => + self.out_handlers[num].0.inject_dial_upgrade_error( + (), + ProtocolsHandlerUpgrErr::Timer + ), + (ProtocolsHandlerUpgrErr::Timer, None) => + self.legacy.inject_dial_upgrade_error((), ProtocolsHandlerUpgrErr::Timer), + (ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)), Some(num)) => + self.out_handlers[num].0.inject_dial_upgrade_error( + (), + ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)) + ), + (ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)), None) => + self.legacy.inject_dial_upgrade_error( + (), + ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)) + ), + (ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(EitherError::A(err))), Some(num)) => + self.out_handlers[num].0.inject_dial_upgrade_error( + (), + ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(err)) + ), + (ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(EitherError::B(err))), None) => + self.legacy.inject_dial_upgrade_error( + (), + ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(err)) + ), + _ => error!("inject_dial_upgrade_error called with bad parameters"), + } + } + + fn connection_keep_alive(&self) -> KeepAlive { + // Iterate over each handler and return the maximum value. + + let mut ret = self.legacy.connection_keep_alive(); + if ret.is_yes() { + return KeepAlive::Yes; + } + + for (handler, _) in &self.in_handlers { + let val = handler.connection_keep_alive(); + if val.is_yes() { + return KeepAlive::Yes; + } + if ret < val { ret = val; } + } + + for (handler, _) in &self.out_handlers { + let val = handler.connection_keep_alive(); + if val.is_yes() { + return KeepAlive::Yes; + } + if ret < val { ret = val; } + } + + ret + } + + fn poll( + &mut self, + cx: &mut Context, + ) -> Poll< + ProtocolsHandlerEvent + > { + for (handler_num, (handler, engine_id)) in self.in_handlers.iter_mut().enumerate() { + while let Poll::Ready(ev) = handler.poll(cx) { + match ev { + ProtocolsHandlerEvent::OutboundSubstreamRequest { .. } => + error!("Incoming substream handler tried to open a substream"), + ProtocolsHandlerEvent::Close(err) => void::unreachable(err), + ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::OpenRequest(_)) => + match self.enabled { + EnabledState::Initial => self.pending_in.push(handler_num), + EnabledState::Enabled => + handler.inject_event(NotifsInHandlerIn::Accept(vec![])), + EnabledState::Disabled => + handler.inject_event(NotifsInHandlerIn::Refuse), + }, + ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Closed) => {}, + ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Notif(message)) => { + // Note that right now the legacy substream has precedence over + // everything. If it is not open, then we consider that nothing is open. + if self.legacy.is_open() { + let msg = NotifsHandlerOut::Notification { + message, + engine_id: *engine_id, + protocol_name: handler.protocol_name().to_owned().into(), + }; + return Poll::Ready(ProtocolsHandlerEvent::Custom(msg)); + } + }, + } + } + } + + for (handler_num, (handler, _)) in self.out_handlers.iter_mut().enumerate() { + while let Poll::Ready(ev) = handler.poll(cx) { + match ev { + ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () } => + return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { + protocol: protocol.map_upgrade(EitherUpgrade::A), + info: Some(handler_num), + }), + ProtocolsHandlerEvent::Close(err) => void::unreachable(err), + + // At the moment we don't actually care whether any notifications protocol + // opens or closes. + // Whether our communications with the remote are open or closed entirely + // depends on the legacy substream, because as long as we are open the user of + // this struct might try to send legacy protocol messages which we need to + // deliver for things to work properly. + ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Open { .. }) => {}, + ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Closed) => {}, + ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Refused) => {}, + } + } + } + + while let Poll::Ready(ev) = self.legacy.poll(cx) { + match ev { + ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () } => + return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { + protocol: protocol.map_upgrade(EitherUpgrade::B), + info: None, + }), + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolOpen { .. }) => + return Poll::Ready(ProtocolsHandlerEvent::Custom( + NotifsHandlerOut::Open + )), + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolClosed { reason }) => + return Poll::Ready(ProtocolsHandlerEvent::Custom( + NotifsHandlerOut::Closed { reason } + )), + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomMessage { message }) => + return Poll::Ready(ProtocolsHandlerEvent::Custom( + NotifsHandlerOut::CustomMessage { message } + )), + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::Clogged { messages }) => + return Poll::Ready(ProtocolsHandlerEvent::Custom( + NotifsHandlerOut::Clogged { messages } + )), + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::ProtocolError { is_severe, error }) => + return Poll::Ready(ProtocolsHandlerEvent::Custom( + NotifsHandlerOut::ProtocolError { is_severe, error } + )), + ProtocolsHandlerEvent::Close(err) => + return Poll::Ready(ProtocolsHandlerEvent::Close(EitherError::B(err))), + } + } + + Poll::Pending + } +} diff --git a/client/network/src/protocol/legacy_proto/handler.rs b/client/network/src/protocol/generic_proto/handler/legacy.rs similarity index 90% rename from client/network/src/protocol/legacy_proto/handler.rs rename to client/network/src/protocol/generic_proto/handler/legacy.rs index e3490993dd46d05549e656b124883f1976a4d062..a2d2fc9246d1c79b761d5e5eb1ce375fba33b49a 100644 --- a/client/network/src/protocol/legacy_proto/handler.rs +++ b/client/network/src/protocol/generic_proto/handler/legacy.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use super::upgrade::{RegisteredProtocol, RegisteredProtocolEvent, RegisteredProtocolSubstream}; +use crate::protocol::generic_proto::upgrade::{RegisteredProtocol, RegisteredProtocolEvent, RegisteredProtocolSubstream}; use bytes::BytesMut; use futures::prelude::*; use futures_timer::Delay; @@ -37,7 +37,7 @@ use std::{pin::Pin, task::{Context, Poll}}; /// /// Every time a connection with a remote starts, an instance of this struct is created and /// sent to a background task dedicated to this connection. Once the connection is established, -/// it is turned into a `CustomProtoHandler`. It then handles all communications that are specific +/// it is turned into a `LegacyProtoHandler`. It then handles all communications that are specific /// to Substrate on that single connection. /// /// Note that there can be multiple instance of this struct simultaneously for same peer. However @@ -87,29 +87,29 @@ use std::{pin::Pin, task::{Context, Poll}}; /// We consider that we are now "closed" if the remote closes all the existing substreams. /// Re-opening it can then be performed by closing all active substream and re-opening one. /// -pub struct CustomProtoHandlerProto { +pub struct LegacyProtoHandlerProto { /// Configuration for the protocol upgrade to negotiate. protocol: RegisteredProtocol, } -impl CustomProtoHandlerProto { - /// Builds a new `CustomProtoHandlerProto`. +impl LegacyProtoHandlerProto { + /// Builds a new `LegacyProtoHandlerProto`. pub fn new(protocol: RegisteredProtocol) -> Self { - CustomProtoHandlerProto { + LegacyProtoHandlerProto { protocol, } } } -impl IntoProtocolsHandler for CustomProtoHandlerProto { - type Handler = CustomProtoHandler; +impl IntoProtocolsHandler for LegacyProtoHandlerProto { + type Handler = LegacyProtoHandler; fn inbound_protocol(&self) -> RegisteredProtocol { self.protocol.clone() } fn into_handler(self, remote_peer_id: &PeerId, connected_point: &ConnectedPoint) -> Self::Handler { - CustomProtoHandler { + LegacyProtoHandler { protocol: self.protocol, endpoint: connected_point.to_endpoint(), remote_peer_id: remote_peer_id.clone(), @@ -123,7 +123,7 @@ impl IntoProtocolsHandler for CustomProtoHandlerProto { } /// The actual handler once the connection has been established. -pub struct CustomProtoHandler { +pub struct LegacyProtoHandler { /// Configuration for the protocol upgrade to negotiate. protocol: RegisteredProtocol, @@ -142,7 +142,7 @@ pub struct CustomProtoHandler { /// /// This queue must only ever be modified to insert elements at the back, or remove the first /// element. - events_queue: SmallVec<[ProtocolsHandlerEvent; 16]>, + events_queue: SmallVec<[ProtocolsHandlerEvent; 16]>, } /// State of the handler. @@ -195,9 +195,9 @@ enum ProtocolState { Poisoned, } -/// Event that can be received by a `CustomProtoHandler`. +/// Event that can be received by a `LegacyProtoHandler`. #[derive(Debug)] -pub enum CustomProtoHandlerIn { +pub enum LegacyProtoHandlerIn { /// The node should start using custom protocols. Enable, @@ -211,9 +211,9 @@ pub enum CustomProtoHandlerIn { }, } -/// Event that can be emitted by a `CustomProtoHandler`. +/// Event that can be emitted by a `LegacyProtoHandler`. #[derive(Debug)] -pub enum CustomProtoHandlerOut { +pub enum LegacyProtoHandlerOut { /// Opened a custom protocol with the remote. CustomProtocolOpen { /// Version of the protocol that has been opened. @@ -248,7 +248,19 @@ pub enum CustomProtoHandlerOut { }, } -impl CustomProtoHandler { +impl LegacyProtoHandler { + /// Returns true if the legacy substream is currently open. + pub fn is_open(&self) -> bool { + match &self.state { + ProtocolState::Init { substreams, .. } => !substreams.is_empty(), + ProtocolState::Opening { .. } => false, + ProtocolState::Normal { substreams, .. } => !substreams.is_empty(), + ProtocolState::Disabled { .. } => false, + ProtocolState::KillAsap => false, + ProtocolState::Poisoned => false, + } + } + /// Enables the handler. fn enable(&mut self) { self.state = match mem::replace(&mut self.state, ProtocolState::Poisoned) { @@ -271,7 +283,7 @@ impl CustomProtoHandler { } } else { - let event = CustomProtoHandlerOut::CustomProtocolOpen { + let event = LegacyProtoHandlerOut::CustomProtocolOpen { version: incoming[0].protocol_version() }; self.events_queue.push(ProtocolsHandlerEvent::Custom(event)); @@ -325,7 +337,7 @@ impl CustomProtoHandler { /// Polls the state for events. Optionally returns an event to produce. #[must_use] fn poll_state(&mut self, cx: &mut Context) - -> Option> { + -> Option> { match mem::replace(&mut self.state, ProtocolState::Poisoned) { ProtocolState::Poisoned => { error!(target: "sub-libp2p", "Handler with {:?} is in poisoned state", @@ -352,7 +364,7 @@ impl CustomProtoHandler { match Pin::new(&mut deadline).poll(cx) { Poll::Ready(()) => { deadline = Delay::new(Duration::from_secs(60)); - let event = CustomProtoHandlerOut::ProtocolError { + let event = LegacyProtoHandlerOut::ProtocolError { is_severe: true, error: "Timeout when opening protocol".to_string().into(), }; @@ -372,7 +384,7 @@ impl CustomProtoHandler { match Pin::new(&mut substream).poll_next(cx) { Poll::Pending => substreams.push(substream), Poll::Ready(Some(Ok(RegisteredProtocolEvent::Message(message)))) => { - let event = CustomProtoHandlerOut::CustomMessage { + let event = LegacyProtoHandlerOut::CustomMessage { message }; substreams.push(substream); @@ -380,7 +392,7 @@ impl CustomProtoHandler { return Some(ProtocolsHandlerEvent::Custom(event)); }, Poll::Ready(Some(Ok(RegisteredProtocolEvent::Clogged { messages }))) => { - let event = CustomProtoHandlerOut::Clogged { + let event = LegacyProtoHandlerOut::Clogged { messages, }; substreams.push(substream); @@ -390,7 +402,7 @@ impl CustomProtoHandler { Poll::Ready(None) => { shutdown.push(substream); if substreams.is_empty() { - let event = CustomProtoHandlerOut::CustomProtocolClosed { + let event = LegacyProtoHandlerOut::CustomProtocolClosed { reason: "All substreams have been closed by the remote".into(), }; self.state = ProtocolState::Disabled { @@ -402,7 +414,7 @@ impl CustomProtoHandler { } Poll::Ready(Some(Err(err))) => { if substreams.is_empty() { - let event = CustomProtoHandlerOut::CustomProtocolClosed { + let event = LegacyProtoHandlerOut::CustomProtocolClosed { reason: format!("Error on the last substream: {:?}", err).into(), }; self.state = ProtocolState::Disabled { @@ -466,7 +478,7 @@ impl CustomProtoHandler { } ProtocolState::Opening { .. } => { - let event = CustomProtoHandlerOut::CustomProtocolOpen { + let event = LegacyProtoHandlerOut::CustomProtocolOpen { version: substream.protocol_version() }; self.events_queue.push(ProtocolsHandlerEvent::Custom(event)); @@ -503,9 +515,9 @@ impl CustomProtoHandler { } } -impl ProtocolsHandler for CustomProtoHandler { - type InEvent = CustomProtoHandlerIn; - type OutEvent = CustomProtoHandlerOut; +impl ProtocolsHandler for LegacyProtoHandler { + type InEvent = LegacyProtoHandlerIn; + type OutEvent = LegacyProtoHandlerOut; type Error = ConnectionKillError; type InboundProtocol = RegisteredProtocol; type OutboundProtocol = RegisteredProtocol; @@ -530,11 +542,11 @@ impl ProtocolsHandler for CustomProtoHandler { self.inject_fully_negotiated(proto); } - fn inject_event(&mut self, message: CustomProtoHandlerIn) { + fn inject_event(&mut self, message: LegacyProtoHandlerIn) { match message { - CustomProtoHandlerIn::Disable => self.disable(), - CustomProtoHandlerIn::Enable => self.enable(), - CustomProtoHandlerIn::SendCustomMessage { message } => + LegacyProtoHandlerIn::Disable => self.disable(), + LegacyProtoHandlerIn::Enable => self.enable(), + LegacyProtoHandlerIn::SendCustomMessage { message } => self.send_message(message), } } @@ -546,7 +558,7 @@ impl ProtocolsHandler for CustomProtoHandler { _ => false, }; - self.events_queue.push(ProtocolsHandlerEvent::Custom(CustomProtoHandlerOut::ProtocolError { + self.events_queue.push(ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::ProtocolError { is_severe, error: Box::new(err), })); @@ -587,9 +599,9 @@ impl ProtocolsHandler for CustomProtoHandler { } } -impl fmt::Debug for CustomProtoHandler { +impl fmt::Debug for LegacyProtoHandler { fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { - f.debug_struct("CustomProtoHandler") + f.debug_struct("LegacyProtoHandler") .finish() } } diff --git a/client/network/src/protocol/generic_proto/handler/notif_in.rs b/client/network/src/protocol/generic_proto/handler/notif_in.rs new file mode 100644 index 0000000000000000000000000000000000000000..4e16fb1af419f4aebb2748b09c477119f1165309 --- /dev/null +++ b/client/network/src/protocol/generic_proto/handler/notif_in.rs @@ -0,0 +1,256 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Implementations of the `IntoProtocolsHandler` and `ProtocolsHandler` traits for ingoing +//! substreams for a single gossiping protocol. +//! +//! > **Note**: Each instance corresponds to a single protocol. In order to support multiple +//! > protocols, you need to create multiple instances and group them. +//! + +use crate::protocol::generic_proto::upgrade::{NotificationsIn, NotificationsInSubstream}; +use bytes::BytesMut; +use futures::prelude::*; +use libp2p::core::{ConnectedPoint, PeerId}; +use libp2p::core::upgrade::{DeniedUpgrade, InboundUpgrade, OutboundUpgrade}; +use libp2p::swarm::{ + ProtocolsHandler, ProtocolsHandlerEvent, + IntoProtocolsHandler, + KeepAlive, + ProtocolsHandlerUpgrErr, + SubstreamProtocol, + NegotiatedSubstream, +}; +use log::{error, warn}; +use smallvec::SmallVec; +use std::{borrow::Cow, fmt, pin::Pin, str, task::{Context, Poll}}; + +/// Implements the `IntoProtocolsHandler` trait of libp2p. +/// +/// Every time a connection with a remote starts, an instance of this struct is created and +/// sent to a background task dedicated to this connection. Once the connection is established, +/// it is turned into a [`NotifsInHandler`]. +pub struct NotifsInHandlerProto { + /// Configuration for the protocol upgrade to negotiate. + in_protocol: NotificationsIn, +} + +/// The actual handler once the connection has been established. +pub struct NotifsInHandler { + /// Configuration for the protocol upgrade to negotiate for inbound substreams. + in_protocol: NotificationsIn, + + /// Substream that is open with the remote. + substream: Option>, + + /// If the substream is opened and closed rapidly, we can emit several `OpenRequest` and + /// `Closed` messages in a row without the handler having time to respond with `Accept` or + /// `Refuse`. + /// + /// In order to keep the state consistent, we increment this variable every time an + /// `OpenRequest` is emitted and decrement it every time an `Accept` or `Refuse` is received. + pending_accept_refuses: usize, + + /// Queue of events to send to the outside. + /// + /// This queue is only ever modified to insert elements at the back, or remove the first + /// element. + events_queue: SmallVec<[ProtocolsHandlerEvent; 16]>, +} + +/// Event that can be received by a `NotifsInHandler`. +#[derive(Debug)] +pub enum NotifsInHandlerIn { + /// Can be sent back as a response to an `OpenRequest`. Contains the status message to send + /// to the remote. + /// + /// After sending this to the handler, the substream is now considered open and `Notif` events + /// can be received. + Accept(Vec), + + /// Can be sent back as a response to an `OpenRequest`. + Refuse, +} + +/// Event that can be emitted by a `NotifsInHandler`. +#[derive(Debug)] +pub enum NotifsInHandlerOut { + /// The remote wants to open a substream. Contains the initial message sent by the remote + /// when the substream has been opened. + /// + /// Every time this event is emitted, a corresponding `Accepted` or `Refused` **must** be sent + /// back even if a `Closed` is received. + OpenRequest(Vec), + + /// The notifications substream has been closed by the remote. In order to avoid race + /// conditions, this does **not** cancel any previously-sent `OpenRequest`. + Closed, + + /// Received a message on the notifications substream. + /// + /// Can only happen after an `Accept` and before a `Closed`. + Notif(BytesMut), +} + +impl NotifsInHandlerProto { + /// Builds a new `NotifsInHandlerProto`. + pub fn new( + protocol_name: impl Into> + ) -> Self { + NotifsInHandlerProto { + in_protocol: NotificationsIn::new(protocol_name), + } + } +} + +impl IntoProtocolsHandler for NotifsInHandlerProto { + type Handler = NotifsInHandler; + + fn inbound_protocol(&self) -> NotificationsIn { + self.in_protocol.clone() + } + + fn into_handler(self, _: &PeerId, _: &ConnectedPoint) -> Self::Handler { + NotifsInHandler { + in_protocol: self.in_protocol, + substream: None, + pending_accept_refuses: 0, + events_queue: SmallVec::new(), + } + } +} + +impl NotifsInHandler { + /// Returns the name of the protocol that we accept. + pub fn protocol_name(&self) -> &[u8] { + self.in_protocol.protocol_name() + } +} + +impl ProtocolsHandler for NotifsInHandler { + type InEvent = NotifsInHandlerIn; + type OutEvent = NotifsInHandlerOut; + type Error = void::Void; + type InboundProtocol = NotificationsIn; + type OutboundProtocol = DeniedUpgrade; + type OutboundOpenInfo = (); + + fn listen_protocol(&self) -> SubstreamProtocol { + SubstreamProtocol::new(self.in_protocol.clone()) + } + + fn inject_fully_negotiated_inbound( + &mut self, + (msg, proto): >::Output + ) { + if self.substream.is_some() { + warn!( + target: "sub-libp2p", + "Received duplicate inbound notifications substream for {:?}", + str::from_utf8(self.in_protocol.protocol_name()), + ); + return; + } + + self.substream = Some(proto); + self.events_queue.push(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::OpenRequest(msg))); + self.pending_accept_refuses = self.pending_accept_refuses + .checked_add(1) + .unwrap_or_else(|| { + error!(target: "sub-libp2p", "Overflow in pending_accept_refuses"); + usize::max_value() + }); + } + + fn inject_fully_negotiated_outbound( + &mut self, + out: >::Output, + _: Self::OutboundOpenInfo + ) { + // We never emit any outgoing substream. + void::unreachable(out) + } + + fn inject_event(&mut self, message: NotifsInHandlerIn) { + self.pending_accept_refuses = match self.pending_accept_refuses.checked_sub(1) { + Some(v) => v, + None => { + error!( + target: "sub-libp2p", + "Inconsistent state: received Accept/Refuse when no pending request exists" + ); + return; + } + }; + + // If we send multiple `OpenRequest`s in a row, we will receive back multiple + // `Accept`/`Refuse` messages. All of them are obsolete except the last one. + if self.pending_accept_refuses != 0 { + return; + } + + match (message, self.substream.as_mut()) { + (NotifsInHandlerIn::Accept(message), Some(sub)) => sub.send_handshake(message), + (NotifsInHandlerIn::Accept(_), None) => {}, + (NotifsInHandlerIn::Refuse, _) => self.substream = None, + } + } + + fn inject_dial_upgrade_error(&mut self, _: (), _: ProtocolsHandlerUpgrErr) { + error!(target: "sub-libp2p", "Received dial upgrade error in inbound-only handler"); + } + + fn connection_keep_alive(&self) -> KeepAlive { + if self.substream.is_some() { + KeepAlive::Yes + } else { + KeepAlive::No + } + } + + fn poll( + &mut self, + cx: &mut Context, + ) -> Poll< + ProtocolsHandlerEvent + > { + // Flush the events queue if necessary. + if !self.events_queue.is_empty() { + let event = self.events_queue.remove(0); + return Poll::Ready(event) + } + + match self.substream.as_mut().map(|s| Stream::poll_next(Pin::new(s), cx)) { + None | Some(Poll::Pending) => {}, + Some(Poll::Ready(Some(Ok(msg)))) => + return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Notif(msg))), + Some(Poll::Ready(None)) | Some(Poll::Ready(Some(Err(_)))) => { + self.substream = None; + return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Closed)); + }, + } + + Poll::Pending + } +} + +impl fmt::Debug for NotifsInHandler { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + f.debug_struct("NotifsInHandler") + .field("substream_open", &self.substream.is_some()) + .finish() + } +} diff --git a/client/network/src/protocol/generic_proto/handler/notif_out.rs b/client/network/src/protocol/generic_proto/handler/notif_out.rs new file mode 100644 index 0000000000000000000000000000000000000000..8c64491d997171df73606c2852765c10c9f3f21b --- /dev/null +++ b/client/network/src/protocol/generic_proto/handler/notif_out.rs @@ -0,0 +1,395 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Implementations of the `IntoProtocolsHandler` and `ProtocolsHandler` traits for outgoing +//! substreams of a single gossiping protocol. +//! +//! > **Note**: Each instance corresponds to a single protocol. In order to support multiple +//! > protocols, you need to create multiple instances and group them. +//! + +use crate::protocol::generic_proto::upgrade::{NotificationsOut, NotificationsOutSubstream, NotificationsHandshakeError}; +use futures::prelude::*; +use libp2p::core::{ConnectedPoint, PeerId}; +use libp2p::core::upgrade::{DeniedUpgrade, InboundUpgrade, OutboundUpgrade}; +use libp2p::swarm::{ + ProtocolsHandler, ProtocolsHandlerEvent, + IntoProtocolsHandler, + KeepAlive, + ProtocolsHandlerUpgrErr, + SubstreamProtocol, + NegotiatedSubstream, +}; +use log::error; +use smallvec::SmallVec; +use std::{borrow::Cow, fmt, mem, pin::Pin, task::{Context, Poll}, time::Duration}; +use wasm_timer::Instant; + +/// Maximum duration to open a substream and receive the handshake message. After that, we +/// consider that we failed to open the substream. +const OPEN_TIMEOUT: Duration = Duration::from_secs(10); +/// After successfully establishing a connection with the remote, we keep the connection open for +/// at least this amount of time in order to give the rest of the code the chance to notify us to +/// open substreams. +const INITIAL_KEEPALIVE_TIME: Duration = Duration::from_secs(5); + +/// Implements the `IntoProtocolsHandler` trait of libp2p. +/// +/// Every time a connection with a remote starts, an instance of this struct is created and +/// sent to a background task dedicated to this connection. Once the connection is established, +/// it is turned into a [`NotifsOutHandler`]. +/// +/// See the documentation of [`NotifsOutHandler`] for more information. +pub struct NotifsOutHandlerProto { + /// Name of the protocol to negotiate. + protocol_name: Cow<'static, [u8]>, +} + +impl NotifsOutHandlerProto { + /// Builds a new [`NotifsOutHandlerProto`]. Will use the given protocol name for the + /// notifications substream. + pub fn new(protocol_name: impl Into>) -> Self { + NotifsOutHandlerProto { + protocol_name: protocol_name.into(), + } + } +} + +impl IntoProtocolsHandler for NotifsOutHandlerProto { + type Handler = NotifsOutHandler; + + fn inbound_protocol(&self) -> DeniedUpgrade { + DeniedUpgrade + } + + fn into_handler(self, _: &PeerId, _: &ConnectedPoint) -> Self::Handler { + NotifsOutHandler { + protocol_name: self.protocol_name, + when_connection_open: Instant::now(), + state: State::Disabled, + events_queue: SmallVec::new(), + } + } +} + +/// Handler for an outbound notification substream. +/// +/// When a connection is established, this handler starts in the "disabled" state, meaning that +/// no substream will be open. +/// +/// One can try open a substream by sending an [`NotifsOutHandlerIn::Enable`] message to the +/// handler. Once done, the handler will try to establish then maintain an outbound substream with +/// the remote for the purpose of sending notifications to it. +pub struct NotifsOutHandler { + /// Name of the protocol to negotiate. + protocol_name: Cow<'static, [u8]>, + + /// Relationship with the node we're connected to. + state: State, + + /// When the connection with the remote has been successfully established. + when_connection_open: Instant, + + /// Queue of events to send to the outside. + /// + /// This queue must only ever be modified to insert elements at the back, or remove the first + /// element. + events_queue: SmallVec<[ProtocolsHandlerEvent; 16]>, +} + +/// Our relationship with the node we're connected to. +enum State { + /// The handler is disabled and idle. No substream is open. + Disabled, + + /// The handler is disabled. A substream is still open and needs to be closed. + /// + /// > **Important**: Having this state means that `poll_close` has been called at least once, + /// > but the `Sink` API is unclear about whether or not the stream can then + /// > be recovered. Because of that, we must never switch from the + /// > `DisabledOpen` state to the `Open` state while keeping the same substream. + DisabledOpen(NotificationsOutSubstream), + + /// The handler is disabled but we are still trying to open a substream with the remote. + /// + /// If the handler gets enabled again, we can immediately switch to `Opening`. + DisabledOpening, + + /// The handler is enabled and we are trying to open a substream with the remote. + Opening { + /// The initial message that we sent. Necessary if we need to re-open a substream. + initial_message: Vec, + }, + + /// The handler is enabled. We have tried opening a substream in the past but the remote + /// refused it. + Refused, + + /// The handler is enabled and substream is open. + Open { + /// Substream that is currently open. + substream: NotificationsOutSubstream, + /// The initial message that we sent. Necessary if we need to re-open a substream. + initial_message: Vec, + }, + + /// Poisoned state. Shouldn't be found in the wild. + Poisoned, +} + +/// Event that can be received by a `NotifsOutHandler`. +#[derive(Debug)] +pub enum NotifsOutHandlerIn { + /// Enables the notifications substream for this node. The handler will try to maintain a + /// substream with the remote. + Enable { + /// Initial message to send to remote nodes when we open substreams. + initial_message: Vec, + }, + + /// Disables the notifications substream for this node. This is the default state. + Disable, + + /// Sends a message on the notifications substream. Ignored if the substream isn't open. + /// + /// It is only valid to send this if the notifications substream has been enabled. + Send(Vec), +} + +/// Event that can be emitted by a `NotifsOutHandler`. +#[derive(Debug)] +pub enum NotifsOutHandlerOut { + /// The notifications substream has been accepted by the remote. + Open { + /// Handshake message sent by the remote after we opened the substream. + handshake: Vec, + }, + + /// The notifications substream has been closed by the remote. + Closed, + + /// We tried to open a notifications substream, but the remote refused it. + /// + /// Can only happen if we're in a closed state. + Refused, +} + +impl NotifsOutHandler { + /// Returns true if the substream is currently open. + pub fn is_open(&self) -> bool { + match &self.state { + State::Disabled => false, + State::DisabledOpening => false, + State::DisabledOpen(_) => true, + State::Opening { .. } => false, + State::Refused => false, + State::Open { .. } => true, + State::Poisoned => false, + } + } + + /// Returns the name of the protocol that we negotiate. + pub fn protocol_name(&self) -> &[u8] { + &self.protocol_name + } +} + +impl ProtocolsHandler for NotifsOutHandler { + type InEvent = NotifsOutHandlerIn; + type OutEvent = NotifsOutHandlerOut; + type Error = void::Void; + type InboundProtocol = DeniedUpgrade; + type OutboundProtocol = NotificationsOut; + type OutboundOpenInfo = (); + + fn listen_protocol(&self) -> SubstreamProtocol { + SubstreamProtocol::new(DeniedUpgrade) + } + + fn inject_fully_negotiated_inbound( + &mut self, + proto: >::Output + ) { + // We should never reach here. `proto` is a `Void`. + void::unreachable(proto) + } + + fn inject_fully_negotiated_outbound( + &mut self, + (handshake_msg, substream): >::Output, + _: () + ) { + match mem::replace(&mut self.state, State::Poisoned) { + State::Opening { initial_message } => { + let ev = NotifsOutHandlerOut::Open { handshake: handshake_msg }; + self.events_queue.push(ProtocolsHandlerEvent::Custom(ev)); + self.state = State::Open { substream, initial_message }; + }, + // If the handler was disabled while we were negotiating the protocol, immediately + // close it. + State::DisabledOpening => self.state = State::DisabledOpen(substream), + + // Any other situation should never happen. + State::Disabled | State::Refused | State::Open { .. } | State::DisabledOpen(_) => + error!("State mismatch in notifications handler: substream already open"), + State::Poisoned => error!("Notifications handler in a poisoned state"), + } + } + + fn inject_event(&mut self, message: NotifsOutHandlerIn) { + match message { + NotifsOutHandlerIn::Enable { initial_message } => { + match mem::replace(&mut self.state, State::Poisoned) { + State::Disabled => { + let proto = NotificationsOut::new(self.protocol_name.clone(), initial_message.clone()); + self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest { + protocol: SubstreamProtocol::new(proto).with_timeout(OPEN_TIMEOUT), + info: (), + }); + self.state = State::Opening { initial_message }; + }, + State::DisabledOpening => self.state = State::Opening { initial_message }, + State::DisabledOpen(mut sub) => { + // As documented above, in this state we have already called `poll_close` + // once on the substream, and it is unclear whether the substream can then + // be recovered. When in doubt, let's drop the existing substream and + // open a new one. + if sub.close().now_or_never().is_none() { + log::warn!( + target: "sub-libp2p", + "Improperly closed outbound notifications substream" + ); + } + + let proto = NotificationsOut::new(self.protocol_name.clone(), initial_message.clone()); + self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest { + protocol: SubstreamProtocol::new(proto).with_timeout(OPEN_TIMEOUT), + info: (), + }); + self.state = State::Opening { initial_message }; + }, + State::Opening { .. } | State::Refused | State::Open { .. } => + error!("Tried to enable notifications handler that was already enabled"), + State::Poisoned => error!("Notifications handler in a poisoned state"), + } + } + + NotifsOutHandlerIn::Disable => { + match mem::replace(&mut self.state, State::Poisoned) { + State::Disabled | State::DisabledOpen(_) | State::DisabledOpening => + error!("Tried to disable notifications handler that was already disabled"), + State::Opening { .. } => self.state = State::DisabledOpening, + State::Refused => self.state = State::Disabled, + State::Open { substream, .. } => self.state = State::DisabledOpen(substream), + State::Poisoned => error!("Notifications handler in a poisoned state"), + } + } + + NotifsOutHandlerIn::Send(msg) => + if let State::Open { substream, .. } = &mut self.state { + if let Some(Ok(_)) = substream.send(msg).now_or_never() { + } else { + log::warn!( + target: "sub-libp2p", + "Failed to push message to queue, dropped it" + ); + } + } else { + // This is an API misuse. + log::warn!( + target: "sub-libp2p", + "Tried to send a notification on a disabled handler" + ); + }, + } + } + + fn inject_dial_upgrade_error(&mut self, _: (), _: ProtocolsHandlerUpgrErr) { + match mem::replace(&mut self.state, State::Poisoned) { + State::Disabled => {}, + State::DisabledOpen(_) | State::Refused | State::Open { .. } => + error!("State mismatch in NotificationsOut"), + State::Opening { .. } => { + self.state = State::Refused; + let ev = NotifsOutHandlerOut::Refused; + self.events_queue.push(ProtocolsHandlerEvent::Custom(ev)); + }, + State::DisabledOpening => self.state = State::Disabled, + State::Poisoned => error!("Notifications handler in a poisoned state"), + } + } + + fn connection_keep_alive(&self) -> KeepAlive { + match self.state { + // We have a small grace period of `INITIAL_KEEPALIVE_TIME` during which we keep the + // connection open no matter what, in order to avoid closing and reopening + // connections all the time. + State::Disabled | State::DisabledOpen(_) | State::DisabledOpening => + KeepAlive::Until(self.when_connection_open + INITIAL_KEEPALIVE_TIME), + State::Opening { .. } | State::Open { .. } => KeepAlive::Yes, + State::Refused | State::Poisoned => KeepAlive::No, + } + } + + fn poll( + &mut self, + cx: &mut Context, + ) -> Poll> { + // Flush the events queue if necessary. + if !self.events_queue.is_empty() { + let event = self.events_queue.remove(0); + return Poll::Ready(event); + } + + match &mut self.state { + State::Open { substream, initial_message } => + match Sink::poll_flush(Pin::new(substream), cx) { + Poll::Pending | Poll::Ready(Ok(())) => {}, + Poll::Ready(Err(_)) => { + // We try to re-open a substream. + let initial_message = mem::replace(initial_message, Vec::new()); + self.state = State::Opening { initial_message: initial_message.clone() }; + let proto = NotificationsOut::new(self.protocol_name.clone(), initial_message); + self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest { + protocol: SubstreamProtocol::new(proto).with_timeout(OPEN_TIMEOUT), + info: (), + }); + return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Closed)); + } + }, + + State::DisabledOpen(sub) => match Sink::poll_close(Pin::new(sub), cx) { + Poll::Pending => {}, + Poll::Ready(Ok(())) | Poll::Ready(Err(_)) => { + self.state = State::Disabled; + return Poll::Ready(ProtocolsHandlerEvent::Custom(NotifsOutHandlerOut::Closed)); + }, + }, + + _ => {} + } + + Poll::Pending + } +} + +impl fmt::Debug for NotifsOutHandler { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + f.debug_struct("NotifsOutHandler") + .field("open", &self.is_open()) + .finish() + } +} diff --git a/client/network/src/protocol/legacy_proto/tests.rs b/client/network/src/protocol/generic_proto/tests.rs similarity index 86% rename from client/network/src/protocol/legacy_proto/tests.rs rename to client/network/src/protocol/generic_proto/tests.rs index 89b0854d9081f82c80e8f0dab8da0e138ebb4fca..b8436e2c7f704bf2c4d5664c6908fce96c80d65c 100644 --- a/client/network/src/protocol/legacy_proto/tests.rs +++ b/client/network/src/protocol/generic_proto/tests.rs @@ -25,8 +25,9 @@ use libp2p::swarm::{PollParameters, NetworkBehaviour, NetworkBehaviourAction}; use libp2p::{PeerId, Multiaddr, Transport}; use rand::seq::SliceRandom; use std::{error, io, task::Context, task::Poll, time::Duration}; -use crate::message::Message; -use crate::protocol::legacy_proto::{LegacyProto, LegacyProtoOut}; +use std::collections::HashSet; +use crate::message::{generic::BlockResponse, Message}; +use crate::protocol::generic_proto::{GenericProto, GenericProtoOut}; use sp_test_primitives::Block; /// Builds two nodes that have each other as bootstrap nodes. @@ -81,7 +82,7 @@ fn build_nodes() -> (Swarm, Swarm) { }); let behaviour = CustomProtoWithAddr { - inner: LegacyProto::new(&b"test"[..], &[1], peerset), + inner: GenericProto::new(&b"test"[..], &[1], peerset), addrs: addrs .iter() .enumerate() @@ -111,12 +112,12 @@ fn build_nodes() -> (Swarm, Swarm) { /// Wraps around the `CustomBehaviour` network behaviour, and adds hardcoded node addresses to it. struct CustomProtoWithAddr { - inner: LegacyProto, + inner: GenericProto, addrs: Vec<(PeerId, Multiaddr)>, } impl std::ops::Deref for CustomProtoWithAddr { - type Target = LegacyProto; + type Target = GenericProto; fn deref(&self) -> &Self::Target { &self.inner @@ -130,8 +131,8 @@ impl std::ops::DerefMut for CustomProtoWithAddr { } impl NetworkBehaviour for CustomProtoWithAddr { - type ProtocolsHandler = ::ProtocolsHandler; - type OutEvent = ::OutEvent; + type ProtocolsHandler = ::ProtocolsHandler; + type OutEvent = ::OutEvent; fn new_handler(&mut self) -> Self::ProtocolsHandler { self.inner.new_handler() @@ -223,11 +224,14 @@ fn two_nodes_transfer_lots_of_packets() { let fut1 = future::poll_fn(move |cx| -> Poll<()> { loop { match ready!(service1.poll_next_unpin(cx)) { - Some(LegacyProtoOut::CustomProtocolOpen { peer_id, .. }) => { + Some(GenericProtoOut::CustomProtocolOpen { peer_id, .. }) => { for n in 0 .. NUM_PACKETS { service1.send_packet( &peer_id, - Message::::ChainSpecific(vec![(n % 256) as u8]).encode() + Message::::BlockResponse(BlockResponse { + id: n as _, + blocks: Vec::new(), + }).encode() ); } }, @@ -240,11 +244,11 @@ fn two_nodes_transfer_lots_of_packets() { let fut2 = future::poll_fn(move |cx| { loop { match ready!(service2.poll_next_unpin(cx)) { - Some(LegacyProtoOut::CustomProtocolOpen { .. }) => {}, - Some(LegacyProtoOut::CustomMessage { message, .. }) => { + Some(GenericProtoOut::CustomProtocolOpen { .. }) => {}, + Some(GenericProtoOut::CustomMessage { message, .. }) => { match Message::::decode(&mut &message[..]).unwrap() { - Message::::ChainSpecific(message) => { - assert_eq!(message.len(), 1); + Message::::BlockResponse(BlockResponse { id: _, blocks }) => { + assert!(blocks.is_empty()); packet_counter += 1; if packet_counter == NUM_PACKETS { return Poll::Ready(()) @@ -270,9 +274,21 @@ fn basic_two_nodes_requests_in_parallel() { // Generate random messages with or without a request id. let mut to_send = { let mut to_send = Vec::new(); + let mut existing_ids = HashSet::new(); for _ in 0..200 { // Note: don't make that number too high or the CPU usage will explode. - let msg = (0..10).map(|_| rand::random::()).collect::>(); - to_send.push(Message::::ChainSpecific(msg)); + let req_id = loop { + let req_id = rand::random::(); + + // ensure uniqueness - odds of randomly sampling collisions + // is unlikely, but possible to cause spurious test failures. + if existing_ids.insert(req_id) { + break req_id; + } + }; + + to_send.push(Message::::BlockResponse( + BlockResponse { id: req_id, blocks: Vec::new() } + )); } to_send }; @@ -285,7 +301,7 @@ fn basic_two_nodes_requests_in_parallel() { let fut1 = future::poll_fn(move |cx| -> Poll<()> { loop { match ready!(service1.poll_next_unpin(cx)) { - Some(LegacyProtoOut::CustomProtocolOpen { peer_id, .. }) => { + Some(GenericProtoOut::CustomProtocolOpen { peer_id, .. }) => { for msg in to_send.drain(..) { service1.send_packet(&peer_id, msg.encode()); } @@ -298,8 +314,8 @@ fn basic_two_nodes_requests_in_parallel() { let fut2 = future::poll_fn(move |cx| { loop { match ready!(service2.poll_next_unpin(cx)) { - Some(LegacyProtoOut::CustomProtocolOpen { .. }) => {}, - Some(LegacyProtoOut::CustomMessage { message, .. }) => { + Some(GenericProtoOut::CustomProtocolOpen { .. }) => {}, + Some(GenericProtoOut::CustomMessage { message, .. }) => { let pos = to_receive.iter().position(|m| m.encode() == message).unwrap(); to_receive.remove(pos); if to_receive.is_empty() { @@ -335,7 +351,7 @@ fn reconnect_after_disconnect() { let mut service1_not_ready = false; match service1.poll_next_unpin(cx) { - Poll::Ready(Some(LegacyProtoOut::CustomProtocolOpen { .. })) => { + Poll::Ready(Some(GenericProtoOut::CustomProtocolOpen { .. })) => { match service1_state { ServiceState::NotConnected => { service1_state = ServiceState::FirstConnec; @@ -347,7 +363,7 @@ fn reconnect_after_disconnect() { ServiceState::FirstConnec | ServiceState::ConnectedAgain => panic!(), } }, - Poll::Ready(Some(LegacyProtoOut::CustomProtocolClosed { .. })) => { + Poll::Ready(Some(GenericProtoOut::CustomProtocolClosed { .. })) => { match service1_state { ServiceState::FirstConnec => service1_state = ServiceState::Disconnected, ServiceState::ConnectedAgain| ServiceState::NotConnected | @@ -359,7 +375,7 @@ fn reconnect_after_disconnect() { } match service2.poll_next_unpin(cx) { - Poll::Ready(Some(LegacyProtoOut::CustomProtocolOpen { .. })) => { + Poll::Ready(Some(GenericProtoOut::CustomProtocolOpen { .. })) => { match service2_state { ServiceState::NotConnected => { service2_state = ServiceState::FirstConnec; @@ -371,7 +387,7 @@ fn reconnect_after_disconnect() { ServiceState::FirstConnec | ServiceState::ConnectedAgain => panic!(), } }, - Poll::Ready(Some(LegacyProtoOut::CustomProtocolClosed { .. })) => { + Poll::Ready(Some(GenericProtoOut::CustomProtocolClosed { .. })) => { match service2_state { ServiceState::FirstConnec => service2_state = ServiceState::Disconnected, ServiceState::ConnectedAgain| ServiceState::NotConnected | diff --git a/client/network/src/protocol/generic_proto/upgrade.rs b/client/network/src/protocol/generic_proto/upgrade.rs new file mode 100644 index 0000000000000000000000000000000000000000..36f826336532619479b94876776c65e8636f8c57 --- /dev/null +++ b/client/network/src/protocol/generic_proto/upgrade.rs @@ -0,0 +1,35 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +pub use self::collec::UpgradeCollec; +pub use self::legacy::{ + RegisteredProtocol, + RegisteredProtocolEvent, + RegisteredProtocolName, + RegisteredProtocolSubstream +}; +pub use self::notifications::{ + NotificationsIn, + NotificationsInSubstream, + NotificationsOut, + NotificationsOutSubstream, + NotificationsHandshakeError, + NotificationsOutError, +}; + +mod collec; +mod legacy; +mod notifications; diff --git a/client/network/src/protocol/generic_proto/upgrade/collec.rs b/client/network/src/protocol/generic_proto/upgrade/collec.rs new file mode 100644 index 0000000000000000000000000000000000000000..f8d199974940fb1abe6d41829d728243dd73af23 --- /dev/null +++ b/client/network/src/protocol/generic_proto/upgrade/collec.rs @@ -0,0 +1,97 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +use futures::prelude::*; +use libp2p::core::upgrade::{InboundUpgrade, ProtocolName, UpgradeInfo}; +use std::{iter::FromIterator, pin::Pin, task::{Context, Poll}, vec}; + +// TODO: move this to libp2p => https://github.com/libp2p/rust-libp2p/issues/1445 + +/// Upgrade that combines multiple upgrades of the same type into one. Supports all the protocols +/// supported by either sub-upgrade. +#[derive(Debug, Clone)] +pub struct UpgradeCollec(pub Vec); + +impl From> for UpgradeCollec { + fn from(list: Vec) -> Self { + UpgradeCollec(list) + } +} + +impl FromIterator for UpgradeCollec { + fn from_iter>(iter: I) -> Self { + UpgradeCollec(iter.into_iter().collect()) + } +} + +impl UpgradeInfo for UpgradeCollec { + type Info = ProtoNameWithUsize; + type InfoIter = vec::IntoIter; + + fn protocol_info(&self) -> Self::InfoIter { + self.0.iter().enumerate() + .flat_map(|(n, p)| + p.protocol_info().into_iter().map(move |i| ProtoNameWithUsize(i, n))) + .collect::>() + .into_iter() + } +} + +impl InboundUpgrade for UpgradeCollec +where + T: InboundUpgrade, +{ + type Output = (T::Output, usize); + type Error = (T::Error, usize); + type Future = FutWithUsize; + + fn upgrade_inbound(mut self, sock: C, info: Self::Info) -> Self::Future { + let fut = self.0.remove(info.1).upgrade_inbound(sock, info.0); + FutWithUsize(fut, info.1) + } +} + +/// Groups a `ProtocolName` with a `usize`. +#[derive(Debug, Clone)] +pub struct ProtoNameWithUsize(T, usize); + +impl ProtocolName for ProtoNameWithUsize { + fn protocol_name(&self) -> &[u8] { + self.0.protocol_name() + } +} + +/// Equivalent to `fut.map_ok(|v| (v, num)).map_err(|e| (e, num))`, where `fut` and `num` are +/// the two fields of this struct. +#[pin_project::pin_project] +pub struct FutWithUsize(#[pin] T, usize); + +impl>, O, E> Future for FutWithUsize { + type Output = Result<(O, usize), (E, usize)>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { + let this = self.project(); + match Future::poll(this.0, cx) { + Poll::Ready(Ok(v)) => Poll::Ready(Ok((v, *this.1))), + Poll::Ready(Err(e)) => Poll::Ready(Err((e, *this.1))), + Poll::Pending => Poll::Pending, + } + } +} diff --git a/client/network/src/protocol/legacy_proto/upgrade.rs b/client/network/src/protocol/generic_proto/upgrade/legacy.rs similarity index 100% rename from client/network/src/protocol/legacy_proto/upgrade.rs rename to client/network/src/protocol/generic_proto/upgrade/legacy.rs diff --git a/client/network/src/protocol/generic_proto/upgrade/notifications.rs b/client/network/src/protocol/generic_proto/upgrade/notifications.rs new file mode 100644 index 0000000000000000000000000000000000000000..ddc07b5d6f3d6b2ffefe8fec47cee78768a1495a --- /dev/null +++ b/client/network/src/protocol/generic_proto/upgrade/notifications.rs @@ -0,0 +1,622 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +/// Notifications protocol. +/// +/// The Substrate notifications protocol consists in the following: +/// +/// - Node A opens a substream to node B and sends a message which contains some protocol-specific +/// higher-level logic. This message is prefixed with a variable-length integer message length. +/// This message can be empty, in which case `0` is sent. +/// - If node B accepts the substream, it sends back a message with the same properties. +/// Afterwards, the sending side of B is closed. +/// - If instead B refuses the connection (which typically happens because no empty slot is +/// available), then it immediately closes the substream without sending back anything. +/// - Node A can then send notifications to B, prefixed with a variable-length integer indicating +/// the length of the message. +/// - Node A closes its writing side if it doesn't want the notifications substream anymore. +/// +/// Notification substreams are unidirectional. If A opens a substream with B, then B is +/// encouraged but not required to open a substream to A as well. +/// + +use bytes::BytesMut; +use futures::{prelude::*, ready}; +use futures_codec::Framed; +use libp2p::core::{UpgradeInfo, InboundUpgrade, OutboundUpgrade, upgrade}; +use log::error; +use std::{borrow::Cow, collections::VecDeque, io, iter, mem, pin::Pin, task::{Context, Poll}}; +use unsigned_varint::codec::UviBytes; + +/// Maximum allowed size of the two handshake messages, in bytes. +const MAX_HANDSHAKE_SIZE: usize = 1024; +/// Maximum number of buffered messages before we consider the remote unresponsive and kill the +/// substream. +const MAX_PENDING_MESSAGES: usize = 256; + +/// Upgrade that accepts a substream, sends back a status message, then becomes a unidirectional +/// stream of messages. +#[derive(Debug, Clone)] +pub struct NotificationsIn { + /// Protocol name to use when negotiating the substream. + protocol_name: Cow<'static, [u8]>, +} + +/// Upgrade that opens a substream, waits for the remote to accept by sending back a status +/// message, then becomes a unidirectional sink of data. +#[derive(Debug, Clone)] +pub struct NotificationsOut { + /// Protocol name to use when negotiating the substream. + protocol_name: Cow<'static, [u8]>, + /// Message to send when we start the handshake. + initial_message: Vec, +} + +/// A substream for incoming notification messages. +/// +/// When creating, this struct starts in a state in which we must first send back a handshake +/// message to the remote. No message will come before this has been done. +#[pin_project::pin_project] +pub struct NotificationsInSubstream { + #[pin] + socket: Framed>>>, + handshake: NotificationsInSubstreamHandshake, +} + +/// State of the handshake sending back process. +enum NotificationsInSubstreamHandshake { + /// Waiting for the user to give us the handshake message. + NotSent, + /// User gave us the handshake message. Trying to push it in the socket. + PendingSend(Vec), + /// Handshake message was pushed in the socket. Still need to flush. + Close, + /// Handshake message successfully sent. + Sent, +} + +/// A substream for outgoing notification messages. +#[pin_project::pin_project] +pub struct NotificationsOutSubstream { + /// Substream where to send messages. + #[pin] + socket: Framed>>>, + /// Queue of messages waiting to be sent. + messages_queue: VecDeque>, + /// If true, we need to flush `socket`. + need_flush: bool, +} + +impl NotificationsIn { + /// Builds a new potential upgrade. + pub fn new(protocol_name: impl Into>) -> Self { + NotificationsIn { + protocol_name: protocol_name.into(), + } + } + + /// Returns the name of the protocol that we accept. + pub fn protocol_name(&self) -> &[u8] { + &self.protocol_name + } +} + +impl UpgradeInfo for NotificationsIn { + type Info = Cow<'static, [u8]>; + type InfoIter = iter::Once; + + fn protocol_info(&self) -> Self::InfoIter { + iter::once(self.protocol_name.clone()) + } +} + +impl InboundUpgrade for NotificationsIn +where TSubstream: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + type Output = (Vec, NotificationsInSubstream); + type Future = Pin> + Send>>; + type Error = NotificationsHandshakeError; + + fn upgrade_inbound( + self, + mut socket: TSubstream, + _: Self::Info, + ) -> Self::Future { + Box::pin(async move { + let initial_message_len = unsigned_varint::aio::read_usize(&mut socket).await?; + if initial_message_len > MAX_HANDSHAKE_SIZE { + return Err(NotificationsHandshakeError::TooLarge { + requested: initial_message_len, + max: MAX_HANDSHAKE_SIZE, + }); + } + + let mut initial_message = vec![0u8; initial_message_len]; + if !initial_message.is_empty() { + socket.read(&mut initial_message).await?; + } + + let substream = NotificationsInSubstream { + socket: Framed::new(socket, UviBytes::default()), + handshake: NotificationsInSubstreamHandshake::NotSent, + }; + + Ok((initial_message, substream)) + }) + } +} + +impl NotificationsInSubstream +where TSubstream: AsyncRead + AsyncWrite, +{ + /// Sends the handshake in order to inform the remote that we accept the substream. + pub fn send_handshake(&mut self, message: impl Into>) { + match self.handshake { + NotificationsInSubstreamHandshake::NotSent => {} + _ => { + error!(target: "sub-libp2p", "Tried to send handshake twice"); + return; + } + } + + self.handshake = NotificationsInSubstreamHandshake::PendingSend(message.into()); + } +} + +impl Stream for NotificationsInSubstream +where TSubstream: AsyncRead + AsyncWrite + Unpin, +{ + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let mut this = self.project(); + + // This `Stream` implementation first tries to send back the handshake if necessary. + loop { + match mem::replace(this.handshake, NotificationsInSubstreamHandshake::Sent) { + NotificationsInSubstreamHandshake::Sent => + return Stream::poll_next(this.socket.as_mut(), cx), + NotificationsInSubstreamHandshake::NotSent => + return Poll::Pending, + NotificationsInSubstreamHandshake::PendingSend(msg) => + match Sink::poll_ready(this.socket.as_mut(), cx) { + Poll::Ready(_) => { + *this.handshake = NotificationsInSubstreamHandshake::Close; + match Sink::start_send(this.socket.as_mut(), io::Cursor::new(msg)) { + Ok(()) => {}, + Err(err) => return Poll::Ready(Some(Err(err))), + } + }, + Poll::Pending => + *this.handshake = NotificationsInSubstreamHandshake::PendingSend(msg), + }, + NotificationsInSubstreamHandshake::Close => + match Sink::poll_close(this.socket.as_mut(), cx)? { + Poll::Ready(()) => + *this.handshake = NotificationsInSubstreamHandshake::Sent, + Poll::Pending => + *this.handshake = NotificationsInSubstreamHandshake::Close, + }, + } + } + } +} + +impl NotificationsOut { + /// Builds a new potential upgrade. + pub fn new(protocol_name: impl Into>, initial_message: impl Into>) -> Self { + let initial_message = initial_message.into(); + if initial_message.len() > MAX_HANDSHAKE_SIZE { + error!(target: "sub-libp2p", "Outbound networking handshake is above allowed protocol limit"); + } + + NotificationsOut { + protocol_name: protocol_name.into(), + initial_message, + } + } +} + +impl UpgradeInfo for NotificationsOut { + type Info = Cow<'static, [u8]>; + type InfoIter = iter::Once; + + fn protocol_info(&self) -> Self::InfoIter { + iter::once(self.protocol_name.clone()) + } +} + +impl OutboundUpgrade for NotificationsOut +where TSubstream: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + type Output = (Vec, NotificationsOutSubstream); + type Future = Pin> + Send>>; + type Error = NotificationsHandshakeError; + + fn upgrade_outbound( + self, + mut socket: TSubstream, + _: Self::Info, + ) -> Self::Future { + Box::pin(async move { + upgrade::write_with_len_prefix(&mut socket, &self.initial_message).await?; + + // Reading handshake. + let handshake_len = unsigned_varint::aio::read_usize(&mut socket).await?; + if handshake_len > MAX_HANDSHAKE_SIZE { + return Err(NotificationsHandshakeError::TooLarge { + requested: handshake_len, + max: MAX_HANDSHAKE_SIZE, + }); + } + + let mut handshake = vec![0u8; handshake_len]; + if !handshake.is_empty() { + socket.read(&mut handshake).await?; + } + + Ok((handshake, NotificationsOutSubstream { + socket: Framed::new(socket, UviBytes::default()), + messages_queue: VecDeque::with_capacity(MAX_PENDING_MESSAGES), + need_flush: false, + })) + }) + } +} + +impl Sink> for NotificationsOutSubstream + where TSubstream: AsyncRead + AsyncWrite + Unpin, +{ + type Error = NotificationsOutError; + + fn poll_ready(self: Pin<&mut Self>, _: &mut Context) -> Poll> { + Poll::Ready(Ok(())) + } + + fn start_send(mut self: Pin<&mut Self>, item: Vec) -> Result<(), Self::Error> { + if self.messages_queue.len() >= MAX_PENDING_MESSAGES { + return Err(NotificationsOutError::Clogged); + } + + self.messages_queue.push_back(item); + Ok(()) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + let mut this = self.project(); + + while !this.messages_queue.is_empty() { + match Sink::poll_ready(this.socket.as_mut(), cx) { + Poll::Ready(Err(err)) => return Poll::Ready(Err(From::from(err))), + Poll::Ready(Ok(())) => { + let msg = this.messages_queue.pop_front() + .expect("checked for !is_empty above; qed"); + Sink::start_send(this.socket.as_mut(), io::Cursor::new(msg))?; + *this.need_flush = true; + }, + Poll::Pending => return Poll::Pending, + } + } + + if *this.need_flush { + match Sink::poll_flush(this.socket.as_mut(), cx) { + Poll::Ready(Err(err)) => return Poll::Ready(Err(From::from(err))), + Poll::Ready(Ok(())) => *this.need_flush = false, + Poll::Pending => return Poll::Pending, + } + } + + Poll::Ready(Ok(())) + } + + fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + ready!(Sink::poll_flush(self.as_mut(), cx))?; + let this = self.project(); + match Sink::poll_close(this.socket, cx) { + Poll::Ready(Ok(())) => Poll::Ready(Ok(())), + Poll::Ready(Err(err)) => Poll::Ready(Err(From::from(err))), + Poll::Pending => Poll::Pending, + } + } +} + +/// Error generated by sending on a notifications out substream. +#[derive(Debug, derive_more::From, derive_more::Display)] +pub enum NotificationsHandshakeError { + /// I/O error on the substream. + Io(io::Error), + + /// Initial message or handshake was too large. + #[display(fmt = "Initial message or handshake was too large: {}", requested)] + TooLarge { + /// Size requested by the remote. + requested: usize, + /// Maximum allowed, + max: usize, + }, + + /// Error while decoding the variable-length integer. + VarintDecode(unsigned_varint::decode::Error), +} + +impl From for NotificationsHandshakeError { + fn from(err: unsigned_varint::io::ReadError) -> Self { + match err { + unsigned_varint::io::ReadError::Io(err) => NotificationsHandshakeError::Io(err), + unsigned_varint::io::ReadError::Decode(err) => NotificationsHandshakeError::VarintDecode(err), + _ => { + log::warn!("Unrecognized varint decoding error"); + NotificationsHandshakeError::Io(From::from(io::ErrorKind::InvalidData)) + } + } + } +} + +/// Error generated by sending on a notifications out substream. +#[derive(Debug, derive_more::From, derive_more::Display)] +pub enum NotificationsOutError { + /// I/O error on the substream. + Io(io::Error), + + /// Remote doesn't process our messages quickly enough. + /// + /// > **Note**: This is not necessarily the remote's fault, and could also be caused by the + /// > local node sending data too quickly. Properly doing back-pressure, however, + /// > would require a deep refactoring effort in Substrate as a whole. + Clogged, +} + +#[cfg(test)] +mod tests { + use super::{NotificationsIn, NotificationsOut}; + + use async_std::net::{TcpListener, TcpStream}; + use futures::{prelude::*, channel::oneshot}; + use libp2p::core::upgrade; + use std::pin::Pin; + + #[test] + fn basic_works() { + const PROTO_NAME: &'static [u8] = b"/test/proto/1"; + let (listener_addr_tx, listener_addr_rx) = oneshot::channel(); + + let client = async_std::task::spawn(async move { + let socket = TcpStream::connect(listener_addr_rx.await.unwrap()).await.unwrap(); + let (handshake, mut substream) = upgrade::apply_outbound( + socket, + NotificationsOut::new(PROTO_NAME, &b"initial message"[..]), + upgrade::Version::V1 + ).await.unwrap(); + + assert_eq!(handshake, b"hello world"); + substream.send(b"test message".to_vec()).await.unwrap(); + }); + + async_std::task::block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener_addr_tx.send(listener.local_addr().unwrap()).unwrap(); + + let (socket, _) = listener.accept().await.unwrap(); + let (initial_message, mut substream) = upgrade::apply_inbound( + socket, + NotificationsIn::new(PROTO_NAME) + ).await.unwrap(); + + assert_eq!(initial_message, b"initial message"); + substream.send_handshake(&b"hello world"[..]); + + let msg = substream.next().await.unwrap().unwrap(); + assert_eq!(msg.as_ref(), b"test message"); + }); + + async_std::task::block_on(client); + } + + #[test] + fn empty_handshake() { + // Check that everything still works when the handshake messages are empty. + + const PROTO_NAME: &'static [u8] = b"/test/proto/1"; + let (listener_addr_tx, listener_addr_rx) = oneshot::channel(); + + let client = async_std::task::spawn(async move { + let socket = TcpStream::connect(listener_addr_rx.await.unwrap()).await.unwrap(); + let (handshake, mut substream) = upgrade::apply_outbound( + socket, + NotificationsOut::new(PROTO_NAME, vec![]), + upgrade::Version::V1 + ).await.unwrap(); + + assert!(handshake.is_empty()); + substream.send(Default::default()).await.unwrap(); + }); + + async_std::task::block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener_addr_tx.send(listener.local_addr().unwrap()).unwrap(); + + let (socket, _) = listener.accept().await.unwrap(); + let (initial_message, mut substream) = upgrade::apply_inbound( + socket, + NotificationsIn::new(PROTO_NAME) + ).await.unwrap(); + + assert!(initial_message.is_empty()); + substream.send_handshake(vec![]); + + let msg = substream.next().await.unwrap().unwrap(); + assert!(msg.as_ref().is_empty()); + }); + + async_std::task::block_on(client); + } + + #[test] + fn refused() { + const PROTO_NAME: &'static [u8] = b"/test/proto/1"; + let (listener_addr_tx, listener_addr_rx) = oneshot::channel(); + + let client = async_std::task::spawn(async move { + let socket = TcpStream::connect(listener_addr_rx.await.unwrap()).await.unwrap(); + let outcome = upgrade::apply_outbound( + socket, + NotificationsOut::new(PROTO_NAME, &b"hello"[..]), + upgrade::Version::V1 + ).await; + + // Despite the protocol negotiation being successfully conducted on the listener + // side, we have to receive an error here because the listener didn't send the + // handshake. + assert!(outcome.is_err()); + }); + + async_std::task::block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener_addr_tx.send(listener.local_addr().unwrap()).unwrap(); + + let (socket, _) = listener.accept().await.unwrap(); + let (initial_msg, substream) = upgrade::apply_inbound( + socket, + NotificationsIn::new(PROTO_NAME) + ).await.unwrap(); + + assert_eq!(initial_msg, b"hello"); + + // We successfully upgrade to the protocol, but then close the substream. + drop(substream); + }); + + async_std::task::block_on(client); + } + + #[test] + fn large_initial_message_refused() { + const PROTO_NAME: &'static [u8] = b"/test/proto/1"; + let (listener_addr_tx, listener_addr_rx) = oneshot::channel(); + + let client = async_std::task::spawn(async move { + let socket = TcpStream::connect(listener_addr_rx.await.unwrap()).await.unwrap(); + let ret = upgrade::apply_outbound( + socket, + // We check that an initial message that is too large gets refused. + NotificationsOut::new(PROTO_NAME, (0..32768).map(|_| 0).collect::>()), + upgrade::Version::V1 + ).await; + assert!(ret.is_err()); + }); + + async_std::task::block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener_addr_tx.send(listener.local_addr().unwrap()).unwrap(); + + let (socket, _) = listener.accept().await.unwrap(); + let ret = upgrade::apply_inbound( + socket, + NotificationsIn::new(PROTO_NAME) + ).await; + assert!(ret.is_err()); + }); + + async_std::task::block_on(client); + } + + #[test] + fn large_handshake_refused() { + const PROTO_NAME: &'static [u8] = b"/test/proto/1"; + let (listener_addr_tx, listener_addr_rx) = oneshot::channel(); + + let client = async_std::task::spawn(async move { + let socket = TcpStream::connect(listener_addr_rx.await.unwrap()).await.unwrap(); + let ret = upgrade::apply_outbound( + socket, + NotificationsOut::new(PROTO_NAME, &b"initial message"[..]), + upgrade::Version::V1 + ).await; + assert!(ret.is_err()); + }); + + async_std::task::block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener_addr_tx.send(listener.local_addr().unwrap()).unwrap(); + + let (socket, _) = listener.accept().await.unwrap(); + let (initial_message, mut substream) = upgrade::apply_inbound( + socket, + NotificationsIn::new(PROTO_NAME) + ).await.unwrap(); + assert_eq!(initial_message, b"initial message"); + + // We check that a handshake that is too large gets refused. + substream.send_handshake((0..32768).map(|_| 0).collect::>()); + let _ = substream.next().await; + }); + + async_std::task::block_on(client); + } + + #[test] + fn buffer_is_full_closes_connection() { + const PROTO_NAME: &'static [u8] = b"/test/proto/1"; + let (listener_addr_tx, listener_addr_rx) = oneshot::channel(); + + let client = async_std::task::spawn(async move { + let socket = TcpStream::connect(listener_addr_rx.await.unwrap()).await.unwrap(); + let (handshake, mut substream) = upgrade::apply_outbound( + socket, + NotificationsOut::new(PROTO_NAME, vec![]), + upgrade::Version::V1 + ).await.unwrap(); + + assert!(handshake.is_empty()); + + // Push an item and flush so that the test works. + substream.send(b"hello world".to_vec()).await.unwrap(); + + for _ in 0..32768 { + // Push an item on the sink without flushing until an error happens because the + // buffer is full. + let message = b"hello world!".to_vec(); + if future::poll_fn(|cx| Sink::poll_ready(Pin::new(&mut substream), cx)).await.is_err() { + return Ok(()); + } + if Sink::start_send(Pin::new(&mut substream), message).is_err() { + return Ok(()); + } + } + + Err(()) + }); + + async_std::task::block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener_addr_tx.send(listener.local_addr().unwrap()).unwrap(); + + let (socket, _) = listener.accept().await.unwrap(); + let (initial_message, mut substream) = upgrade::apply_inbound( + socket, + NotificationsIn::new(PROTO_NAME) + ).await.unwrap(); + + assert!(initial_message.is_empty()); + substream.send_handshake(vec![]); + + // Process one message so that the handshake and all works. + let _ = substream.next().await.unwrap().unwrap(); + + client.await.unwrap(); + }); + } +} diff --git a/client/network/src/protocol/light_client_handler.rs b/client/network/src/protocol/light_client_handler.rs index 77cf71408d6c49f78511cb6c6936820eaa126c1e..a141e134fca051a40c67bdf84c4662edcda60ae5 100644 --- a/client/network/src/protocol/light_client_handler.rs +++ b/client/network/src/protocol/light_client_handler.rs @@ -127,7 +127,7 @@ impl Config { let mut v = Vec::new(); v.extend_from_slice(b"/"); v.extend_from_slice(id.as_bytes()); - v.extend_from_slice(b"/light/1"); + v.extend_from_slice(b"/light/2"); self.protocol = v.into(); self } @@ -350,7 +350,7 @@ where , response: api::v1::light::Response ) -> Result, Error> { - log::trace!("response {} from {}", response.id, peer); + log::trace!("response from {}", peer); use api::v1::light::response::Response; match response.response { Some(Response::RemoteCallResponse(response)) => @@ -419,12 +419,10 @@ where fn on_remote_call_request ( &mut self , peer: &PeerId - , request_id: u64 , request: &api::v1::light::RemoteCallRequest ) -> Result { - log::trace!("remote call request {} from {} ({} at {:?})", - request_id, + log::trace!("remote call request from {} ({} at {:?})", peer, request.method, request.block); @@ -434,8 +432,7 @@ where let proof = match self.chain.execution_proof(&block, &request.method, &request.data) { Ok((_, proof)) => proof, Err(e) => { - log::trace!("remote call request {} from {} ({} at {:?}) failed with: {}", - request_id, + log::trace!("remote call request from {} ({} at {:?}) failed with: {}", peer, request.method, request.block, @@ -449,13 +446,12 @@ where api::v1::light::response::Response::RemoteCallResponse(r) }; - Ok(api::v1::light::Response { id: request_id, response: Some(response) }) + Ok(api::v1::light::Response { response: Some(response) }) } fn on_remote_read_request ( &mut self , peer: &PeerId - , request_id: u64 , request: &api::v1::light::RemoteReadRequest ) -> Result { @@ -464,19 +460,17 @@ where return Err(Error::BadRequest("remote read request without keys")) } - log::trace!("remote read request {} from {} ({} at {:?})", - request_id, + log::trace!("remote read request from {} ({} at {:?})", peer, fmt_keys(request.keys.first(), request.keys.last()), request.block); let block = Decode::decode(&mut request.block.as_ref())?; - let proof = match self.chain.read_proof(&block, &request.keys) { + let proof = match self.chain.read_proof(&block, &mut request.keys.iter().map(AsRef::as_ref)) { Ok(proof) => proof, Err(error) => { - log::trace!("remote read request {} from {} ({} at {:?}) failed with: {}", - request_id, + log::trace!("remote read request from {} ({} at {:?}) failed with: {}", peer, fmt_keys(request.keys.first(), request.keys.last()), request.block, @@ -490,13 +484,12 @@ where api::v1::light::response::Response::RemoteReadResponse(r) }; - Ok(api::v1::light::Response { id: request_id, response: Some(response) }) + Ok(api::v1::light::Response { response: Some(response) }) } fn on_remote_read_child_request ( &mut self , peer: &PeerId - , request_id: u64 , request: &api::v1::light::RemoteReadChildRequest ) -> Result { @@ -505,8 +498,7 @@ where return Err(Error::BadRequest("remove read child request without keys")) } - log::trace!("remote read child request {} from {} ({} {} at {:?})", - request_id, + log::trace!("remote read child request from {} ({} {} at {:?})", peer, request.storage_key.to_hex::(), fmt_keys(request.keys.first(), request.keys.last()), @@ -516,11 +508,15 @@ where let proof = if let Some(info) = ChildInfo::resolve_child_info(request.child_type, &request.child_info[..]) { - match self.chain.read_child_proof(&block, &request.storage_key, info, &request.keys) { + match self.chain.read_child_proof( + &block, + &request.storage_key, + info, + &mut request.keys.iter().map(AsRef::as_ref) + ) { Ok(proof) => proof, Err(error) => { - log::trace!("remote read child request {} from {} ({} {} at {:?}) failed with: {}", - request_id, + log::trace!("remote read child request from {} ({} {} at {:?}) failed with: {}", peer, request.storage_key.to_hex::(), fmt_keys(request.keys.first(), request.keys.last()), @@ -530,8 +526,7 @@ where } } } else { - log::trace!("remote read child request {} from {} ({} {} at {:?}) failed with: {}", - request_id, + log::trace!("remote read child request from {} ({} {} at {:?}) failed with: {}", peer, request.storage_key.to_hex::(), fmt_keys(request.keys.first(), request.keys.last()), @@ -546,25 +541,23 @@ where api::v1::light::response::Response::RemoteReadResponse(r) }; - Ok(api::v1::light::Response { id: request_id, response: Some(response) }) + Ok(api::v1::light::Response { response: Some(response) }) } fn on_remote_header_request ( &mut self , peer: &PeerId - , request_id: u64 , request: &api::v1::light::RemoteHeaderRequest ) -> Result { - log::trace!("remote header proof request {} from {} ({:?})", request_id, peer, request.block); + log::trace!("remote header proof request from {} ({:?})", peer, request.block); let block = Decode::decode(&mut request.block.as_ref())?; let (header, proof) = match self.chain.header_proof(block) { Ok((header, proof)) => (header.encode(), proof), Err(error) => { - log::trace!("remote header proof request {} from {} ({:?}) failed with: {}", - request_id, + log::trace!("remote header proof request from {} ({:?}) failed with: {}", peer, request.block, error); @@ -577,18 +570,16 @@ where api::v1::light::response::Response::RemoteHeaderResponse(r) }; - Ok(api::v1::light::Response { id: request_id, response: Some(response) }) + Ok(api::v1::light::Response { response: Some(response) }) } fn on_remote_changes_request ( &mut self , peer: &PeerId - , request_id: u64 , request: &api::v1::light::RemoteChangesRequest ) -> Result { - log::trace!("remote changes proof request {} from {} for key {} ({:?}..{:?})", - request_id, + log::trace!("remote changes proof request from {} for key {} ({:?}..{:?})", peer, if !request.storage_key.is_empty() { format!("{} : {}", request.storage_key.to_hex::(), request.key.to_hex::()) @@ -613,8 +604,7 @@ where let proof = match self.chain.key_changes_proof(first, last, min, max, storage_key.as_ref(), &key) { Ok(proof) => proof, Err(error) => { - log::trace!("remote changes proof request {} from {} for key {} ({:?}..{:?}) failed with: {}", - request_id, + log::trace!("remote changes proof request from {} for key {} ({:?}..{:?}) failed with: {}", peer, if let Some(sk) = storage_key { format!("{} : {}", sk.0.to_hex::(), key.0.to_hex::()) @@ -646,7 +636,7 @@ where api::v1::light::response::Response::RemoteChangesResponse(r) }; - Ok(api::v1::light::Response { id: request_id, response: Some(response) }) + Ok(api::v1::light::Response { response: Some(response) }) } } @@ -697,29 +687,29 @@ where match event { // An incoming request from remote has been received. Event::Request(request, mut stream) => { - log::trace!("incoming request {} from {}", peer, request.id); + log::trace!("incoming request from {}", peer); let result = match &request.request { Some(api::v1::light::request::Request::RemoteCallRequest(r)) => - self.on_remote_call_request(&peer, request.id, r), + self.on_remote_call_request(&peer, r), Some(api::v1::light::request::Request::RemoteReadRequest(r)) => - self.on_remote_read_request(&peer, request.id, r), + self.on_remote_read_request(&peer, r), Some(api::v1::light::request::Request::RemoteHeaderRequest(r)) => - self.on_remote_header_request(&peer, request.id, r), + self.on_remote_header_request(&peer, r), Some(api::v1::light::request::Request::RemoteReadChildRequest(r)) => - self.on_remote_read_child_request(&peer, request.id, r), + self.on_remote_read_child_request(&peer, r), Some(api::v1::light::request::Request::RemoteChangesRequest(r)) => - self.on_remote_changes_request(&peer, request.id, r), + self.on_remote_changes_request(&peer, r), None => { - log::debug!("ignoring request {} without request data from peer {}", request.id, peer); + log::debug!("ignoring request without request data from peer {}", peer); return } }; match result { Ok(response) => { - log::trace!("enqueueing response {} for peer {}", response.id, peer); + log::trace!("enqueueing response for peer {}", peer); let mut data = Vec::new(); if let Err(e) = response.encode(&mut data) { - log::debug!("error encoding response {} for peer {}: {}", response.id, peer, e) + log::debug!("error encoding response for peer {}: {}", peer, e) } else { let future = async move { if let Err(e) = write_one(&mut stream, data).await { @@ -733,16 +723,15 @@ where self.remove_peer(&peer); self.peerset.report_peer(peer, ReputationChange::new(-(1 << 12), "bad request")) } - Err(e) => log::debug!("error handling request {} from peer {}: {}", request.id, peer, e) + Err(e) => log::debug!("error handling request from peer {}: {}", peer, e) } } // A response to one of our own requests has been received. - Event::Response(response) => { - let id = response.id; + Event::Response(id, response) => { if let Some(request) = self.outstanding.remove(&id) { // We first just check if the response originates from the expected peer. if request.peer != peer { - log::debug!("was expecting response {} from {} instead of {}", id, request.peer, peer); + log::debug!("was expecting response from {} instead of {}", request.peer, peer); self.outstanding.insert(id, request); self.remove_peer(&peer); self.peerset.report_peer(peer, ReputationChange::new_fatal("response from unexpected peer")); @@ -836,16 +825,17 @@ where } }; if let Some(peer) = available_peer { - let id = self.next_request_id(); - let rq = serialize_request(id, &request.request); + let rq = serialize_request(&request.request); let mut buf = Vec::with_capacity(rq.encoded_len()); if let Err(e) = rq.encode(&mut buf) { - log::debug!("failed to serialize request {}: {}", id, e); + log::debug!("failed to serialize request: {}", e); send_reply(Err(ClientError::RemoteFetchFailed), request.request) } else { + let id = self.next_request_id(); log::trace!("sending request {} to peer {}", id, peer); let protocol = OutboundProtocol { request: buf, + request_id: id, max_data_size: self.config.max_data_size, protocol: self.config.protocol.clone(), }; @@ -918,7 +908,7 @@ fn retries(request: &Request) -> usize { rc.unwrap_or(0) } -fn serialize_request(id: u64, request: &Request) -> api::v1::light::Request { +fn serialize_request(request: &Request) -> api::v1::light::Request { let request = match request { Request::Header { request, .. } => { let r = api::v1::light::RemoteHeaderRequest { block: request.block.encode() }; @@ -962,7 +952,7 @@ fn serialize_request(id: u64, request: &Request) -> api::v1::light: } }; - api::v1::light::Request { id, request: Some(request) } + api::v1::light::Request { request: Some(request) } } fn send_reply(result: Result, ClientError>, request: Request) { @@ -1004,7 +994,7 @@ pub enum Event { /// Incoming request from remote and substream to use for the response. Request(api::v1::light::Request, T), /// Incoming response from remote. - Response(api::v1::light::Response), + Response(u64, api::v1::light::Response), } /// Substream upgrade protocol. @@ -1054,6 +1044,8 @@ where pub struct OutboundProtocol { /// The serialized protobuf request. request: Vec, + /// Local identifier for the request. Used to associate it with a response. + request_id: u64, /// The max. request length in bytes. max_data_size: usize, /// The protocol to use for upgrade negotiation. @@ -1082,7 +1074,7 @@ where write_one(&mut s, &self.request).await?; let vec = read_one(&mut s, self.max_data_size).await?; api::v1::light::Response::decode(&vec[..]) - .map(Event::Response) + .map(|r| Event::Response(self.request_id, r)) .map_err(|e| { ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e)) }) @@ -1308,53 +1300,6 @@ mod tests { assert_eq!(0, behaviour.outstanding.len()); } - #[test] - fn disconnects_from_peer_on_response_with_wrong_id() { - let peer = PeerId::random(); - let pset = peerset(); - let mut behaviour = make_behaviour(true, pset.1, make_config()); - - behaviour.inject_connected(peer.clone(), empty_dialer()); - assert_eq!(1, behaviour.peers.len()); - - let chan = oneshot::channel(); - let request = fetcher::RemoteCallRequest { - block: Default::default(), - header: dummy_header(), - method: "test".into(), - call_data: vec![], - retry_count: Some(1), - }; - behaviour.request(Request::Call { request, sender: chan.0 }).unwrap(); - - assert_eq!(1, behaviour.pending_requests.len()); - assert_eq!(0, behaviour.outstanding.len()); - poll(&mut behaviour); // Make progress - assert_eq!(0, behaviour.pending_requests.len()); - assert_eq!(1, behaviour.outstanding.len()); - - // Construct response with bogus ID - let response = { - let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; - api::v1::light::Response { - id: 2365789, - response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), - } - }; - - // Make sure our bogus ID is really not used. - assert!(!behaviour.outstanding.keys().any(|id| id == &response.id)); - - behaviour.inject_node_event(peer.clone(), Event::Response(response)); - assert!(behaviour.peers.is_empty()); - - poll(&mut behaviour); // More progress - - // The request should be back in the pending queue - assert_eq!(1, behaviour.pending_requests.len()); - assert_eq!(0, behaviour.outstanding.len()); - } - #[test] fn disconnects_from_peer_on_incorrect_response() { let peer = PeerId::random(); @@ -1386,12 +1331,11 @@ mod tests { let response = { let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; api::v1::light::Response { - id: request_id, response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), } }; - behaviour.inject_node_event(peer.clone(), Event::Response(response)); + behaviour.inject_node_event(peer.clone(), Event::Response(request_id, response)); assert!(behaviour.peers.is_empty()); poll(&mut behaviour); // More progress @@ -1416,12 +1360,11 @@ mod tests { let response = { let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; api::v1::light::Response { - id: 2347895932, response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), } }; - behaviour.inject_node_event(peer.clone(), Event::Response(response)); + behaviour.inject_node_event(peer.clone(), Event::Response(2347895932, response)); assert!(behaviour.peers.is_empty()); poll(&mut behaviour); @@ -1459,12 +1402,11 @@ mod tests { let response = { let r = api::v1::light::RemoteReadResponse { proof: empty_proof() }; // Not a RemoteCallResponse! api::v1::light::Response { - id: request_id, response: Some(api::v1::light::response::Response::RemoteReadResponse(r)), } }; - behaviour.inject_node_event(peer.clone(), Event::Response(response)); + behaviour.inject_node_event(peer.clone(), Event::Response(request_id, response)); assert!(behaviour.peers.is_empty()); poll(&mut behaviour); // More progress @@ -1513,11 +1455,10 @@ mod tests { let response = { let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; api::v1::light::Response { - id: request_id, response: Some(api::v1::light::response::Response::RemoteCallResponse(r)) } }; - behaviour.inject_node_event(responding_peer, Event::Response(response.clone())); + behaviour.inject_node_event(responding_peer, Event::Response(request_id, response.clone())); assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { .. })); assert_matches!(chan.1.try_recv(), Ok(None)) } @@ -1527,11 +1468,10 @@ mod tests { let response = { let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; api::v1::light::Response { - id: request_id, response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), } }; - behaviour.inject_node_event(responding_peer, Event::Response(response)); + behaviour.inject_node_event(responding_peer, Event::Response(request_id, response)); assert_matches!(poll(&mut behaviour), Poll::Pending); assert_matches!(chan.1.try_recv(), Ok(Some(Err(ClientError::RemoteFetchFailed)))) } @@ -1551,28 +1491,24 @@ mod tests { proof: empty_proof() }; api::v1::light::Response { - id: 1, response: Some(api::v1::light::response::Response::RemoteHeaderResponse(r)), } } Request::Read{..} => { let r = api::v1::light::RemoteReadResponse { proof: empty_proof() }; api::v1::light::Response { - id: 1, response: Some(api::v1::light::response::Response::RemoteReadResponse(r)), } } Request::ReadChild{..} => { let r = api::v1::light::RemoteReadResponse { proof: empty_proof() }; api::v1::light::Response { - id: 1, response: Some(api::v1::light::response::Response::RemoteReadResponse(r)), } } Request::Call{..} => { let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; api::v1::light::Response { - id: 1, response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), } } @@ -1584,7 +1520,6 @@ mod tests { roots_proof: empty_proof() }; api::v1::light::Response { - id: 1, response: Some(api::v1::light::response::Response::RemoteChangesResponse(r)), } } @@ -1599,7 +1534,7 @@ mod tests { assert_eq!(1, behaviour.outstanding.len()); assert_eq!(1, *behaviour.outstanding.keys().next().unwrap()); - behaviour.inject_node_event(peer.clone(), Event::Response(response)); + behaviour.inject_node_event(peer.clone(), Event::Response(1, response)); poll(&mut behaviour); diff --git a/client/network/src/protocol/light_dispatch.rs b/client/network/src/protocol/light_dispatch.rs index 738847dbaf3cf2229503e53855f2d18318ffad5b..aff220b6e03a6cdf41389d881e62f73497cea003 100644 --- a/client/network/src/protocol/light_dispatch.rs +++ b/client/network/src/protocol/light_dispatch.rs @@ -30,7 +30,7 @@ use sp_blockchain::Error as ClientError; use sc_client_api::{FetchChecker, RemoteHeaderRequest, RemoteCallRequest, RemoteReadRequest, RemoteChangesRequest, ChangesProof, RemoteReadChildRequest, RemoteBodyRequest, StorageProof}; -use crate::message::{self, BlockAttributes, Direction, FromBlock, RequestId}; +use crate::protocol::message::{self, BlockAttributes, Direction, FromBlock, RequestId}; use libp2p::PeerId; use crate::config::Roles; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index a2261b2059110af0049e520a093f0acf262512c1..a12c26da2e47eec14ae136ede05e6094d9e0c715 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -219,9 +219,6 @@ pub mod generic { FinalityProofResponse(FinalityProofResponse), /// Batch of consensus protocol messages. ConsensusBatch(Vec), - /// Chain-specific message. - #[codec(index = "255")] - ChainSpecific(Vec), } impl Message { @@ -246,7 +243,6 @@ pub mod generic { Message::FinalityProofRequest(_) => "FinalityProofRequest", Message::FinalityProofResponse(_) => "FinalityProofResponse", Message::ConsensusBatch(_) => "ConsensusBatch", - Message::ChainSpecific(_) => "ChainSpecific", } } } diff --git a/client/network/src/protocol/schema/api.v1.proto b/client/network/src/protocol/schema/api.v1.proto index 73128c53de46635a1af74cd78c6f19087028af7a..ccbf49d666115ea50fbe08885c9242c891d805a1 100644 --- a/client/network/src/protocol/schema/api.v1.proto +++ b/client/network/src/protocol/schema/api.v1.proto @@ -14,31 +14,27 @@ enum Direction { // Request block data from a peer. message BlockRequest { - // Unique request id. - uint64 id = 1; // Bits of block data to request. - uint32 fields = 2; + uint32 fields = 1; // Start from this block. oneof from_block { // Start with given hash. - bytes hash = 3; + bytes hash = 2; // Start with given block number. - bytes number = 4; + bytes number = 3; } // End at this block. An implementation defined maximum is used when unspecified. - bytes to_block = 5; // optional + bytes to_block = 4; // optional // Sequence direction. - Direction direction = 6; + Direction direction = 5; // Maximum number of blocks to return. An implementation defined maximum is used when unspecified. - uint32 max_blocks = 7; // optional + uint32 max_blocks = 6; // optional } // Response to `BlockRequest` message BlockResponse { - // Id of a request this response was made for. - uint64 id = 1; // Block data for the requested sequence. - repeated BlockData blocks = 2; + repeated BlockData blocks = 1; } // Block data sent in the response. diff --git a/client/network/src/protocol/schema/light.v1.proto b/client/network/src/protocol/schema/light.v1.proto index b9aee67b5ee24ff6614e3db2a0c711988a8d03c1..1c98d49730cf98dcafb9999056682f6c40efb101 100644 --- a/client/network/src/protocol/schema/light.v1.proto +++ b/client/network/src/protocol/schema/light.v1.proto @@ -14,26 +14,22 @@ message Pair { // Enumerate all possible light client request messages. message Request { - // Unique request id. - uint64 id = 1; oneof request { - RemoteCallRequest remote_call_request = 2; - RemoteReadRequest remote_read_request = 3; - RemoteHeaderRequest remote_header_request = 4; - RemoteReadChildRequest remote_read_child_request = 5; - RemoteChangesRequest remote_changes_request = 6; + RemoteCallRequest remote_call_request = 1; + RemoteReadRequest remote_read_request = 2; + RemoteHeaderRequest remote_header_request = 3; + RemoteReadChildRequest remote_read_child_request = 4; + RemoteChangesRequest remote_changes_request = 5; } } // Enumerate all possible light client response messages. message Response { - /// Id of a request this response was made for. - uint64 id = 1; oneof response { - RemoteCallResponse remote_call_response = 2; - RemoteReadResponse remote_read_response = 3; - RemoteHeaderResponse remote_header_response = 4; - RemoteChangesResponse remote_changes_response = 6; + RemoteCallResponse remote_call_response = 1; + RemoteReadResponse remote_read_response = 2; + RemoteHeaderResponse remote_header_response = 3; + RemoteChangesResponse remote_changes_response = 4; } } diff --git a/client/network/src/protocol/specialization.rs b/client/network/src/protocol/specialization.rs deleted file mode 100644 index 6ffa335c8c330fb1689acf7176c8442dd31d4e8a..0000000000000000000000000000000000000000 --- a/client/network/src/protocol/specialization.rs +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2017-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Specializations of the substrate network protocol to allow more complex forms of communication. - -pub use crate::protocol::event::{DhtEvent, Event}; - -use crate::protocol::Context; -use libp2p::PeerId; -use sp_runtime::traits::Block as BlockT; - -/// A specialization of the substrate network protocol. Handles events and sends messages. -pub trait NetworkSpecialization: Send + Sync + 'static { - /// Get the current specialization-status. - fn status(&self) -> Vec; - - /// Called when a peer successfully handshakes. - fn on_connect(&mut self, ctx: &mut dyn Context, who: PeerId, status: crate::message::Status); - - /// Called when a peer is disconnected. If the peer ID is unknown, it should be ignored. - fn on_disconnect(&mut self, ctx: &mut dyn Context, who: PeerId); - - /// Called when a network-specific message arrives. - fn on_message( - &mut self, - ctx: &mut dyn Context, - who: PeerId, - message: Vec - ); - - /// Called periodically to maintain peers and handle timeouts. - fn maintain_peers(&mut self, _ctx: &mut dyn Context) { } - - /// Called when a block is _imported_ at the head of the chain (not during major sync). - /// Not guaranteed to be called for every block, but will be most of the after major sync. - fn on_block_imported(&mut self, _ctx: &mut dyn Context, _hash: B::Hash, _header: &B::Header) { } -} - -/// A specialization that does nothing. -#[derive(Clone)] -pub struct DummySpecialization; - -impl NetworkSpecialization for DummySpecialization { - fn status(&self) -> Vec { - vec![] - } - - fn on_connect( - &mut self, - _ctx: &mut dyn Context, - _peer_id: PeerId, - _status: crate::message::Status - ) {} - - fn on_disconnect(&mut self, _ctx: &mut dyn Context, _peer_id: PeerId) {} - - fn on_message( - &mut self, - _ctx: &mut dyn Context, - _peer_id: PeerId, - _message: Vec, - ) {} -} - -/// Construct a simple protocol that is composed of several sub protocols. -/// Each "sub protocol" needs to implement `Specialization` and needs to provide a `new()` function. -/// For more fine grained implementations, this macro is not usable. -/// -/// # Example -/// -/// ```nocompile -/// construct_simple_protocol! { -/// pub struct MyProtocol where Block = MyBlock { -/// consensus_gossip: ConsensusGossip, -/// other_protocol: MyCoolStuff, -/// } -/// } -/// ``` -/// -/// You can also provide an optional parameter after `where Block = MyBlock`, so it looks like -/// `where Block = MyBlock, Status = consensus_gossip`. This will instruct the implementation to -/// use the `status()` function from the `ConsensusGossip` protocol. By default, `status()` returns -/// an empty vector. -#[macro_export] -macro_rules! construct_simple_protocol { - ( - $( #[ $attr:meta ] )* - pub struct $protocol:ident where - Block = $block:ident - $( , Status = $status_protocol_name:ident )* - { - $( $sub_protocol_name:ident : $sub_protocol:ident $( <$protocol_block:ty> )*, )* - } - ) => { - $( #[$attr] )* - pub struct $protocol { - $( $sub_protocol_name: $sub_protocol $( <$protocol_block> )*, )* - } - - impl $protocol { - /// Instantiate a node protocol handler. - pub fn new() -> Self { - Self { - $( $sub_protocol_name: $sub_protocol::new(), )* - } - } - } - - impl $crate::specialization::NetworkSpecialization<$block> for $protocol { - fn status(&self) -> Vec { - $( - let status = self.$status_protocol_name.status(); - - if !status.is_empty() { - return status; - } - )* - - Vec::new() - } - - fn on_connect( - &mut self, - _ctx: &mut $crate::Context<$block>, - _who: $crate::PeerId, - _status: $crate::StatusMessage<$block> - ) { - $( self.$sub_protocol_name.on_connect(_ctx, _who, _status); )* - } - - fn on_disconnect(&mut self, _ctx: &mut $crate::Context<$block>, _who: $crate::PeerId) { - $( self.$sub_protocol_name.on_disconnect(_ctx, _who); )* - } - - fn on_message( - &mut self, - _ctx: &mut $crate::Context<$block>, - _who: $crate::PeerId, - _message: Vec, - ) { - $( self.$sub_protocol_name.on_message(_ctx, _who, _message); )* - } - - fn maintain_peers(&mut self, _ctx: &mut $crate::Context<$block>) { - $( self.$sub_protocol_name.maintain_peers(_ctx); )* - } - - fn on_block_imported( - &mut self, - _ctx: &mut $crate::Context<$block>, - _hash: <$block as $crate::BlockT>::Hash, - _header: &<$block as $crate::BlockT>::Header - ) { - $( self.$sub_protocol_name.on_block_imported(_ctx, _hash, _header); )* - } - } - } -} diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index 88e663c904bb1175704593db5427442c4a2ee7b8..04afc5d918471514ac2ece465f305531273f6452 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -35,7 +35,7 @@ use sp_consensus::{BlockOrigin, BlockStatus, }; use crate::{ config::{Roles, BoxFinalityProofRequestBuilder}, - message::{self, generic::FinalityProofRequest, BlockAnnounce, BlockAttributes, BlockRequest, BlockResponse, + protocol::message::{self, generic::FinalityProofRequest, BlockAnnounce, BlockAttributes, BlockRequest, BlockResponse, FinalityProofResponse}, }; use either::Either; @@ -751,7 +751,7 @@ impl ChainSync { | PeerSyncState::DownloadingFinalityProof(..) => Vec::new() } } else { - // When request.is_none() just accept blocks + // When request.is_none() this is a block announcement. Just accept blocks. blocks.into_iter().map(|b| { IncomingBlock { hash: b.hash, @@ -1167,8 +1167,7 @@ impl ChainSync { } /// Restart the sync process. - fn restart<'a>(&'a mut self) -> impl Iterator), BadPeer>> + 'a - { + fn restart<'a>(&'a mut self) -> impl Iterator), BadPeer>> + 'a { self.queue_blocks.clear(); self.blocks.clear(); let info = self.client.info(); @@ -1203,6 +1202,27 @@ impl ChainSync { fn is_already_downloading(&self, hash: &B::Hash) -> bool { self.peers.iter().any(|(_, p)| p.state == PeerSyncState::DownloadingStale(*hash)) } + + /// Return some key metrics. + pub(crate) fn metrics(&self) -> Metrics { + use std::convert::TryInto; + Metrics { + queued_blocks: self.queue_blocks.len().try_into().unwrap_or(std::u32::MAX), + fork_targets: self.fork_targets.len().try_into().unwrap_or(std::u32::MAX), + finality_proofs: self.extra_finality_proofs.metrics(), + justifications: self.extra_justifications.metrics(), + _priv: () + } + } +} + +#[derive(Debug)] +pub(crate) struct Metrics { + pub(crate) queued_blocks: u32, + pub(crate) fork_targets: u32, + pub(crate) finality_proofs: extra_requests::Metrics, + pub(crate) justifications: extra_requests::Metrics, + _priv: () } /// Request the ancestry for a block. Sends a request for header and justification for the given diff --git a/client/network/src/protocol/sync/blocks.rs b/client/network/src/protocol/sync/blocks.rs index 974935f765e6c09a9c390b2b5aa17e5813bb9fed..31b798ace28a54b207df5d43e625fb2d0c9170f2 100644 --- a/client/network/src/protocol/sync/blocks.rs +++ b/client/network/src/protocol/sync/blocks.rs @@ -14,7 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use std::mem; use std::cmp; use std::ops::Range; use std::collections::{HashMap, BTreeMap}; @@ -22,7 +21,7 @@ use std::collections::hash_map::Entry; use log::trace; use libp2p::PeerId; use sp_runtime::traits::{Block as BlockT, NumberFor, One}; -use crate::message; +use crate::protocol::message; /// Block data with origin. #[derive(Debug, Clone, PartialEq, Eq)] @@ -104,8 +103,7 @@ impl BlockCollection { common: NumberFor, max_parallel: u32, max_ahead: u32, - ) -> Option>> - { + ) -> Option>> { if peer_best <= common { // Bail out early return None; @@ -165,20 +163,20 @@ impl BlockCollection { pub fn drain(&mut self, from: NumberFor) -> Vec> { let mut drained = Vec::new(); let mut ranges = Vec::new(); - { - let mut prev = from; - for (start, range_data) in &mut self.blocks { - match range_data { - &mut BlockRangeState::Complete(ref mut blocks) if *start <= prev => { - prev = *start + (blocks.len() as u32).into(); - let mut blocks = mem::replace(blocks, Vec::new()); - drained.append(&mut blocks); - ranges.push(*start); - }, - _ => break, - } + + let mut prev = from; + for (start, range_data) in &mut self.blocks { + match range_data { + &mut BlockRangeState::Complete(ref mut blocks) if *start <= prev => { + prev = *start + (blocks.len() as u32).into(); + // Remove all elements from `blocks` and add them to `drained` + drained.append(blocks); + ranges.push(*start); + }, + _ => break, } } + for r in ranges { self.blocks.remove(&r); } diff --git a/client/network/src/protocol/sync/extra_requests.rs b/client/network/src/protocol/sync/extra_requests.rs index 38c250cddf26d920d9e2f2e2e08eed19ac53a3e4..81b12a1a704aab9b17737c4a03190bef519ff3c5 100644 --- a/client/network/src/protocol/sync/extra_requests.rs +++ b/client/network/src/protocol/sync/extra_requests.rs @@ -53,6 +53,15 @@ pub(crate) struct ExtraRequests { request_type_name: &'static str, } +#[derive(Debug)] +pub(crate) struct Metrics { + pub(crate) pending_requests: u32, + pub(crate) active_requests: u32, + pub(crate) importing_requests: u32, + pub(crate) failed_requests: u32, + _priv: () +} + impl ExtraRequests { pub(crate) fn new(request_type_name: &'static str) -> Self { ExtraRequests { @@ -240,6 +249,18 @@ impl ExtraRequests { pub(crate) fn pending_requests(&self) -> impl Iterator> { self.pending_requests.iter() } + + /// Get some key metrics. + pub(crate) fn metrics(&self) -> Metrics { + use std::convert::TryInto; + Metrics { + pending_requests: self.pending_requests.len().try_into().unwrap_or(std::u32::MAX), + active_requests: self.active_requests.len().try_into().unwrap_or(std::u32::MAX), + failed_requests: self.failed_requests.len().try_into().unwrap_or(std::u32::MAX), + importing_requests: self.importing_requests.len().try_into().unwrap_or(std::u32::MAX), + _priv: () + } + } } /// Matches peers with pending extra requests. diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 8d990282981bb795ce65db10ef636de943847156..a220a009f3fafb075b8b406c2d0ecc4cd82f192d 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -25,7 +25,7 @@ //! The methods of the [`NetworkService`] are implemented by sending a message over a channel, //! which is then processed by [`NetworkWorker::poll`]. -use std::{collections::{HashMap, HashSet}, fs, marker::PhantomData, io, path::Path}; +use std::{borrow::Cow, collections::{HashMap, HashSet}, fs, marker::PhantomData, io, path::Path, str}; use std::sync::{Arc, atomic::{AtomicBool, AtomicUsize, Ordering}}; use std::pin::Pin; use std::task::Poll; @@ -39,15 +39,15 @@ use libp2p::swarm::{NetworkBehaviour, SwarmBuilder, SwarmEvent}; use parking_lot::Mutex; use sc_peerset::PeersetHandle; use sp_runtime::{traits::{Block as BlockT, NumberFor}, ConsensusEngineId}; +use prometheus_endpoint::{Registry, Counter, CounterVec, Gauge, GaugeVec, Opts, U64, register, PrometheusError}; use crate::{behaviour::{Behaviour, BehaviourOut}, config::{parse_str_addr, parse_addr}}; -use crate::{NetworkState, NetworkStateNotConnectedPeer, NetworkStatePeer}; use crate::{transport, config::NonReservedPeerMode, ReputationChange}; use crate::config::{Params, TransportConfig}; use crate::error::Error; -use crate::protocol::{self, Protocol, Context, PeerInfo}; +use crate::network_state::{NetworkState, NotConnectedPeer as NetworkStateNotConnectedPeer, Peer as NetworkStatePeer}; +use crate::protocol::{self, Protocol, PeerInfo}; use crate::protocol::{event::Event, light_dispatch::{AlwaysBadChecker, RequestData}}; -use crate::protocol::specialization::NetworkSpecialization; use crate::protocol::sync::SyncState; /// Minimum Requirements for a Hash within Networking @@ -101,7 +101,7 @@ impl ReportHandle { } /// Substrate network service. Handles network IO and manages connectivity. -pub struct NetworkService, H: ExHashT> { +pub struct NetworkService { /// Number of peers we're connected to. num_connected: Arc, /// The local external addresses. @@ -116,19 +116,19 @@ pub struct NetworkService, H: E /// nodes it should be connected to or not. peerset: PeersetHandle, /// Channel that sends messages to the actual worker. - to_worker: mpsc::UnboundedSender>, + to_worker: mpsc::UnboundedSender>, /// Marker to pin the `H` generic. Serves no purpose except to not break backwards /// compatibility. _marker: PhantomData, } -impl, H: ExHashT> NetworkWorker { +impl NetworkWorker { /// Creates the network service. /// /// Returns a `NetworkWorker` that implements `Future` and must be regularly polled in order /// for the network processing to advance. From it, you can extract a `NetworkService` using /// `worker.service()`. The `NetworkService` can be shared through the codebase. - pub fn new(params: Params) -> Result, Error> { + pub fn new(params: Params) -> Result, Error> { let (to_worker, from_worker) = mpsc::unbounded(); if let Some(ref path) = params.network_config.net_config_path { @@ -205,17 +205,17 @@ impl, H: ExHashT> NetworkWorker }, params.chain.clone(), checker.clone(), - params.specialization, params.transaction_pool, params.finality_proof_provider.clone(), params.finality_proof_request_builder, params.protocol_id.clone(), peerset_config, - params.block_announce_validator + params.block_announce_validator, + params.metrics_registry.as_ref() )?; // Build the swarm. - let (mut swarm, bandwidth): (Swarm::, _) = { + let (mut swarm, bandwidth): (Swarm::, _) = { let user_agent = format!( "{} ({})", params.network_config.client_version, @@ -263,14 +263,14 @@ impl, H: ExHashT> NetworkWorker // Listen on multiaddresses. for addr in ¶ms.network_config.listen_addresses { - if let Err(err) = Swarm::::listen_on(&mut swarm, addr.clone()) { + if let Err(err) = Swarm::::listen_on(&mut swarm, addr.clone()) { warn!(target: "sub-libp2p", "Can't listen on {} because: {:?}", addr, err) } } // Add external addresses. for addr in ¶ms.network_config.public_addresses { - Swarm::::add_external_address(&mut swarm, addr.clone()); + Swarm::::add_external_address(&mut swarm, addr.clone()); } let external_addresses = Arc::new(Mutex::new(Vec::new())); @@ -296,6 +296,10 @@ impl, H: ExHashT> NetworkWorker from_worker, light_client_rqs: params.on_demand.and_then(|od| od.extract_receiver()), event_streams: Vec::new(), + metrics: match params.metrics_registry { + Some(registry) => Some(Metrics::register(®istry)?), + None => None + } }) } @@ -351,13 +355,13 @@ impl, H: ExHashT> NetworkWorker /// Return a `NetworkService` that can be shared through the code base and can be used to /// manipulate the worker. - pub fn service(&self) -> &Arc> { + pub fn service(&self) -> &Arc> { &self.service } /// You must call this when a new block is imported by the client. - pub fn on_block_imported(&mut self, hash: B::Hash, header: B::Header, data: Vec, is_best: bool) { - self.network_service.user_protocol_mut().on_block_imported(hash, &header, data, is_best); + pub fn on_block_imported(&mut self, header: B::Header, data: Vec, is_best: bool) { + self.network_service.user_protocol_mut().on_block_imported(&header, data, is_best); } /// You must call this when a new block is finalized by the client. @@ -415,9 +419,9 @@ impl, H: ExHashT> NetworkWorker }; NetworkState { - peer_id: Swarm::::local_peer_id(&swarm).to_base58(), - listened_addresses: Swarm::::listeners(&swarm).cloned().collect(), - external_addresses: Swarm::::external_addresses(&swarm).cloned().collect(), + peer_id: Swarm::::local_peer_id(&swarm).to_base58(), + listened_addresses: Swarm::::listeners(&swarm).cloned().collect(), + external_addresses: Swarm::::external_addresses(&swarm).cloned().collect(), average_download_per_sec: self.service.bandwidth.average_download_per_sec(), average_upload_per_sec: self.service.bandwidth.average_upload_per_sec(), connected_peers, @@ -446,7 +450,7 @@ impl, H: ExHashT> NetworkWorker } } -impl, H: ExHashT> NetworkService { +impl NetworkService { /// Writes a message on an open notifications channel. Has no effect if the notifications /// channel with this protocol name is closed. /// @@ -490,9 +494,11 @@ impl, H: ExHashT> NetworkServic pub fn register_notifications_protocol( &self, engine_id: ConsensusEngineId, + protocol_name: impl Into>, ) { let _ = self.to_worker.unbounded_send(ServiceToWorkerMsg::RegisterNotifProtocol { engine_id, + protocol_name: protocol_name.into(), }); } @@ -543,15 +549,6 @@ impl, H: ExHashT> NetworkServic .unbounded_send(ServiceToWorkerMsg::RequestJustification(hash.clone(), number)); } - /// Execute a closure with the chain-specific network specialization. - pub fn with_spec(&self, f: F) - where F: FnOnce(&mut S, &mut dyn Context) + Send + 'static - { - let _ = self - .to_worker - .unbounded_send(ServiceToWorkerMsg::ExecuteWithSpec(Box::new(f))); - } - /// Are we in the process of downloading the chain? pub fn is_major_syncing(&self) -> bool { self.is_major_syncing.load(Ordering::Relaxed) @@ -639,8 +636,8 @@ impl, H: ExHashT> NetworkServic } } -impl, H: ExHashT> sp_consensus::SyncOracle - for NetworkService +impl sp_consensus::SyncOracle + for NetworkService { fn is_major_syncing(&mut self) -> bool { NetworkService::is_major_syncing(self) @@ -651,8 +648,8 @@ impl, H: ExHashT> sp_consensus: } } -impl<'a, B: BlockT + 'static, S: NetworkSpecialization, H: ExHashT> sp_consensus::SyncOracle - for &'a NetworkService +impl<'a, B: BlockT + 'static, H: ExHashT> sp_consensus::SyncOracle + for &'a NetworkService { fn is_major_syncing(&mut self) -> bool { NetworkService::is_major_syncing(self) @@ -672,10 +669,9 @@ pub trait NetworkStateInfo { fn local_peer_id(&self) -> PeerId; } -impl NetworkStateInfo for NetworkService +impl NetworkStateInfo for NetworkService where B: sp_runtime::traits::Block, - S: NetworkSpecialization, H: ExHashT, { /// Returns the local external addresses. @@ -692,12 +688,11 @@ impl NetworkStateInfo for NetworkService /// Messages sent from the `NetworkService` to the `NetworkWorker`. /// /// Each entry corresponds to a method of `NetworkService`. -enum ServiceToWorkerMsg> { +enum ServiceToWorkerMsg { PropagateExtrinsic(H), PropagateExtrinsics, RequestJustification(B::Hash, NumberFor), AnnounceBlock(B::Hash, Vec), - ExecuteWithSpec(Box) + Send>), GetValue(record::Key), PutValue(record::Key, Vec), AddKnownAddress(PeerId, Multiaddr), @@ -710,6 +705,7 @@ enum ServiceToWorkerMsg> { }, RegisterNotifProtocol { engine_id: ConsensusEngineId, + protocol_name: Cow<'static, [u8]>, }, DisconnectPeer(PeerId), } @@ -718,7 +714,7 @@ enum ServiceToWorkerMsg> { /// /// You are encouraged to poll this in a separate background thread or task. #[must_use = "The NetworkWorker must be polled in order for the network to work"] -pub struct NetworkWorker, H: ExHashT> { +pub struct NetworkWorker { /// Updated by the `NetworkWorker` and loaded by the `NetworkService`. external_addresses: Arc>>, /// Updated by the `NetworkWorker` and loaded by the `NetworkService`. @@ -726,20 +722,123 @@ pub struct NetworkWorker, H: Ex /// Updated by the `NetworkWorker` and loaded by the `NetworkService`. is_major_syncing: Arc, /// The network service that can be extracted and shared through the codebase. - service: Arc>, + service: Arc>, /// The *actual* network. - network_service: Swarm, + network_service: Swarm, /// The import queue that was passed as initialization. import_queue: Box>, /// Messages from the `NetworkService` and that must be processed. - from_worker: mpsc::UnboundedReceiver>, + from_worker: mpsc::UnboundedReceiver>, /// Receiver for queries from the light client that must be processed. light_client_rqs: Option>>, /// Senders for events that happen on the network. event_streams: Vec>, + /// Prometheus network metrics. + metrics: Option, +} + +struct Metrics { + // This list is ordered alphabetically + connections: Gauge, + import_queue_blocks_submitted: Counter, + import_queue_finality_proofs_submitted: Counter, + import_queue_justifications_submitted: Counter, + is_major_syncing: Gauge, + kbuckets_num_nodes: Gauge, + network_per_sec_bytes: GaugeVec, + notifications_total: CounterVec, + num_event_stream_channels: Gauge, + opened_notification_streams: GaugeVec, + peers_count: Gauge, + peerset_num_discovered: Gauge, + peerset_num_requested: Gauge, + random_kademalia_queries_total: Counter, +} + +impl Metrics { + fn register(registry: &Registry) -> Result { + Ok(Self { + // This list is ordered alphabetically + connections: register(Gauge::new( + "sub_libp2p_connections", "Number of libp2p connections" + )?, registry)?, + import_queue_blocks_submitted: register(Counter::new( + "import_queue_blocks_submitted", + "Number of blocks submitted to the import queue.", + )?, registry)?, + import_queue_finality_proofs_submitted: register(Counter::new( + "import_queue_finality_proofs_submitted", + "Number of finality proofs submitted to the import queue.", + )?, registry)?, + import_queue_justifications_submitted: register(Counter::new( + "import_queue_justifications_submitted", + "Number of justifications submitted to the import queue.", + )?, registry)?, + is_major_syncing: register(Gauge::new( + "sub_libp2p_is_major_syncing", "Whether the node is performing a major sync or not.", + )?, registry)?, + kbuckets_num_nodes: register(Gauge::new( + "sub_libp2p_kbuckets_num_nodes", "Number of nodes in the Kademlia k-buckets" + )?, registry)?, + network_per_sec_bytes: register(GaugeVec::new( + Opts::new( + "sub_libp2p_network_per_sec_bytes", + "Average bandwidth usage per second" + ), + &["direction"] + )?, registry)?, + notifications_total: register(CounterVec::new( + Opts::new( + "sub_libp2p_notifications_total", + "Number of notification received from all nodes" + ), + &["direction", "protocol"] + )?, registry)?, + num_event_stream_channels: register(Gauge::new( + "sub_libp2p_num_event_stream_channels", + "Number of internal active channels that broadcast network events", + )?, registry)?, + opened_notification_streams: register(GaugeVec::new( + Opts::new( + "sub_libp2p_opened_notification_streams", + "Number of open notification substreams" + ), + &["protocol"] + )?, registry)?, + peers_count: register(Gauge::new( + "sub_libp2p_peers_count", "Number of network gossip peers", + )?, registry)?, + peerset_num_discovered: register(Gauge::new( + "sub_libp2p_peerset_num_discovered", "Number of nodes stored in the peerset manager", + )?, registry)?, + peerset_num_requested: register(Gauge::new( + "sub_libp2p_peerset_num_requested", "Number of nodes that the peerset manager wants us to be connected to", + )?, registry)?, + random_kademalia_queries_total: register(Counter::new( + "sub_libp2p_random_kademalia_queries_total", "Number of random Kademlia queries started", + )?, registry)?, + }) + } + + fn update_with_network_event(&self, event: &Event) { + match event { + Event::NotificationStreamOpened { engine_id, .. } => { + self.opened_notification_streams.with_label_values(&[&engine_id_to_string(&engine_id)]).inc(); + }, + Event::NotificationStreamClosed { engine_id, .. } => { + self.opened_notification_streams.with_label_values(&[&engine_id_to_string(&engine_id)]).dec(); + }, + Event::NotificationsReceived { messages, .. } => { + for (engine_id, _) in messages { + self.notifications_total.with_label_values(&["in", &engine_id_to_string(&engine_id)]).inc(); + } + }, + _ => {} + } + } } -impl, H: ExHashT> Future for NetworkWorker { +impl Future for NetworkWorker { type Output = Result<(), io::Error>; fn poll(mut self: Pin<&mut Self>, cx: &mut std::task::Context) -> Poll { @@ -766,11 +865,6 @@ impl, H: ExHashT> Future for Ne }; match msg { - ServiceToWorkerMsg::ExecuteWithSpec(task) => { - let protocol = this.network_service.user_protocol_mut(); - let (mut context, spec) = protocol.specialization_lock(); - task(spec, &mut context); - }, ServiceToWorkerMsg::AnnounceBlock(hash, data) => this.network_service.user_protocol_mut().announce_block(hash, data), ServiceToWorkerMsg::RequestJustification(hash, number) => @@ -789,10 +883,15 @@ impl, H: ExHashT> Future for Ne this.network_service.user_protocol_mut().set_sync_fork_request(peer_ids, &hash, number), ServiceToWorkerMsg::EventStream(sender) => this.event_streams.push(sender), - ServiceToWorkerMsg::WriteNotification { message, engine_id, target } => - this.network_service.user_protocol_mut().write_notification(target, engine_id, message), - ServiceToWorkerMsg::RegisterNotifProtocol { engine_id } => { - let events = this.network_service.user_protocol_mut().register_notifications_protocol(engine_id); + ServiceToWorkerMsg::WriteNotification { message, engine_id, target } => { + if let Some(metrics) = this.metrics.as_ref() { + metrics.notifications_total.with_label_values(&["out", &engine_id_to_string(&engine_id)]).inc(); + } + this.network_service.user_protocol_mut().write_notification(target, engine_id, message) + }, + ServiceToWorkerMsg::RegisterNotifProtocol { engine_id, protocol_name } => { + let events = this.network_service.user_protocol_mut() + .register_notifications_protocol(engine_id, protocol_name); for event in events { this.event_streams.retain(|sender| sender.unbounded_send(event.clone()).is_ok()); } @@ -810,18 +909,47 @@ impl, H: ExHashT> Future for Ne match poll_value { Poll::Pending => break, - Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::BlockImport(origin, blocks))) => - this.import_queue.import_blocks(origin, blocks), - Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::JustificationImport(origin, hash, nb, justification))) => - this.import_queue.import_justification(origin, hash, nb, justification), - Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::FinalityProofImport(origin, hash, nb, proof))) => - this.import_queue.import_finality_proof(origin, hash, nb, proof), - Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::Event(ev))) => - this.event_streams.retain(|sender| sender.unbounded_send(ev.clone()).is_ok()), - Poll::Ready(SwarmEvent::Connected(peer_id)) => - trace!(target: "sub-libp2p", "Libp2p => Connected({:?})", peer_id), - Poll::Ready(SwarmEvent::Disconnected(peer_id)) => - trace!(target: "sub-libp2p", "Libp2p => Disconnected({:?})", peer_id), + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::BlockImport(origin, blocks))) => { + if let Some(metrics) = this.metrics.as_ref() { + metrics.import_queue_blocks_submitted.inc(); + } + this.import_queue.import_blocks(origin, blocks); + }, + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::JustificationImport(origin, hash, nb, justification))) => { + if let Some(metrics) = this.metrics.as_ref() { + metrics.import_queue_justifications_submitted.inc(); + } + this.import_queue.import_justification(origin, hash, nb, justification); + }, + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::FinalityProofImport(origin, hash, nb, proof))) => { + if let Some(metrics) = this.metrics.as_ref() { + metrics.import_queue_finality_proofs_submitted.inc(); + } + this.import_queue.import_finality_proof(origin, hash, nb, proof); + }, + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RandomKademliaStarted)) => { + if let Some(metrics) = this.metrics.as_ref() { + metrics.random_kademalia_queries_total.inc(); + } + }, + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::Event(ev))) => { + this.event_streams.retain(|sender| sender.unbounded_send(ev.clone()).is_ok()); + if let Some(metrics) = this.metrics.as_ref() { + metrics.update_with_network_event(&ev); + } + }, + Poll::Ready(SwarmEvent::Connected(peer_id)) => { + trace!(target: "sub-libp2p", "Libp2p => Connected({:?})", peer_id); + if let Some(metrics) = this.metrics.as_ref() { + metrics.connections.inc(); + } + }, + Poll::Ready(SwarmEvent::Disconnected(peer_id)) => { + trace!(target: "sub-libp2p", "Libp2p => Disconnected({:?})", peer_id); + if let Some(metrics) = this.metrics.as_ref() { + metrics.connections.dec(); + } + }, Poll::Ready(SwarmEvent::NewListenAddr(addr)) => trace!(target: "sub-libp2p", "Libp2p => NewListenAddr({})", addr), Poll::Ready(SwarmEvent::ExpiredListenAddr(addr)) => @@ -833,35 +961,58 @@ impl, H: ExHashT> Future for Ne }; } + let num_connected_peers = this.network_service.user_protocol_mut().num_connected_peers(); + // Update the variables shared with the `NetworkService`. - this.num_connected.store(this.network_service.user_protocol_mut().num_connected_peers(), Ordering::Relaxed); + this.num_connected.store(num_connected_peers, Ordering::Relaxed); { - let external_addresses = Swarm::::external_addresses(&this.network_service).cloned().collect(); + let external_addresses = Swarm::::external_addresses(&this.network_service).cloned().collect(); *this.external_addresses.lock() = external_addresses; } - this.is_major_syncing.store(match this.network_service.user_protocol_mut().sync_state() { + + let is_major_syncing = match this.network_service.user_protocol_mut().sync_state() { SyncState::Idle => false, SyncState::Downloading => true, - }, Ordering::Relaxed); + }; + + this.is_major_syncing.store(is_major_syncing, Ordering::Relaxed); + + if let Some(metrics) = this.metrics.as_ref() { + metrics.network_per_sec_bytes.with_label_values(&["in"]).set(this.service.bandwidth.average_download_per_sec()); + metrics.network_per_sec_bytes.with_label_values(&["out"]).set(this.service.bandwidth.average_upload_per_sec()); + metrics.is_major_syncing.set(is_major_syncing as u64); + metrics.kbuckets_num_nodes.set(this.network_service.num_kbuckets_entries() as u64); + metrics.num_event_stream_channels.set(this.event_streams.len() as u64); + metrics.peers_count.set(num_connected_peers as u64); + metrics.peerset_num_discovered.set(this.network_service.user_protocol().num_discovered_peers() as u64); + metrics.peerset_num_requested.set(this.network_service.user_protocol().requested_peers().count() as u64); + } Poll::Pending } } -impl, H: ExHashT> Unpin for NetworkWorker { +impl Unpin for NetworkWorker { +} + +/// Turns a `ConsensusEngineId` into a representable string. +fn engine_id_to_string(id: &ConsensusEngineId) -> Cow { + if let Ok(s) = std::str::from_utf8(&id[..]) { + Cow::Borrowed(s) + } else { + Cow::Owned(format!("{:?}", id)) + } } /// The libp2p swarm, customized for our needs. -type Swarm = libp2p::swarm::Swarm< - Behaviour ->; +type Swarm = libp2p::swarm::Swarm>; // Implementation of `import_queue::Link` trait using the available local variables. -struct NetworkLink<'a, B: BlockT, S: NetworkSpecialization, H: ExHashT> { - protocol: &'a mut Swarm, +struct NetworkLink<'a, B: BlockT, H: ExHashT> { + protocol: &'a mut Swarm, } -impl<'a, B: BlockT, S: NetworkSpecialization, H: ExHashT> Link for NetworkLink<'a, B, S, H> { +impl<'a, B: BlockT, H: ExHashT> Link for NetworkLink<'a, B, H> { fn blocks_processed( &mut self, imported: usize, diff --git a/client/network/test/Cargo.toml b/client/network/test/Cargo.toml index 339279c704828376d460e17f6480005a1849f023..7fec4f4da8b47ccca2df942dadf729e8357583dd 100644 --- a/client/network/test/Cargo.toml +++ b/client/network/test/Cargo.toml @@ -1,30 +1,31 @@ [package] description = "Integration tests for Substrate network protocol" name = "sc-network-test" -version = "0.8.0" +version = "0.8.0-dev" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" +publish = false +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sc-network = { version = "0.8", path = "../" } +sc-network = { version = "0.8.0-alpha.2", path = "../" } log = "0.4.8" parking_lot = "0.10.0" -futures = "0.1.29" -futures03 = { package = "futures", version = "0.3.1", features = ["compat"] } +futures = "0.3.1" futures-timer = "3.0.1" rand = "0.7.2" -libp2p = { version = "0.16.0", default-features = false, features = ["libp2p-websocket"] } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } -sc-client = { version = "0.8", path = "../../" } -sc-client-api = { version = "2.0.0", path = "../../api" } -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sc-block-builder = { version = "0.8", path = "../../block-builder" } -sp-consensus-babe = { version = "0.8", path = "../../../primitives/consensus/babe" } +libp2p = { version = "0.16.2", default-features = false, features = ["libp2p-websocket"] } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/common" } +sc-client = { version = "0.8.0-alpha.2", path = "../../" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../../api" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sc-block-builder = { version = "0.8.0-alpha.2", path = "../../block-builder" } +sp-consensus-babe = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/babe" } env_logger = "0.7.0" -substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } -substrate-test-runtime = { version = "2.0.0", path = "../../../test-utils/runtime" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } +substrate-test-runtime = { version = "2.0.0-dev", path = "../../../test-utils/runtime" } tempfile = "3.1.0" -tokio = "0.1.22" diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index c15bd3365503f733801e6c3e3e066e8be1c5149a..0add6c63d5ac974fae8277ea6d44f5a76b6b2604 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -21,11 +21,11 @@ mod block_import; #[cfg(test)] mod sync; -use std::{collections::HashMap, pin::Pin, sync::Arc, marker::PhantomData}; +use std::{collections::HashMap, pin::Pin, sync::Arc, marker::PhantomData, task::{Poll, Context as FutureContext}}; use libp2p::build_multiaddr; use log::trace; -use sc_network::FinalityProofProvider; +use sc_network::config::FinalityProofProvider; use sp_blockchain::{ Result as ClientResult, well_known_cache_keys::{self, Id as CacheKeyId}, Info as BlockchainInfo, }; @@ -35,7 +35,7 @@ use sc_client_api::{ FinalityNotification, backend::{TransactionFor, AuxStore, Backend, Finalizer}, }; -use sc_block_builder::BlockBuilder; +use sc_block_builder::{BlockBuilder, BlockBuilderProvider}; use sc_client::LongestChain; use sc_network::config::Roles; use sp_consensus::block_validation::DefaultBlockAnnounceValidator; @@ -46,23 +46,19 @@ use sp_consensus::block_import::{BlockImport, ImportResult}; use sp_consensus::Error as ConsensusError; use sp_consensus::{BlockOrigin, ForkChoiceStrategy, BlockImportParams, BlockCheckParams, JustificationImport}; use futures::prelude::*; -use futures03::{Future as _, FutureExt as _, TryFutureExt as _, StreamExt as _, TryStreamExt as _}; use sc_network::{NetworkWorker, NetworkStateInfo, NetworkService, ReportHandle, config::ProtocolId}; use sc_network::config::{NetworkConfiguration, TransportConfig, BoxFinalityProofRequestBuilder}; use libp2p::PeerId; use parking_lot::Mutex; use sp_core::H256; -use sc_network::ProtocolConfig; +use sc_network::config::{ProtocolConfig, TransactionPool}; use sp_runtime::generic::{BlockId, OpaqueDigestItemId}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; use sp_runtime::Justification; -use sc_network::TransactionPool; -use sc_network::specialization::NetworkSpecialization; use substrate_test_runtime_client::{self, AccountKeyring}; pub use substrate_test_runtime_client::runtime::{Block, Extrinsic, Hash, Transfer}; pub use substrate_test_runtime_client::{TestClient, TestClientBuilder, TestClientBuilderExt}; -pub use sc_network::specialization::DummySpecialization; type AuthorityId = sp_consensus_babe::AuthorityId; @@ -178,7 +174,7 @@ impl PeersClient { } } -pub struct Peer> { +pub struct Peer { pub data: D, client: PeersClient, /// We keep a copy of the verifier so that we can invoke it for locally-generated blocks, @@ -189,12 +185,12 @@ pub struct Peer> { block_import: BlockImportAdapter<()>, select_chain: Option>, backend: Option>, - network: NetworkWorker::Hash>, - imported_blocks_stream: Box, Error = ()> + Send>, - finality_notification_stream: Box, Error = ()> + Send>, + network: NetworkWorker::Hash>, + imported_blocks_stream: Pin> + Send>>, + finality_notification_stream: Pin> + Send>>, } -impl> Peer { +impl Peer { /// Get this peer ID. pub fn id(&self) -> PeerId { self.network.service().local_peer_id() @@ -240,7 +236,7 @@ impl> Peer { where F: FnMut(BlockBuilder) -> Block { let best_hash = self.client.info().best_hash; - self.generate_blocks_at(BlockId::Hash(best_hash), count, origin, edit_block) + self.generate_blocks_at(BlockId::Hash(best_hash), count, origin, edit_block, false) } /// Add blocks to the peer -- edit the block before adding. The chain will @@ -250,7 +246,8 @@ impl> Peer { at: BlockId, count: usize, origin: BlockOrigin, - mut edit_block: F + mut edit_block: F, + headers_only: bool, ) -> H256 where F: FnMut(BlockBuilder) -> Block { let full_client = self.client.as_full() .expect("blocks could only be generated by full clients"); @@ -275,7 +272,7 @@ impl> Peer { origin, header.clone(), None, - Some(block.extrinsics) + if headers_only { None } else { Some(block.extrinsics) }, ).unwrap(); let cache = if let Some(cache) = cache { cache.into_iter().collect() @@ -283,7 +280,7 @@ impl> Peer { Default::default() }; self.block_import.import_block(import_block, cache).expect("block_import failed"); - self.network.on_block_imported(hash, header, Vec::new(), true); + self.network.on_block_imported(header, Vec::new(), true); at = hash; } @@ -297,28 +294,46 @@ impl> Peer { self.push_blocks_at(BlockId::Hash(best_hash), count, with_tx) } + /// Push blocks to the peer (simplified: with or without a TX) + pub fn push_headers(&mut self, count: usize) -> H256 { + let best_hash = self.client.info().best_hash; + self.generate_tx_blocks_at(BlockId::Hash(best_hash), count, false, true) + } + /// Push blocks to the peer (simplified: with or without a TX) starting from /// given hash. pub fn push_blocks_at(&mut self, at: BlockId, count: usize, with_tx: bool) -> H256 { + self.generate_tx_blocks_at(at, count, with_tx, false) + } + + /// Push blocks/headers to the peer (simplified: with or without a TX) starting from + /// given hash. + fn generate_tx_blocks_at(&mut self, at: BlockId, count: usize, with_tx: bool, headers_only:bool) -> H256 { let mut nonce = 0; if with_tx { - self.generate_blocks_at(at, count, BlockOrigin::File, |mut builder| { - let transfer = Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Alice.into(), - amount: 1, - nonce, - }; - builder.push(transfer.into_signed_tx()).unwrap(); - nonce = nonce + 1; - builder.build().unwrap().block - }) + self.generate_blocks_at( + at, + count, + BlockOrigin::File, |mut builder| { + let transfer = Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Alice.into(), + amount: 1, + nonce, + }; + builder.push(transfer.into_signed_tx()).unwrap(); + nonce = nonce + 1; + builder.build().unwrap().block + }, + headers_only + ) } else { self.generate_blocks_at( at, count, BlockOrigin::File, |builder| builder.build().unwrap().block, + headers_only, ) } } @@ -336,7 +351,7 @@ impl> Peer { } /// Get a reference to the network service. - pub fn network_service(&self) -> &Arc::Hash>> { + pub fn network_service(&self) -> &Arc::Hash>> { &self.network.service() } @@ -392,16 +407,6 @@ impl TransactionPool for EmptyTransactionPool { fn transaction(&self, _h: &Hash) -> Option { None } } -pub trait SpecializationFactory { - fn create() -> Self; -} - -impl SpecializationFactory for DummySpecialization { - fn create() -> DummySpecialization { - DummySpecialization - } -} - /// Implements `BlockImport` for any `Transaction`. Internally the transaction is /// "converted", aka the field is set to `None`. /// @@ -522,7 +527,6 @@ impl VerifierAdapter { } pub trait TestNetFactory: Sized { - type Specialization: NetworkSpecialization + SpecializationFactory; type Verifier: 'static + Verifier; type PeerData: Default; @@ -536,9 +540,9 @@ pub trait TestNetFactory: Sized { ) -> Self::Verifier; /// Get reference to peer. - fn peer(&mut self, i: usize) -> &mut Peer; - fn peers(&self) -> &Vec>; - fn mut_peers>)>( + fn peer(&mut self, i: usize) -> &mut Peer; + fn peers(&self) -> &Vec>; + fn mut_peers>)>( &mut self, closure: F, ); @@ -636,8 +640,8 @@ pub trait TestNetFactory: Sized { transaction_pool: Arc::new(EmptyTransactionPool), protocol_id: ProtocolId::from(&b"test-protocol-name"[..]), import_queue, - specialization: self::SpecializationFactory::create(), - block_announce_validator: Box::new(DefaultBlockAnnounceValidator::new(client.clone())) + block_announce_validator: Box::new(DefaultBlockAnnounceValidator::new(client.clone())), + metrics_registry: None, }).unwrap(); self.mut_peers(|peers| { @@ -645,10 +649,8 @@ pub trait TestNetFactory: Sized { peer.network.add_known_address(network.service().local_peer_id(), listen_addr.clone()); } - let imported_blocks_stream = Box::new(client.import_notification_stream() - .map(|v| Ok::<_, ()>(v)).compat().fuse()); - let finality_notification_stream = Box::new(client.finality_notification_stream() - .map(|v| Ok::<_, ()>(v)).compat().fuse()); + let imported_blocks_stream = Box::pin(client.import_notification_stream().fuse()); + let finality_notification_stream = Box::pin(client.finality_notification_stream().fuse()); peers.push(Peer { data, @@ -712,8 +714,8 @@ pub trait TestNetFactory: Sized { transaction_pool: Arc::new(EmptyTransactionPool), protocol_id: ProtocolId::from(&b"test-protocol-name"[..]), import_queue, - specialization: self::SpecializationFactory::create(), - block_announce_validator: Box::new(DefaultBlockAnnounceValidator::new(client.clone())) + block_announce_validator: Box::new(DefaultBlockAnnounceValidator::new(client.clone())), + metrics_registry: None, }).unwrap(); self.mut_peers(|peers| { @@ -721,10 +723,8 @@ pub trait TestNetFactory: Sized { peer.network.add_known_address(network.service().local_peer_id(), listen_addr.clone()); } - let imported_blocks_stream = Box::new(client.import_notification_stream() - .map(|v| Ok::<_, ()>(v)).compat().fuse()); - let finality_notification_stream = Box::new(client.finality_notification_stream() - .map(|v| Ok::<_, ()>(v)).compat().fuse()); + let imported_blocks_stream = Box::pin(client.import_notification_stream().fuse()); + let finality_notification_stream = Box::pin(client.finality_notification_stream().fuse()); peers.push(Peer { data, @@ -743,48 +743,71 @@ pub trait TestNetFactory: Sized { /// Polls the testnet until all nodes are in sync. /// /// Must be executed in a task context. - fn poll_until_sync(&mut self) -> Async<()> { - self.poll(); + fn poll_until_sync(&mut self, cx: &mut FutureContext) -> Poll<()> { + self.poll(cx); // Return `NotReady` if there's a mismatch in the highest block number. let mut highest = None; for peer in self.peers().iter() { if peer.is_major_syncing() || peer.network.num_queued_blocks() != 0 { - return Async::NotReady + return Poll::Pending } if peer.network.num_sync_requests() != 0 { - return Async::NotReady + return Poll::Pending } match (highest, peer.client.info().best_hash) { (None, b) => highest = Some(b), (Some(ref a), ref b) if a == b => {}, - (Some(_), _) => return Async::NotReady, + (Some(_), _) => return Poll::Pending } } - Async::Ready(()) + Poll::Ready(()) + } + + /// Polls the testnet until theres' no activiy of any kind. + /// + /// Must be executed in a task context. + fn poll_until_idle(&mut self, cx: &mut FutureContext) -> Poll<()> { + self.poll(cx); + + for peer in self.peers().iter() { + if peer.is_major_syncing() || peer.network.num_queued_blocks() != 0 { + return Poll::Pending + } + if peer.network.num_sync_requests() != 0 { + return Poll::Pending + } + } + Poll::Ready(()) } /// Blocks the current thread until we are sync'ed. /// - /// Calls `poll_until_sync` repeatedly with the runtime passed as parameter. - fn block_until_sync(&mut self, runtime: &mut tokio::runtime::current_thread::Runtime) { - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| Ok(self.poll_until_sync()))).unwrap(); + /// Calls `poll_until_sync` repeatedly. + fn block_until_sync(&mut self) { + futures::executor::block_on(futures::future::poll_fn::<(), _>(|cx| self.poll_until_sync(cx))); + } + + /// Blocks the current thread until there are no pending packets. + /// + /// Calls `poll_until_idle` repeatedly with the runtime passed as parameter. + fn block_until_idle(&mut self) { + futures::executor::block_on(futures::future::poll_fn::<(), _>(|cx| self.poll_until_idle(cx))); } /// Polls the testnet. Processes all the pending actions and returns `NotReady`. - fn poll(&mut self) { + fn poll(&mut self, cx: &mut FutureContext) { self.mut_peers(|peers| { for peer in peers { trace!(target: "sync", "-- Polling {}", peer.id()); - futures03::future::poll_fn(|cx| Pin::new(&mut peer.network).poll(cx)) - .map(|item| Ok::<_, ()>(item)) - .compat().poll().unwrap(); + if let Poll::Ready(res) = Pin::new(&mut peer.network).poll(cx) { + res.unwrap(); + } trace!(target: "sync", "-- Polling complete {}", peer.id()); // We poll `imported_blocks_stream`. - while let Ok(Async::Ready(Some(notification))) = peer.imported_blocks_stream.poll() { + while let Poll::Ready(Some(notification)) = peer.imported_blocks_stream.as_mut().poll_next(cx) { peer.network.on_block_imported( - notification.hash, notification.header, Vec::new(), true, @@ -793,7 +816,7 @@ pub trait TestNetFactory: Sized { // We poll `finality_notification_stream`, but we only take the last event. let mut last = None; - while let Ok(Async::Ready(Some(item))) = peer.finality_notification_stream.poll() { + while let Poll::Ready(Some(item)) = peer.finality_notification_stream.as_mut().poll_next(cx) { last = Some(item); } if let Some(notification) = last { @@ -805,11 +828,10 @@ pub trait TestNetFactory: Sized { } pub struct TestNet { - peers: Vec>, + peers: Vec>, } impl TestNetFactory for TestNet { - type Specialization = DummySpecialization; type Verifier = PassThroughVerifier; type PeerData = (); @@ -826,15 +848,15 @@ impl TestNetFactory for TestNet { PassThroughVerifier(false) } - fn peer(&mut self, i: usize) -> &mut Peer<(), Self::Specialization> { + fn peer(&mut self, i: usize) -> &mut Peer<()> { &mut self.peers[i] } - fn peers(&self) -> &Vec> { + fn peers(&self) -> &Vec> { &self.peers } - fn mut_peers>)>(&mut self, closure: F) { + fn mut_peers>)>(&mut self, closure: F) { closure(&mut self.peers); } } @@ -858,7 +880,6 @@ impl JustificationImport for ForceFinalized { pub struct JustificationTestNet(TestNet); impl TestNetFactory for JustificationTestNet { - type Specialization = DummySpecialization; type Verifier = PassThroughVerifier; type PeerData = (); @@ -870,17 +891,16 @@ impl TestNetFactory for JustificationTestNet { self.0.make_verifier(client, config, peer_data) } - fn peer(&mut self, i: usize) -> &mut Peer { + fn peer(&mut self, i: usize) -> &mut Peer { self.0.peer(i) } - fn peers(&self) -> &Vec> { + fn peers(&self) -> &Vec> { self.0.peers() } fn mut_peers>, + &mut Vec>, )>(&mut self, closure: F) { self.0.mut_peers(closure) } diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 210a4fb38bb68949edfd44925e8feb0aca13b898..388257516832002de97775bbeb21c6696eaae38c 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -16,14 +16,12 @@ use sc_network::config::Roles; use sp_consensus::BlockOrigin; -use futures03::TryFutureExt as _; use std::time::Duration; -use tokio::runtime::current_thread; +use futures::executor::block_on; use super::*; fn test_ancestor_search_when_common_is(n: usize) { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(3); net.peer(0).push_blocks(n, false); @@ -34,7 +32,7 @@ fn test_ancestor_search_when_common_is(n: usize) { net.peer(1).push_blocks(100, false); net.peer(2).push_blocks(100, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); let peer1 = &net.peers()[1]; assert!(net.peers()[0].blockchain_canon_equals(peer1)); } @@ -42,24 +40,22 @@ fn test_ancestor_search_when_common_is(n: usize) { #[test] fn sync_peers_works() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(3); - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); for peer in 0..3 { if net.peer(peer).num_peers() != 2 { - return Ok(Async::NotReady) + return Poll::Pending } } - Ok(Async::Ready(())) - })).unwrap(); + Poll::Ready(()) + })); } #[test] fn sync_cycle_from_offline_to_syncing_to_offline() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(3); for peer in 0..3 { // Offline, and not major syncing. @@ -71,51 +67,50 @@ fn sync_cycle_from_offline_to_syncing_to_offline() { net.peer(2).push_blocks(100, false); // Block until all nodes are online and nodes 0 and 1 and major syncing. - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); for peer in 0..3 { // Online if net.peer(peer).is_offline() { - return Ok(Async::NotReady) + return Poll::Pending } if peer < 2 { // Major syncing. if net.peer(peer).blocks_count() < 100 && !net.peer(peer).is_major_syncing() { - return Ok(Async::NotReady) + return Poll::Pending } } } - Ok(Async::Ready(())) - })).unwrap(); + Poll::Ready(()) + })); // Block until all nodes are done syncing. - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); for peer in 0..3 { if net.peer(peer).is_major_syncing() { - return Ok(Async::NotReady) + return Poll::Pending } } - Ok(Async::Ready(())) - })).unwrap(); + Poll::Ready(()) + })); // Now drop nodes 1 and 2, and check that node 0 is offline. net.peers.remove(2); net.peers.remove(1); - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); if !net.peer(0).is_offline() { - Ok(Async::NotReady) + Poll::Pending } else { - Ok(Async::Ready(())) + Poll::Ready(()) } - })).unwrap(); + })); } #[test] fn syncing_node_not_major_syncing_when_disconnected() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(3); // Generate blocks. @@ -125,36 +120,35 @@ fn syncing_node_not_major_syncing_when_disconnected() { assert!(!net.peer(1).is_major_syncing()); // Check that we switch to major syncing. - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); if !net.peer(1).is_major_syncing() { - Ok(Async::NotReady) + Poll::Pending } else { - Ok(Async::Ready(())) + Poll::Ready(()) } - })).unwrap(); + })); // Destroy two nodes, and check that we switch to non-major syncing. net.peers.remove(2); net.peers.remove(0); - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); if net.peer(0).is_major_syncing() { - Ok(Async::NotReady) + Poll::Pending } else { - Ok(Async::Ready(())) + Poll::Ready(()) } - })).unwrap(); + })); } #[test] fn sync_from_two_peers_works() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(3); net.peer(1).push_blocks(100, false); net.peer(2).push_blocks(100, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); let peer1 = &net.peers()[1]; assert!(net.peers()[0].blockchain_canon_equals(peer1)); assert!(!net.peer(0).is_major_syncing()); @@ -163,12 +157,11 @@ fn sync_from_two_peers_works() { #[test] fn sync_from_two_peers_with_ancestry_search_works() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(3); net.peer(0).push_blocks(10, true); net.peer(1).push_blocks(100, false); net.peer(2).push_blocks(100, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); let peer1 = &net.peers()[1]; assert!(net.peers()[0].blockchain_canon_equals(peer1)); } @@ -176,14 +169,13 @@ fn sync_from_two_peers_with_ancestry_search_works() { #[test] fn ancestry_search_works_when_backoff_is_one() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(3); net.peer(0).push_blocks(1, false); net.peer(1).push_blocks(2, false); net.peer(2).push_blocks(2, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); let peer1 = &net.peers()[1]; assert!(net.peers()[0].blockchain_canon_equals(peer1)); } @@ -191,14 +183,13 @@ fn ancestry_search_works_when_backoff_is_one() { #[test] fn ancestry_search_works_when_ancestor_is_genesis() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(3); net.peer(0).push_blocks(13, true); net.peer(1).push_blocks(100, false); net.peer(2).push_blocks(100, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); let peer1 = &net.peers()[1]; assert!(net.peers()[0].blockchain_canon_equals(peer1)); } @@ -221,10 +212,9 @@ fn ancestry_search_works_when_common_is_hundred() { #[test] fn sync_long_chain_works() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(2); net.peer(1).push_blocks(500, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); let peer1 = &net.peers()[1]; assert!(net.peers()[0].blockchain_canon_equals(peer1)); } @@ -232,18 +222,17 @@ fn sync_long_chain_works() { #[test] fn sync_no_common_longer_chain_fails() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(3); net.peer(0).push_blocks(20, true); net.peer(1).push_blocks(20, false); - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); if net.peer(0).is_major_syncing() { - Ok(Async::NotReady) + Poll::Pending } else { - Ok(Async::Ready(())) + Poll::Ready(()) } - })).unwrap(); + })); let peer1 = &net.peers()[1]; assert!(!net.peers()[0].blockchain_canon_equals(peer1)); } @@ -251,10 +240,9 @@ fn sync_no_common_longer_chain_fails() { #[test] fn sync_justifications() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = JustificationTestNet::new(3); net.peer(0).push_blocks(20, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); // there's currently no justification for block #10 assert_eq!(net.peer(0).client().justification(&BlockId::Number(10)).unwrap(), None); @@ -274,26 +262,25 @@ fn sync_justifications() { net.peer(1).request_justification(&h2.hash().into(), 15); net.peer(1).request_justification(&h3.hash().into(), 20); - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); for height in (10..21).step_by(5) { if net.peer(0).client().justification(&BlockId::Number(height)).unwrap() != Some(Vec::new()) { - return Ok(Async::NotReady); + return Poll::Pending; } if net.peer(1).client().justification(&BlockId::Number(height)).unwrap() != Some(Vec::new()) { - return Ok(Async::NotReady); + return Poll::Pending; } } - Ok(Async::Ready(())) - })).unwrap(); + Poll::Ready(()) + })); } #[test] fn sync_justifications_across_forks() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = JustificationTestNet::new(3); // we push 5 blocks net.peer(0).push_blocks(5, false); @@ -303,30 +290,29 @@ fn sync_justifications_across_forks() { // peer 1 will only see the longer fork. but we'll request justifications // for both and finalize the small fork instead. - net.block_until_sync(&mut runtime); + net.block_until_sync(); net.peer(0).client().finalize_block(BlockId::Hash(f1_best), Some(Vec::new()), true).unwrap(); net.peer(1).request_justification(&f1_best, 10); net.peer(1).request_justification(&f2_best, 11); - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); if net.peer(0).client().justification(&BlockId::Number(10)).unwrap() == Some(Vec::new()) && net.peer(1).client().justification(&BlockId::Number(10)).unwrap() == Some(Vec::new()) { - Ok(Async::Ready(())) + Poll::Ready(()) } else { - Ok(Async::NotReady) + Poll::Pending } - })).unwrap(); + })); } #[test] fn sync_after_fork_works() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(3); net.peer(0).push_blocks(30, false); net.peer(1).push_blocks(30, false); @@ -340,7 +326,7 @@ fn sync_after_fork_works() { net.peer(2).push_blocks(1, false); // peer 1 has the best chain - net.block_until_sync(&mut runtime); + net.block_until_sync(); let peer1 = &net.peers()[1]; assert!(net.peers()[0].blockchain_canon_equals(peer1)); (net.peers()[1].blockchain_canon_equals(peer1)); @@ -350,7 +336,6 @@ fn sync_after_fork_works() { #[test] fn syncs_all_forks() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(4); net.peer(0).push_blocks(2, false); net.peer(1).push_blocks(2, false); @@ -358,7 +343,7 @@ fn syncs_all_forks() { net.peer(0).push_blocks(2, true); net.peer(1).push_blocks(4, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); // Check that all peers have all of the blocks. assert_eq!(9, net.peer(0).blocks_count()); assert_eq!(9, net.peer(1).blocks_count()); @@ -367,12 +352,11 @@ fn syncs_all_forks() { #[test] fn own_blocks_are_announced() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(3); - net.block_until_sync(&mut runtime); // connect'em + net.block_until_sync(); // connect'em net.peer(0).generate_blocks(1, BlockOrigin::Own, |builder| builder.build().unwrap().block); - net.block_until_sync(&mut runtime); + net.block_until_sync(); assert_eq!(net.peer(0).client.info().best_number, 1); assert_eq!(net.peer(1).client.info().best_number, 1); @@ -384,7 +368,6 @@ fn own_blocks_are_announced() { #[test] fn blocks_are_not_announced_by_light_nodes() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(0); // full peer0 is connected to light peer @@ -397,7 +380,7 @@ fn blocks_are_not_announced_by_light_nodes() { // Sync between 0 and 1. net.peer(0).push_blocks(1, false); assert_eq!(net.peer(0).client.info().best_number, 1); - net.block_until_sync(&mut runtime); + net.block_until_sync(); assert_eq!(net.peer(1).client.info().best_number, 1); // Add another node and remove node 0. @@ -405,18 +388,17 @@ fn blocks_are_not_announced_by_light_nodes() { net.peers.remove(0); // Poll for a few seconds and make sure 1 and 2 (now 0 and 1) don't sync together. - let mut delay = futures_timer::Delay::new(Duration::from_secs(5)).unit_error().compat(); - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| { - net.poll(); - delay.poll().map_err(|_| ()) - })).unwrap(); + let mut delay = futures_timer::Delay::new(Duration::from_secs(5)); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); + Pin::new(&mut delay).poll(cx) + })); assert_eq!(net.peer(1).client.info().best_number, 0); } #[test] fn can_sync_small_non_best_forks() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(2); net.peer(0).push_blocks(30, false); net.peer(1).push_blocks(30, false); @@ -435,14 +417,14 @@ fn can_sync_small_non_best_forks() { assert!(net.peer(1).client().header(&BlockId::Hash(small_hash)).unwrap().is_none()); // poll until the two nodes connect, otherwise announcing the block will not work - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); if net.peer(0).num_peers() == 0 { - Ok(Async::NotReady) + Poll::Pending } else { - Ok(Async::Ready(())) + Poll::Ready(()) } - })).unwrap(); + })); // synchronization: 0 synced to longer chain and 1 didn't sync to small chain. @@ -455,32 +437,31 @@ fn can_sync_small_non_best_forks() { // after announcing, peer 1 downloads the block. - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); assert!(net.peer(0).client().header(&BlockId::Hash(small_hash)).unwrap().is_some()); if net.peer(1).client().header(&BlockId::Hash(small_hash)).unwrap().is_none() { - return Ok(Async::NotReady) + return Poll::Pending } - Ok(Async::Ready(())) - })).unwrap(); - net.block_until_sync(&mut runtime); + Poll::Ready(()) + })); + net.block_until_sync(); let another_fork = net.peer(0).push_blocks_at(BlockId::Number(35), 2, true); net.peer(0).announce_block(another_fork, Vec::new()); - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); if net.peer(1).client().header(&BlockId::Hash(another_fork)).unwrap().is_none() { - return Ok(Async::NotReady) + return Poll::Pending } - Ok(Async::Ready(())) - })).unwrap(); + Poll::Ready(()) + })); } #[test] fn can_not_sync_from_light_peer() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); // given the network with 1 full nodes (#0) and 1 light node (#1) let mut net = TestNet::new(1); @@ -490,7 +471,7 @@ fn can_not_sync_from_light_peer() { net.peer(0).push_blocks(1, false); // and let the light client sync from this node - net.block_until_sync(&mut runtime); + net.block_until_sync(); // ensure #0 && #1 have the same best block let full0_info = net.peer(0).client.info(); @@ -504,29 +485,28 @@ fn can_not_sync_from_light_peer() { net.peers.remove(0); // ensure that the #2 (now #1) fails to sync block #1 even after 5 seconds - let mut test_finished = futures_timer::Delay::new(Duration::from_secs(5)).unit_error().compat(); - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { - net.poll(); - test_finished.poll().map_err(|_| ()) - })).unwrap(); + let mut test_finished = futures_timer::Delay::new(Duration::from_secs(5)); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); + Pin::new(&mut test_finished).poll(cx) + })); } #[test] fn light_peer_imports_header_from_announce() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); - fn import_with_announce(net: &mut TestNet, runtime: &mut current_thread::Runtime, hash: H256) { + fn import_with_announce(net: &mut TestNet, hash: H256) { net.peer(0).announce_block(hash, Vec::new()); - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); if net.peer(1).client().header(&BlockId::Hash(hash)).unwrap().is_some() { - Ok(Async::Ready(())) + Poll::Ready(()) } else { - Ok(Async::NotReady) + Poll::Pending } - })).unwrap(); + })); } // given the network with 1 full nodes (#0) and 1 light node (#1) @@ -534,21 +514,20 @@ fn light_peer_imports_header_from_announce() { net.add_light_peer(&Default::default()); // let them connect to each other - net.block_until_sync(&mut runtime); + net.block_until_sync(); // check that NEW block is imported from announce message let new_hash = net.peer(0).push_blocks(1, false); - import_with_announce(&mut net, &mut runtime, new_hash); + import_with_announce(&mut net, new_hash); // check that KNOWN STALE block is imported from announce message let known_stale_hash = net.peer(0).push_blocks_at(BlockId::Number(0), 1, true); - import_with_announce(&mut net, &mut runtime, known_stale_hash); + import_with_announce(&mut net, known_stale_hash); } #[test] fn can_sync_explicit_forks() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(2); net.peer(0).push_blocks(30, false); net.peer(1).push_blocks(30, false); @@ -568,14 +547,14 @@ fn can_sync_explicit_forks() { assert!(net.peer(1).client().header(&BlockId::Hash(small_hash)).unwrap().is_none()); // poll until the two nodes connect, otherwise announcing the block will not work - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); if net.peer(0).num_peers() == 0 || net.peer(1).num_peers() == 0 { - Ok(Async::NotReady) + Poll::Pending } else { - Ok(Async::Ready(())) + Poll::Ready(()) } - })).unwrap(); + })); // synchronization: 0 synced to longer chain and 1 didn't sync to small chain. @@ -589,21 +568,20 @@ fn can_sync_explicit_forks() { net.peer(1).set_sync_fork_request(vec![first_peer_id], small_hash, small_number); // peer 1 downloads the block. - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); assert!(net.peer(0).client().header(&BlockId::Hash(small_hash)).unwrap().is_some()); if net.peer(1).client().header(&BlockId::Hash(small_hash)).unwrap().is_none() { - return Ok(Async::NotReady) + return Poll::Pending } - Ok(Async::Ready(())) - })).unwrap(); + Poll::Ready(()) + })); } #[test] fn syncs_header_only_forks() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(0); let config = ProtocolConfig::default(); net.add_full_peer_with_states(&config, None); @@ -616,7 +594,7 @@ fn syncs_header_only_forks() { let small_number = net.peer(0).client().info().best_number; net.peer(1).push_blocks(4, false); - net.block_until_sync(&mut runtime); + net.block_until_sync(); // Peer 1 will sync the small fork even though common block state is missing assert_eq!(9, net.peer(0).blocks_count()); assert_eq!(9, net.peer(1).blocks_count()); @@ -624,19 +602,18 @@ fn syncs_header_only_forks() { // Request explicit header-only sync request for the ancient fork. let first_peer_id = net.peer(0).id(); net.peer(1).set_sync_fork_request(vec![first_peer_id], small_hash, small_number); - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { - net.poll(); + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); if net.peer(1).client().header(&BlockId::Hash(small_hash)).unwrap().is_none() { - return Ok(Async::NotReady) + return Poll::Pending } - Ok(Async::Ready(())) - })).unwrap(); + Poll::Ready(()) + })); } #[test] fn does_not_sync_announced_old_best_block() { let _ = ::env_logger::try_init(); - let mut runtime = current_thread::Runtime::new().unwrap(); let mut net = TestNet::new(3); let old_hash = net.peer(0).push_blocks(1, false); @@ -645,18 +622,38 @@ fn does_not_sync_announced_old_best_block() { net.peer(1).push_blocks(20, true); net.peer(0).announce_block(old_hash, Vec::new()); - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { + block_on(futures::future::poll_fn::<(), _>(|cx| { // poll once to import announcement - net.poll(); - Ok(Async::Ready(())) - })).unwrap(); + net.poll(cx); + Poll::Ready(()) + })); assert!(!net.peer(1).is_major_syncing()); net.peer(0).announce_block(old_hash_with_parent, Vec::new()); - runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { + block_on(futures::future::poll_fn::<(), _>(|cx| { // poll once to import announcement - net.poll(); - Ok(Async::Ready(())) - })).unwrap(); + net.poll(cx); + Poll::Ready(()) + })); assert!(!net.peer(1).is_major_syncing()); } + +#[test] +fn full_sync_requires_block_body() { + // Check that we don't sync headers-only in full mode. + let _ = ::env_logger::try_init(); + let mut net = TestNet::new(2); + + net.peer(0).push_headers(1); + // Wait for nodes to connect + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); + if net.peer(0).num_peers() == 0 || net.peer(1).num_peers() == 0 { + Poll::Pending + } else { + Poll::Ready(()) + } + })); + net.block_until_idle(); + assert_eq!(net.peer(1).client.info().best_number, 0); +} diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml index 29e6dca2a7f95bbad8422186586ac4305b9453cd..9ea0372969b666ee22ad6944a93260e9395b2516 100644 --- a/client/offchain/Cargo.toml +++ b/client/offchain/Cargo.toml @@ -1,41 +1,43 @@ [package] description = "Substrate offchain workers" name = "sc-offchain" -version = "2.0.0" +version = "2.0.0-alpha.3" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] bytes = "0.5" -sc-client-api = { version = "2.0.0", path = "../api" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../api" } +sp-api = { version = "2.0.0-alpha.2", path = "../../primitives/api" } fnv = "1.0.6" futures = "0.3.1" futures-timer = "3.0.1" log = "0.4.8" threadpool = "1.7" num_cpus = "1.10" -sp-offchain = { version = "2.0.0", path = "../../primitives/offchain" } -codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } +sp-offchain = { version = "2.0.0-alpha.2", path = "../../primitives/offchain" } +codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } parking_lot = "0.10.0" -sp-core = { version = "2.0.0", path = "../../primitives/core" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } rand = "0.7.2" -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sc-network = { version = "0.8", path = "../network" } -sc-keystore = { version = "2.0.0", path = "../keystore" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } +sc-network = { version = "0.8.0-alpha.2", path = "../network" } +sc-keystore = { version = "2.0.0-alpha.2", path = "../keystore" } [target.'cfg(not(target_os = "unknown"))'.dependencies] hyper = "0.13.2" hyper-rustls = "0.19" [dev-dependencies] -sc-client-db = { version = "0.8", default-features = true, path = "../db/" } +sc-client-db = { version = "0.8.0-alpha.2", default-features = true, path = "../db/" } env_logger = "0.7.0" -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } tokio = "0.2" -sc-transaction-pool = { version = "2.0.0", path = "../../client/transaction-pool" } -sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } +sc-transaction-pool = { version = "2.0.0-alpha.2", path = "../../client/transaction-pool" } +sp-transaction-pool = { version = "2.0.0-alpha.2", path = "../../primitives/transaction-pool" } [features] default = [] diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 7fc9671fcbcd826c85b69c9414fbaa34e56797da..27a7f508459ba1bbc3cc4ba0eef0ed2a7c5f9770 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -169,6 +169,7 @@ mod tests { use substrate_test_runtime_client::runtime::Block; use sc_transaction_pool::{BasicPool, FullChainApi}; use sp_transaction_pool::{TransactionPool, InPoolTransaction}; + use sc_client_api::ExecutorProvider; struct MockNetworkStateInfo(); diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index c17ffdc385ba93a1475181272a8669c99c74edf2..9e76b8015afcadf897faf017a5ba935270cd5eef 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -3,13 +3,16 @@ description = "Connectivity manager based on reputation" homepage = "http://parity.io" license = "GPL-3.0" name = "sc-peerset" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" +repository = "https://github.com/paritytech/substrate/" +documentation = "https://docs.rs/sc-peerset" + [dependencies] futures = "0.3.1" -libp2p = { version = "0.16.0", default-features = false } +libp2p = { version = "0.16.2", default-features = false } log = "0.4.8" serde_json = "1.0.41" wasm-timer = "0.2" diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index fb91f1fcf69e09d4c7fec2949e2ee586184b588d..87ed2336aea9c65da74aafbc53216ce0a6ffaff7 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -21,18 +21,22 @@ mod peersstate; use std::{collections::{HashSet, HashMap}, collections::VecDeque}; use futures::{prelude::*, channel::mpsc}; -use libp2p::PeerId; use log::{debug, error, trace}; use serde_json::json; -use std::{pin::Pin, task::Context, task::Poll}; +use std::{pin::Pin, task::{Context, Poll}, time::Duration}; use wasm_timer::Instant; +pub use libp2p::PeerId; + /// We don't accept nodes whose reputation is under this value. const BANNED_THRESHOLD: i32 = 82 * (i32::min_value() / 100); /// Reputation change for a node when we get disconnected from it. const DISCONNECT_REPUTATION_CHANGE: i32 = -256; /// Reserved peers group ID const RESERVED_NODES: &'static str = "reserved"; +/// Amount of time between the moment we disconnect from a node and the moment we remove it from +/// the list. +const FORGET_AFTER: Duration = Duration::from_secs(3600); #[derive(Debug)] enum Action { @@ -45,7 +49,7 @@ enum Action { RemoveFromPriorityGroup(String, PeerId), } -/// Shared handle to the peer set manager (PSM). Distributed around the code. +/// Description of a reputation adjustment for a node. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ReputationChange { /// Reputation delta. @@ -309,10 +313,11 @@ impl Peerset { /// Updates the value of `self.latest_time_update` and performs all the updates that happen /// over time, such as reputation increases for staying connected. fn update_time(&mut self) { + let now = Instant::now(); + // We basically do `(now - self.latest_update).as_secs()`, except that by the way we do it // we know that we're not going to miss seconds because of rounding to integers. let secs_diff = { - let now = Instant::now(); let elapsed_latest = self.latest_time_update - self.created; let elapsed_now = now - self.created; self.latest_time_update = now; @@ -344,10 +349,16 @@ impl Peerset { peer.set_reputation(after) } peersstate::Peer::NotConnected(mut peer) => { - let before = peer.reputation(); - let after = reput_tick(before); - trace!(target: "peerset", "Fleeting {}: {} -> {}", peer_id, before, after); - peer.set_reputation(after) + if peer.reputation() == 0 && + peer.last_connected_or_discovered() + FORGET_AFTER < now + { + peer.forget_peer(); + } else { + let before = peer.reputation(); + let after = reput_tick(before); + trace!(target: "peerset", "Fleeting {}: {} -> {}", peer_id, before, after); + peer.set_reputation(after) + } } peersstate::Peer::Unknown(_) => unreachable!("We iterate over known peers; qed") }; @@ -413,7 +424,10 @@ impl Peerset { let not_connected = match self.data.peer(&peer_id) { // If we're already connected, don't answer, as the docs mention. peersstate::Peer::Connected(_) => return, - peersstate::Peer::NotConnected(entry) => entry, + peersstate::Peer::NotConnected(mut entry) => { + entry.bump_last_connected_or_discovered(); + entry + }, peersstate::Peer::Unknown(entry) => entry.discover(), }; @@ -504,6 +518,11 @@ impl Peerset { }) } + /// Returns the number of peers that we have discovered. + pub fn num_discovered_peers(&self) -> usize { + self.data.peers().len() + } + /// Returns priority group by id. pub fn get_priority_group(&self, group_id: &str) -> Option> { self.data.get_priority_group(group_id) diff --git a/client/peerset/src/peersstate.rs b/client/peerset/src/peersstate.rs index 96a6698734b744436cc846fcc772e2edb53b221a..843ec0a36006f436e3d9b2b9c83175ec632566e5 100644 --- a/client/peerset/src/peersstate.rs +++ b/client/peerset/src/peersstate.rs @@ -17,8 +17,9 @@ //! Contains the state storage behind the peerset. use libp2p::PeerId; +use log::{error, warn}; use std::{borrow::Cow, collections::{HashSet, HashMap}}; -use log::warn; +use wasm_timer::Instant; /// State storage behind the peerset. /// @@ -69,7 +70,9 @@ struct Node { impl Default for Node { fn default() -> Node { Node { - connection_state: ConnectionState::NotConnected, + connection_state: ConnectionState::NotConnected { + last_connected: Instant::now(), + }, reputation: 0, } } @@ -83,7 +86,11 @@ enum ConnectionState { /// We are connected through an outgoing connection. Out, /// We are not connected to this node. - NotConnected, + NotConnected { + /// When we were last connected to the node, or if we were never connected when we + /// discovered it. + last_connected: Instant, + }, } impl ConnectionState { @@ -92,7 +99,7 @@ impl ConnectionState { match self { ConnectionState::In => true, ConnectionState::Out => true, - ConnectionState::NotConnected => false, + ConnectionState::NotConnected { .. } => false, } } } @@ -137,7 +144,7 @@ impl PeersState { /// Returns the list of all the peers we know of. // Note: this method could theoretically return a `Peer`, but implementing that // isn't simple. - pub fn peers(&self) -> impl Iterator { + pub fn peers(&self) -> impl ExactSizeIterator { self.nodes.keys() } @@ -212,11 +219,13 @@ impl PeersState { match node.connection_state { ConnectionState::In => self.num_in -= 1, ConnectionState::Out => self.num_out -= 1, - ConnectionState::NotConnected => + ConnectionState::NotConnected { .. } => debug_assert!(false, "State inconsistency: disconnecting a disconnected node") } } - node.connection_state = ConnectionState::NotConnected; + node.connection_state = ConnectionState::NotConnected { + last_connected: Instant::now(), + }; } else { warn!(target: "peerset", "Attempting to disconnect unknown peer {}", peer_id); } @@ -292,7 +301,7 @@ impl PeersState { match peer.connection_state { ConnectionState::In => self.num_in += 1, ConnectionState::Out => self.num_out += 1, - ConnectionState::NotConnected => {}, + ConnectionState::NotConnected { .. } => {}, } } } @@ -305,7 +314,7 @@ impl PeersState { match peer.connection_state { ConnectionState::In => self.num_in -= 1, ConnectionState::Out => self.num_out -= 1, - ConnectionState::NotConnected => {}, + ConnectionState::NotConnected { .. } => {}, } } } @@ -467,6 +476,45 @@ impl<'a> NotConnectedPeer<'a> { self.peer_id.into_owned() } + /// Bumps the value that `last_connected_or_discovered` would return to now, even if we + /// didn't connect or disconnect. + pub fn bump_last_connected_or_discovered(&mut self) { + let state = match self.state.nodes.get_mut(&*self.peer_id) { + Some(s) => s, + None => return, + }; + + if let ConnectionState::NotConnected { last_connected } = &mut state.connection_state { + *last_connected = Instant::now(); + } + } + + /// Returns when we were last connected to this peer, or when we discovered it if we were + /// never connected. + /// + /// Guaranteed to be earlier than calling `Instant::now()` after the function returns. + pub fn last_connected_or_discovered(&self) -> Instant { + let state = match self.state.nodes.get(&*self.peer_id) { + Some(s) => s, + None => { + error!( + target: "peerset", + "State inconsistency with {}; not connected after borrow", + self.peer_id + ); + return Instant::now(); + } + }; + + match state.connection_state { + ConnectionState::NotConnected { last_connected } => last_connected, + _ => { + error!(target: "peerset", "State inconsistency with {}", self.peer_id); + Instant::now() + } + } + } + /// Tries to set the peer as connected as an outgoing connection. /// /// If there are enough slots available, switches the node to "connected" and returns `Ok`. If @@ -518,6 +566,22 @@ impl<'a> NotConnectedPeer<'a> { pub fn add_reputation(&mut self, modifier: i32) { self.state.add_reputation(&self.peer_id, modifier) } + + /// Un-discovers the peer. Removes it from the list. + pub fn forget_peer(self) -> UnknownPeer<'a> { + if self.state.nodes.remove(&*self.peer_id).is_none() { + error!( + target: "peerset", + "State inconsistency with {} when forgetting peer", + self.peer_id + ); + } + + UnknownPeer { + parent: self.state, + peer_id: self.peer_id, + } + } } /// A peer that we have never heard of. @@ -533,7 +597,9 @@ impl<'a> UnknownPeer<'a> { /// values using the `NotConnectedPeer` that this method returns. pub fn discover(self) -> NotConnectedPeer<'a> { self.parent.nodes.insert(self.peer_id.clone().into_owned(), Node { - connection_state: ConnectionState::NotConnected, + connection_state: ConnectionState::NotConnected { + last_connected: Instant::now(), + }, reputation: 0, }); diff --git a/client/rpc-api/Cargo.toml b/client/rpc-api/Cargo.toml index b9f4cbb0159ce5e059499c38829860aa7cef5599..0f3b72e519ba5cbff72236a885deb54101347594 100644 --- a/client/rpc-api/Cargo.toml +++ b/client/rpc-api/Cargo.toml @@ -1,12 +1,15 @@ [package] name = "sc-rpc-api" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Substrate RPC interfaces." [dependencies] -codec = { package = "parity-scale-codec", version = "1.0.0" } +codec = { package = "parity-scale-codec", version = "1.2.0" } derive_more = "0.99.2" futures = { version = "0.3.1", features = ["compat"] } jsonrpc-core = "14.0.3" @@ -15,10 +18,10 @@ jsonrpc-derive = "14.0.3" jsonrpc-pubsub = "14.0.3" log = "0.4.8" parking_lot = "0.10.0" -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-version = { version = "2.0.0", path = "../../primitives/version" } -sp-runtime = { path = "../../primitives/runtime" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +sp-version = { version = "2.0.0-alpha.2", path = "../../primitives/version" } +sp-runtime = { path = "../../primitives/runtime" , version = "2.0.0-alpha.2"} serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" -sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } -sp-rpc = { version = "2.0.0", path = "../../primitives/rpc" } +sp-transaction-pool = { version = "2.0.0-alpha.2", path = "../../primitives/transaction-pool" } +sp-rpc = { version = "2.0.0-alpha.2", path = "../../primitives/rpc" } diff --git a/client/rpc-api/src/chain/mod.rs b/client/rpc-api/src/chain/mod.rs index 0c270a3f70528e4b772a639cca98f395a85b8e2b..2ab3851d37663a328e37b4f1d8998d103905d9f3 100644 --- a/client/rpc-api/src/chain/mod.rs +++ b/client/rpc-api/src/chain/mod.rs @@ -54,6 +54,18 @@ pub trait ChainApi { #[rpc(name = "chain_getFinalizedHead", alias("chain_getFinalisedHead"))] fn finalized_head(&self) -> Result; + /// All head subscription + #[pubsub(subscription = "chain_allHead", subscribe, name = "chain_subscribeAllHeads")] + fn subscribe_all_heads(&self, metadata: Self::Metadata, subscriber: Subscriber

); + + /// Unsubscribe from all head subscription. + #[pubsub(subscription = "chain_allHead", unsubscribe, name = "chain_unsubscribeAllHeads")] + fn unsubscribe_all_heads( + &self, + metadata: Option, + id: SubscriptionId, + ) -> RpcResult; + /// New head subscription #[pubsub( subscription = "chain_newHead", @@ -76,7 +88,7 @@ pub trait ChainApi { id: SubscriptionId, ) -> RpcResult; - /// New head subscription + /// Finalized head subscription #[pubsub( subscription = "chain_finalizedHead", subscribe, @@ -85,7 +97,7 @@ pub trait ChainApi { )] fn subscribe_finalized_heads(&self, metadata: Self::Metadata, subscriber: Subscriber
); - /// Unsubscribe from new head subscription. + /// Unsubscribe from finalized head subscription. #[pubsub( subscription = "chain_finalizedHead", unsubscribe, diff --git a/client/rpc-servers/Cargo.toml b/client/rpc-servers/Cargo.toml index 6b0d3f4adacf8bc3d5d07b28082455916080df07..79d2984c2293e0649456ce5453f0dd719cfe77ad 100644 --- a/client/rpc-servers/Cargo.toml +++ b/client/rpc-servers/Cargo.toml @@ -1,9 +1,12 @@ [package] name = "sc-rpc-server" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Substrate RPC servers." [dependencies] jsonrpc-core = "14.0.3" @@ -11,7 +14,7 @@ pubsub = { package = "jsonrpc-pubsub", version = "14.0.3" } log = "0.4.8" serde = "1.0.101" serde_json = "1.0.41" -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } [target.'cfg(not(target_os = "unknown"))'.dependencies] http = { package = "jsonrpc-http-server", version = "14.0.3" } diff --git a/client/rpc/Cargo.toml b/client/rpc/Cargo.toml index d09617ebc1658405e87fa127827987d6ddee9d94..9a8be85becf59f61fc914ab6c707dd976714a0d2 100644 --- a/client/rpc/Cargo.toml +++ b/client/rpc/Cargo.toml @@ -1,41 +1,44 @@ [package] name = "sc-rpc" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Substrate Client RPC" [dependencies] -sc-rpc-api = { version = "0.8", path = "../rpc-api" } -sc-client-api = { version = "2.0.0", path = "../api" } -sc-client = { version = "0.8", path = "../" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } -codec = { package = "parity-scale-codec", version = "1.0.0" } +sc-rpc-api = { version = "0.8.0-alpha.2", path = "../rpc-api" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../api" } +sc-client = { version = "0.8.0-alpha.2", path = "../" } +sp-api = { version = "2.0.0-alpha.2", path = "../../primitives/api" } +codec = { package = "parity-scale-codec", version = "1.2.0" } futures = { version = "0.3.1", features = ["compat"] } jsonrpc-pubsub = "14.0.3" log = "0.4.8" -sp-core = { version = "2.0.0", path = "../../primitives/core" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } rpc = { package = "jsonrpc-core", version = "14.0.3" } -sp-version = { version = "2.0.0", path = "../../primitives/version" } +sp-version = { version = "2.0.0-alpha.2", path = "../../primitives/version" } serde_json = "1.0.41" -sp-session = { version = "2.0.0", path = "../../primitives/session" } -sp-offchain = { version = "2.0.0", path = "../../primitives/offchain" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sp-rpc = { version = "2.0.0", path = "../../primitives/rpc" } -sp-state-machine = { version = "0.8", path = "../../primitives/state-machine" } -sc-executor = { version = "0.8", path = "../executor" } -sc-keystore = { version = "2.0.0", path = "../keystore" } -sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } +sp-session = { version = "2.0.0-alpha.2", path = "../../primitives/session" } +sp-offchain = { version = "2.0.0-alpha.2", path = "../../primitives/offchain" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } +sp-rpc = { version = "2.0.0-alpha.2", path = "../../primitives/rpc" } +sp-state-machine = { version = "0.8.0-alpha.2", path = "../../primitives/state-machine" } +sc-executor = { version = "0.8.0-alpha.2", path = "../executor" } +sc-keystore = { version = "2.0.0-alpha.2", path = "../keystore" } +sp-transaction-pool = { version = "2.0.0-alpha.2", path = "../../primitives/transaction-pool" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../primitives/blockchain" } hash-db = { version = "0.15.2", default-features = false } parking_lot = "0.10.0" [dev-dependencies] assert_matches = "1.3.0" futures01 = { package = "futures", version = "0.1.29" } -sc-network = { version = "0.8", path = "../network" } +sc-network = { version = "0.8.0-alpha.2", path = "../network" } rustc-hex = "2.0.1" -sp-io = { version = "2.0.0", path = "../../primitives/io" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } +sp-io = { version = "2.0.0-alpha.2", path = "../../primitives/io" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } tokio = "0.1.22" -sc-transaction-pool = { version = "2.0.0", path = "../transaction-pool" } +sc-transaction-pool = { version = "2.0.0-alpha.2", path = "../transaction-pool" } diff --git a/client/rpc/src/author/mod.rs b/client/rpc/src/author/mod.rs index 06bdcf883c6891548717cd4d44824e0816ba0fd9..80a3a4349ed82f5fa2ebe04d96cf24d28e18ae5e 100644 --- a/client/rpc/src/author/mod.rs +++ b/client/rpc/src/author/mod.rs @@ -22,8 +22,7 @@ mod tests; use std::{sync::Arc, convert::TryInto}; use log::warn; -use sc_client::Client; -use sp_blockchain::Error as ClientError; +use sp_blockchain::{Error as ClientError, HeaderBackend}; use rpc::futures::{ Sink, Future, @@ -36,7 +35,7 @@ use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId}; use codec::{Encode, Decode}; use sp_core::{Bytes, traits::BareCryptoStorePtr}; use sp_api::ProvideRuntimeApi; -use sp_runtime::{generic, traits}; +use sp_runtime::generic; use sp_transaction_pool::{ TransactionPool, InPoolTransaction, TransactionStatus, BlockHash, TxHash, TransactionFor, error::IntoPoolError, @@ -48,9 +47,9 @@ pub use sc_rpc_api::author::*; use self::error::{Error, FutureResult, Result}; /// Authoring API -pub struct Author { +pub struct Author { /// Substrate client - client: Arc>, + client: Arc, /// Transactions pool pool: Arc

, /// Subscriptions manager @@ -59,10 +58,10 @@ pub struct Author { keystore: BareCryptoStorePtr, } -impl Author { +impl Author { /// Create new instance of Authoring API. pub fn new( - client: Arc>, + client: Arc, pool: Arc

, subscriptions: Subscriptions, keystore: BareCryptoStorePtr, @@ -76,18 +75,11 @@ impl Author { } } -impl AuthorApi, BlockHash

> - for Author::Block, RA> -where - B: sc_client_api::backend::Backend<

::Block> + Send + Sync + 'static, - E: sc_client::CallExecutor<

::Block> + Send + Sync + 'static, - P: TransactionPool + Sync + Send + 'static, - P::Block: traits::Block, - P::Error: 'static, - RA: Send + Sync + 'static, - Client: ProvideRuntimeApi, - as ProvideRuntimeApi>::Api: - SessionKeys, +impl AuthorApi, BlockHash

> for Author + where + P: TransactionPool + Sync + Send + 'static, + Client: HeaderBackend + ProvideRuntimeApi + Send + Sync + 'static, + Client::Api: SessionKeys, { type Metadata = crate::metadata::Metadata; @@ -105,7 +97,7 @@ where } fn rotate_keys(&self) -> Result { - let best_block_hash = self.client.chain_info().best_hash; + let best_block_hash = self.client.info().best_hash; self.client.runtime_api().generate_session_keys( &generic::BlockId::Hash(best_block_hash), None, @@ -113,7 +105,7 @@ where } fn has_session_keys(&self, session_keys: Bytes) -> Result { - let best_block_hash = self.client.chain_info().best_hash; + let best_block_hash = self.client.info().best_hash; let keys = self.client.runtime_api().decode_session_keys( &generic::BlockId::Hash(best_block_hash), session_keys.to_vec(), @@ -133,7 +125,7 @@ where Ok(xt) => xt, Err(err) => return Box::new(result(Err(err.into()))), }; - let best_block_hash = self.client.chain_info().best_hash; + let best_block_hash = self.client.info().best_hash; Box::new(self.pool .submit_one(&generic::BlockId::hash(best_block_hash), xt) .compat() @@ -176,7 +168,7 @@ where xt: Bytes, ) { let submit = || -> Result<_> { - let best_block_hash = self.client.chain_info().best_hash; + let best_block_hash = self.client.info().best_hash; let dxt = TransactionFor::

::decode(&mut &xt[..]) .map_err(error::Error::from)?; Ok( diff --git a/client/rpc/src/author/tests.rs b/client/rpc/src/author/tests.rs index 41bfc46d388bd5e64b3efc63f5645394916a1db1..3093cd9d3b759b08d887eb10165e26f697a9a073 100644 --- a/client/rpc/src/author/tests.rs +++ b/client/rpc/src/author/tests.rs @@ -25,8 +25,8 @@ use sp_core::{ }; use rpc::futures::Stream as _; use substrate_test_runtime_client::{ - self, AccountKeyring, runtime::{Extrinsic, Transfer, SessionKeys, RuntimeApi, Block}, - DefaultTestClientBuilderExt, TestClientBuilderExt, Backend, Client, Executor, + self, AccountKeyring, runtime::{Extrinsic, Transfer, SessionKeys, Block}, + DefaultTestClientBuilderExt, TestClientBuilderExt, Backend, Client, }; use sc_transaction_pool::{BasicPool, FullChainApi}; use tokio::runtime; @@ -75,7 +75,7 @@ impl Default for TestSetup { } impl TestSetup { - fn author(&self) -> Author { + fn author(&self) -> Author> { Author { client: self.client.clone(), pool: self.pool.clone(), diff --git a/client/rpc/src/chain/chain_full.rs b/client/rpc/src/chain/chain_full.rs index ff732368fe9d219f42cbe83ab94ebe47a14c2da3..ea562d47748c50aeb6e1ba110773fc568ce84c25 100644 --- a/client/rpc/src/chain/chain_full.rs +++ b/client/rpc/src/chain/chain_full.rs @@ -20,37 +20,39 @@ use std::sync::Arc; use rpc::futures::future::result; use sc_rpc_api::Subscriptions; -use sc_client_api::{CallExecutor, backend::Backend}; -use sc_client::Client; +use sc_client_api::{BlockchainEvents, BlockBody}; use sp_runtime::{generic::{BlockId, SignedBlock}, traits::{Block as BlockT}}; use super::{ChainBackend, client_err, error::FutureResult}; +use std::marker::PhantomData; +use sp_blockchain::HeaderBackend; /// Blockchain API backend for full nodes. Reads all the data from local database. -pub struct FullChain { +pub struct FullChain { /// Substrate client. - client: Arc>, + client: Arc, /// Current subscriptions. subscriptions: Subscriptions, + /// phantom member to pin the block type + _phantom: PhantomData, } -impl FullChain { +impl FullChain { /// Create new Chain API RPC handler. - pub fn new(client: Arc>, subscriptions: Subscriptions) -> Self { + pub fn new(client: Arc, subscriptions: Subscriptions) -> Self { Self { client, subscriptions, + _phantom: PhantomData, } } } -impl ChainBackend for FullChain where +impl ChainBackend for FullChain where Block: BlockT + 'static, - B: Backend + Send + Sync + 'static, - E: CallExecutor + Send + Sync + 'static, - RA: Send + Sync + 'static, + Client: BlockBody + HeaderBackend + BlockchainEvents + 'static, { - fn client(&self) -> &Arc> { + fn client(&self) -> &Arc { &self.client } @@ -60,7 +62,7 @@ impl ChainBackend for FullChain) -> FutureResult> { Box::new(result(self.client - .header(&BlockId::Hash(self.unwrap_or_best(hash))) + .header(BlockId::Hash(self.unwrap_or_best(hash))) .map_err(client_err) )) } diff --git a/client/rpc/src/chain/chain_light.rs b/client/rpc/src/chain/chain_light.rs index 3e26bd24bb090be405614d28c90359001a59758a..b258c8dd3bc2561f65e10c6045cb7d2c1762840e 100644 --- a/client/rpc/src/chain/chain_light.rs +++ b/client/rpc/src/chain/chain_light.rs @@ -22,7 +22,7 @@ use rpc::futures::future::{result, Future, Either}; use sc_rpc_api::Subscriptions; use sc_client::{ - Client, light::{fetcher::{Fetcher, RemoteBodyRequest}, blockchain::RemoteBlockchain}, + light::{fetcher::{Fetcher, RemoteBodyRequest}, blockchain::RemoteBlockchain}, }; use sp_runtime::{ generic::{BlockId, SignedBlock}, @@ -30,12 +30,14 @@ use sp_runtime::{ }; use super::{ChainBackend, client_err, error::FutureResult}; +use sp_blockchain::HeaderBackend; +use sc_client_api::BlockchainEvents; /// Blockchain API backend for light nodes. Reads all the data from local /// database, if available, or fetches it from remote node otherwise. -pub struct LightChain { +pub struct LightChain { /// Substrate client. - client: Arc>, + client: Arc, /// Current subscriptions. subscriptions: Subscriptions, /// Remote blockchain reference @@ -44,10 +46,10 @@ pub struct LightChain { fetcher: Arc, } -impl> LightChain { +impl> LightChain { /// Create new Chain API RPC handler. pub fn new( - client: Arc>, + client: Arc, subscriptions: Subscriptions, remote_blockchain: Arc>, fetcher: Arc, @@ -61,14 +63,12 @@ impl> LightChain } } -impl ChainBackend for LightChain where +impl ChainBackend for LightChain where Block: BlockT + 'static, - B: sc_client_api::backend::Backend + Send + Sync + 'static, - E: sc_client::CallExecutor + Send + Sync + 'static, - RA: Send + Sync + 'static, + Client: BlockchainEvents + HeaderBackend + Send + Sync + 'static, F: Fetcher + Send + Sync + 'static, { - fn client(&self) -> &Arc> { + fn client(&self) -> &Arc { &self.client } diff --git a/client/rpc/src/chain/mod.rs b/client/rpc/src/chain/mod.rs index a2971983c793f18f9858b53211f0525cd977fd87..e7a927e780627b5d89eb76bc869b39ea760f43c3 100644 --- a/client/rpc/src/chain/mod.rs +++ b/client/rpc/src/chain/mod.rs @@ -32,7 +32,7 @@ use rpc::{ use sc_rpc_api::Subscriptions; use sc_client::{ - self, Client, BlockchainEvents, + self, BlockchainEvents, light::{fetcher::Fetcher, blockchain::RemoteBlockchain}, }; use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId}; @@ -45,16 +45,17 @@ use sp_runtime::{ use self::error::{Result, Error, FutureResult}; pub use sc_rpc_api::chain::*; +use sp_blockchain::HeaderBackend; +use sc_client_api::BlockBody; /// Blockchain backend API -trait ChainBackend: Send + Sync + 'static +trait ChainBackend: Send + Sync + 'static where Block: BlockT + 'static, - B: sc_client_api::backend::Backend + Send + Sync + 'static, - E: sc_client::CallExecutor + Send + Sync + 'static, + Client: HeaderBackend + BlockchainEvents + 'static, { /// Get client reference. - fn client(&self) -> &Arc>; + fn client(&self) -> &Arc; /// Get subscriptions reference. fn subscriptions(&self) -> &Subscriptions; @@ -62,7 +63,7 @@ trait ChainBackend: Send + Sync + 'static /// Tries to unwrap passed block hash, or uses best block hash otherwise. fn unwrap_or_best(&self, hash: Option) -> Block::Hash { match hash.into() { - None => self.client().chain_info().best_hash, + None => self.client().info().best_hash, Some(hash) => hash, } } @@ -81,9 +82,9 @@ trait ChainBackend: Send + Sync + 'static number: Option>>, ) -> Result> { Ok(match number { - None => Some(self.client().chain_info().best_hash), + None => Some(self.client().info().best_hash), Some(num_or_hex) => self.client() - .header(&BlockId::number(num_or_hex.to_number()?)) + .header(BlockId::number(num_or_hex.to_number()?)) .map_err(client_err)? .map(|h| h.hash()), }) @@ -91,10 +92,36 @@ trait ChainBackend: Send + Sync + 'static /// Get hash of the last finalized block in the canon chain. fn finalized_head(&self) -> Result { - Ok(self.client().chain_info().finalized_hash) + Ok(self.client().info().finalized_hash) } - /// New head subscription + /// All new head subscription + fn subscribe_all_heads( + &self, + _metadata: crate::metadata::Metadata, + subscriber: Subscriber, + ) { + subscribe_headers( + self.client(), + self.subscriptions(), + subscriber, + || self.client().info().best_hash, + || self.client().import_notification_stream() + .map(|notification| Ok::<_, ()>(notification.header)) + .compat(), + ) + } + + /// Unsubscribe from all head subscription. + fn unsubscribe_all_heads( + &self, + _metadata: Option, + id: SubscriptionId, + ) -> RpcResult { + Ok(self.subscriptions().cancel(id)) + } + + /// New best head subscription fn subscribe_new_heads( &self, _metadata: crate::metadata::Metadata, @@ -104,7 +131,7 @@ trait ChainBackend: Send + Sync + 'static self.client(), self.subscriptions(), subscriber, - || self.client().chain_info().best_hash, + || self.client().info().best_hash, || self.client().import_notification_stream() .filter(|notification| future::ready(notification.is_new_best)) .map(|notification| Ok::<_, ()>(notification.header)) @@ -112,7 +139,7 @@ trait ChainBackend: Send + Sync + 'static ) } - /// Unsubscribe from new head subscription. + /// Unsubscribe from new best head subscription. fn unsubscribe_new_heads( &self, _metadata: Option, @@ -121,7 +148,7 @@ trait ChainBackend: Send + Sync + 'static Ok(self.subscriptions().cancel(id)) } - /// New head subscription + /// Finalized head subscription fn subscribe_finalized_heads( &self, _metadata: crate::metadata::Metadata, @@ -131,14 +158,14 @@ trait ChainBackend: Send + Sync + 'static self.client(), self.subscriptions(), subscriber, - || self.client().chain_info().finalized_hash, + || self.client().info().finalized_hash, || self.client().finality_notification_stream() .map(|notification| Ok::<_, ()>(notification.header)) .compat(), ) } - /// Unsubscribe from new head subscription. + /// Unsubscribe from finalized head subscription. fn unsubscribe_finalized_heads( &self, _metadata: Option, @@ -149,15 +176,13 @@ trait ChainBackend: Send + Sync + 'static } /// Create new state API that works on full node. -pub fn new_full( - client: Arc>, +pub fn new_full( + client: Arc, subscriptions: Subscriptions, -) -> Chain +) -> Chain where Block: BlockT + 'static, - B: sc_client_api::backend::Backend + Send + Sync + 'static, - E: sc_client::CallExecutor + Send + Sync + 'static + Clone, - RA: Send + Sync + 'static, + Client: BlockBody + HeaderBackend + BlockchainEvents + 'static, { Chain { backend: Box::new(self::chain_full::FullChain::new(client, subscriptions)), @@ -165,17 +190,15 @@ pub fn new_full( } /// Create new state API that works on light node. -pub fn new_light>( - client: Arc>, +pub fn new_light>( + client: Arc, subscriptions: Subscriptions, remote_blockchain: Arc>, fetcher: Arc, -) -> Chain +) -> Chain where Block: BlockT + 'static, - B: sc_client_api::backend::Backend + Send + Sync + 'static, - E: sc_client::CallExecutor + Send + Sync + 'static + Clone, - RA: Send + Sync + 'static, + Client: BlockBody + HeaderBackend + BlockchainEvents + 'static, F: Send + Sync + 'static, { Chain { @@ -189,15 +212,15 @@ pub fn new_light>( } /// Chain API with subscriptions support. -pub struct Chain { - backend: Box>, +pub struct Chain { + backend: Box>, } -impl ChainApi, Block::Hash, Block::Header, SignedBlock> for Chain where - Block: BlockT + 'static, - B: sc_client_api::backend::Backend + Send + Sync + 'static, - E: sc_client::CallExecutor + Send + Sync + 'static, - RA: Send + Sync + 'static +impl ChainApi, Block::Hash, Block::Header, SignedBlock> for + Chain + where + Block: BlockT + 'static, + Client: HeaderBackend + BlockchainEvents + 'static, { type Metadata = crate::metadata::Metadata; @@ -229,6 +252,14 @@ impl ChainApi, Block::Hash, Block::Header, Sig self.backend.finalized_head() } + fn subscribe_all_heads(&self, metadata: Self::Metadata, subscriber: Subscriber) { + self.backend.subscribe_all_heads(metadata, subscriber) + } + + fn unsubscribe_all_heads(&self, metadata: Option, id: SubscriptionId) -> RpcResult { + self.backend.unsubscribe_all_heads(metadata, id) + } + fn subscribe_new_heads(&self, metadata: Self::Metadata, subscriber: Subscriber) { self.backend.subscribe_new_heads(metadata, subscriber) } @@ -247,16 +278,15 @@ impl ChainApi, Block::Hash, Block::Header, Sig } /// Subscribe to new headers. -fn subscribe_headers( - client: &Arc>, +fn subscribe_headers( + client: &Arc, subscriptions: &Subscriptions, subscriber: Subscriber, best_block_hash: G, stream: F, ) where Block: BlockT + 'static, - B: sc_client_api::backend::Backend + Send + Sync + 'static, - E: sc_client::CallExecutor + Send + Sync + 'static, + Client: HeaderBackend + 'static, F: FnOnce() -> S, G: FnOnce() -> Block::Hash, ERR: ::std::fmt::Debug, @@ -264,7 +294,7 @@ fn subscribe_headers( { subscriptions.add(subscriber, |sink| { // send current head right at the start. - let header = client.header(&BlockId::Hash(best_block_hash())) + let header = client.header(BlockId::Hash(best_block_hash())) .map_err(client_err) .and_then(|header| { header.ok_or_else(|| "Best header missing.".to_owned().into()) diff --git a/client/rpc/src/chain/tests.rs b/client/rpc/src/chain/tests.rs index eb05639018781d3bd7a320d92219f48a30372877..02e4d2f16337bad25bf2ff29629ce2c3a876c612 100644 --- a/client/rpc/src/chain/tests.rs +++ b/client/rpc/src/chain/tests.rs @@ -195,6 +195,35 @@ fn should_notify_about_latest_block() { let remote = core.executor(); let (subscriber, id, transport) = Subscriber::new_test("test"); + { + let mut client = Arc::new(substrate_test_runtime_client::new()); + let api = new_full(client.clone(), Subscriptions::new(Arc::new(remote))); + + api.subscribe_all_heads(Default::default(), subscriber); + + // assert id assigned + assert_eq!(core.block_on(id), Ok(Ok(SubscriptionId::Number(1)))); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + } + + // assert initial head sent. + let (notification, next) = core.block_on(transport.into_future()).unwrap(); + assert!(notification.is_some()); + // assert notification sent to transport + let (notification, next) = core.block_on(next.into_future()).unwrap(); + assert!(notification.is_some()); + // no more notifications on this channel + assert_eq!(core.block_on(next.into_future()).unwrap().0, None); +} + +#[test] +fn should_notify_about_best_block() { + let mut core = ::tokio::runtime::Runtime::new().unwrap(); + let remote = core.executor(); + let (subscriber, id, transport) = Subscriber::new_test("test"); + { let mut client = Arc::new(substrate_test_runtime_client::new()); let api = new_full(client.clone(), Subscriptions::new(Arc::new(remote))); diff --git a/client/rpc/src/state/mod.rs b/client/rpc/src/state/mod.rs index 8f621cc8afc9667fed9c15d79db7d9ffa69248cd..82568866ee3ba4df6686d9201783f7cf8cfa673e 100644 --- a/client/rpc/src/state/mod.rs +++ b/client/rpc/src/state/mod.rs @@ -27,26 +27,26 @@ use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId}; use rpc::{Result as RpcResult, futures::{Future, future::result}}; use sc_rpc_api::Subscriptions; -use sc_client::{Client, CallExecutor, light::{blockchain::RemoteBlockchain, fetcher::Fetcher}}; +use sc_client::{light::{blockchain::RemoteBlockchain, fetcher::Fetcher}}; use sp_core::{Bytes, storage::{StorageKey, StorageData, StorageChangeSet}}; use sp_version::RuntimeVersion; use sp_runtime::traits::Block as BlockT; -use sp_api::{Metadata, ProvideRuntimeApi}; +use sp_api::{Metadata, ProvideRuntimeApi, CallApiAt}; use self::error::{Error, FutureResult}; pub use sc_rpc_api::state::*; +use sc_client_api::{ExecutorProvider, StorageProvider, BlockchainEvents, Backend}; +use sp_blockchain::{HeaderMetadata, HeaderBackend}; const STORAGE_KEYS_PAGED_MAX_COUNT: u32 = 1000; /// State backend API. -pub trait StateBackend: Send + Sync + 'static +pub trait StateBackend: Send + Sync + 'static where Block: BlockT + 'static, - B: sc_client_api::backend::Backend + Send + Sync + 'static, - E: sc_client::CallExecutor + Send + Sync + 'static, - RA: Send + Sync + 'static, + Client: Send + Sync + 'static, { /// Call runtime method at given block. fn call( @@ -194,18 +194,18 @@ pub trait StateBackend: Send + Sync + 'static } /// Create new state API that works on full node. -pub fn new_full( - client: Arc>, +pub fn new_full( + client: Arc, subscriptions: Subscriptions, -) -> State +) -> State where Block: BlockT + 'static, - B: sc_client_api::backend::Backend + Send + Sync + 'static, - E: CallExecutor + Send + Sync + 'static + Clone, - RA: Send + Sync + 'static, - Client: ProvideRuntimeApi, - as ProvideRuntimeApi>::Api: - Metadata, + BE: Backend + 'static, + Client: ExecutorProvider + StorageProvider + HeaderBackend + + HeaderMetadata + BlockchainEvents + + CallApiAt + + ProvideRuntimeApi + Send + Sync + 'static, + Client::Api: Metadata, { State { backend: Box::new(self::state_full::FullState::new(client, subscriptions)), @@ -213,17 +213,19 @@ pub fn new_full( } /// Create new state API that works on light node. -pub fn new_light>( - client: Arc>, +pub fn new_light>( + client: Arc, subscriptions: Subscriptions, remote_blockchain: Arc>, fetcher: Arc, -) -> State +) -> State where Block: BlockT + 'static, - B: sc_client_api::backend::Backend + Send + Sync + 'static, - E: CallExecutor + Send + Sync + 'static + Clone, - RA: Send + Sync + 'static, + BE: Backend + 'static, + Client: ExecutorProvider + StorageProvider + + HeaderMetadata + + ProvideRuntimeApi + HeaderBackend + BlockchainEvents + + Send + Sync + 'static, F: Send + Sync + 'static, { State { @@ -237,16 +239,14 @@ pub fn new_light>( } /// State API with subscriptions support. -pub struct State { - backend: Box>, +pub struct State { + backend: Box>, } -impl StateApi for State +impl StateApi for State where Block: BlockT + 'static, - B: sc_client_api::backend::Backend + Send + Sync + 'static, - E: CallExecutor + Send + Sync + 'static + Clone, - RA: Send + Sync + 'static, + Client: Send + Sync + 'static, { type Metadata = crate::metadata::Metadata; diff --git a/client/rpc/src/state/state_full.rs b/client/rpc/src/state/state_full.rs index 3d5613626e0447f28b4ced86a2f3138daed5737c..b7589d2aefecafc9daa41e2122461c21f53c93d2 100644 --- a/client/rpc/src/state/state_full.rs +++ b/client/rpc/src/state/state_full.rs @@ -26,12 +26,8 @@ use rpc::{Result as RpcResult, futures::{stream, Future, Sink, Stream, future::r use sc_rpc_api::Subscriptions; use sc_client_api::backend::Backend; -use sp_blockchain::{ - Result as ClientResult, Error as ClientError, HeaderMetadata, CachedHeaderMetadata -}; -use sc_client::{ - Client, CallExecutor, BlockchainEvents -}; +use sp_blockchain::{Result as ClientResult, Error as ClientError, HeaderMetadata, CachedHeaderMetadata, HeaderBackend}; +use sc_client::BlockchainEvents; use sp_core::{ Bytes, storage::{well_known_keys, StorageKey, StorageData, StorageChangeSet, ChildInfo}, }; @@ -40,9 +36,11 @@ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, NumberFor, SaturatedConversion}, }; -use sp_api::{Metadata, ProvideRuntimeApi}; +use sp_api::{Metadata, ProvideRuntimeApi, CallApiAt}; use super::{StateBackend, error::{FutureResult, Error, Result}, client_err, child_resolution_error}; +use std::marker::PhantomData; +use sc_client_api::{CallExecutor, StorageProvider, ExecutorProvider}; /// Ranges to query in state_queryStorage. struct QueryStorageRange { @@ -59,25 +57,27 @@ struct QueryStorageRange { } /// State API backend for full nodes. -pub struct FullState { - client: Arc>, +pub struct FullState { + client: Arc, subscriptions: Subscriptions, + _phantom: PhantomData<(BE, Block)> } -impl FullState +impl FullState where + BE: Backend, + Client: StorageProvider + HeaderBackend + + HeaderMetadata, Block: BlockT + 'static, - B: Backend + Send + Sync + 'static, - E: CallExecutor + Send + Sync + 'static + Clone, { /// Create new state API backend for full nodes. - pub fn new(client: Arc>, subscriptions: Subscriptions) -> Self { - Self { client, subscriptions } + pub fn new(client: Arc, subscriptions: Subscriptions) -> Self { + Self { client, subscriptions, _phantom: PhantomData } } /// Returns given block hash or best block hash if None is passed. fn block_or_best(&self, hash: Option) -> ClientResult { - Ok(hash.unwrap_or_else(|| self.client.chain_info().best_hash)) + Ok(hash.unwrap_or_else(|| self.client.info().best_hash)) } /// Splits the `query_storage` block range into 'filtered' and 'unfiltered' subranges. @@ -212,14 +212,14 @@ impl FullState } } -impl StateBackend for FullState where +impl StateBackend for FullState where Block: BlockT + 'static, - B: Backend + Send + Sync + 'static, - E: CallExecutor + Send + Sync + 'static + Clone, - RA: Send + Sync + 'static, - Client: ProvideRuntimeApi, - as ProvideRuntimeApi>::Api: - Metadata, + BE: Backend + 'static, + Client: ExecutorProvider + StorageProvider + HeaderBackend + + HeaderMetadata + BlockchainEvents + + CallApiAt + ProvideRuntimeApi + + Send + Sync + 'static, + Client::Api: Metadata, { fn call( &self, @@ -424,7 +424,7 @@ impl StateBackend for FullState StateBackend for FullState>; /// State API backend for light nodes. -pub struct LightState, B, E, RA> { - client: Arc>, +pub struct LightState, Client> { + client: Arc, subscriptions: Subscriptions, version_subscriptions: SimpleSubscriptions, storage_subscriptions: Arc>>, @@ -134,16 +133,14 @@ impl SharedRequests for SimpleSubscriptions where } } -impl + 'static, B, E, RA> LightState +impl + 'static, Client> LightState where Block: BlockT, - B: Backend + Send + Sync + 'static, - E: CallExecutor + Send + Sync + 'static + Clone, - RA: Send + Sync + 'static, + Client: HeaderBackend + Send + Sync + 'static, { /// Create new state API backend for light nodes. pub fn new( - client: Arc>, + client: Arc, subscriptions: Subscriptions, remote_blockchain: Arc>, fetcher: Arc, @@ -164,16 +161,14 @@ impl + 'static, B, E, RA> LightState) -> Block::Hash { - hash.unwrap_or_else(|| self.client.chain_info().best_hash) + hash.unwrap_or_else(|| self.client.info().best_hash) } } -impl StateBackend for LightState +impl StateBackend for LightState where Block: BlockT, - B: Backend + Send + Sync + 'static, - E: CallExecutor + Send + Sync + 'static + Clone, - RA: Send + Sync + 'static, + Client: BlockchainEvents + HeaderBackend + Send + Sync + 'static, F: Fetcher + 'static { fn call( @@ -241,7 +236,7 @@ impl StateBackend for LightState::hash(&storage.0)))) + result(Ok(maybe_storage.map(|storage| HashFor::::hash(&storage.0)))) ) ) } @@ -302,7 +297,7 @@ impl StateBackend for LightState::hash(&storage.0)))) + result(Ok(maybe_storage.map(|storage| HashFor::::hash(&storage.0)))) ) ) } diff --git a/client/rpc/src/system/tests.rs b/client/rpc/src/system/tests.rs index a280110093b0bbf99c0e0a76fdd0e2e0f35aad8b..4487566e44cfd9fc18b41112d9ec66555c728196 100644 --- a/client/rpc/src/system/tests.rs +++ b/client/rpc/src/system/tests.rs @@ -69,7 +69,7 @@ fn api>>(sync: T) -> System { let _ = sender.send(peers); } Request::NetworkState(sender) => { - let _ = sender.send(serde_json::to_value(&sc_network::NetworkState { + let _ = sender.send(serde_json::to_value(&sc_network::network_state::NetworkState { peer_id: String::new(), listened_addresses: Default::default(), external_addresses: Default::default(), @@ -223,8 +223,8 @@ fn system_peers() { fn system_network_state() { let res = wait_receiver(api(None).system_network_state()); assert_eq!( - serde_json::from_value::(res).unwrap(), - sc_network::NetworkState { + serde_json::from_value::(res).unwrap(), + sc_network::network_state::NetworkState { peer_id: String::new(), listened_addresses: Default::default(), external_addresses: Default::default(), diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml index 1e781aa695634c239cf24acf7f7cddf789ad9f27..bfc08eb705aa4454c4f8e3e5ee82d0d31d46864b 100644 --- a/client/service/Cargo.toml +++ b/client/service/Cargo.toml @@ -1,9 +1,12 @@ [package] name = "sc-service" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Substrate service. Starts a thread that spins up the network, client, and extrinsic pool. Manages communication between them." [features] default = ["rocksdb"] @@ -30,37 +33,36 @@ serde = "1.0.101" serde_json = "1.0.41" sysinfo = "0.9.5" target_info = "0.1.0" -sc-keystore = { version = "2.0.0", path = "../keystore" } -sp-io = { version = "2.0.0", path = "../../primitives/io" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-session = { version = "2.0.0", path = "../../primitives/session" } -sp-application-crypto = { version = "2.0.0", path = "../../primitives/application-crypto" } -sp-consensus = { version = "0.8", path = "../../primitives/consensus/common" } -sc-network = { version = "0.8", path = "../network" } -sc-chain-spec = { version = "2.0.0", path = "../chain-spec" } -sc-client-api = { version = "2.0.0", path = "../api" } -sc-client = { version = "0.8", path = "../" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } -sc-client-db = { version = "0.8", path = "../db" } -codec = { package = "parity-scale-codec", version = "1.0.0" } -sc-executor = { version = "0.8", path = "../executor" } -sc-transaction-pool = { version = "2.0.0", path = "../transaction-pool" } -sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } -sc-rpc-server = { version = "2.0.0", path = "../rpc-servers" } -sc-rpc = { version = "2.0.0", path = "../rpc" } -sc-telemetry = { version = "2.0.0", path = "../telemetry" } -sc-offchain = { version = "2.0.0", path = "../offchain" } -parity-multiaddr = { package = "parity-multiaddr", version = "0.7.1" } -grafana-data-source = { version = "0.8", path = "../../utils/grafana-data-source" } -sc-tracing = { version = "2.0.0", path = "../tracing" } +sc-keystore = { version = "2.0.0-alpha.2", path = "../keystore" } +sp-io = { version = "2.0.0-alpha.2", path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../primitives/blockchain" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +sp-session = { version = "2.0.0-alpha.2", path = "../../primitives/session" } +sp-application-crypto = { version = "2.0.0-alpha.2", path = "../../primitives/application-crypto" } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../primitives/consensus/common" } +sc-network = { version = "0.8.0-alpha.2", path = "../network" } +sc-chain-spec = { version = "2.0.0-alpha.2", path = "../chain-spec" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../api" } +sc-client = { version = "0.8.0-alpha.2", path = "../" } +sp-api = { version = "2.0.0-alpha.2", path = "../../primitives/api" } +sc-client-db = { version = "0.8.0-alpha.2", path = "../db" } +codec = { package = "parity-scale-codec", version = "1.2.0" } +sc-executor = { version = "0.8.0-alpha.2", path = "../executor" } +sc-transaction-pool = { version = "2.0.0-alpha.2", path = "../transaction-pool" } +sp-transaction-pool = { version = "2.0.0-alpha.2", path = "../../primitives/transaction-pool" } +sc-rpc-server = { version = "2.0.0-alpha.2", path = "../rpc-servers" } +sc-rpc = { version = "2.0.0-alpha.2", path = "../rpc" } +sc-telemetry = { version = "2.0.0-alpha.2", path = "../telemetry" } +sc-offchain = { version = "2.0.0-alpha.2", path = "../offchain" } +parity-multiaddr = { package = "parity-multiaddr", version = "0.5.0" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" , version = "0.8.0-alpha.2"} +sc-tracing = { version = "2.0.0-alpha.2", path = "../tracing" } tracing = "0.1.10" parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } [dev-dependencies] -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } -sp-consensus-babe = { version = "0.8", path = "../../primitives/consensus/babe" } -grandpa = { version = "0.8", package = "sc-finality-grandpa", path = "../finality-grandpa" } -grandpa-primitives = { version = "2.0.0", package = "sp-finality-grandpa", path = "../../primitives/finality-grandpa" } -tokio = { version = "0.2", features = ["rt-core"] } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } +sp-consensus-babe = { version = "0.8.0-alpha.2", path = "../../primitives/consensus/babe" } +grandpa = { version = "0.8.0-alpha.2", package = "sc-finality-grandpa", path = "../finality-grandpa" } +grandpa-primitives = { version = "2.0.0-alpha.2", package = "sp-finality-grandpa", path = "../../primitives/finality-grandpa" } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index c67551afa35564b7f91d39ac2317f1361f2496d4..096f492e6468b32a5c29a122c4caaf4e66833d22 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -15,14 +15,15 @@ // along with Substrate. If not, see . use crate::{Service, NetworkStatus, NetworkState, error::Error, DEFAULT_PROTOCOL_ID, MallocSizeOfWasm}; -use crate::{SpawnTaskHandle, start_rpc_servers, build_network_future, TransactionPoolAdapter}; +use crate::{TaskManagerBuilder, start_rpc_servers, build_network_future, TransactionPoolAdapter}; use crate::status_sinks; -use crate::config::{Configuration, DatabaseConfig, KeystoreConfig}; +use crate::config::{Configuration, DatabaseConfig, KeystoreConfig, PrometheusConfig}; use sc_client_api::{ self, BlockchainEvents, backend::RemoteBackend, light::RemoteBlockchain, execution_extensions::ExtensionsFactory, + ExecutorProvider, CallExecutor }; use sc_client::Client; use sc_chain_spec::{RuntimeGenesis, Extension}; @@ -30,21 +31,20 @@ use sp_consensus::import_queue::ImportQueue; use futures::{ Future, FutureExt, StreamExt, channel::mpsc, - future::{select, ready} + future::ready, }; use sc_keystore::{Store as Keystore}; use log::{info, warn, error}; -use sc_network::{FinalityProofProvider, OnDemand, NetworkService, NetworkStateInfo}; -use sc_network::{config::BoxFinalityProofRequestBuilder, specialization::NetworkSpecialization}; +use sc_network::config::{FinalityProofProvider, OnDemand, BoxFinalityProofRequestBuilder}; +use sc_network::{NetworkService, NetworkStateInfo}; use parking_lot::{Mutex, RwLock}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{ - Block as BlockT, NumberFor, SaturatedConversion, HasherFor, + Block as BlockT, NumberFor, SaturatedConversion, HashFor, UniqueSaturatedInto, }; use sp_api::ProvideRuntimeApi; use sc_executor::{NativeExecutor, NativeExecutionDispatch}; use std::{ - borrow::Cow, io::{Read, Write, Seek}, marker::PhantomData, sync::Arc, pin::Pin }; @@ -53,7 +53,44 @@ use sysinfo::{get_current_pid, ProcessExt, System, SystemExt}; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; use sp_transaction_pool::{MaintainedTransactionPool, ChainEvent}; use sp_blockchain; -use grafana_data_source::{self, record_metrics}; +use prometheus_endpoint::{register, Gauge, U64, F64, Registry, PrometheusError, Opts, GaugeVec}; + +struct ServiceMetrics { + block_height_number: GaugeVec, + ready_transactions_number: Gauge, + memory_usage_bytes: Gauge, + cpu_usage_percentage: Gauge, + network_per_sec_bytes: GaugeVec, + node_roles: Gauge, +} + +impl ServiceMetrics { + fn register(registry: &Registry) -> Result { + Ok(Self { + block_height_number: register(GaugeVec::new( + Opts::new("block_height_number", "Height of the chain"), + &["status"] + )?, registry)?, + ready_transactions_number: register(Gauge::new( + "ready_transactions_number", "Number of transactions in the ready queue", + )?, registry)?, + memory_usage_bytes: register(Gauge::new( + "memory_usage_bytes", "Node memory usage", + )?, registry)?, + cpu_usage_percentage: register(Gauge::new( + "cpu_usage_percentage", "Node CPU usage", + )?, registry)?, + network_per_sec_bytes: register(GaugeVec::new( + Opts::new("network_per_sec_bytes", "Networking bytes per second"), + &["direction"] + )?, registry)?, + node_roles: register(Gauge::new( + "node_roles", "The roles the node is running as", + )?, registry)?, + + }) + } +} pub type BackgroundTask = Pin + Send>>; @@ -66,7 +103,6 @@ pub type BackgroundTask = Pin + Send>>; /// /// - [`with_select_chain`](ServiceBuilder::with_select_chain) /// - [`with_import_queue`](ServiceBuilder::with_import_queue) -/// - [`with_network_protocol`](ServiceBuilder::with_network_protocol) /// - [`with_finality_proof_provider`](ServiceBuilder::with_finality_proof_provider) /// - [`with_transaction_pool`](ServiceBuilder::with_transaction_pool) /// @@ -76,18 +112,18 @@ pub type BackgroundTask = Pin + Send>>; /// generics is done when you call `build`. /// pub struct ServiceBuilder + TExPool, TRpc, Backend> { config: Configuration, pub (crate) client: Arc, backend: Arc, + tasks_builder: TaskManagerBuilder, keystore: Arc>, fetcher: Option, select_chain: Option, pub (crate) import_queue: TImpQu, finality_proof_request_builder: Option, finality_proof_provider: Option, - network_protocol: TNetP, transaction_pool: Arc, rpc_extensions: TRpc, remote_backend: Option>>, @@ -123,19 +159,19 @@ pub type TLightClient = Client< /// Light client backend type. pub type TLightBackend = sc_client::light::backend::Backend< sc_client_db::light::LightStorage, - HasherFor, + HashFor, >; /// Light call executor type. pub type TLightCallExecutor = sc_client::light::call_executor::GenesisCallExecutor< sc_client::light::backend::Backend< sc_client_db::light::LightStorage, - HasherFor + HashFor >, sc_client::LocalCallExecutor< sc_client::light::backend::Backend< sc_client_db::light::LightStorage, - HasherFor + HashFor >, NativeExecutor >, @@ -145,6 +181,7 @@ type TFullParts = ( TFullClient, Arc>, Arc>, + TaskManagerBuilder, ); /// Creates a new full client for the given config. @@ -176,6 +213,8 @@ fn new_full_parts( KeystoreConfig::None => return Err("No keystore config provided!".into()), }; + let tasks_builder = TaskManagerBuilder::new(); + let executor = NativeExecutor::::new( config.wasm_method, config.default_heap_pages, @@ -223,13 +262,14 @@ fn new_full_parts( fork_blocks, bad_blocks, extensions, + config.prometheus_config.as_ref().map(|config| config.registry.clone()), )? }; - Ok((client, backend, keystore)) + Ok((client, backend, keystore, tasks_builder)) } -impl ServiceBuilder<(), (), TGen, TCSExt, (), (), (), (), (), (), (), (), (), ()> +impl ServiceBuilder<(), (), TGen, TCSExt, (), (), (), (), (), (), (), (), ()> where TGen: RuntimeGenesis, TCSExt: Extension { /// Start the service builder with a configuration. pub fn new_full( @@ -247,10 +287,9 @@ where TGen: RuntimeGenesis, TCSExt: Extension { Arc>, (), (), - (), TFullBackend, >, Error> { - let (client, backend, keystore) = new_full_parts(&config)?; + let (client, backend, keystore, tasks_builder) = new_full_parts(&config)?; let client = Arc::new(client); @@ -259,12 +298,12 @@ where TGen: RuntimeGenesis, TCSExt: Extension { client, backend, keystore, + tasks_builder, fetcher: None, select_chain: None, import_queue: (), finality_proof_request_builder: None, finality_proof_provider: None, - network_protocol: (), transaction_pool: Arc::new(()), rpc_extensions: Default::default(), remote_backend: None, @@ -289,9 +328,10 @@ where TGen: RuntimeGenesis, TCSExt: Extension { Arc>, (), (), - (), TLightBackend, >, Error> { + let tasks_builder = TaskManagerBuilder::new(); + let keystore = match &config.keystore { KeystoreConfig::Path { path, password } => Keystore::open( path.clone(), @@ -331,26 +371,27 @@ where TGen: RuntimeGenesis, TCSExt: Extension { executor.clone(), ), ); - let fetcher = Arc::new(sc_network::OnDemand::new(fetch_checker)); + let fetcher = Arc::new(sc_network::config::OnDemand::new(fetch_checker)); let backend = sc_client::light::new_light_backend(light_blockchain); let remote_blockchain = backend.remote_blockchain(); let client = Arc::new(sc_client::light::new_light( backend.clone(), config.expect_chain_spec(), executor, + config.prometheus_config.as_ref().map(|config| config.registry.clone()), )?); Ok(ServiceBuilder { config, client, backend, + tasks_builder, keystore, fetcher: Some(fetcher.clone()), select_chain: None, import_queue: (), finality_proof_request_builder: None, finality_proof_provider: None, - network_protocol: (), transaction_pool: Arc::new(()), rpc_extensions: Default::default(), remote_backend: Some(remote_blockchain), @@ -360,9 +401,9 @@ where TGen: RuntimeGenesis, TCSExt: Extension { } } -impl +impl ServiceBuilder { + TExPool, TRpc, Backend> { /// Returns a reference to the client that was stored in this builder. pub fn client(&self) -> &Arc { @@ -410,20 +451,20 @@ impl, &Arc ) -> Result, Error> ) -> Result, Error> { + TExPool, TRpc, Backend>, Error> { let select_chain = select_chain_builder(&self.config, &self.backend)?; Ok(ServiceBuilder { config: self.config, client: self.client, backend: self.backend, + tasks_builder: self.tasks_builder, keystore: self.keystore, fetcher: self.fetcher, select_chain, import_queue: self.import_queue, finality_proof_request_builder: self.finality_proof_request_builder, finality_proof_provider: self.finality_proof_provider, - network_protocol: self.network_protocol, transaction_pool: self.transaction_pool, rpc_extensions: self.rpc_extensions, remote_backend: self.remote_backend, @@ -437,7 +478,7 @@ impl, &Arc) -> Result ) -> Result, Error> { + TExPool, TRpc, Backend>, Error> { self.with_opt_select_chain(|cfg, b| builder(cfg, b).map(Option::Some)) } @@ -447,7 +488,7 @@ impl, Arc, Option, Arc) -> Result ) -> Result, Error> + TExPool, TRpc, Backend>, Error> where TSc: Clone { let import_queue = builder( &self.config, @@ -460,40 +501,13 @@ impl( - self, - network_protocol_builder: impl FnOnce(&Configuration) -> Result - ) -> Result, Error> { - let network_protocol = network_protocol_builder(&self.config)?; - - Ok(ServiceBuilder { - config: self.config, - client: self.client, - backend: self.backend, - keystore: self.keystore, - fetcher: self.fetcher, - select_chain: self.select_chain, - import_queue: self.import_queue, - finality_proof_request_builder: self.finality_proof_request_builder, - finality_proof_provider: self.finality_proof_provider, - network_protocol, transaction_pool: self.transaction_pool, rpc_extensions: self.rpc_extensions, remote_backend: self.remote_backend, @@ -517,7 +531,6 @@ impl>, - TNetP, TExPool, TRpc, Backend, @@ -528,13 +541,13 @@ impl>, - TNetP, TExPool, TRpc, Backend, @@ -578,7 +590,7 @@ impl, ) -> Result<(UImpQu, Option), Error> ) -> Result, Error> + TExPool, TRpc, Backend>, Error> where TSc: Clone, TFchr: Clone { let (import_queue, fprb) = builder( &self.config, @@ -593,13 +605,13 @@ impl, ) -> Result<(UImpQu, UFprb), Error> ) -> Result, Error> + TExPool, TRpc, Backend>, Error> where TSc: Clone, TFchr: Clone { self.with_import_queue_and_opt_fprb(|cfg, cl, b, f, sc, tx| builder(cfg, cl, b, f, sc, tx) @@ -637,7 +649,7 @@ impl, ) -> Result<(UExPool, Option), Error> ) -> Result, Error> + UExPool, TRpc, Backend>, Error> where TSc: Clone, TFchr: Clone { let (transaction_pool, background_task) = transaction_pool_builder( self.config.transaction_pool.clone(), @@ -652,6 +664,7 @@ impl Result, ) -> Result, Error> + TExPool, URpc, Backend>, Error> where TSc: Clone, TFchr: Clone { let rpc_extensions = rpc_ext_builder(&self)?; @@ -681,13 +693,13 @@ impl, to: Option>, - json: bool + binary: bool ) -> Pin>>>; /// Performs a revert of `blocks` blocks. @@ -733,7 +745,7 @@ pub trait ServiceBuilderCommand { ) -> Pin> + Send>>; } -impl +impl ServiceBuilder< TBl, TRtApi, @@ -745,7 +757,6 @@ ServiceBuilder< TImpQu, BoxFinalityProofRequestBuilder, Arc>, - TNetP, TExPool, TRpc, TBackend, @@ -766,7 +777,6 @@ ServiceBuilder< TExec: 'static + sc_client::CallExecutor + Send + Sync + Clone, TSc: Clone, TImpQu: 'static + ImportQueue, - TNetP: NetworkSpecialization, TExPool: MaintainedTransactionPool::Hash> + MallocSizeOfWasm + 'static, TRpc: sc_rpc::RpcExtension + Clone, { @@ -783,18 +793,21 @@ ServiceBuilder< Client, TSc, NetworkStatus, - NetworkService::Hash>, + NetworkService::Hash>, TExPool, sc_offchain::OffchainWorkers< Client, TBackend::OffchainStorage, TBl >, - >, Error> { + >, Error> + where TExec: CallExecutor, + { let ServiceBuilder { marker: _, mut config, client, + tasks_builder, fetcher: on_demand, backend, keystore, @@ -802,7 +815,6 @@ ServiceBuilder< import_queue, finality_proof_request_builder, finality_proof_provider, - network_protocol, transaction_pool, rpc_extensions, remote_backend, @@ -815,12 +827,6 @@ ServiceBuilder< config.dev_key_seed.clone().map(|s| vec![s]).unwrap_or_default(), )?; - let (signal, exit) = exit_future::signal(); - - // List of asynchronous tasks to spawn. We collect them, then spawn them all at once. - let (to_spawn_tx, to_spawn_rx) = - mpsc::unbounded::<(Pin + Send>>, Cow<'static, str>)>(); - // A side-channel for essential tasks to communicate shutdown. let (essential_failed_tx, essential_failed_rx) = mpsc::unbounded(); @@ -845,7 +851,7 @@ ServiceBuilder< imports_external_transactions: !config.roles.is_light(), pool: transaction_pool.clone(), client: client.clone(), - executor: SpawnTaskHandle { sender: to_spawn_tx.clone(), on_exit: exit.clone() }, + executor: tasks_builder.spawn_handle(), }); let protocol_id = { @@ -867,11 +873,9 @@ ServiceBuilder< let network_params = sc_network::config::Params { roles: config.roles, executor: { - let to_spawn_tx = to_spawn_tx.clone(); + let spawn_handle = tasks_builder.spawn_handle(); Some(Box::new(move |fut| { - if let Err(e) = to_spawn_tx.unbounded_send((fut, From::from("libp2p-node"))) { - error!("Failed to spawn libp2p background task: {:?}", e); - } + spawn_handle.spawn("libp2p-node", fut); })) }, network_config: config.network.clone(), @@ -882,8 +886,8 @@ ServiceBuilder< transaction_pool: transaction_pool_adapter.clone() as _, import_queue, protocol_id, - specialization: network_protocol, block_announce_validator, + metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()) }; let has_bootnodes = !network_params.network_config.boot_nodes.is_empty(); @@ -903,20 +907,19 @@ ServiceBuilder< _ => None, }; + let spawn_handle = tasks_builder.spawn_handle(); + // Spawn background tasks which were stacked during the // service building. for (title, background_task) in background_tasks { - let _ = to_spawn_tx.unbounded_send(( - background_task, - title.into(), - )); + spawn_handle.spawn(title, background_task); } { // block notifications let txpool = Arc::downgrade(&transaction_pool); let offchain = offchain_workers.as_ref().map(Arc::downgrade); - let to_spawn_tx_ = to_spawn_tx.clone(); + let notifications_spawn_handle = tasks_builder.spawn_handle(); let network_state_info: Arc = network.clone(); let is_validator = config.roles.is_authority(); @@ -938,15 +941,14 @@ ServiceBuilder< let offchain = offchain.as_ref().and_then(|o| o.upgrade()); match offchain { Some(offchain) if is_new_best => { - let future = offchain.on_block_imported( - &header, - network_state_info.clone(), - is_validator, + notifications_spawn_handle.spawn( + "offchain-on-block", + offchain.on_block_imported( + &header, + network_state_info.clone(), + is_validator, + ), ); - let _ = to_spawn_tx_.unbounded_send(( - Box::pin(future), - From::from("offchain-on-block"), - )); }, Some(_) => log::debug!( target: "sc_offchain", @@ -959,20 +961,19 @@ ServiceBuilder< let txpool = txpool.upgrade(); if let Some(txpool) = txpool.as_ref() { - let future = txpool.maintain(event); - let _ = to_spawn_tx_.unbounded_send(( - Box::pin(future), - From::from("txpool-maintain") - )); + notifications_spawn_handle.spawn( + "txpool-maintain", + txpool.maintain(event), + ); } ready(()) }); - let _ = to_spawn_tx.unbounded_send(( - Box::pin(select(events, exit.clone()).map(drop)), - From::from("txpool-and-offchain-notif"), - )); + spawn_handle.spawn( + "txpool-and-offchain-notif", + events, + ); } { @@ -992,12 +993,24 @@ ServiceBuilder< ready(()) }); - let _ = to_spawn_tx.unbounded_send(( - Box::pin(select(events, exit.clone()).map(drop)), - From::from("telemetry-on-block"), - )); + spawn_handle.spawn( + "telemetry-on-block", + events, + ); } + // Prometheus metrics + let metrics = if let Some(PrometheusConfig { port, registry }) = config.prometheus_config.clone() { + let metrics = ServiceMetrics::register(®istry)?; + metrics.node_roles.set(u64::from(config.roles.bits())); + spawn_handle.spawn( + "prometheus-endpoint", + prometheus_endpoint::init_prometheus(port, registry).map(drop) + ); + Some(metrics) + } else { + None + }; // Periodically notify the telemetry. let transaction_pool_ = transaction_pool.clone(); let client_ = client.clone(); @@ -1014,6 +1027,8 @@ ServiceBuilder< let finalized_number: u64 = info.chain.finalized_number.saturated_into::(); let bandwidth_download = net_status.average_download_per_sec; let bandwidth_upload = net_status.average_upload_per_sec; + let best_seen_block = net_status.best_seen_block + .map(|num: NumberFor| num.unique_saturated_into() as u64); // get cpu usage and memory usage of this process let (cpu_usage, memory) = if let Some(self_pid) = self_pid { @@ -1037,38 +1052,42 @@ ServiceBuilder< "finalized_hash" => ?info.chain.finalized_hash, "bandwidth_download" => bandwidth_download, "bandwidth_upload" => bandwidth_upload, - "used_state_cache_size" => info.usage.as_ref().map(|usage| usage.memory.state_cache).unwrap_or(0), - "used_db_cache_size" => info.usage.as_ref().map(|usage| usage.memory.database_cache).unwrap_or(0), - "disk_read_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_read).unwrap_or(0), - "disk_write_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_written).unwrap_or(0), - ); - #[cfg(not(target_os = "unknown"))] - let memory_transaction_pool = parity_util_mem::malloc_size(&*transaction_pool_); - #[cfg(target_os = "unknown")] - let memory_transaction_pool = 0; - let _ = record_metrics!( - "peers" => num_peers, - "height" => best_number, - "txcount" => txpool_status.ready, - "cpu" => cpu_usage, - "memory" => memory, - "finalized_height" => finalized_number, - "bandwidth_download" => bandwidth_download, - "bandwidth_upload" => bandwidth_upload, - "used_state_cache_size" => info.usage.as_ref().map(|usage| usage.memory.state_cache).unwrap_or(0), - "used_db_cache_size" => info.usage.as_ref().map(|usage| usage.memory.database_cache).unwrap_or(0), - "disk_read_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_read).unwrap_or(0), - "disk_write_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_written).unwrap_or(0), - "memory_transaction_pool" => memory_transaction_pool, + "used_state_cache_size" => info.usage.as_ref() + .map(|usage| usage.memory.state_cache.as_bytes()) + .unwrap_or(0), + "used_db_cache_size" => info.usage.as_ref() + .map(|usage| usage.memory.database_cache.as_bytes()) + .unwrap_or(0), + "disk_read_per_sec" => info.usage.as_ref() + .map(|usage| usage.io.bytes_read) + .unwrap_or(0), + "disk_write_per_sec" => info.usage.as_ref() + .map(|usage| usage.io.bytes_written) + .unwrap_or(0), ); + if let Some(metrics) = metrics.as_ref() { + metrics.memory_usage_bytes.set(memory); + metrics.cpu_usage_percentage.set(f64::from(cpu_usage)); + metrics.ready_transactions_number.set(txpool_status.ready as u64); + + metrics.network_per_sec_bytes.with_label_values(&["download"]).set(net_status.average_download_per_sec); + metrics.network_per_sec_bytes.with_label_values(&["upload"]).set(net_status.average_upload_per_sec); + + metrics.block_height_number.with_label_values(&["finalized"]).set(finalized_number); + metrics.block_height_number.with_label_values(&["best"]).set(best_number); + + if let Some(best_seen_block) = best_seen_block { + metrics.block_height_number.with_label_values(&["sync_target"]).set(best_seen_block); + } + } ready(()) }); - let _ = to_spawn_tx.unbounded_send(( - Box::pin(select(tel_task, exit.clone()).map(drop)), - From::from("telemetry-periodic-send"), - )); + spawn_handle.spawn( + "telemetry-periodic-send", + tel_task, + ); // Periodically send the network state to the telemetry. let (netstat_tx, netstat_rx) = mpsc::unbounded::<(NetworkStatus<_>, NetworkState)>(); @@ -1081,10 +1100,10 @@ ServiceBuilder< ); ready(()) }); - let _ = to_spawn_tx.unbounded_send(( - Box::pin(select(tel_task_2, exit.clone()).map(drop)), - From::from("telemetry-periodic-network-state"), - )); + spawn_handle.spawn( + "telemetry-periodic-network-state", + tel_task_2, + ); // RPC let (system_rpc_tx, system_rpc_rx) = mpsc::unbounded(); @@ -1098,10 +1117,7 @@ ServiceBuilder< properties: chain_spec.properties().clone(), }; - let subscriptions = sc_rpc::Subscriptions::new(Arc::new(SpawnTaskHandle { - sender: to_spawn_tx.clone(), - on_exit: exit.clone() - })); + let subscriptions = sc_rpc::Subscriptions::new(Arc::new(tasks_builder.spawn_handle())); let (chain, state) = if let (Some(remote_backend), Some(on_demand)) = (remote_backend.as_ref(), on_demand.as_ref()) { @@ -1159,18 +1175,17 @@ ServiceBuilder< let rpc_handlers = gen_handler(); let rpc = start_rpc_servers(&config, gen_handler)?; - - let _ = to_spawn_tx.unbounded_send(( - Box::pin(select(build_network_future( + spawn_handle.spawn( + "network-worker", + build_network_future( config.roles, network_mut, client.clone(), network_status_sinks.clone(), system_rpc_rx, has_bootnodes, - ), exit.clone()).map(drop)), - From::from("network-worker"), - )); + ), + ); let telemetry_connection_sinks: Arc>>> = Default::default(); @@ -1211,21 +1226,14 @@ ServiceBuilder< }); ready(()) }); - let _ = to_spawn_tx.unbounded_send((Box::pin(select( - future, exit.clone() - ).map(drop)), From::from("telemetry-worker"))); - telemetry - }); - // Grafana data source - if let Some(port) = config.grafana_port { - let future = select( - grafana_data_source::run_server(port).boxed(), - exit.clone() - ).map(drop); + spawn_handle.spawn( + "telemetry-worker", + future, + ); - let _ = to_spawn_tx.unbounded_send((Box::pin(future), From::from("grafana-server"))); - } + telemetry + }); // Instrumentation if let Some(tracing_targets) = config.tracing_targets.as_ref() { @@ -1240,21 +1248,13 @@ ServiceBuilder< Ok(Service { client, + task_manager: tasks_builder.into_task_manager(config.task_executor.ok_or(Error::TaskExecutorRequired)?), network, network_status_sinks, select_chain, transaction_pool, - exit, - signal: Some(signal), essential_failed_tx, essential_failed_rx, - to_spawn_tx, - to_spawn_rx, - task_executor: if let Some(exec) = config.task_executor { - exec - } else { - return Err(Error::TaskExecutorRequired); - }, rpc_handlers, _rpc: rpc, _telemetry: telemetry, @@ -1262,6 +1262,7 @@ ServiceBuilder< _telemetry_on_connect_sinks: telemetry_connection_sinks.clone(), keystore, marker: PhantomData::, + prometheus_registry: config.prometheus_config.map(|config| config.registry) }) } } diff --git a/client/service/src/chain_ops.rs b/client/service/src/chain_ops.rs index 3d77d9c815e4f5c70e6e6c52e3eb2d392f0afcfb..350aac91758134ac9c809ffcba945f0b89591ada 100644 --- a/client/service/src/chain_ops.rs +++ b/client/service/src/chain_ops.rs @@ -22,24 +22,20 @@ use crate::error::Error; use sc_chain_spec::{ChainSpec, RuntimeGenesis, Extension}; use log::{warn, info}; use futures::{future, prelude::*}; -use sp_runtime::{ - BuildStorage, BenchmarkResults, - traits::{ - Block as BlockT, NumberFor, One, Zero, Header, SaturatedConversion - } +use sp_runtime::traits::{ + Block as BlockT, NumberFor, One, Zero, Header, SaturatedConversion }; use sp_runtime::generic::{BlockId, SignedBlock}; use codec::{Decode, Encode, IoReader}; -use sc_client::{Client, ExecutionStrategy, StateMachine, LocalCallExecutor}; -#[cfg(feature = "rocksdb")] -use sc_client_db::BenchmarkingState; -use sp_consensus::import_queue::{IncomingBlock, Link, BlockImportError, BlockImportResult, ImportQueue}; -use sp_consensus::BlockOrigin; -use sc_executor::{NativeExecutor, NativeExecutionDispatch, WasmExecutionMethod}; +use sc_client::{Client, LocalCallExecutor}; +use sp_consensus::{ + BlockOrigin, + import_queue::{IncomingBlock, Link, BlockImportError, BlockImportResult, ImportQueue}, +}; +use sc_executor::{NativeExecutor, NativeExecutionDispatch}; use std::{io::{Read, Write, Seek}, pin::Pin}; - -use sc_network::message; +use sc_client_api::BlockBody; /// Build a chain spec json pub fn build_spec(spec: ChainSpec, raw: bool) -> error::Result where @@ -49,67 +45,14 @@ pub fn build_spec(spec: ChainSpec, raw: bool) -> error::Result ( - spec: ChainSpec, - strategy: ExecutionStrategy, - wasm_method: WasmExecutionMethod, - pallet: String, - extrinsic: String, - steps: u32, - repeat: u32, -) -> error::Result<()> where - TBl: BlockT, - TExecDisp: NativeExecutionDispatch + 'static, - G: RuntimeGenesis, - E: Extension, -{ - let genesis_storage = spec.build_storage()?; - let mut changes = Default::default(); - let state = BenchmarkingState::::new(genesis_storage)?; - let executor = NativeExecutor::::new( - wasm_method, - None, // heap pages - ); - let result = StateMachine::<_, _, NumberFor, _>::new( - &state, - None, - &mut changes, - &executor, - "Benchmark_dispatch_benchmark", - &(&pallet, &extrinsic, steps, repeat).encode(), - Default::default(), - ).execute(strategy).map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?; - let results = > as Decode>::decode(&mut &result[..]).unwrap_or(None); - if let Some(results) = results { - // Print benchmark metadata - println!("Pallet: {:?}, Extrinsic: {:?}, Steps: {:?}, Repeat: {:?}", pallet, extrinsic, steps, repeat); - // Print the table header - results[0].0.iter().for_each(|param| print!("{:?},", param.0)); - print!("time\n"); - // Print the values - results.iter().for_each(|result| { - let parameters = &result.0; - parameters.iter().for_each(|param| print!("{:?},", param.1)); - print!("{:?}\n", result.1); - }); - info!("Done."); - } else { - info!("No Results."); - } - Ok(()) -} - - impl< TBl, TRtApi, TGen, TCSExt, TBackend, - TExecDisp, TFchr, TSc, TImpQu, TFprb, TFpp, TNetP, + TExecDisp, TFchr, TSc, TImpQu, TFprb, TFpp, TExPool, TRpc, Backend > ServiceBuilderCommand for ServiceBuilder< TBl, TRtApi, TGen, TCSExt, Client>, TBl, TRtApi>, - TFchr, TSc, TImpQu, TFprb, TFpp, TNetP, TExPool, TRpc, Backend + TFchr, TSc, TImpQu, TFprb, TFpp, TExPool, TRpc, Backend > where TBl: BlockT, TBackend: 'static + sc_client_api::backend::Backend + Send, @@ -197,21 +140,13 @@ impl< Ok(signed) => { let (header, extrinsics) = signed.block.deconstruct(); let hash = header.hash(); - let block = message::BlockData:: { - hash, - justification: signed.justification, - header: Some(header), - body: Some(extrinsics), - receipt: None, - message_queue: None - }; // import queue handles verification and importing it into the client queue.import_blocks(BlockOrigin::File, vec![ IncomingBlock:: { - hash: block.hash, - header: block.header, - body: block.body, - justification: block.justification, + hash, + header: Some(header), + body: Some(extrinsics), + justification: signed.justification, origin: None, allow_missing_state: false, import_existing: force, @@ -269,7 +204,7 @@ impl< mut output: impl Write + 'static, from: NumberFor, to: Option>, - json: bool + binary: bool ) -> Pin>>> { let client = self.client; let mut block = from; @@ -296,7 +231,7 @@ impl< if !wrote_header { info!("Exporting blocks from #{} to #{}", block, last); - if !json { + if binary { let last_: u64 = last.saturated_into::(); let block_: u64 = block.saturated_into::(); let len: u64 = last_ - block_ + 1; @@ -307,13 +242,13 @@ impl< match client.block(&BlockId::number(block))? { Some(block) => { - if json { + if binary { + output.write_all(&block.encode())?; + } else { serde_json::to_writer(&mut output, &block) .map_err(|e| format!("Error writing JSON: {}", e))?; - } else { - output.write_all(&block.encode())?; } - }, + }, // Reached end of the chain. None => return std::task::Poll::Ready(Ok(())), } diff --git a/client/service/src/config.rs b/client/service/src/config.rs index 2099c600df15e3282774e13e9a558f734bef7b6c..6400712cad3b9ccd4b6a2b2539f45bf49b3e188a 100644 --- a/client/service/src/config.rs +++ b/client/service/src/config.rs @@ -27,6 +27,7 @@ use sc_chain_spec::{ChainSpec, NoExtension}; use sp_core::crypto::Protected; use target_info::Target; use sc_telemetry::TelemetryEndpoints; +use prometheus_endpoint::Registry; /// Executable version. Used to pass version information from the root crate. #[derive(Clone)] @@ -93,8 +94,8 @@ pub struct Configuration { pub rpc_ws_max_connections: Option, /// CORS settings for HTTP & WS servers. `None` if all origins are allowed. pub rpc_cors: Option>, - /// Grafana data source http port. `None` if disabled. - pub grafana_port: Option, + /// Prometheus endpoint configuration. `None` if disabled. + pub prometheus_config: Option, /// Telemetry service URL. `None` if disabled. pub telemetry_endpoints: Option, /// External WASM transport for the telemetry. If `Some`, when connection to a telemetry @@ -165,6 +166,28 @@ pub enum DatabaseConfig { Custom(Arc), } +/// Configuration of the Prometheus endpoint. +#[derive(Clone)] +pub struct PrometheusConfig { + /// Port to use. + pub port: SocketAddr, + /// A metrics registry to use. Useful for setting the metric prefix. + pub registry: Registry, +} + +impl PrometheusConfig { + /// Create a new config using the default registry. + /// + /// The default registry prefixes metrics with `substrate`. + pub fn new_with_default_registry(port: SocketAddr) -> Self { + Self { + port, + registry: Registry::new_custom(Some("substrate".into()), None) + .expect("this can only fail if the prefix is empty") + } + } +} + impl Default for Configuration { /// Create a default config fn default() -> Self { @@ -190,7 +213,7 @@ impl Default for Configuration { rpc_ws: None, rpc_ws_max_connections: None, rpc_cors: Some(vec![]), - grafana_port: None, + prometheus_config: None, telemetry_endpoints: None, telemetry_external_transport: None, default_heap_pages: None, @@ -207,7 +230,7 @@ impl Default for Configuration { impl Configuration { /// Create a default config using `VersionInfo` - pub fn new(version: &VersionInfo) -> Self { + pub fn from_version(version: &VersionInfo) -> Self { let mut config = Configuration::default(); config.impl_name = version.name; config.impl_version = version.version; @@ -254,6 +277,28 @@ impl Configuration { pub fn expect_database(&self) -> &DatabaseConfig { self.database.as_ref().expect("database must be specified") } + + /// Returns a string displaying the node role, special casing the sentry mode + /// (returning `SENTRY`), since the node technically has an `AUTHORITY` role but + /// doesn't participate. + pub fn display_role(&self) -> String { + if self.sentry_mode { + "SENTRY".to_string() + } else { + self.roles.to_string() + } + } + + /// Use in memory keystore config when it is not required at all. + /// + /// This function returns an error if the keystore is already set to something different than + /// `KeystoreConfig::None`. + pub fn use_in_memory_keystore(&mut self) -> Result<(), String> { + match &mut self.keystore { + cfg @ KeystoreConfig::None => { *cfg = KeystoreConfig::InMemory; Ok(()) }, + _ => Err("Keystore config specified when it should not be!".into()), + } + } } /// Returns platform info diff --git a/client/service/src/error.rs b/client/service/src/error.rs index 059e1c19e490d6861339e278a274f5c060382950..5a78a1878923080ee05d2c5c46568356b8a71a59 100644 --- a/client/service/src/error.rs +++ b/client/service/src/error.rs @@ -53,6 +53,12 @@ impl<'a> From<&'a str> for Error { } } +impl From for Error { + fn from(e: prometheus_endpoint::PrometheusError) -> Self { + Error::Other(format!("Prometheus error: {}", e)) + } +} + impl std::error::Error for Error { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index a2e53616aba94ad39d7c096e4b67fe5ad4f90fc7..db56c141db0b5bb7c6765a2ff8f53ec8039592e1 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -26,6 +26,7 @@ pub mod error; mod builder; mod status_sinks; +mod task_manager; use std::{borrow::Cow, io, pin::Pin}; use std::marker::PhantomData; @@ -37,18 +38,14 @@ use std::task::{Poll, Context}; use parking_lot::Mutex; use sc_client::Client; -use exit_future::Signal; use futures::{ Future, FutureExt, Stream, StreamExt, - future::select, channel::mpsc, + channel::mpsc, compat::*, sink::SinkExt, task::{Spawn, FutureObj, SpawnError}, }; -use sc_network::{ - NetworkService, NetworkState, specialization::NetworkSpecialization, - PeerId, ReportHandle, -}; +use sc_network::{NetworkService, network_state::NetworkState, PeerId, ReportHandle}; use log::{log, warn, debug, error, Level}; use codec::{Encode, Decode}; use sp_runtime::generic::BlockId; @@ -71,7 +68,9 @@ pub use sc_executor::NativeExecutionDispatch; #[doc(hidden)] pub use std::{ops::Deref, result::Result, sync::Arc}; #[doc(hidden)] -pub use sc_network::{FinalityProofProvider, OnDemand, config::BoxFinalityProofRequestBuilder}; +pub use sc_network::config::{FinalityProofProvider, OnDemand, BoxFinalityProofRequestBuilder}; +pub use task_manager::{TaskManagerBuilder, SpawnTaskHandle}; +use task_manager::TaskManager; const DEFAULT_PROTOCOL_ID: &str = "sup"; @@ -88,28 +87,18 @@ impl MallocSizeOfWasm for T {} /// Substrate service. pub struct Service { client: Arc, + task_manager: TaskManager, select_chain: Option, network: Arc, /// Sinks to propagate network status updates. /// For each element, every time the `Interval` fires we push an element on the sender. network_status_sinks: Arc>>, transaction_pool: Arc, - /// A future that resolves when the service has exited, this is useful to - /// make sure any internally spawned futures stop when the service does. - exit: exit_future::Exit, - /// A signal that makes the exit future above resolve, fired on service drop. - signal: Option, /// Send a signal when a spawned essential task has concluded. The next time /// the service future is polled it should complete with an error. essential_failed_tx: mpsc::UnboundedSender<()>, /// A receiver for spawned essential-tasks concluding. essential_failed_rx: mpsc::UnboundedReceiver<()>, - /// Sender for futures that must be spawned as background tasks. - to_spawn_tx: mpsc::UnboundedSender<(Pin + Send>>, Cow<'static, str>)>, - /// Receiver for futures that must be spawned as background tasks. - to_spawn_rx: mpsc::UnboundedReceiver<(Pin + Send>>, Cow<'static, str>)>, - /// How to spawn background tasks. - task_executor: Arc + Send>>) + Send + Sync>, rpc_handlers: sc_rpc_server::RpcHandler, _rpc: Box, _telemetry: Option, @@ -117,49 +106,10 @@ pub struct Service { _offchain_workers: Option>, keystore: sc_keystore::KeyStorePtr, marker: PhantomData, + prometheus_registry: Option, } -/// Alias for a an implementation of `futures::future::Executor`. -pub type TaskExecutor = Arc; - -/// An handle for spawning tasks in the service. -#[derive(Clone)] -pub struct SpawnTaskHandle { - sender: mpsc::UnboundedSender<(Pin + Send>>, Cow<'static, str>)>, - on_exit: exit_future::Exit, -} - -impl SpawnTaskHandle { - /// Spawns the given task with the given name. - pub fn spawn(&self, name: impl Into>, task: impl Future + Send + 'static) { - let on_exit = self.on_exit.clone(); - let future = async move { - futures::pin_mut!(task); - let _ = select(on_exit, task).await; - }; - if self.sender.unbounded_send((Box::pin(future), name.into())).is_err() { - error!("Failed to send task to spawn over channel"); - } - } -} - -impl Spawn for SpawnTaskHandle { - fn spawn_obj(&self, future: FutureObj<'static, ()>) - -> Result<(), SpawnError> { - let future = select(self.on_exit.clone(), future).map(drop); - self.sender.unbounded_send((Box::pin(future), From::from("unnamed"))) - .map_err(|_| SpawnError::shutdown()) - } -} - -type Boxed01Future01 = Box + Send + 'static>; - -impl futures01::future::Executor for SpawnTaskHandle { - fn execute(&self, future: Boxed01Future01) -> Result<(), futures01::future::ExecuteError>{ - self.spawn("unnamed", future.compat().map(drop)); - Ok(()) - } -} +impl Unpin for Service {} /// Abstraction over a Substrate service. pub trait AbstractService: 'static + Future> + @@ -176,8 +126,6 @@ pub trait AbstractService: 'static + Future> + type SelectChain: sp_consensus::SelectChain; /// Transaction pool. type TransactionPool: TransactionPool + MallocSizeOfWasm; - /// Network specialization. - type NetworkSpecialization: NetworkSpecialization; /// Get event stream for telemetry connection established events. fn telemetry_on_connect_stream(&self) -> futures::channel::mpsc::UnboundedReceiver<()>; @@ -218,7 +166,7 @@ pub trait AbstractService: 'static + Future> + /// Get shared network instance. fn network(&self) - -> Arc::Hash>>; + -> Arc::Hash>>; /// Returns a receiver that periodically receives a status of the network. fn network_status(&self, interval: Duration) -> mpsc::UnboundedReceiver<(NetworkStatus, NetworkState)>; @@ -227,21 +175,24 @@ pub trait AbstractService: 'static + Future> + fn transaction_pool(&self) -> Arc; /// Get a handle to a future that will resolve on exit. + #[deprecated(note = "Use `spawn_task`/`spawn_essential_task` instead, those functions will attach on_exit signal.")] fn on_exit(&self) -> ::exit_future::Exit; + + /// Get the prometheus metrics registry, if available. + fn prometheus_registry(&self) -> Option; } -impl AbstractService for +impl AbstractService for Service, TSc, NetworkStatus, - NetworkService, TExPool, TOc> + NetworkService, TExPool, TOc> where - TBl: BlockT + Unpin, + TBl: BlockT, TBackend: 'static + sc_client_api::backend::Backend, TExec: 'static + sc_client::CallExecutor + Send + Sync + Clone, TRtApi: 'static + Send + Sync, TSc: sp_consensus::SelectChain + 'static + Clone + Send + Unpin, TExPool: 'static + TransactionPool + MallocSizeOfWasm, TOc: 'static + Send + Sync, - TNetSpec: NetworkSpecialization, { type Block = TBl; type Backend = TBackend; @@ -249,7 +200,6 @@ where type RuntimeApi = TRtApi; type SelectChain = TSc; type TransactionPool = TExPool; - type NetworkSpecialization = TNetSpec; fn telemetry_on_connect_stream(&self) -> futures::channel::mpsc::UnboundedReceiver<()> { let (sink, stream) = futures::channel::mpsc::unbounded(); @@ -266,12 +216,7 @@ where } fn spawn_task(&self, name: impl Into>, task: impl Future + Send + 'static) { - let on_exit = self.on_exit(); - let task = async move { - futures::pin_mut!(task); - let _ = select(on_exit, task).await; - }; - let _ = self.to_spawn_tx.unbounded_send((Box::pin(task), name.into())); + self.task_manager.spawn(name, task) } fn spawn_essential_task(&self, name: impl Into>, task: impl Future + Send + 'static) { @@ -282,20 +227,12 @@ where error!("Essential task failed. Shutting down service."); let _ = essential_failed.send(()); }); - let on_exit = self.on_exit(); - let task = async move { - futures::pin_mut!(essential_task); - let _ = select(on_exit, essential_task).await; - }; - let _ = self.to_spawn_tx.unbounded_send((Box::pin(task), name.into())); + let _ = self.spawn_task(name, essential_task); } fn spawn_task_handle(&self) -> SpawnTaskHandle { - SpawnTaskHandle { - sender: self.to_spawn_tx.clone(), - on_exit: self.on_exit(), - } + self.task_manager.spawn_handle() } fn rpc_query(&self, mem: &RpcSession, request: &str) -> Pin> + Send>> { @@ -315,7 +252,7 @@ where } fn network(&self) - -> Arc::Hash>> + -> Arc::Hash>> { self.network.clone() } @@ -331,11 +268,15 @@ where } fn on_exit(&self) -> exit_future::Exit { - self.exit.clone() + self.task_manager.on_exit() + } + + fn prometheus_registry(&self) -> Option { + self.prometheus_registry.clone() } } -impl Future for +impl Future for Service { type Output = Result<(), Error>; @@ -352,9 +293,7 @@ impl Future for } } - while let Poll::Ready(Some((task_to_spawn, name))) = Pin::new(&mut this.to_spawn_rx).poll_next(cx) { - (this.task_executor)(Box::pin(futures_diagnose::diagnose(name, task_to_spawn))); - } + this.task_manager.process_receiver(cx); // The service future never ends. Poll::Pending @@ -368,7 +307,7 @@ impl Spawn for &self, future: FutureObj<'static, ()> ) -> Result<(), SpawnError> { - self.to_spawn_tx.unbounded_send((Box::pin(future), From::from("unnamed"))) + self.task_manager.scheduler().unbounded_send((Box::pin(future), From::from("unnamed"))) .map_err(|_| SpawnError::shutdown()) } } @@ -379,11 +318,10 @@ impl Spawn for fn build_network_future< B: BlockT, C: sc_client::BlockchainEvents, - S: sc_network::specialization::NetworkSpecialization, H: sc_network::ExHashT > ( roles: Roles, - mut network: sc_network::NetworkWorker, + mut network: sc_network::NetworkWorker, client: Arc, status_sinks: Arc, NetworkState)>>>, mut rpc_rx: mpsc::UnboundedReceiver>, @@ -397,7 +335,7 @@ fn build_network_future< // We poll `imported_blocks_stream`. while let Poll::Ready(Some(notification)) = Pin::new(&mut imported_blocks_stream).poll_next(cx) { - network.on_block_imported(notification.hash, notification.header, Vec::new(), notification.is_new_best); + network.on_block_imported(notification.header, Vec::new(), notification.is_new_best); } // We poll `finality_notification_stream`, but we only take the last event. @@ -523,13 +461,26 @@ pub struct NetworkStatus { pub average_upload_per_sec: u64, } -impl Drop for - Service -{ - fn drop(&mut self) { - debug!(target: "service", "Substrate service shutdown"); - if let Some(signal) = self.signal.take() { - let _ = signal.fire(); +#[cfg(not(target_os = "unknown"))] +// Wrapper for HTTP and WS servers that makes sure they are properly shut down. +mod waiting { + pub struct HttpServer(pub Option); + impl Drop for HttpServer { + fn drop(&mut self) { + if let Some(server) = self.0.take() { + server.close_handle().close(); + server.wait(); + } + } + } + + pub struct WsServer(pub Option); + impl Drop for WsServer { + fn drop(&mut self) { + if let Some(server) = self.0.take() { + server.close_handle().close(); + let _ = server.wait(); + } } } } @@ -562,7 +513,7 @@ fn start_rpc_servers sc_rpc_server::RpcHandler sc_rpc_server::RpcHandler sc_network::TransactionPool for +impl sc_network::config::TransactionPool for TransactionPoolAdapter where - C: sc_network::ClientHandle + Send + Sync, + C: sc_network::config::Client + Send + Sync, Pool: 'static + TransactionPool, B: BlockT, H: std::hash::Hash + Eq + sp_runtime::traits::Member + sp_runtime::traits::MaybeSerialize, diff --git a/client/service/src/status_sinks.rs b/client/service/src/status_sinks.rs index de5fe865736af94911deb806f30eb6f64199e613..8e189be157be5234882d234fc64ed9709a3918a7 100644 --- a/client/service/src/status_sinks.rs +++ b/client/service/src/status_sinks.rs @@ -122,28 +122,17 @@ mod tests { let (tx, rx) = mpsc::unbounded(); status_sinks.push(Duration::from_millis(100), tx); - let mut runtime = tokio::runtime::Runtime::new().unwrap(); - let mut val_order = 5; - runtime.spawn(futures::future::poll_fn(move |cx| { - status_sinks.poll(cx, || { val_order += 1; val_order }); - Poll::<()>::Pending - })); - - let done = rx - .into_future() - .then(|(item, rest)| { - assert_eq!(item, Some(6)); - rest.into_future() - }) - .then(|(item, rest)| { - assert_eq!(item, Some(7)); - rest.into_future() - }) - .map(|(item, _)| { - assert_eq!(item, Some(8)); - }); - runtime.block_on(done); + futures::executor::block_on(futures::future::select( + futures::future::poll_fn(move |cx| { + status_sinks.poll(cx, || { val_order += 1; val_order }); + Poll::<()>::Pending + }), + Box::pin(async { + let items: Vec = rx.take(3).collect().await; + assert_eq!(items, [6, 7, 8]); + }) + )); } } diff --git a/client/service/src/task_manager.rs b/client/service/src/task_manager.rs new file mode 100644 index 0000000000000000000000000000000000000000..d7041e44b9c1e17eaa2d82e0e8a80467ddcffcb3 --- /dev/null +++ b/client/service/src/task_manager.rs @@ -0,0 +1,190 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +//! Substrate service tasks management module. + +use std::{ + result::Result, sync::Arc, + task::{Poll, Context}, + borrow::Cow, pin::Pin, +}; +use exit_future::Signal; +use log::{debug, error}; +use futures::{ + Future, FutureExt, Stream, + future::select, channel::mpsc, + compat::*, + task::{Spawn, FutureObj, SpawnError}, +}; + +/// Type alias for service task executor (usually runtime). +pub type ServiceTaskExecutor = Arc + Send>>) + Send + Sync>; + +/// Type alias for the task scheduler. +pub type TaskScheduler = mpsc::UnboundedSender<(Pin + Send>>, Cow<'static, str>)>; + +/// Helper struct to setup background tasks execution for service. +pub struct TaskManagerBuilder { + /// A future that resolves when the service has exited, this is useful to + /// make sure any internally spawned futures stop when the service does. + on_exit: exit_future::Exit, + /// A signal that makes the exit future above resolve, fired on service drop. + signal: Option, + /// Sender for futures that must be spawned as background tasks. + to_spawn_tx: TaskScheduler, + /// Receiver for futures that must be spawned as background tasks. + to_spawn_rx: mpsc::UnboundedReceiver<(Pin + Send>>, Cow<'static, str>)>, +} + +impl TaskManagerBuilder { + /// New asynchronous task manager setup. + pub fn new() -> Self { + let (signal, on_exit) = exit_future::signal(); + let (to_spawn_tx, to_spawn_rx) = mpsc::unbounded(); + Self { + on_exit, + signal: Some(signal), + to_spawn_tx, + to_spawn_rx, + } + } + + /// Get spawn handle. + /// + /// Tasks spawned through this handle will get scheduled once + /// service is up and running. + pub fn spawn_handle(&self) -> SpawnTaskHandle { + SpawnTaskHandle { + on_exit: self.on_exit.clone(), + sender: self.to_spawn_tx.clone(), + } + } + + /// Convert into actual task manager from initial setup. + pub(crate) fn into_task_manager(self, executor: ServiceTaskExecutor) -> TaskManager { + let TaskManagerBuilder { + on_exit, + signal, + to_spawn_rx, + to_spawn_tx + } = self; + TaskManager { + on_exit, + signal, + to_spawn_tx, + to_spawn_rx, + executor, + } + } +} + +/// An handle for spawning tasks in the service. +#[derive(Clone)] +pub struct SpawnTaskHandle { + sender: TaskScheduler, + on_exit: exit_future::Exit, +} + +impl SpawnTaskHandle { + /// Spawns the given task with the given name. + pub fn spawn(&self, name: impl Into>, task: impl Future + Send + 'static) { + let on_exit = self.on_exit.clone(); + let future = async move { + futures::pin_mut!(task); + let _ = select(on_exit, task).await; + }; + if self.sender.unbounded_send((Box::pin(future), name.into())).is_err() { + error!("Failed to send task to spawn over channel"); + } + } +} + +impl Spawn for SpawnTaskHandle { + fn spawn_obj(&self, future: FutureObj<'static, ()>) + -> Result<(), SpawnError> { + let future = select(self.on_exit.clone(), future).map(drop); + self.sender.unbounded_send((Box::pin(future), From::from("unnamed"))) + .map_err(|_| SpawnError::shutdown()) + } +} + +type Boxed01Future01 = Box + Send + 'static>; + +impl futures01::future::Executor for SpawnTaskHandle { + fn execute(&self, future: Boxed01Future01) -> Result<(), futures01::future::ExecuteError>{ + self.spawn("unnamed", future.compat().map(drop)); + Ok(()) + } +} + +/// Helper struct to manage background/async tasks in Service. +pub struct TaskManager { + /// A future that resolves when the service has exited, this is useful to + /// make sure any internally spawned futures stop when the service does. + on_exit: exit_future::Exit, + /// A signal that makes the exit future above resolve, fired on service drop. + signal: Option, + /// Sender for futures that must be spawned as background tasks. + to_spawn_tx: TaskScheduler, + /// Receiver for futures that must be spawned as background tasks. + to_spawn_rx: mpsc::UnboundedReceiver<(Pin + Send>>, Cow<'static, str>)>, + /// How to spawn background tasks. + executor: ServiceTaskExecutor, +} + +impl TaskManager { + /// Spawn background/async task, which will be aware on exit signal. + pub(super) fn spawn(&self, name: impl Into>, task: impl Future + Send + 'static) { + let on_exit = self.on_exit.clone(); + let future = async move { + futures::pin_mut!(task); + let _ = select(on_exit, task).await; + }; + if self.to_spawn_tx.unbounded_send((Box::pin(future), name.into())).is_err() { + error!("Failed to send task to spawn over channel"); + } + } + + pub(super) fn spawn_handle(&self) -> SpawnTaskHandle { + SpawnTaskHandle { + on_exit: self.on_exit.clone(), + sender: self.to_spawn_tx.clone(), + } + } + + /// Get sender where background/async tasks can be sent. + pub(super) fn scheduler(&self) -> TaskScheduler { + self.to_spawn_tx.clone() + } + + /// Process background task receiver. + pub(super) fn process_receiver(&mut self, cx: &mut Context) { + while let Poll::Ready(Some((task_to_spawn, name))) = Pin::new(&mut self.to_spawn_rx).poll_next(cx) { + (self.executor)(Box::pin(futures_diagnose::diagnose(name, task_to_spawn))); + } + } + + /// Clone on exit signal. + pub(super) fn on_exit(&self) -> exit_future::Exit { + self.on_exit.clone() + } +} + +impl Drop for TaskManager { + fn drop(&mut self) { + debug!(target: "service", "Tasks manager shutdown"); + if let Some(signal) = self.signal.take() { + let _ = signal.fire(); + } + } +} diff --git a/client/service/test/Cargo.toml b/client/service/test/Cargo.toml index c9dbe97464aa135a12dc2928eb2262478152315d..f27f8803300b30d86f6ad605541bed438788ffb7 100644 --- a/client/service/test/Cargo.toml +++ b/client/service/test/Cargo.toml @@ -1,9 +1,12 @@ [package] name = "sc-service-test" -version = "2.0.0" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +publish = false +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] tempfile = "3.1.0" @@ -13,10 +16,10 @@ log = "0.4.8" env_logger = "0.7.0" fdlimit = "0.1.1" futures = { version = "0.3.1", features = ["compat"] } -sc-service = { version = "0.8.0", default-features = false, path = "../../service" } -sc-network = { version = "0.8", path = "../../network" } -sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } -sc-client = { version = "0.8", path = "../../" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-transaction-pool = { version = "2.0.0", path = "../../../primitives/transaction-pool" } +sc-service = { version = "0.8.0-alpha.2", default-features = false, path = "../../service" } +sc-network = { version = "0.8.0-alpha.2", path = "../../network" } +sp-consensus = { version = "0.8.0-alpha.2", path = "../../../primitives/consensus/common" } +sc-client = { version = "0.8.0-alpha.2", path = "../../" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-transaction-pool = { version = "2.0.0-alpha.2", path = "../../../primitives/transaction-pool" } diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index b65bccc151823af8bc6db72702de57d97467a032..03cccbe4a053ad8b1a5b51a23035a0ac942355a2 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -141,7 +141,7 @@ fn node_config ( { let root = root.path().join(format!("node-{}", index)); - let config_path = Some(String::from(root.join("network").to_str().unwrap())); + let config_path = Some(root.join("network")); let net_config_path = config_path.clone(); let network_config = NetworkConfiguration { @@ -199,7 +199,7 @@ fn node_config ( rpc_ws: None, rpc_ws_max_connections: None, rpc_cors: None, - grafana_port: None, + prometheus_config: None, telemetry_endpoints: None, telemetry_external_transport: None, default_heap_pages: None, diff --git a/client/src/call_executor.rs b/client/src/call_executor.rs index 18ad5b113e983f98b94e384b5432357fe2e1d4e6..b5206d3c461a85f3ed41a3e12feeea5b296485fc 100644 --- a/client/src/call_executor.rs +++ b/client/src/call_executor.rs @@ -17,7 +17,7 @@ use std::{sync::Arc, panic::UnwindSafe, result, cell::RefCell}; use codec::{Encode, Decode}; use sp_runtime::{ - generic::BlockId, traits::{Block as BlockT, HasherFor, NumberFor}, + generic::BlockId, traits::{Block as BlockT, HashFor, NumberFor}, }; use sp_state_machine::{ self, OverlayedChanges, Ext, ExecutionManager, StateMachine, ExecutionStrategy, @@ -80,7 +80,6 @@ where let changes_trie = backend::changes_tries_state_at_block( id, self.backend.changes_trie_storage() )?; - // make sure to destroy state before exiting this function let state = self.backend.state_at(*id)?; let return_data = StateMachine::new( &state, @@ -93,12 +92,9 @@ where ).execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( strategy.get_manager(), None, - ); - { - let _lock = self.backend.get_import_lock().read(); - self.backend.destroy_state(state)?; - } - Ok(return_data?.into_encoded()) + )?; + + Ok(return_data.into_encoded()) } fn contextual_call< @@ -138,9 +134,8 @@ where let changes_trie_state = backend::changes_tries_state_at_block(at, self.backend.changes_trie_storage())?; let mut storage_transaction_cache = storage_transaction_cache.map(|c| c.borrow_mut()); - // make sure to destroy state before exiting this function let mut state = self.backend.state_at(*at)?; - let result = match recorder { + match recorder { Some(recorder) => state.as_trie_backend() .ok_or_else(|| Box::new(sp_state_machine::ExecutionError::UnableToGenerateProof) @@ -176,18 +171,15 @@ where ) .with_storage_transaction_cache(storage_transaction_cache.as_mut().map(|c| &mut **c)) .execute_using_consensus_failure_handler(execution_manager, native_call) - }; - { - let _lock = self.backend.get_import_lock().read(); - self.backend.destroy_state(state)?; - } - result.map_err(Into::into) + }.map_err(Into::into) } fn runtime_version(&self, id: &BlockId) -> sp_blockchain::Result { let mut overlay = OverlayedChanges::default(); - let changes_trie_state = backend::changes_tries_state_at_block(id, self.backend.changes_trie_storage())?; - // make sure to destroy state before exiting this function + let changes_trie_state = backend::changes_tries_state_at_block( + id, + self.backend.changes_trie_storage(), + )?; let state = self.backend.state_at(*id)?; let mut cache = StorageTransactionCache::::default(); let mut ext = Ext::new( @@ -197,17 +189,13 @@ where changes_trie_state, None, ); - let version = self.executor.runtime_version(&mut ext); - { - let _lock = self.backend.get_import_lock().read(); - self.backend.destroy_state(state)?; - } - version.map_err(|e| sp_blockchain::Error::VersionInvalid(format!("{:?}", e)).into()) + self.executor.runtime_version(&mut ext) + .map_err(|e| sp_blockchain::Error::VersionInvalid(format!("{:?}", e)).into()) } - fn prove_at_trie_state>>( + fn prove_at_trie_state>>( &self, - trie_state: &sp_state_machine::TrieBackend>, + trie_state: &sp_state_machine::TrieBackend>, overlay: &mut OverlayedChanges, method: &str, call_data: &[u8] diff --git a/client/src/cht.rs b/client/src/cht.rs index 1435b77ec592e21d1a05be901662653dc6fcfbc1..de67280632302395e2363f6d309b2fecbf75d9ad 100644 --- a/client/src/cht.rs +++ b/client/src/cht.rs @@ -331,8 +331,8 @@ pub fn decode_cht_value(value: &[u8]) -> Option { #[cfg(test)] mod tests { - use sp_core::Blake2Hasher; use substrate_test_runtime_client::runtime::Header; + use sp_runtime::traits::BlakeTwo256; use super::*; #[test] @@ -398,7 +398,7 @@ mod tests { #[test] fn compute_root_works() { - assert!(compute_root::( + assert!(compute_root::( SIZE as _, 42, ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))) @@ -409,7 +409,7 @@ mod tests { #[test] #[should_panic] fn build_proof_panics_when_querying_wrong_block() { - assert!(build_proof::( + assert!(build_proof::( SIZE as _, 0, vec![(SIZE * 1000) as u64], @@ -420,7 +420,7 @@ mod tests { #[test] fn build_proof_works() { - assert!(build_proof::( + assert!(build_proof::( SIZE as _, 0, vec![(SIZE / 2) as u64], diff --git a/client/src/client.rs b/client/src/client.rs index e9a8f1228c5e19e15ba21d6702301342b623b2cf..699e3320ff4b7dda7137d6fadec1a4b786d5b9bb 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -34,7 +34,7 @@ use sp_runtime::{ Justification, BuildStorage, generic::{BlockId, SignedBlock, DigestItem}, traits::{ - Block as BlockT, Header as HeaderT, Zero, NumberFor, HasherFor, SaturatedConversion, One, + Block as BlockT, Header as HeaderT, Zero, NumberFor, HashFor, SaturatedConversion, One, DigestFor, }, }; @@ -60,13 +60,14 @@ use sp_api::{ CallApiAt, ConstructRuntimeApi, Core as CoreApi, ApiExt, ApiRef, ProvideRuntimeApi, CallApiAtParams, }; -use sc_block_builder::BlockBuilderApi; +use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider}; pub use sc_client_api::{ backend::{ self, BlockImportOperation, PrunableStateChangesTrieStorage, ClientImportOperation, Finalizer, ImportSummary, NewBlockState, - changes_tries_state_at_block, + changes_tries_state_at_block, StorageProvider, + LockImportRun, }, client::{ ImportNotifications, FinalityNotification, FinalityNotifications, BlockImportNotification, @@ -75,15 +76,17 @@ pub use sc_client_api::{ }, execution_extensions::{ExecutionExtensions, ExecutionStrategies}, notifications::{StorageNotifications, StorageEventStream}, - CallExecutor, + CallExecutor, ExecutorProvider, ProofProvider, }; use sp_blockchain::Error; +use prometheus_endpoint::Registry; use crate::{ call_executor::LocalCallExecutor, light::{call_executor::prove_execution, fetcher::ChangesProof}, in_mem, genesis, cht, block_rules::{BlockRules, LookupResult as BlockLookupResult}, }; +use crate::client::backend::KeyIterator; /// Substrate Client pub struct Client where Block: BlockT { @@ -99,46 +102,6 @@ pub struct Client where Block: BlockT { _phantom: PhantomData, } -/// An `Iterator` that iterates keys in a given block under a prefix. -pub struct KeyIterator<'a, State, Block> { - state: State, - prefix: Option<&'a StorageKey>, - current_key: Vec, - _phantom: PhantomData, -} - -impl <'a, State, Block> KeyIterator<'a, State, Block> { - fn new(state: State, prefix: Option<&'a StorageKey>, current_key: Vec) -> Self { - Self { - state, - prefix, - current_key, - _phantom: PhantomData, - } - } -} - -impl<'a, State, Block> Iterator for KeyIterator<'a, State, Block> where - Block: BlockT, - State: StateBackend>, -{ - type Item = StorageKey; - - fn next(&mut self) -> Option { - let next_key = self.state - .next_storage_key(&self.current_key) - .ok() - .flatten()?; - if let Some(prefix) = self.prefix { - if !next_key.starts_with(&prefix.0[..]) { - return None; - } - } - self.current_key = next_key.clone(); - Some(StorageKey(next_key)) - } -} - // used in importing a block, where additional changes are made after the runtime // executed. enum PrePostHeader { @@ -171,6 +134,7 @@ pub fn new_in_mem( executor: E, genesis_storage: &S, keystore: Option, + prometheus_registry: Option, ) -> sp_blockchain::Result, LocalCallExecutor, E>, @@ -181,7 +145,7 @@ pub fn new_in_mem( S: BuildStorage, Block: BlockT, { - new_with_backend(Arc::new(in_mem::Backend::new()), executor, genesis_storage, keystore) + new_with_backend(Arc::new(in_mem::Backend::new()), executor, genesis_storage, keystore, prometheus_registry) } /// Create a client with the explicitly provided backend. @@ -191,6 +155,7 @@ pub fn new_with_backend( executor: E, build_genesis_storage: &S, keystore: Option, + prometheus_registry: Option, ) -> sp_blockchain::Result, Block, RA>> where E: CodeExecutor + RuntimeInfo, @@ -207,6 +172,7 @@ pub fn new_with_backend( Default::default(), Default::default(), extensions, + prometheus_registry, ) } @@ -218,6 +184,61 @@ impl BlockOf for Client where type Type = Block; } +impl LockImportRun for Client + where + B: backend::Backend, + E: CallExecutor, + Block: BlockT, +{ + fn lock_import_and_run(&self, f: F) -> Result + where + F: FnOnce(&mut ClientImportOperation) -> Result, + Err: From, + { + let inner = || { + let _import_lock = self.backend.get_import_lock().write(); + + let mut op = ClientImportOperation { + op: self.backend.begin_operation()?, + notify_imported: None, + notify_finalized: Vec::new(), + }; + + let r = f(&mut op)?; + + let ClientImportOperation { op, notify_imported, notify_finalized } = op; + self.backend.commit_operation(op)?; + self.notify_finalized(notify_finalized)?; + + if let Some(notify_imported) = notify_imported { + self.notify_imported(notify_imported)?; + } + + Ok(r) + }; + + let result = inner(); + *self.importing_block.write() = None; + + result + } +} + +impl LockImportRun for &Client + where + Block: BlockT, + B: backend::Backend, + E: CallExecutor, +{ + fn lock_import_and_run(&self, f: F) -> Result + where + F: FnOnce(&mut ClientImportOperation) -> Result, + Err: From, + { + (**self).lock_import_and_run(f) + } +} + impl Client where B: backend::Backend, E: CallExecutor, @@ -231,6 +252,7 @@ impl Client where fork_blocks: ForkBlocks, bad_blocks: BadBlocks, execution_extensions: ExecutionExtensions, + _prometheus_registry: Option, ) -> sp_blockchain::Result { if backend.blockchain().header(BlockId::Number(Zero::zero()))?.is_none() { let genesis_storage = build_genesis_storage.build_storage()?; @@ -264,119 +286,14 @@ impl Client where }) } - /// Get a reference to the execution extensions. - pub fn execution_extensions(&self) -> &ExecutionExtensions { - &self.execution_extensions - } - /// Get a reference to the state at a given block. pub fn state_at(&self, block: &BlockId) -> sp_blockchain::Result { self.backend.state_at(*block) } - /// Given a `BlockId` and a key prefix, return the matching storage keys in that block. - pub fn storage_keys(&self, id: &BlockId, key_prefix: &StorageKey) -> sp_blockchain::Result> { - let keys = self.state_at(id)?.keys(&key_prefix.0).into_iter().map(StorageKey).collect(); - Ok(keys) - } - - /// Given a `BlockId` and a key prefix, return the matching child storage keys and values in that block. - pub fn storage_pairs(&self, id: &BlockId, key_prefix: &StorageKey) - -> sp_blockchain::Result> - { - let state = self.state_at(id)?; - let keys = state - .keys(&key_prefix.0) - .into_iter() - .map(|k| { - let d = state.storage(&k).ok().flatten().unwrap_or_default(); - (StorageKey(k), StorageData(d)) - }) - .collect(); - Ok(keys) - } - - /// Given a `BlockId` and a key prefix, return a `KeyIterator` iterates matching storage keys in that block. - pub fn storage_keys_iter<'a>( - &self, - id: &BlockId, - prefix: Option<&'a StorageKey>, - start_key: Option<&StorageKey> - ) -> sp_blockchain::Result> { - let state = self.state_at(id)?; - let start_key = start_key - .or(prefix) - .map(|key| key.0.clone()) - .unwrap_or_else(Vec::new); - Ok(KeyIterator::new(state, prefix, start_key)) - } - - /// Given a `BlockId` and a key, return the value under the key in that block. - pub fn storage(&self, id: &BlockId, key: &StorageKey) - -> sp_blockchain::Result> - { - Ok(self.state_at(id)? - .storage(&key.0).map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? - .map(StorageData) - ) - } - - /// Given a `BlockId` and a key, return the value under the hash in that block. - pub fn storage_hash(&self, id: &BlockId, key: &StorageKey) - -> sp_blockchain::Result> - { - Ok(self.state_at(id)? - .storage_hash(&key.0).map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? - ) - } - - /// Given a `BlockId`, a key prefix, and a child storage key, return the matching child storage keys. - pub fn child_storage_keys( - &self, - id: &BlockId, - child_storage_key: &StorageKey, - child_info: ChildInfo, - key_prefix: &StorageKey - ) -> sp_blockchain::Result> { - let keys = self.state_at(id)? - .child_keys(&child_storage_key.0, child_info, &key_prefix.0) - .into_iter() - .map(StorageKey) - .collect(); - Ok(keys) - } - - /// Given a `BlockId`, a key and a child storage key, return the value under the key in that block. - pub fn child_storage( - &self, - id: &BlockId, - storage_key: &StorageKey, - child_info: ChildInfo, - key: &StorageKey - ) -> sp_blockchain::Result> { - Ok(self.state_at(id)? - .child_storage(&storage_key.0, child_info, &key.0) - .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? - .map(StorageData)) - } - - /// Given a `BlockId`, a key and a child storage key, return the hash under the key in that block. - pub fn child_storage_hash( - &self, - id: &BlockId, - storage_key: &StorageKey, - child_info: ChildInfo, - key: &StorageKey - ) -> sp_blockchain::Result> { - Ok(self.state_at(id)? - .child_storage_hash(&storage_key.0, child_info, &key.0) - .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? - ) - } - /// Get the code at a given block. pub fn code_at(&self, id: &BlockId) -> sp_blockchain::Result> { - Ok(self.storage(id, &StorageKey(well_known_keys::CODE.to_vec()))? + Ok(StorageProvider::storage(self, id, &StorageKey(well_known_keys::CODE.to_vec()))? .expect("None is returned if there's no value stored for the given key;\ ':code' key is always defined; qed").0) } @@ -386,57 +303,6 @@ impl Client where self.executor.runtime_version(id) } - /// Get call executor reference. - pub fn executor(&self) -> &E { - &self.executor - } - - /// Reads storage value at a given block + key, returning read proof. - pub fn read_proof(&self, id: &BlockId, keys: I) -> sp_blockchain::Result where - I: IntoIterator, - I::Item: AsRef<[u8]>, - { - self.state_at(id) - .and_then(|state| prove_read(state, keys) - .map_err(Into::into)) - } - - /// Reads child storage value at a given block + storage_key + key, returning - /// read proof. - pub fn read_child_proof( - &self, - id: &BlockId, - storage_key: &[u8], - child_info: ChildInfo, - keys: I, - ) -> sp_blockchain::Result where - I: IntoIterator, - I::Item: AsRef<[u8]>, - { - self.state_at(id) - .and_then(|state| prove_child_read(state, storage_key, child_info, keys) - .map_err(Into::into)) - } - - /// Execute a call to a contract on top of state in a block of given hash - /// AND returning execution proof. - /// - /// No changes are made. - pub fn execution_proof(&self, - id: &BlockId, - method: &str, - call_data: &[u8] - ) -> sp_blockchain::Result<(Vec, StorageProof)> { - let state = self.state_at(id)?; - let header = self.prepare_environment_block(id)?; - prove_execution(state, header, &self.executor, method, call_data) - } - - /// Reads given header and generates CHT-based header proof. - pub fn header_proof(&self, id: &BlockId) -> sp_blockchain::Result<(Block::Header, StorageProof)> { - self.header_proof_with_cht_size(id, cht::size()) - } - /// Get block hash by number. pub fn block_hash(&self, block_number: <::Header as HeaderT>::Number @@ -462,7 +328,7 @@ impl Client where Some(old_current_num) }); let headers = cht_range.map(|num| self.block_hash(num)); - let proof = cht::build_proof::, _, _>( + let proof = cht::build_proof::, _, _>( cht_size, cht_num, std::iter::once(block_num), @@ -471,112 +337,6 @@ impl Client where Ok((header, proof)) } - /// Get longest range within [first; last] that is possible to use in `key_changes` - /// and `key_changes_proof` calls. - /// Range could be shortened from the beginning if some changes tries have been pruned. - /// Returns Ok(None) if changes tries are not supported. - pub fn max_key_changes_range( - &self, - first: NumberFor, - last: BlockId, - ) -> sp_blockchain::Result, BlockId)>> { - let last_number = self.backend.blockchain().expect_block_number_from_id(&last)?; - let last_hash = self.backend.blockchain().expect_block_hash_from_id(&last)?; - if first > last_number { - return Err(sp_blockchain::Error::ChangesTrieAccessFailed("Invalid changes trie range".into())); - } - - let (storage, configs) = match self.require_changes_trie(first, last_hash, false).ok() { - Some((storage, configs)) => (storage, configs), - None => return Ok(None), - }; - - let first_available_changes_trie = configs.last().map(|config| config.0); - match first_available_changes_trie { - Some(first_available_changes_trie) => { - let oldest_unpruned = storage.oldest_pruned_digest_range_end(); - let first = std::cmp::max(first_available_changes_trie, oldest_unpruned); - Ok(Some((first, last))) - }, - None => Ok(None) - } - } - - /// Get pairs of (block, extrinsic) where key has been changed at given blocks range. - /// Works only for runtimes that are supporting changes tries. - /// - /// Changes are returned in descending order (i.e. last block comes first). - pub fn key_changes( - &self, - first: NumberFor, - last: BlockId, - storage_key: Option<&StorageKey>, - key: &StorageKey - ) -> sp_blockchain::Result, u32)>> { - let last_number = self.backend.blockchain().expect_block_number_from_id(&last)?; - let last_hash = self.backend.blockchain().expect_block_hash_from_id(&last)?; - let (storage, configs) = self.require_changes_trie(first, last_hash, true)?; - - let mut result = Vec::new(); - let best_number = self.backend.blockchain().info().best_number; - for (config_zero, config_end, config) in configs { - let range_first = ::std::cmp::max(first, config_zero + One::one()); - let range_anchor = match config_end { - Some((config_end_number, config_end_hash)) => if last_number > config_end_number { - ChangesTrieAnchorBlockId { hash: config_end_hash, number: config_end_number } - } else { - ChangesTrieAnchorBlockId { hash: convert_hash(&last_hash), number: last_number } - }, - None => ChangesTrieAnchorBlockId { hash: convert_hash(&last_hash), number: last_number }, - }; - - let config_range = ChangesTrieConfigurationRange { - config: &config, - zero: config_zero.clone(), - end: config_end.map(|(config_end_number, _)| config_end_number), - }; - let result_range: Vec<(NumberFor, u32)> = key_changes::, _>( - config_range, - storage.storage(), - range_first, - &range_anchor, - best_number, - storage_key.as_ref().map(|x| &x.0[..]), - &key.0) - .and_then(|r| r.map(|r| r.map(|(block, tx)| (block, tx))).collect::>()) - .map_err(|err| sp_blockchain::Error::ChangesTrieAccessFailed(err))?; - result.extend(result_range); - } - - Ok(result) - } - - /// Get proof for computation of (block, extrinsic) pairs where key has been changed at given blocks range. - /// `min` is the hash of the first block, which changes trie root is known to the requester - when we're using - /// changes tries from ascendants of this block, we should provide proofs for changes tries roots - /// `max` is the hash of the last block known to the requester - we can't use changes tries from descendants - /// of this block. - /// Works only for runtimes that are supporting changes tries. - pub fn key_changes_proof( - &self, - first: Block::Hash, - last: Block::Hash, - min: Block::Hash, - max: Block::Hash, - storage_key: Option<&StorageKey>, - key: &StorageKey, - ) -> sp_blockchain::Result> { - self.key_changes_proof_with_cht_size( - first, - last, - min, - max, - storage_key, - key, - cht::size(), - ) - } - /// Does the same work as `key_changes_proof`, but assumes that CHTs are of passed size. pub fn key_changes_proof_with_cht_size( &self, @@ -589,12 +349,12 @@ impl Client where cht_size: NumberFor, ) -> sp_blockchain::Result> { struct AccessedRootsRecorder<'a, Block: BlockT> { - storage: &'a dyn ChangesTrieStorage, NumberFor>, + storage: &'a dyn ChangesTrieStorage, NumberFor>, min: NumberFor, required_roots_proofs: Mutex, Block::Hash>>, }; - impl<'a, Block: BlockT> ChangesTrieRootsStorage, NumberFor> for + impl<'a, Block: BlockT> ChangesTrieRootsStorage, NumberFor> for AccessedRootsRecorder<'a, Block> { fn build_anchor(&self, hash: Block::Hash) @@ -621,11 +381,11 @@ impl Client where } } - impl<'a, Block: BlockT> ChangesTrieStorage, NumberFor> for + impl<'a, Block: BlockT> ChangesTrieStorage, NumberFor> for AccessedRootsRecorder<'a, Block> { fn as_roots_storage(&self) - -> &dyn sp_state_machine::ChangesTrieRootsStorage, NumberFor> + -> &dyn sp_state_machine::ChangesTrieRootsStorage, NumberFor> { self } @@ -669,7 +429,7 @@ impl Client where zero: config_zero, end: config_end.map(|(config_end_number, _)| config_end_number), }; - let proof_range = key_changes_proof::, _>( + let proof_range = key_changes_proof::, _>( config_range, &recording_storage, first_number, @@ -736,7 +496,7 @@ impl Client where .map(|block| block.and_then(|block| block.digest().log(DigestItem::as_changes_trie_root).cloned())) ); - let proof = cht::build_proof::, _, _>( + let proof = cht::build_proof::, _, _>( cht_size, cht_num, blocks, @@ -808,66 +568,6 @@ impl Client where ) } - /// Create a new block, built on top of `parent`. - /// - /// When proof recording is enabled, all accessed trie nodes are saved. - /// These recorded trie nodes can be used by a third party to proof the - /// output of this block builder without having access to the full storage. - pub fn new_block_at>( - &self, - parent: &BlockId, - inherent_digests: DigestFor, - record_proof: R, - ) -> sp_blockchain::Result> where - E: Clone + Send + Sync, - RA: Send + Sync, - Self: ProvideRuntimeApi, - >::Api: BlockBuilderApi + - ApiExt> - { - sc_block_builder::BlockBuilder::new( - self, - self.expect_block_hash_from_id(parent)?, - self.expect_block_number_from_id(parent)?, - record_proof.into(), - inherent_digests, - &self.backend - ) - } - - /// Lock the import lock, and run operations inside. - pub fn lock_import_and_run(&self, f: F) -> Result where - F: FnOnce(&mut ClientImportOperation) -> Result, - Err: From, - { - let inner = || { - let _import_lock = self.backend.get_import_lock().write(); - - let mut op = ClientImportOperation { - op: self.backend.begin_operation()?, - notify_imported: None, - notify_finalized: Vec::new(), - }; - - let r = f(&mut op)?; - - let ClientImportOperation { op, notify_imported, notify_finalized } = op; - self.backend.commit_operation(op)?; - self.notify_finalized(notify_finalized)?; - - if let Some(notify_imported) = notify_imported { - self.notify_imported(notify_imported)?; - } - - Ok(r) - }; - - let result = inner(); - *self.importing_block.write() = None; - - result - } - /// Apply a checked and validated block to an operation. If a justification is provided /// then `finalized` *must* be true. fn apply_block( @@ -1130,15 +830,7 @@ impl Client where &state, changes_trie_state.as_ref(), *parent_hash, - ); - - { - let _lock = self.backend.get_import_lock().read(); - self.backend.destroy_state(state)?; - } - - // Make sure to consume the error, only after we have destroyed the state. - let gen_storage_changes = gen_storage_changes?; + )?; if import_block.header.state_root() != &gen_storage_changes.transaction_storage_root @@ -1340,19 +1032,8 @@ impl Client where } /// Get block justification set by id. - pub fn justification(&self, id: &BlockId) -> sp_blockchain::Result> { - self.backend.blockchain().justification(*id) - } - - /// Get full block by id. - pub fn block(&self, id: &BlockId) - -> sp_blockchain::Result>> - { - Ok(match (self.header(id)?, self.body(id)?, self.justification(id)?) { - (Some(header), Some(extrinsics), justification) => - Some(SignedBlock { block: Block::new(header, extrinsics), justification }), - _ => None, - }) + pub fn justification(&self, id: &BlockId) -> sp_blockchain::Result> { + self.backend.blockchain().justification(*id) } /// Gets the uncles of the block with `target_hash` going back `max_generation` ancestors. @@ -1399,6 +1080,286 @@ impl Client where } } +impl ProofProvider for Client where + B: backend::Backend, + E: CallExecutor, + Block: BlockT, +{ + fn read_proof( + &self, + id: &BlockId, + keys: &mut dyn Iterator, + ) -> sp_blockchain::Result { + self.state_at(id) + .and_then(|state| prove_read(state, keys) + .map_err(Into::into)) + } + + fn read_child_proof( + &self, + id: &BlockId, + storage_key: &[u8], + child_info: ChildInfo, + keys: &mut dyn Iterator, + ) -> sp_blockchain::Result { + self.state_at(id) + .and_then(|state| prove_child_read(state, storage_key, child_info, keys) + .map_err(Into::into)) + } + + fn execution_proof( + &self, + id: &BlockId, + method: &str, + call_data: &[u8] + ) -> sp_blockchain::Result<(Vec, StorageProof)> { + let state = self.state_at(id)?; + let header = self.prepare_environment_block(id)?; + prove_execution(state, header, &self.executor, method, call_data) + } + + fn header_proof(&self, id: &BlockId) -> sp_blockchain::Result<(Block::Header, StorageProof)> { + self.header_proof_with_cht_size(id, cht::size()) + } + + fn key_changes_proof( + &self, + first: Block::Hash, + last: Block::Hash, + min: Block::Hash, + max: Block::Hash, + storage_key: Option<&StorageKey>, + key: &StorageKey, + ) -> sp_blockchain::Result> { + self.key_changes_proof_with_cht_size( + first, + last, + min, + max, + storage_key, + key, + cht::size(), + ) + } +} + + +impl BlockBuilderProvider for Client + where + B: backend::Backend + Send + Sync + 'static, + E: CallExecutor + Send + Sync + 'static, + Block: BlockT, + Self: ChainHeaderBackend + ProvideRuntimeApi, + >::Api: ApiExt> + + BlockBuilderApi, +{ + fn new_block_at>( + &self, + parent: &BlockId, + inherent_digests: DigestFor, + record_proof: R, + ) -> sp_blockchain::Result> { + sc_block_builder::BlockBuilder::new( + self, + self.expect_block_hash_from_id(parent)?, + self.expect_block_number_from_id(parent)?, + record_proof.into(), + inherent_digests, + &self.backend + ) + } +} + +impl ExecutorProvider for Client where + B: backend::Backend, + E: CallExecutor, + Block: BlockT, +{ + type Executor = E; + + fn executor(&self) -> &Self::Executor { + &self.executor + } + + fn execution_extensions(&self) -> &ExecutionExtensions { + &self.execution_extensions + } +} + +impl StorageProvider for Client where + B: backend::Backend, + E: CallExecutor, + Block: BlockT, +{ + fn storage_keys(&self, id: &BlockId, key_prefix: &StorageKey) -> sp_blockchain::Result> { + let keys = self.state_at(id)?.keys(&key_prefix.0).into_iter().map(StorageKey).collect(); + Ok(keys) + } + + fn storage_pairs(&self, id: &BlockId, key_prefix: &StorageKey) + -> sp_blockchain::Result> + { + let state = self.state_at(id)?; + let keys = state + .keys(&key_prefix.0) + .into_iter() + .map(|k| { + let d = state.storage(&k).ok().flatten().unwrap_or_default(); + (StorageKey(k), StorageData(d)) + }) + .collect(); + Ok(keys) + } + + + fn storage_keys_iter<'a>( + &self, + id: &BlockId, + prefix: Option<&'a StorageKey>, + start_key: Option<&StorageKey> + ) -> sp_blockchain::Result> { + let state = self.state_at(id)?; + let start_key = start_key + .or(prefix) + .map(|key| key.0.clone()) + .unwrap_or_else(Vec::new); + Ok(KeyIterator::new(state, prefix, start_key)) + } + + + fn storage(&self, id: &BlockId, key: &StorageKey) -> sp_blockchain::Result> + { + Ok(self.state_at(id)? + .storage(&key.0).map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? + .map(StorageData) + ) + } + + + fn storage_hash(&self, id: &BlockId, key: &StorageKey) -> sp_blockchain::Result> + { + Ok(self.state_at(id)? + .storage_hash(&key.0).map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? + ) + } + + + fn child_storage_keys( + &self, + id: &BlockId, + child_storage_key: &StorageKey, + child_info: ChildInfo, + key_prefix: &StorageKey + ) -> sp_blockchain::Result> { + let keys = self.state_at(id)? + .child_keys(&child_storage_key.0, child_info, &key_prefix.0) + .into_iter() + .map(StorageKey) + .collect(); + Ok(keys) + } + + + fn child_storage( + &self, + id: &BlockId, + storage_key: &StorageKey, + child_info: ChildInfo, + key: &StorageKey + ) -> sp_blockchain::Result> { + Ok(self.state_at(id)? + .child_storage(&storage_key.0, child_info, &key.0) + .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? + .map(StorageData)) + } + + + fn child_storage_hash( + &self, + id: &BlockId, + storage_key: &StorageKey, + child_info: ChildInfo, + key: &StorageKey + ) -> sp_blockchain::Result> { + Ok(self.state_at(id)? + .child_storage_hash(&storage_key.0, child_info, &key.0) + .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? + ) + } + + fn max_key_changes_range( + &self, + first: NumberFor, + last: BlockId, + ) -> sp_blockchain::Result, BlockId)>> { + let last_number = self.backend.blockchain().expect_block_number_from_id(&last)?; + let last_hash = self.backend.blockchain().expect_block_hash_from_id(&last)?; + if first > last_number { + return Err(sp_blockchain::Error::ChangesTrieAccessFailed("Invalid changes trie range".into())); + } + + let (storage, configs) = match self.require_changes_trie(first, last_hash, false).ok() { + Some((storage, configs)) => (storage, configs), + None => return Ok(None), + }; + + let first_available_changes_trie = configs.last().map(|config| config.0); + match first_available_changes_trie { + Some(first_available_changes_trie) => { + let oldest_unpruned = storage.oldest_pruned_digest_range_end(); + let first = std::cmp::max(first_available_changes_trie, oldest_unpruned); + Ok(Some((first, last))) + }, + None => Ok(None) + } + } + + fn key_changes( + &self, + first: NumberFor, + last: BlockId, + storage_key: Option<&StorageKey>, + key: &StorageKey + ) -> sp_blockchain::Result, u32)>> { + let last_number = self.backend.blockchain().expect_block_number_from_id(&last)?; + let last_hash = self.backend.blockchain().expect_block_hash_from_id(&last)?; + let (storage, configs) = self.require_changes_trie(first, last_hash, true)?; + + let mut result = Vec::new(); + let best_number = self.backend.blockchain().info().best_number; + for (config_zero, config_end, config) in configs { + let range_first = ::std::cmp::max(first, config_zero + One::one()); + let range_anchor = match config_end { + Some((config_end_number, config_end_hash)) => if last_number > config_end_number { + ChangesTrieAnchorBlockId { hash: config_end_hash, number: config_end_number } + } else { + ChangesTrieAnchorBlockId { hash: convert_hash(&last_hash), number: last_number } + }, + None => ChangesTrieAnchorBlockId { hash: convert_hash(&last_hash), number: last_number }, + }; + + let config_range = ChangesTrieConfigurationRange { + config: &config, + zero: config_zero.clone(), + end: config_end.map(|(config_end_number, _)| config_end_number), + }; + let result_range: Vec<(NumberFor, u32)> = key_changes::, _>( + config_range, + storage.storage(), + range_first, + &range_anchor, + best_number, + storage_key.as_ref().map(|x| &x.0[..]), + &key.0) + .and_then(|r| r.map(|r| r.map(|(block, tx)| (block, tx))).collect::>()) + .map_err(|err| sp_blockchain::Error::ChangesTrieAccessFailed(err))?; + result.extend(result_range); + } + + Ok(result) + } +} + impl HeaderMetadata for Client where B: backend::Backend, E: CallExecutor, @@ -1823,7 +1784,9 @@ where fn best_block_header(&self) -> sp_blockchain::Result<::Header> { let info = self.backend.blockchain().info(); let import_lock = self.backend.get_import_lock(); - let best_hash = self.backend.blockchain().best_containing(info.best_hash, None, import_lock)? + let best_hash = self.backend + .blockchain() + .best_containing(info.best_hash, None, import_lock)? .unwrap_or(info.best_hash); Ok(self.backend.blockchain().header(BlockId::Hash(best_hash))? @@ -1876,6 +1839,15 @@ impl BlockBody for Client ) -> sp_blockchain::Result::Extrinsic>>> { self.body(id) } + + fn block(&self, id: &BlockId) -> sp_blockchain::Result>> + { + Ok(match (self.header(id)?, self.body(id)?, self.justification(id)?) { + (Some(header), Some(extrinsics), justification) => + Some(SignedBlock { block: Block::new(header, extrinsics), justification }), + _ => None, + }) + } } impl backend::AuxStore for Client diff --git a/client/src/genesis.rs b/client/src/genesis.rs index fccdd71817e1fe0fae978bef8a52d166d4aeb1f1..0eecc6cdae8908188b48291edeac9ad3da4c0422 100644 --- a/client/src/genesis.rs +++ b/client/src/genesis.rs @@ -53,7 +53,7 @@ mod tests { runtime::{Hash, Transfer, Block, BlockNumber, Header, Digest}, AccountKeyring, Sr25519Keyring, }; - use sp_core::Blake2Hasher; + use sp_runtime::traits::BlakeTwo256; use hex_literal::*; native_executor_instance!( @@ -67,7 +67,7 @@ mod tests { } fn construct_block( - backend: &InMemoryBackend, + backend: &InMemoryBackend, number: BlockNumber, parent_hash: Hash, state_root: Hash, @@ -78,7 +78,7 @@ mod tests { let transactions = txs.into_iter().map(|tx| tx.into_signed_tx()).collect::>(); let iter = transactions.iter().map(Encode::encode); - let extrinsics_root = Layout::::ordered_trie_root(iter).into(); + let extrinsics_root = Layout::::ordered_trie_root(iter).into(); let mut header = Header { parent_hash, @@ -132,7 +132,7 @@ mod tests { (vec![].and(&Block { header, extrinsics: transactions }), hash) } - fn block1(genesis_hash: Hash, backend: &InMemoryBackend) -> (Vec, Hash) { + fn block1(genesis_hash: Hash, backend: &InMemoryBackend) -> (Vec, Hash) { construct_block( backend, 1, diff --git a/client/src/in_mem.rs b/client/src/in_mem.rs index dcff8102aeb6d497d72f367c5c33d0eb3a726685..bdbfdbc7ec8f03cba799075dd84274860b018ab3 100644 --- a/client/src/in_mem.rs +++ b/client/src/in_mem.rs @@ -24,7 +24,7 @@ use sp_core::offchain::storage::{ InMemOffchainStorage as OffchainStorage }; use sp_runtime::generic::BlockId; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor, HasherFor}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor, HashFor}; use sp_runtime::{Justification, Storage}; use sp_state_machine::{ ChangesTrieTransaction, InMemoryBackend, Backend as StateBackend, StorageCollection, @@ -462,8 +462,8 @@ impl sc_client_api::light::Storage for Blockchain pub struct BlockImportOperation { pending_block: Option>, pending_cache: HashMap>, - old_state: InMemoryBackend>, - new_state: Option>>, + old_state: InMemoryBackend>, + new_state: Option>>, aux: Vec<(Vec, Option>)>, finalized_blocks: Vec<(BlockId, Option)>, set_head: Option>, @@ -472,7 +472,7 @@ pub struct BlockImportOperation { impl backend::BlockImportOperation for BlockImportOperation where Block::Hash: Ord, { - type State = InMemoryBackend>; + type State = InMemoryBackend>; fn state(&self) -> sp_blockchain::Result> { Ok(Some(&self.old_state)) @@ -499,7 +499,7 @@ impl backend::BlockImportOperation for BlockImportOperatio fn update_db_storage( &mut self, - update: > as StateBackend>>::Transaction, + update: > as StateBackend>>::Transaction, ) -> sp_blockchain::Result<()> { self.new_state = Some(self.old_state.update(update)); Ok(()) @@ -507,7 +507,7 @@ impl backend::BlockImportOperation for BlockImportOperatio fn update_changes_trie( &mut self, - _update: ChangesTrieTransaction, NumberFor>, + _update: ChangesTrieTransaction, NumberFor>, ) -> sp_blockchain::Result<()> { Ok(()) } @@ -564,7 +564,7 @@ impl backend::BlockImportOperation for BlockImportOperatio /// > **Warning**: Doesn't support all the features necessary for a proper database. Only use this /// > struct for testing purposes. Do **NOT** use in production. pub struct Backend where Block::Hash: Ord { - states: RwLock>>>, + states: RwLock>>>, blockchain: Blockchain, import_lock: RwLock<()>, } @@ -599,7 +599,7 @@ impl backend::AuxStore for Backend where Block::Hash: Ord impl backend::Backend for Backend where Block::Hash: Ord { type BlockImportOperation = BlockImportOperation; type Blockchain = Blockchain; - type State = InMemoryBackend>; + type State = InMemoryBackend>; type OffchainStorage = OffchainStorage; fn begin_operation(&self) -> sp_blockchain::Result { diff --git a/client/src/lib.rs b/client/src/lib.rs index d97246d478c2b6df0eb13128c4ceba23d9ccd6b0..9c6393314f06a0a57fe2c75c981ba1f938ad383d 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -47,7 +47,6 @@ //! ``` //! use std::sync::Arc; //! use sc_client::{Client, in_mem::Backend, LocalCallExecutor}; -//! use sp_core::Blake2Hasher; //! use sp_runtime::Storage; //! use sc_executor::{NativeExecutor, WasmExecutionMethod}; //! @@ -69,6 +68,7 @@ //! Default::default(), //! Default::default(), //! Default::default(), +//! None, //! ); //! ``` //! @@ -98,7 +98,7 @@ pub use crate::{ client::{ new_with_backend, new_in_mem, - BlockBody, ImportNotifications, FinalityNotifications, BlockchainEvents, + BlockBody, ImportNotifications, FinalityNotifications, BlockchainEvents, LockImportRun, BlockImportNotification, Client, ClientInfo, ExecutionStrategies, FinalityNotification, LongestChain, BlockOf, ProvideUncles, BadBlocks, ForkBlocks, apply_aux, }, diff --git a/client/src/light/backend.rs b/client/src/light/backend.rs index ad9f43587e4cdba28e50a79b3237a703100bdbde..6b5f9263009a2d370d814a545e2a3cde700f0123 100644 --- a/client/src/light/backend.rs +++ b/client/src/light/backend.rs @@ -31,7 +31,7 @@ use sp_state_machine::{ StorageCollection, ChildStorageCollection, }; use sp_runtime::{generic::BlockId, Justification, Storage}; -use sp_runtime::traits::{Block as BlockT, NumberFor, Zero, Header, HasherFor}; +use sp_runtime::traits::{Block as BlockT, NumberFor, Zero, Header, HashFor}; use crate::in_mem::check_genesis_storage; use sp_blockchain::{Error as ClientError, Result as ClientResult}; use sc_client_api::{ @@ -65,7 +65,7 @@ pub struct ImportOperation { aux_ops: Vec<(Vec, Option>)>, finalized_blocks: Vec>, set_head: Option>, - storage_update: Option>>, + storage_update: Option>>, changes_trie_config_update: Option>, _phantom: std::marker::PhantomData, } @@ -111,7 +111,7 @@ impl AuxStore for Backend { } } -impl ClientBackend for Backend> +impl ClientBackend for Backend> where Block: BlockT, S: BlockchainStorage, @@ -119,7 +119,7 @@ impl ClientBackend for Backend> { type BlockImportOperation = ImportOperation; type Blockchain = Blockchain; - type State = GenesisOrUnavailableState>; + type State = GenesisOrUnavailableState>; type OffchainStorage = InMemOffchainStorage; fn begin_operation(&self) -> ClientResult { @@ -238,7 +238,7 @@ impl ClientBackend for Backend> } } -impl RemoteBackend for Backend> +impl RemoteBackend for Backend> where Block: BlockT, S: BlockchainStorage + 'static, @@ -262,7 +262,7 @@ impl BlockImportOperation for ImportOperation S: BlockchainStorage, Block::Hash: Ord, { - type State = GenesisOrUnavailableState>; + type State = GenesisOrUnavailableState>; fn state(&self) -> ClientResult> { // None means 'locally-stateless' backend @@ -287,7 +287,7 @@ impl BlockImportOperation for ImportOperation fn update_db_storage( &mut self, - _update: >>::Transaction, + _update: >>::Transaction, ) -> ClientResult<()> { // we're not storing anything locally => ignore changes Ok(()) @@ -295,7 +295,7 @@ impl BlockImportOperation for ImportOperation fn update_changes_trie( &mut self, - _update: ChangesTrieTransaction, NumberFor>, + _update: ChangesTrieTransaction, NumberFor>, ) -> ClientResult<()> { // we're not storing anything locally => ignore changes Ok(()) @@ -515,10 +515,10 @@ impl StateBackend for GenesisOrUnavailableState #[cfg(test)] mod tests { - use sp_core::Blake2Hasher; use substrate_test_runtime_client::{self, runtime::Block}; use sc_client_api::backend::NewBlockState; use crate::light::blockchain::tests::{DummyBlockchain, DummyStorage}; + use sp_runtime::traits::BlakeTwo256; use super::*; #[test] @@ -526,7 +526,9 @@ mod tests { let def = Default::default(); let header0 = substrate_test_runtime_client::runtime::Header::new(0, def, def, def, Default::default()); - let backend: Backend<_, Blake2Hasher> = Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new()))); + let backend: Backend<_, BlakeTwo256> = Backend::new( + Arc::new(DummyBlockchain::new(DummyStorage::new())), + ); let mut op = backend.begin_operation().unwrap(); op.set_block_data(header0, None, None, NewBlockState::Final).unwrap(); op.reset_storage(Default::default()).unwrap(); @@ -540,7 +542,9 @@ mod tests { #[test] fn unavailable_state_is_created_when_genesis_state_is_unavailable() { - let backend: Backend<_, Blake2Hasher> = Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new()))); + let backend: Backend<_, BlakeTwo256> = Backend::new( + Arc::new(DummyBlockchain::new(DummyStorage::new())), + ); match backend.state_at(BlockId::Number(0)).unwrap() { GenesisOrUnavailableState::Unavailable => (), diff --git a/client/src/light/call_executor.rs b/client/src/light/call_executor.rs index 01a93c78219bc836a6c9dc89506b66aa55889cf1..cae5d5a0aa8e290b3a372b20eb8212076965dc3c 100644 --- a/client/src/light/call_executor.rs +++ b/client/src/light/call_executor.rs @@ -23,7 +23,7 @@ use std::{ use codec::{Encode, Decode}; use sp_core::{convert_hash, NativeOrEncoded, traits::CodeExecutor}; use sp_runtime::{ - generic::BlockId, traits::{One, Block as BlockT, Header as HeaderT, HasherFor}, + generic::BlockId, traits::{One, Block as BlockT, Header as HeaderT, HashFor}, }; use sp_externalities::Extensions; use sp_state_machine::{ @@ -40,7 +40,7 @@ use sp_blockchain::{Error as ClientError, Result as ClientResult}; use sc_client_api::{ backend::RemoteBackend, light::RemoteCallRequest, - call_executor::CallExecutor + call_executor::CallExecutor, }; use sc_executor::{RuntimeVersion, NativeVersion}; @@ -153,9 +153,9 @@ impl CallExecutor for } } - fn prove_at_trie_state>>( + fn prove_at_trie_state>>( &self, - _state: &sp_state_machine::TrieBackend>, + _state: &sp_state_machine::TrieBackend>, _changes: &mut OverlayedChanges, _method: &str, _call_data: &[u8], @@ -181,7 +181,7 @@ pub fn prove_execution( ) -> ClientResult<(Vec, StorageProof)> where Block: BlockT, - S: StateBackend>, + S: StateBackend>, E: CallExecutor, { let trie_state = state.as_trie_backend() @@ -285,9 +285,11 @@ mod tests { runtime::{Header, Digest, Block}, TestClient, ClientBlockImportExt, }; use sc_executor::{NativeExecutor, WasmExecutionMethod}; - use sp_core::{Blake2Hasher, H256}; + use sp_core::H256; use sc_client_api::backend::{Backend, NewBlockState}; use crate::in_mem::Backend as InMemBackend; + use sc_client_api::ProofProvider; + use sp_runtime::traits::BlakeTwo256; struct DummyCallExecutor; @@ -342,9 +344,9 @@ mod tests { unreachable!() } - fn prove_at_trie_state>>( + fn prove_at_trie_state>>( &self, - _trie_state: &sp_state_machine::TrieBackend>, + _trie_state: &sp_state_machine::TrieBackend>, _overlay: &mut OverlayedChanges, _method: &str, _call_data: &[u8] @@ -375,7 +377,7 @@ mod tests { ).unwrap(); // check remote execution proof locally - let local_result = check_execution_proof::<_, _, Blake2Hasher>( + let local_result = check_execution_proof::<_, _, BlakeTwo256>( &local_executor(), &RemoteCallRequest { block: substrate_test_runtime_client::runtime::Hash::default(), @@ -402,7 +404,7 @@ mod tests { ).unwrap(); // check remote execution proof locally - let execution_result = check_execution_proof_with_make_header::<_, _, Blake2Hasher, _>( + let execution_result = check_execution_proof_with_make_header::<_, _, BlakeTwo256, _>( &local_executor(), &RemoteCallRequest { block: substrate_test_runtime_client::runtime::Hash::default(), diff --git a/client/src/light/fetcher.rs b/client/src/light/fetcher.rs index 9df6a38630684f8f545f2a4f1175fa3b21d386f6..87e21e3c0e10e6709283296febe391ad670705a6 100644 --- a/client/src/light/fetcher.rs +++ b/client/src/light/fetcher.rs @@ -341,20 +341,21 @@ pub mod tests { }; use sp_consensus::BlockOrigin; - use crate::in_mem::{Blockchain as InMemoryBlockchain}; + use crate::in_mem::Blockchain as InMemoryBlockchain; use crate::light::fetcher::{FetchChecker, LightDataChecker, RemoteHeaderRequest}; use crate::light::blockchain::tests::{DummyStorage, DummyBlockchain}; - use sp_core::{blake2_256, Blake2Hasher, ChangesTrieConfiguration, H256}; + use sp_core::{blake2_256, ChangesTrieConfiguration, H256}; use sp_core::storage::{well_known_keys, StorageKey, ChildInfo}; - use sp_runtime::generic::BlockId; + use sp_runtime::{generic::BlockId, traits::BlakeTwo256}; use sp_state_machine::Backend; use super::*; + use sc_client_api::{StorageProvider, ProofProvider}; const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); type TestChecker = LightDataChecker< NativeExecutor, - Blake2Hasher, + BlakeTwo256, Block, DummyStorage, >; @@ -378,7 +379,7 @@ pub mod tests { .and_then(|v| Decode::decode(&mut &v.0[..]).ok()).unwrap(); let remote_read_proof = remote_client.read_proof( &remote_block_id, - &[well_known_keys::HEAP_PAGES], + &mut std::iter::once(well_known_keys::HEAP_PAGES), ).unwrap(); // check remote read proof locally @@ -426,7 +427,7 @@ pub mod tests { &remote_block_id, b":child_storage:default:child1", CHILD_INFO_1, - &[b"key1"], + &mut std::iter::once("key1".as_bytes()), ).unwrap(); // check locally @@ -464,7 +465,7 @@ pub mod tests { // check remote read proof locally let local_storage = InMemoryBlockchain::::new(); - let local_cht_root = cht::compute_root::(4, 0, local_headers_hashes).unwrap(); + let local_cht_root = cht::compute_root::(4, 0, local_headers_hashes).unwrap(); if insert_cht { local_storage.insert_cht_root(1, local_cht_root); } @@ -478,7 +479,7 @@ pub mod tests { fn header_with_computed_extrinsics_root(extrinsics: Vec) -> Header { use sp_trie::{TrieConfiguration, trie_types::Layout}; let iter = extrinsics.iter().map(Encode::encode); - let extrinsics_root = Layout::::ordered_trie_root(iter); + let extrinsics_root = Layout::::ordered_trie_root(iter); // only care about `extrinsics_root` Header::new(0, extrinsics_root, H256::zero(), H256::zero(), Default::default()) @@ -624,7 +625,7 @@ pub mod tests { ).unwrap(); // prepare local checker, having a root of changes trie CHT#0 - let local_cht_root = cht::compute_root::(4, 0, remote_roots.iter().cloned().map(|ct| Ok(Some(ct)))).unwrap(); + let local_cht_root = cht::compute_root::(4, 0, remote_roots.iter().cloned().map(|ct| Ok(Some(ct)))).unwrap(); let mut local_storage = DummyStorage::new(); local_storage.changes_tries_cht_roots.insert(0, local_cht_root); let local_checker = TestChecker::new( @@ -731,7 +732,7 @@ pub mod tests { // we're testing this test case here: // (1, 4, dave.clone(), vec![(4, 0), (1, 1), (1, 0)]), let (remote_client, remote_roots, _) = prepare_client_with_key_changes(); - let local_cht_root = cht::compute_root::( + let local_cht_root = cht::compute_root::( 4, 0, remote_roots.iter().cloned().map(|ct| Ok(Some(ct)))).unwrap(); let dave = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec(); let dave = StorageKey(dave); diff --git a/client/src/light/mod.rs b/client/src/light/mod.rs index d65fdef71193db17135fc324f3bb3bdeead46409..07816b3b3569252e6ab15023db74eabde75d7577 100644 --- a/client/src/light/mod.rs +++ b/client/src/light/mod.rs @@ -26,8 +26,9 @@ use std::sync::Arc; use sc_executor::RuntimeInfo; use sp_core::traits::CodeExecutor; use sp_runtime::BuildStorage; -use sp_runtime::traits::{Block as BlockT, HasherFor}; +use sp_runtime::traits::{Block as BlockT, HashFor}; use sp_blockchain::Result as ClientResult; +use prometheus_endpoint::Registry; use crate::call_executor::LocalCallExecutor; use crate::client::Client; @@ -45,7 +46,7 @@ pub fn new_light_blockchain>(storage: S) -> A } /// Create an instance of light client backend. -pub fn new_light_backend(blockchain: Arc>) -> Arc>> +pub fn new_light_backend(blockchain: Arc>) -> Arc>> where B: BlockT, S: BlockchainStorage, @@ -55,15 +56,16 @@ pub fn new_light_backend(blockchain: Arc>) -> Arc( - backend: Arc>>, + backend: Arc>>, genesis_storage: &GS, code_executor: E, + prometheus_registry: Option, ) -> ClientResult< Client< - Backend>, + Backend>, GenesisCallExecutor< - Backend>, - LocalCallExecutor>, E> + Backend>, + LocalCallExecutor>, E> >, B, RA @@ -84,6 +86,7 @@ pub fn new_light( Default::default(), Default::default(), Default::default(), + prometheus_registry, ) } @@ -91,7 +94,7 @@ pub fn new_light( pub fn new_fetch_checker>( blockchain: Arc>, executor: E, -) -> LightDataChecker, B, S> +) -> LightDataChecker, B, S> where E: CodeExecutor, { diff --git a/client/state-db/Cargo.toml b/client/state-db/Cargo.toml index 1bfa13693780a5e549d8d1fa82c5436ab54e4c98..7c7823b534ea27f2542d23ebeecf3a8642172303 100644 --- a/client/state-db/Cargo.toml +++ b/client/state-db/Cargo.toml @@ -1,15 +1,21 @@ [package] name = "sc-state-db" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "State database maintenance. Handles canonicalization and pruning in the database." [dependencies] parking_lot = "0.10.0" log = "0.4.8" -sp-core = { version = "2.0.0", path = "../../primitives/core" } -codec = { package = "parity-scale-codec", version = "1.0.0", features = ["derive"] } +sc-client-api = { version = "2.0.0-alpha.2", path = "../api" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } +parity-util-mem = "0.5.1" +parity-util-mem-derive = "0.1.0" [dev-dependencies] env_logger = "0.7.0" diff --git a/client/state-db/src/lib.rs b/client/state-db/src/lib.rs index 0eab640de84e898e41c229890effbe6975d214e0..49b1a59285e11eb332bb099b7839f8b7383f5439 100644 --- a/client/state-db/src/lib.rs +++ b/client/state-db/src/lib.rs @@ -31,7 +31,8 @@ mod noncanonical; mod pruning; -#[cfg(test)] mod test; +#[cfg(test)] +mod test; use std::fmt; use parking_lot::RwLock; @@ -40,6 +41,8 @@ use std::collections::{HashMap, hash_map::Entry}; use noncanonical::NonCanonicalOverlay; use pruning::RefWindow; use log::trace; +use parity_util_mem::{MallocSizeOf, malloc_size}; +use sc_client_api::{StateDbMemoryInfo, MemorySize}; const PRUNING_MODE: &[u8] = b"mode"; const PRUNING_MODE_ARCHIVE: &[u8] = b"archive"; @@ -120,7 +123,6 @@ pub struct ChangeSet { pub deleted: Vec, } - /// A set of changes to the backing database. #[derive(Default, Debug, Clone)] pub struct CommitSet { @@ -196,8 +198,11 @@ struct StateDbSync { pinned: HashMap, } -impl StateDbSync { - pub fn new(mode: PruningMode, db: &D) -> Result, Error> { +impl StateDbSync { + fn new( + mode: PruningMode, + db: &D, + ) -> Result, Error> { trace!(target: "state-db", "StateDb settings: {:?}", mode); // Check that settings match @@ -234,7 +239,13 @@ impl StateDbSync { } } - pub fn insert_block(&mut self, hash: &BlockHash, number: u64, parent_hash: &BlockHash, mut changeset: ChangeSet) -> Result, Error> { + fn insert_block( + &mut self, + hash: &BlockHash, + number: u64, + parent_hash: &BlockHash, + mut changeset: ChangeSet, + ) -> Result, Error> { let mut meta = ChangeSet::default(); if number == 0 { // Save pruning mode when writing first block. @@ -247,7 +258,7 @@ impl StateDbSync { // write changes immediately Ok(CommitSet { data: changeset, - meta: meta, + meta, }) }, PruningMode::Constrained(_) | PruningMode::ArchiveCanonical => { @@ -260,7 +271,10 @@ impl StateDbSync { } } - pub fn canonicalize_block(&mut self, hash: &BlockHash) -> Result, Error> { + fn canonicalize_block( + &mut self, + hash: &BlockHash, + ) -> Result, Error> { let mut commit = CommitSet::default(); if self.mode == PruningMode::ArchiveAll { return Ok(commit) @@ -280,18 +294,23 @@ impl StateDbSync { Ok(commit) } - pub fn best_canonical(&self) -> Option { + fn best_canonical(&self) -> Option { return self.non_canonical.last_canonicalized_block_number() } - pub fn is_pruned(&self, hash: &BlockHash, number: u64) -> bool { + fn is_pruned(&self, hash: &BlockHash, number: u64) -> bool { match self.mode { PruningMode::ArchiveAll => false, PruningMode::ArchiveCanonical | PruningMode::Constrained(_) => { if self.best_canonical().map(|c| number > c).unwrap_or(true) { !self.non_canonical.have_block(hash) } else { - self.pruning.as_ref().map_or(false, |pruning| number < pruning.pending() || !pruning.have_block(hash)) + self.pruning + .as_ref() + .map_or( + false, + |pruning| number < pruning.pending() || !pruning.have_block(hash), + ) } } } @@ -320,7 +339,7 @@ impl StateDbSync { /// Revert all non-canonical blocks with the best block number. /// Returns a database commit or `None` if not possible. /// For archive an empty commit set is returned. - pub fn revert_one(&mut self) -> Option> { + fn revert_one(&mut self) -> Option> { match self.mode { PruningMode::ArchiveAll => { Some(CommitSet::default()) @@ -331,7 +350,7 @@ impl StateDbSync { } } - pub fn pin(&mut self, hash: &BlockHash) -> Result<(), PinError> { + fn pin(&mut self, hash: &BlockHash) -> Result<(), PinError> { match self.mode { PruningMode::ArchiveAll => Ok(()), PruningMode::ArchiveCanonical | PruningMode::Constrained(_) => { @@ -340,7 +359,7 @@ impl StateDbSync { { let refs = self.pinned.entry(hash.clone()).or_default(); if *refs == 0 { - trace!(target: "state-db", "Pinned block: {:?}", hash); + trace!(target: "state-db-pin", "Pinned block: {:?}", hash); self.non_canonical.pin(hash); } *refs += 1; @@ -352,16 +371,16 @@ impl StateDbSync { } } - pub fn unpin(&mut self, hash: &BlockHash) { + fn unpin(&mut self, hash: &BlockHash) { match self.pinned.entry(hash.clone()) { Entry::Occupied(mut entry) => { *entry.get_mut() -= 1; if *entry.get() == 0 { - trace!(target: "state-db", "Unpinned block: {:?}", hash); + trace!(target: "state-db-pin", "Unpinned block: {:?}", hash); entry.remove(); self.non_canonical.unpin(hash); } else { - trace!(target: "state-db", "Releasing reference for {:?}", hash); + trace!(target: "state-db-pin", "Releasing reference for {:?}", hash); } }, Entry::Vacant(_) => {}, @@ -377,12 +396,14 @@ impl StateDbSync { db.get(key.as_ref()).map_err(|e| Error::Db(e)) } - pub fn apply_pending(&mut self) { + fn apply_pending(&mut self) { self.non_canonical.apply_pending(); if let Some(pruning) = &mut self.pruning { pruning.apply_pending(); } - trace!(target: "forks", "First available: {:?} ({}), Last canon: {:?} ({}), Best forks: {:?}", + trace!( + target: "forks", + "First available: {:?} ({}), Last canon: {:?} ({}), Best forks: {:?}", self.pruning.as_ref().and_then(|p| p.next_hash()), self.pruning.as_ref().map(|p| p.pending()).unwrap_or(0), self.non_canonical.last_canonicalized_hash(), @@ -391,12 +412,20 @@ impl StateDbSync { ); } - pub fn revert_pending(&mut self) { + fn revert_pending(&mut self) { if let Some(pruning) = &mut self.pruning { pruning.revert_pending(); } self.non_canonical.revert_pending(); } + + fn memory_info(&self) -> StateDbMemoryInfo { + StateDbMemoryInfo { + non_canonical: MemorySize::from_bytes(malloc_size(&self.non_canonical)), + pruning: self.pruning.as_ref().map(|p| MemorySize::from_bytes(malloc_size(p))), + pinned: MemorySize::from_bytes(malloc_size(&self.pinned)), + } + } } /// State DB maintenance. See module description. @@ -405,21 +434,33 @@ pub struct StateDb { db: RwLock>, } -impl StateDb { +impl StateDb { /// Creates a new instance. Does not expect any metadata in the database. - pub fn new(mode: PruningMode, db: &D) -> Result, Error> { + pub fn new( + mode: PruningMode, + db: &D, + ) -> Result, Error> { Ok(StateDb { db: RwLock::new(StateDbSync::new(mode, db)?) }) } /// Add a new non-canonical block. - pub fn insert_block(&self, hash: &BlockHash, number: u64, parent_hash: &BlockHash, changeset: ChangeSet) -> Result, Error> { + pub fn insert_block( + &self, + hash: &BlockHash, + number: u64, + parent_hash: &BlockHash, + changeset: ChangeSet, + ) -> Result, Error> { self.db.write().insert_block(hash, number, parent_hash, changeset) } /// Finalize a previously inserted block. - pub fn canonicalize_block(&self, hash: &BlockHash) -> Result, Error> { + pub fn canonicalize_block( + &self, + hash: &BlockHash, + ) -> Result, Error> { self.db.write().canonicalize_block(hash) } @@ -466,6 +507,11 @@ impl StateDb { pub fn revert_pending(&self) { self.db.write().revert_pending(); } + + /// Returns the current memory statistics of this instance. + pub fn memory_info(&self) -> StateDbMemoryInfo { + self.db.read().memory_info() + } } #[cfg(test)] diff --git a/client/state-db/src/noncanonical.rs b/client/state-db/src/noncanonical.rs index db2f58fa8981d2231a750f8c68472c92f278ff87..6a34523b66fff9e510518a4db4a884b5a693daf4 100644 --- a/client/state-db/src/noncanonical.rs +++ b/client/state-db/src/noncanonical.rs @@ -30,6 +30,7 @@ const NON_CANONICAL_JOURNAL: &[u8] = b"noncanonical_journal"; const LAST_CANONICAL: &[u8] = b"last_canonical"; /// See module documentation. +#[derive(parity_util_mem_derive::MallocSizeOf)] pub struct NonCanonicalOverlay { last_canonicalized: Option<(BlockHash, u64)>, levels: VecDeque>>, @@ -55,6 +56,7 @@ fn to_journal_key(block: u64, index: u64) -> Vec { } #[cfg_attr(test, derive(PartialEq, Debug))] +#[derive(parity_util_mem_derive::MallocSizeOf)] struct BlockOverlay { hash: BlockHash, journal_key: Vec, @@ -99,8 +101,10 @@ fn discard_descendants( let mut discarded = Vec::new(); if let Some(level) = levels.get_mut(index) { *level = level.drain(..).filter_map(|overlay| { - let parent = parents.get(&overlay.hash).expect("there is a parent entry for each entry in levels; qed").clone(); - if parent == *hash { + let parent = parents.get(&overlay.hash) + .expect("there is a parent entry for each entry in levels; qed"); + + if parent == hash { discarded.push(overlay.hash.clone()); if pinned.contains_key(&overlay.hash) { // save to be discarded later. @@ -375,7 +379,7 @@ impl NonCanonicalOverlay { None } - /// Check if the block is in the canonicalization queue. + /// Check if the block is in the canonicalization queue. pub fn have_block(&self, hash: &BlockHash) -> bool { (self.parents.contains_key(hash) || self.pending_insertions.contains(hash)) && !self.pending_canonicalizations.contains(hash) @@ -436,7 +440,7 @@ impl NonCanonicalOverlay { while let Some(hash) = parent { let refs = self.pinned.entry(hash.clone()).or_default(); if *refs == 0 { - trace!(target: "state-db", "Pinned non-canon block: {:?}", hash); + trace!(target: "state-db-pin", "Pinned non-canon block: {:?}", hash); } *refs += 1; parent = self.parents.get(hash); @@ -455,7 +459,7 @@ impl NonCanonicalOverlay { if *entry.get() == 0 { entry.remove(); if let Some(inserted) = self.pinned_insertions.remove(&hash) { - trace!(target: "state-db", "Discarding unpinned non-canon block: {:?}", hash); + trace!(target: "state-db-pin", "Discarding unpinned non-canon block: {:?}", hash); discard_values(&mut self.values, inserted); self.parents.remove(&hash); } diff --git a/client/state-db/src/pruning.rs b/client/state-db/src/pruning.rs index 71d018087b5903cfdb8ccc24999612fcfb12c02f..6cf5f260060f500432ff306eb2e3f3c6a69f9089 100644 --- a/client/state-db/src/pruning.rs +++ b/client/state-db/src/pruning.rs @@ -31,6 +31,7 @@ const LAST_PRUNED: &[u8] = b"last_pruned"; const PRUNING_JOURNAL: &[u8] = b"pruning_journal"; /// See module documentation. +#[derive(parity_util_mem_derive::MallocSizeOf)] pub struct RefWindow { /// A queue of keys that should be deleted for each block in the pruning window. death_rows: VecDeque>, @@ -46,7 +47,7 @@ pub struct RefWindow { pending_prunings: usize, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, parity_util_mem_derive::MallocSizeOf)] struct DeathRow { hash: BlockHash, journal_key: Vec, diff --git a/client/telemetry/Cargo.toml b/client/telemetry/Cargo.toml index 10f4b188af6b994f5f7297022ae84c53db5d9a30..248c90ec68623f0320815fbf25511d07304dc31d 100644 --- a/client/telemetry/Cargo.toml +++ b/client/telemetry/Cargo.toml @@ -1,10 +1,14 @@ [package] name = "sc-telemetry" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] description = "Telemetry utils" edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +documentation = "https://docs.rs/sc-telemetry" + [dependencies] bytes = "0.5" @@ -12,7 +16,7 @@ parking_lot = "0.10.0" futures = "0.3.1" futures-timer = "3.0.1" wasm-timer = "0.2.0" -libp2p = { version = "0.16.0", default-features = false, features = ["libp2p-websocket"] } +libp2p = { version = "0.16.2", default-features = false, features = ["libp2p-websocket"] } log = "0.4.8" pin-project = "0.4.6" rand = "0.7.2" diff --git a/client/tracing/Cargo.toml b/client/tracing/Cargo.toml index b476db6a0113d73167196b22f839ec2d0844e282..ad988f0607953fb921d443c15315741dd1820054 100644 --- a/client/tracing/Cargo.toml +++ b/client/tracing/Cargo.toml @@ -1,9 +1,12 @@ [package] name = "sc-tracing" -version = "2.0.0" +version = "2.0.0-alpha.3" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Instrumentation implementation for substrate." [dependencies] erased-serde = "0.3.9" @@ -14,8 +17,7 @@ serde_json = "1.0.41" slog = { version = "2.5.2", features = ["nested-values"] } tracing-core = "0.1.7" -sc-telemetry = { version = "2.0.0", path = "../telemetry" } -grafana-data-source = { version = "0.8", path = "../../utils/grafana-data-source" } +sc-telemetry = { version = "2.0.0-alpha.2", path = "../telemetry" } [dev-dependencies] tracing = "0.1.10" diff --git a/client/tracing/src/lib.rs b/client/tracing/src/lib.rs index cd301041d39af4854522e3aa49608b93b3d28bd0..c00bca9275eec7fa2bf99050b6413c99f846bc78 100644 --- a/client/tracing/src/lib.rs +++ b/client/tracing/src/lib.rs @@ -34,7 +34,7 @@ //! let span = tracing::span!(tracing::Level::INFO, "my_span_name", my_number = 10, a_key = "a value"); //! let _guard = span.enter(); //! ``` -//! Currently we provide `Log` (default), `Telemetry` and `Grafana` variants for `Receiver` +//! Currently we provide `Log` (default), `Telemetry` variants for `Receiver` use std::collections::HashMap; use std::fmt; @@ -53,7 +53,6 @@ use tracing_core::{ subscriber::Subscriber }; -use grafana_data_source::{self, record_metrics}; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; /// Used to configure how to receive the metrics @@ -63,8 +62,6 @@ pub enum TracingReceiver { Log, /// Output to telemetry Telemetry, - /// Output to Grafana - Grafana, } impl Default for TracingReceiver { @@ -255,7 +252,6 @@ impl ProfilingSubscriber { match self.receiver { TracingReceiver::Log => print_log(span_datum), TracingReceiver::Telemetry => send_telemetry(span_datum), - TracingReceiver::Grafana => send_grafana(span_datum), } } } @@ -291,9 +287,3 @@ fn send_telemetry(span_datum: SpanDatum) { ); } -fn send_grafana(span_datum: SpanDatum) { - let name = format!("{}::{}", span_datum.target, span_datum.name); - if let Err(e) = record_metrics!(&name => span_datum.overall_time.as_nanos(),) { - log::warn!("Unable to send metrics to grafana: {:?}", e); - } -} diff --git a/client/transaction-pool/Cargo.toml b/client/transaction-pool/Cargo.toml index 7ccc98a9c09e92f63815e76f3bddc61b8d2c35a9..3d80e06c955f75317fe03df0a8800e27fff7bf0d 100644 --- a/client/transaction-pool/Cargo.toml +++ b/client/transaction-pool/Cargo.toml @@ -1,29 +1,32 @@ [package] name = "sc-transaction-pool" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Substrate transaction pool implementation." [dependencies] -codec = { package = "parity-scale-codec", version = "1.0.0" } +codec = { package = "parity-scale-codec", version = "1.2.0" } derive_more = "0.99.2" futures = { version = "0.3.1", features = ["compat"] } futures-diagnose = "1.0" log = "0.4.8" parking_lot = "0.10.0" wasm-timer = "0.2" -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-api = { version = "2.0.0", path = "../../primitives/api" } -sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } -sc-transaction-graph = { version = "2.0.0", path = "./graph" } -sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } -sc-client-api = { version = "2.0.0", path = "../api" } -sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +sp-api = { version = "2.0.0-alpha.2", path = "../../primitives/api" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime" } +sc-transaction-graph = { version = "2.0.0-alpha.2", path = "./graph" } +sp-transaction-pool = { version = "2.0.0-alpha.2", path = "../../primitives/transaction-pool" } +sc-client-api = { version = "2.0.0-alpha.2", path = "../api" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../primitives/blockchain" } futures-timer = "2.0" parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } [dev-dependencies] -sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } -substrate-test-runtime-transaction-pool = { version = "2.0.0", path = "../../test-utils/runtime/transaction-pool" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } +sp-keyring = { version = "2.0.0-alpha.2", path = "../../primitives/keyring" } +substrate-test-runtime-transaction-pool = { version = "2.0.0-dev", path = "../../test-utils/runtime/transaction-pool" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } diff --git a/client/transaction-pool/graph/Cargo.toml b/client/transaction-pool/graph/Cargo.toml index daec970a69493a5e9f49272c5a3bd3cae0bd9954..10846acfea18825bcb2215f49e0d22c785bbaf94 100644 --- a/client/transaction-pool/graph/Cargo.toml +++ b/client/transaction-pool/graph/Cargo.toml @@ -1,9 +1,12 @@ [package] name = "sc-transaction-graph" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Generic Transaction Pool" [dependencies] derive_more = "0.99.2" @@ -12,17 +15,17 @@ log = "0.4.8" parking_lot = "0.10.0" serde = { version = "1.0.101", features = ["derive"] } wasm-timer = "0.2" -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-transaction-pool = { version = "2.0.0", path = "../../../primitives/transaction-pool" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../../primitives/blockchain" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +sp-transaction-pool = { version = "2.0.0-alpha.2", path = "../../../primitives/transaction-pool" } parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } linked-hash-map = "0.5.2" [dev-dependencies] assert_matches = "1.3.0" -codec = { package = "parity-scale-codec", version = "1.0.0" } -substrate-test-runtime = { version = "2.0.0", path = "../../../test-utils/runtime" } +codec = { package = "parity-scale-codec", version = "1.2.0" } +substrate-test-runtime = { version = "2.0.0-dev", path = "../../../test-utils/runtime" } criterion = "0.3" [[bench]] diff --git a/client/transaction-pool/graph/benches/basics.rs b/client/transaction-pool/graph/benches/basics.rs index 54bbe930b393bd37a94cb648272f1a66199cb691..6f5d39f09f5c540a31762cd129e89f48bb4c66d1 100644 --- a/client/transaction-pool/graph/benches/basics.rs +++ b/client/transaction-pool/graph/benches/basics.rs @@ -113,7 +113,11 @@ impl ChainApi for TestApi { } fn uxt(transfer: Transfer) -> Extrinsic { - Extrinsic::Transfer(transfer, Default::default()) + Extrinsic::Transfer { + transfer, + signature: Default::default(), + exhaust_resources_when_not_first: false, + } } fn bench_configured(pool: Pool, number: u64) { diff --git a/client/transaction-pool/graph/src/pool.rs b/client/transaction-pool/graph/src/pool.rs index 12601bb8d21be1e3349cf4eb2e6879215ac983a9..f0bf17dcb8dd2ecb2091dc9a799c06e8ac90f879 100644 --- a/client/transaction-pool/graph/src/pool.rs +++ b/client/transaction-pool/graph/src/pool.rs @@ -524,7 +524,11 @@ mod tests { } fn uxt(transfer: Transfer) -> Extrinsic { - Extrinsic::Transfer(transfer, Default::default()) + Extrinsic::Transfer { + transfer, + signature: Default::default(), + exhaust_resources_when_not_first: false, + } } fn pool() -> Pool { diff --git a/client/transaction-pool/src/api.rs b/client/transaction-pool/src/api.rs index 84e06cc33e4f5c83e001dce41d68e882fd2657c7..d14f532435f3f2fe2bdcf1a5a853013d3f00aa90 100644 --- a/client/transaction-pool/src/api.rs +++ b/client/transaction-pool/src/api.rs @@ -27,7 +27,6 @@ use sc_client_api::{ light::{Fetcher, RemoteCallRequest, RemoteBodyRequest}, BlockBody, }; -use sp_core::Hasher; use sp_runtime::{ generic::BlockId, traits::{self, Block as BlockT, BlockIdTo, Header as HeaderT, Hash as HashT}, transaction_validity::TransactionValidity, @@ -120,7 +119,7 @@ impl sc_transaction_graph::ChainApi for FullChainApi) -> (Self::Hash, usize) { ex.using_encoded(|x| { - (traits::HasherFor::::hash(x), x.len()) + ( as traits::Hash>::hash(x), x.len()) }) } } diff --git a/client/transaction-pool/src/lib.rs b/client/transaction-pool/src/lib.rs index a69323553fa8f53c33021e482ac2eedfd79a881c..7ee73b862ad464bfcc869548a83e0cb1b26a6440 100644 --- a/client/transaction-pool/src/lib.rs +++ b/client/transaction-pool/src/lib.rs @@ -16,6 +16,7 @@ //! Substrate transaction pool implementation. +#![recursion_limit="256"] #![warn(missing_docs)] #![warn(unused_extern_crates)] diff --git a/client/transaction-pool/src/revalidation.rs b/client/transaction-pool/src/revalidation.rs index dbf8a2935427113916c78ceffcb8bde742da78db..b915f1fe71935e3af5f66fc9bddf788ba153e015 100644 --- a/client/transaction-pool/src/revalidation.rs +++ b/client/transaction-pool/src/revalidation.rs @@ -113,7 +113,9 @@ async fn batch_revalidate( } pool.validated_pool().remove_invalid(&invalid_hashes); - pool.resubmit(revalidated); + if revalidated.len() > 0 { + pool.resubmit(revalidated); + } } impl RevalidationWorker { @@ -149,6 +151,7 @@ impl RevalidationWorker { } else { for xt in &to_queue { extrinsics.remove(xt); + self.members.remove(xt); } } left -= to_queue.len(); @@ -163,6 +166,10 @@ impl RevalidationWorker { queued_exts } + fn len(&self) -> usize { + self.block_ordered.iter().map(|b| b.1.len()).sum() + } + fn push(&mut self, worker_payload: WorkerPayload) { // we don't add something that already scheduled for revalidation let transactions = worker_payload.transactions; @@ -170,7 +177,15 @@ impl RevalidationWorker { for ext_hash in transactions { // we don't add something that already scheduled for revalidation - if self.members.contains_key(&ext_hash) { continue; } + if self.members.contains_key(&ext_hash) { + log::debug!( + target: "txpool", + "[{:?}] Skipped adding for revalidation: Already there.", + ext_hash, + ); + + continue; + } self.block_ordered.entry(block_number) .and_modify(|value| { value.insert(ext_hash.clone()); }) @@ -198,7 +213,18 @@ impl RevalidationWorker { futures::select! { _ = interval.next() => { let next_batch = this.prepare_batch(); + let batch_len = next_batch.len(); + batch_revalidate(this.pool.clone(), this.api.clone(), this.best_block, next_batch).await; + + if batch_len > 0 || this.len() > 0 { + log::debug!( + target: "txpool", + "Revalidated {} transactions. Left in the queue for revalidation: {}.", + batch_len, + this.len(), + ); + } }, workload = from_queue.next() => { match workload { @@ -264,6 +290,10 @@ where /// If queue configured without background worker, this will resolve after /// revalidation is actually done. pub async fn revalidate_later(&self, at: NumberFor, transactions: Vec>) { + if transactions.len() > 0 { + log::debug!(target: "txpool", "Added {} transactions to revalidation queue", transactions.len()); + } + if let Some(ref to_worker) = self.background { if let Err(e) = to_worker.unbounded_send(WorkerPayload { at, transactions }) { log::warn!(target: "txpool", "Failed to update background worker: {:?}", e); diff --git a/client/transaction-pool/src/testing/pool.rs b/client/transaction-pool/src/testing/pool.rs index 37b80df9e1be42d0d9f5c12c1055e4032a4d92af..d9f54ede94a4454174acbdc99b33bf5fa5dd4b58 100644 --- a/client/transaction-pool/src/testing/pool.rs +++ b/client/transaction-pool/src/testing/pool.rs @@ -267,6 +267,34 @@ fn should_not_retain_invalid_hashes_from_retracted() { assert_eq!(pool.status().ready, 0); } +#[test] +fn should_revalidate_transaction_multiple_times() { + let xt = uxt(Alice, 209); + + let (pool, _guard) = maintained_pool(); + + block_on(pool.submit_one(&BlockId::number(0), xt.clone())).expect("1. Imported"); + assert_eq!(pool.status().ready, 1); + + pool.api.push_block(1, vec![xt.clone()]); + + // maintenance is in background + block_on(pool.maintain(block_event(1))); + block_on(futures_timer::Delay::new(BACKGROUND_REVALIDATION_INTERVAL*2)); + + block_on(pool.submit_one(&BlockId::number(0), xt.clone())).expect("1. Imported"); + assert_eq!(pool.status().ready, 1); + + pool.api.push_block(2, vec![]); + pool.api.add_invalid(&xt); + + // maintenance is in background + block_on(pool.maintain(block_event(2))); + block_on(futures_timer::Delay::new(BACKGROUND_REVALIDATION_INTERVAL*2)); + + assert_eq!(pool.status().ready, 0); +} + #[test] fn should_push_watchers_during_maintaince() { fn alice_uxt(nonce: u64) -> Extrinsic { @@ -384,7 +412,7 @@ fn fork_aware_finalization() { let mut canon_watchers = vec![]; let from_alice = uxt(Alice, 1); - let from_dave = uxt(Dave, 1); + let from_dave = uxt(Dave, 2); let from_bob = uxt(Bob, 1); let from_charlie = uxt(Charlie, 1); pool.api.increment_nonce(Alice.into()); @@ -405,6 +433,7 @@ fn fork_aware_finalization() { let watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_alice.clone())).expect("1. Imported"); let header = pool.api.push_block(2, vec![from_alice.clone()]); canon_watchers.push((watcher, header.hash())); + assert_eq!(pool.status().ready, 1); let event = ChainEvent::NewBlock { id: BlockId::Number(2), @@ -414,6 +443,7 @@ fn fork_aware_finalization() { }; b1 = header.hash(); block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); let event = ChainEvent::Finalized { hash: b1 }; block_on(pool.maintain(event)); } @@ -423,6 +453,7 @@ fn fork_aware_finalization() { let header = pool.api.push_fork_block_with_parent(b1, vec![from_dave.clone()]); from_dave_watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_dave.clone())) .expect("1. Imported"); + assert_eq!(pool.status().ready, 1); let event = ChainEvent::NewBlock { id: BlockId::Hash(header.hash()), is_new_best: true, @@ -431,11 +462,13 @@ fn fork_aware_finalization() { }; c2 = header.hash(); block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); } // block D2 { from_bob_watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_bob.clone())).expect("1. Imported"); + assert_eq!(pool.status().ready, 1); let header = pool.api.push_fork_block_with_parent(c2, vec![from_bob.clone()]); let event = ChainEvent::NewBlock { @@ -446,11 +479,13 @@ fn fork_aware_finalization() { }; d2 = header.hash(); block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); } // block C1 { let watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_charlie.clone())).expect("1.Imported"); + assert_eq!(pool.status().ready, 1); let header = pool.api.push_block(3, vec![from_charlie.clone()]); canon_watchers.push((watcher, header.hash())); @@ -461,6 +496,7 @@ fn fork_aware_finalization() { retracted: vec![c2, d2], }; block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 2); let event = ChainEvent::Finalized { hash: header.hash() }; block_on(pool.maintain(event)); } @@ -469,6 +505,7 @@ fn fork_aware_finalization() { { let xt = uxt(Eve, 0); let w = block_on(pool.submit_and_watch(&BlockId::number(1), xt.clone())).expect("1. Imported"); + assert_eq!(pool.status().ready, 3); let header = pool.api.push_block(4, vec![xt.clone()]); canon_watchers.push((w, header.hash())); @@ -480,6 +517,7 @@ fn fork_aware_finalization() { }; d1 = header.hash(); block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 2); let event = ChainEvent::Finalized { hash: d1 }; block_on(pool.maintain(event)); } @@ -488,7 +526,7 @@ fn fork_aware_finalization() { // block e1 { - let header = pool.api.push_block(5, vec![from_dave]); + let header = pool.api.push_block(5, vec![from_dave, from_bob]); e1 = header.hash(); let event = ChainEvent::NewBlock { id: BlockId::Hash(header.hash()), @@ -497,6 +535,7 @@ fn fork_aware_finalization() { retracted: vec![] }; block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); block_on(pool.maintain(ChainEvent::Finalized { hash: e1 })); } @@ -515,15 +554,8 @@ fn fork_aware_finalization() { assert_eq!(stream.next(), Some(TransactionStatus::Ready)); assert_eq!(stream.next(), Some(TransactionStatus::InBlock(c2.clone()))); assert_eq!(stream.next(), Some(TransactionStatus::Retracted(c2))); - - // can be either Ready, or InBlock, depending on which event comes first - assert_eq!( - match stream.next() { - Some(TransactionStatus::Ready) => stream.next(), - val @ _ => val, - }, - Some(TransactionStatus::InBlock(e1)), - ); + assert_eq!(stream.next(), Some(TransactionStatus::Ready)); + assert_eq!(stream.next(), Some(TransactionStatus::InBlock(e1))); assert_eq!(stream.next(), Some(TransactionStatus::Finalized(e1.clone()))); assert_eq!(stream.next(), None); } @@ -533,6 +565,10 @@ fn fork_aware_finalization() { assert_eq!(stream.next(), Some(TransactionStatus::Ready)); assert_eq!(stream.next(), Some(TransactionStatus::InBlock(d2.clone()))); assert_eq!(stream.next(), Some(TransactionStatus::Retracted(d2))); + assert_eq!(stream.next(), Some(TransactionStatus::Ready)); + assert_eq!(stream.next(), Some(TransactionStatus::InBlock(e1))); + assert_eq!(stream.next(), Some(TransactionStatus::Finalized(e1.clone()))); + assert_eq!(stream.next(), None); } } diff --git a/docs/CODEOWNERS b/docs/CODEOWNERS index 9304f278534bbe4faa0bc7523c7a27bb99781ce5..701df299a35453ce71a037150d46d329847d5f80 100644 --- a/docs/CODEOWNERS +++ b/docs/CODEOWNERS @@ -75,3 +75,6 @@ # Authority discovery /client/authority-discovery/ @mxinden /frame/authority-discovery/ @mxinden + +# Prometheus endpoint +/utils/prometheus/ @mxinden diff --git a/docs/CONTRIBUTING.adoc b/docs/CONTRIBUTING.adoc index c83b686b097e6a486a22cd846b5b25e373e88642..cdd9809fff864cdb1b26321d76567ac0152233e5 100644 --- a/docs/CONTRIBUTING.adoc +++ b/docs/CONTRIBUTING.adoc @@ -27,7 +27,7 @@ Merging pull requests once CI is successful: . Once a PR is ready for review please add the https://github.com/paritytech/substrate/pulls?q=is%3Apr+is%3Aopen+label%3AA0-pleasereview[`pleasereview`] label. Generally PRs should sit with this label for 48 hours in order to garner feedback. It may be merged before if all relevant parties had a look at it. . If the first review is not an approval, swap `A0-pleasereview` to any label `[A3, A7]` to indicate that the PR has received some feedback, but needs further work. For example. https://github.com/paritytech/substrate/labels/A3-inprogress[`A3-inprogress`] is a general indicator that the PR is work in progress and https://github.com/paritytech/substrate/labels/A4-gotissues[`A4-gotissues`] means that it has significant problems that need fixing. Once the work is done, change the label back to `A0-pleasereview`. You might end up swapping a few times back and forth to climb up the A label group. Once a PR is https://github.com/paritytech/substrate/labels/A8-looksgood[`A8-looksgood`], it is ready to merge. -. PRs that break the external API must be tagged with https://github.com/paritytech/substrate/labels/B2-breaksapi[`breaksapi`], when it changes the SRML or consensus of running system with https://github.com/paritytech/substrate/labels/B3-breaksconsensus[`breaksconsensus`] +. PRs that break the external API must be tagged with https://github.com/paritytech/substrate/labels/B2-breaksapi[`breaksapi`], when it changes the FRAME or consensus of running system with https://github.com/paritytech/substrate/labels/B3-breaksconsensus[`breaksconsensus`] . No PR should be merged until all reviews' comments are addressed. *Reviewing pull requests*: diff --git a/docs/README.adoc b/docs/README.adoc index bbc2713fbeb798620bfb4fad22ae52940a066313..8d762fee05f8789f164b0c5a263e334c28668176 100644 --- a/docs/README.adoc +++ b/docs/README.adoc @@ -26,7 +26,7 @@ Substrate is designed for use in one of three ways: **2. Modular**: By hacking together pallets built with Substrate FRAME into a new runtime and possibly altering or reconfiguring the Substrate client's block authoring logic. This affords you a very large amount of freedom over your blockchain's logic, letting you change data types, add or remove modules, and crucially, add your own modules. Much can be changed without touching the block authoring logic (since it is generic). If this is the case, then the existing Substrate binary can be used for block authoring and syncing. If the block authoring logic needs to be tweaked, then a new, altered block authoring binary must be built as a separate project and used by validators. This is how the Polkadot relay chain is built and should suffice for almost all circumstances in the near to mid-term. -**3. Generic**: The entire SRML can be ignored and the entire runtime designed and implemented from scratch. If desired, this can be done in a language other than Rust, provided it can target WebAssembly. If the runtime can be made compatible with the existing client's block authoring logic, then you can simply construct a new genesis block from your Wasm blob and launch your chain with the existing Rust-based Substrate client. If not, then you'll need to alter the client's block authoring logic accordingly. This is probably a useless option for most projects right now, but provides complete flexibility allowing for a long-term, far-reaching upgrade path for the Substrate paradigm. +**3. Generic**: The entire FRAME can be ignored and the entire runtime designed and implemented from scratch. If desired, this can be done in a language other than Rust, provided it can target WebAssembly. If the runtime can be made compatible with the existing client's block authoring logic, then you can simply construct a new genesis block from your Wasm blob and launch your chain with the existing Rust-based Substrate client. If not, then you'll need to alter the client's block authoring logic accordingly. This is probably a useless option for most projects right now, but provides complete flexibility allowing for a long-term, far-reaching upgrade path for the Substrate paradigm. === The Basics of Substrate diff --git a/frame/assets/Cargo.toml b/frame/assets/Cargo.toml index cd6b41c4c7a3c5523b0622c968b7f1fd4b9c055b..5b0b5f096a91fdfea93956119879c8873d714de9 100644 --- a/frame/assets/Cargo.toml +++ b/frame/assets/Cargo.toml @@ -1,24 +1,27 @@ [package] name = "pallet-assets" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME asset management pallet" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } # Needed for various traits. In our case, `OnFinalize`. -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/runtime" } # Needed for type-safe access to storage DB. -frame-support = { version = "2.0.0", default-features = false, path = "../support" } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../support" } # `system` module provides us with all sorts of useful stuff and macros depend on it being around. -frame-system = { version = "2.0.0", default-features = false, path = "../system" } +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0", path = "../../primitives/core" } -sp-std = { version = "2.0.0", path = "../../primitives/std" } -sp-io = { version = "2.0.0", path = "../../primitives/io" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +sp-std = { version = "2.0.0-alpha.2", path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.2", path = "../../primitives/io" } [features] default = ["std"] diff --git a/frame/assets/src/lib.rs b/frame/assets/src/lib.rs index 8cdc9b9cc0fa8338670f4d7ab68db3689fa56af9..042ff89913417ac710a20f348662ea6a2d10eceb 100644 --- a/frame/assets/src/lib.rs +++ b/frame/assets/src/lib.rs @@ -265,9 +265,9 @@ mod tests { pub enum Origin for Test where system = frame_system {} } - // For testing the module, we construct most of a mock runtime. This means + // For testing the pallet, we construct most of a mock runtime. This means // first constructing a configuration type (`Test`) which `impl`s each of the - // configuration traits of modules we want to use. + // configuration traits of pallets we want to use. #[derive(Clone, Eq, PartialEq)] pub struct Test; parameter_types! { @@ -295,7 +295,7 @@ mod tests { type ModuleToIndex = (); type AccountData = (); type OnNewAccount = (); - type OnReapAccount = (); + type OnKilledAccount = (); } impl Trait for Test { type Event = (); diff --git a/frame/aura/Cargo.toml b/frame/aura/Cargo.toml index e982a02b47c8db39d755af9739123c4f909562a0..77abfe4ae1f48aa062c50d4d516e62dd4042631d 100644 --- a/frame/aura/Cargo.toml +++ b/frame/aura/Cargo.toml @@ -1,25 +1,28 @@ [package] name = "pallet-aura" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME AURA consensus pallet" [dependencies] -sp-application-crypto = { version = "2.0.0", default-features = false, path = "../../primitives/application-crypto" } -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-inherents = { version = "2.0.0", default-features = false, path = "../../primitives/inherents" } -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } +sp-application-crypto = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/application-crypto" } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +sp-inherents = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/inherents" } +sp-core = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/std" } serde = { version = "1.0.101", optional = true } -pallet-session = { version = "2.0.0", default-features = false, path = "../session" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -sp-io ={ path = "../../primitives/io", default-features = false } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -sp-consensus-aura = { path = "../../primitives/consensus/aura", default-features = false} -frame-system = { version = "2.0.0", default-features = false, path = "../system" } -sp-timestamp = { version = "2.0.0", default-features = false, path = "../../primitives/timestamp" } -pallet-timestamp = { version = "2.0.0", default-features = false, path = "../timestamp" } +pallet-session = { version = "2.0.0-alpha.2", default-features = false, path = "../session" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/runtime" } +sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.2"} +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../support" } +sp-consensus-aura = { path = "../../primitives/consensus/aura", default-features = false, version = "0.8.0-alpha.2"} +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../system" } +sp-timestamp = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/timestamp" } +pallet-timestamp = { version = "2.0.0-alpha.2", default-features = false, path = "../timestamp" } [dev-dependencies] diff --git a/frame/aura/src/mock.rs b/frame/aura/src/mock.rs index 5b3ccd7745f8cc9cd261c45ebe7a8e21ca691176..05a161ee49c3d6c9ac7c51028d31f1dd9d193bc8 100644 --- a/frame/aura/src/mock.rs +++ b/frame/aura/src/mock.rs @@ -63,7 +63,7 @@ impl frame_system::Trait for Test { type ModuleToIndex = (); type AccountData = (); type OnNewAccount = (); - type OnReapAccount = (); + type OnKilledAccount = (); } impl pallet_timestamp::Trait for Test { diff --git a/frame/authority-discovery/Cargo.toml b/frame/authority-discovery/Cargo.toml index d8c72683cac786067793c4883d167f567dc15c18..c82d1eab588b39fcf27f5924ca5471a8b4b616fd 100644 --- a/frame/authority-discovery/Cargo.toml +++ b/frame/authority-discovery/Cargo.toml @@ -1,25 +1,28 @@ [package] name = "pallet-authority-discovery" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet for authority discovery" [dependencies] -sp-authority-discovery = { version = "2.0.0", default-features = false, path = "../../primitives/authority-discovery" } -sp-application-crypto = { version = "2.0.0", default-features = false, path = "../../primitives/application-crypto" } -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } +sp-authority-discovery = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/authority-discovery" } +sp-application-crypto = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/application-crypto" } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +sp-core = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/std" } serde = { version = "1.0.101", optional = true } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } -pallet-session = { version = "2.0.0", features = ["historical" ], path = "../session", default-features = false } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } +sp-io = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/io" } +pallet-session = { version = "2.0.0-alpha.2", features = ["historical" ], path = "../session", default-features = false } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../system" } [dev-dependencies] -sp-staking = { version = "2.0.0", default-features = false, path = "../../primitives/staking" } +sp-staking = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/staking" } [features] default = ["std"] diff --git a/frame/authority-discovery/src/lib.rs b/frame/authority-discovery/src/lib.rs index 77906e1bfe34f7e12bf9c7cb03978f71128906bb..8ee4931e488c3740174706480188dfbbdd5b1857 100644 --- a/frame/authority-discovery/src/lib.rs +++ b/frame/authority-discovery/src/lib.rs @@ -159,7 +159,7 @@ mod tests { type ModuleToIndex = (); type AccountData = (); type OnNewAccount = (); - type OnReapAccount = (); + type OnKilledAccount = (); } impl_outer_origin! { diff --git a/frame/authorship/Cargo.toml b/frame/authorship/Cargo.toml index 124d66c1bc33f25cfab28fb3a28dc3f262c182df..2a01cdbd8ad85b22245bf68b8505b4c8be846d6c 100644 --- a/frame/authorship/Cargo.toml +++ b/frame/authorship/Cargo.toml @@ -1,21 +1,23 @@ [package] name = "pallet-authorship" -version = "2.0.0" -description = "Block and Uncle Author tracking for the SRML" +version = "2.0.0-alpha.3" +description = "Block and Uncle Author tracking for the FRAME" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" [dependencies] -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-inherents = { version = "2.0.0", default-features = false, path = "../../primitives/inherents" } -sp-authorship = { version = "2.0.0", default-features = false, path = "../../primitives/authorship" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } -sp-io ={ path = "../../primitives/io", default-features = false } +sp-core = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/core" } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +sp-inherents = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/inherents" } +sp-authorship = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/authorship" } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../system" } +sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.2"} impl-trait-for-tuples = "0.1.3" [features] diff --git a/frame/authorship/src/lib.rs b/frame/authorship/src/lib.rs index a26437134bfae5c0378df39ae0a185ffdfe64612..a7b448cf9a91f8d0d73698ecd9b21e631a6f8d5e 100644 --- a/frame/authorship/src/lib.rs +++ b/frame/authorship/src/lib.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Authorship tracking for SRML runtimes. +//! Authorship tracking for FRAME runtimes. //! //! This tracks the current author of the block and recent uncles. @@ -440,7 +440,7 @@ mod tests { type ModuleToIndex = (); type AccountData = (); type OnNewAccount = (); - type OnReapAccount = (); + type OnKilledAccount = (); } parameter_types! { diff --git a/frame/babe/Cargo.toml b/frame/babe/Cargo.toml index 5e7764862a66b4c58f5b9fcb57e58a6325d97659..7e39e5bed5c29a607ef3faf077ab1964d3921c57 100644 --- a/frame/babe/Cargo.toml +++ b/frame/babe/Cargo.toml @@ -1,32 +1,35 @@ [package] name = "pallet-babe" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Consensus extension module for BABE consensus. Collects on-chain randomness from VRF outputs and manages epoch transitions." [dependencies] hex-literal = "0.2.1" -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true } -sp-inherents = { version = "2.0.0", default-features = false, path = "../../primitives/inherents" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -sp-staking = { version = "2.0.0", default-features = false, path = "../../primitives/staking" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } -pallet-timestamp = { version = "2.0.0", default-features = false, path = "../timestamp" } -sp-timestamp = { version = "2.0.0", default-features = false, path = "../../primitives/timestamp" } -pallet-session = { version = "2.0.0", default-features = false, path = "../session" } -sp-consensus-babe = { version = "0.8", default-features = false, path = "../../primitives/consensus/babe" } -sp-io ={ path = "../../primitives/io", default-features = false } +sp-inherents = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/inherents" } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/staking" } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../system" } +pallet-timestamp = { version = "2.0.0-alpha.2", default-features = false, path = "../timestamp" } +sp-timestamp = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/timestamp" } +pallet-session = { version = "2.0.0-alpha.2", default-features = false, path = "../session" } +sp-consensus-babe = { version = "0.8.0-alpha.2", default-features = false, path = "../../primitives/consensus/babe" } +sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.2"} [dev-dependencies] lazy_static = "1.4.0" parking_lot = "0.10.0" -sp-version = { version = "2.0.0", default-features = false, path = "../../primitives/version" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -substrate-test-runtime = { version = "2.0.0", path = "../../test-utils/runtime" } +sp-version = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/version" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +substrate-test-runtime = { version = "2.0.0-dev", path = "../../test-utils/runtime" } [features] default = ["std"] diff --git a/frame/babe/src/lib.rs b/frame/babe/src/lib.rs index e4e37a4b7ac1f36b32764e03954d5a413feea730..4dc9304fa8467f7e2bf36dcda500fb8fda37962e 100644 --- a/frame/babe/src/lib.rs +++ b/frame/babe/src/lib.rs @@ -159,7 +159,7 @@ decl_storage! { } decl_module! { - /// The BABE SRML module + /// The BABE Pallet pub struct Module for enum Call where origin: T::Origin { /// The number of **slots** that an epoch takes. We couple sessions to /// epochs, i.e. we start a new session once the new epoch begins. @@ -248,7 +248,6 @@ impl pallet_session::ShouldEndSession for Module { /// A BABE equivocation offence report. /// /// When a validator released two or more blocks at the same slot. -#[allow(dead_code)] struct BabeEquivocationOffence { /// A babe slot number in which this incident happened. slot: u64, diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index efb5570c1db7b3d9da7fe95db6ad8799e153d4f2..2ec083728e82ceb8488fa9c24efb7223ae5f450e 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -15,17 +15,15 @@ // along with Substrate. If not, see . //! Test utilities -#![allow(dead_code, unused_imports)] use super::{Trait, Module, GenesisConfig}; -use sp_consensus_babe::AuthorityId; use sp_runtime::{ - traits::IdentityLookup, Perbill, PerThing, testing::{Header, UintAuthorityId}, impl_opaque_keys, + traits::IdentityLookup, Perbill, testing::{Header, UintAuthorityId}, impl_opaque_keys, }; use sp_version::RuntimeVersion; use frame_support::{impl_outer_origin, parameter_types, weights::Weight}; use sp_io; -use sp_core::{H256, Blake2Hasher}; +use sp_core::H256; impl_outer_origin!{ pub enum Origin for Test where system = frame_system {} @@ -68,7 +66,7 @@ impl frame_system::Trait for Test { type ModuleToIndex = (); type AccountData = (); type OnNewAccount = (); - type OnReapAccount = (); + type OnKilledAccount = (); } impl_opaque_keys! { diff --git a/frame/babe/src/tests.rs b/frame/babe/src/tests.rs index 976a264d7ba7bb24f29c556ec92822a728598075..84f8166b10431a9d107cf82d831d4aaa3230dfe1 100644 --- a/frame/babe/src/tests.rs +++ b/frame/babe/src/tests.rs @@ -17,7 +17,7 @@ //! Consensus extension module tests for BABE consensus. use super::*; -use mock::{new_test_ext, Babe, Test}; +use mock::{new_test_ext, Babe, System}; use sp_runtime::{traits::OnFinalize, testing::{Digest, DigestItem}}; use pallet_session::ShouldEndSession; @@ -66,8 +66,6 @@ fn check_module() { }) } -type System = frame_system::Module; - #[test] fn first_block_epoch_zero_start() { new_test_ext(vec![0, 1, 2, 3]).execute_with(|| { diff --git a/frame/balances/Cargo.toml b/frame/balances/Cargo.toml index 871290b182fdd0ec6e177ac0f1b1cf9557929289..6bb551c9c6fea03a747e700e5aac6f0d8156f300 100644 --- a/frame/balances/Cargo.toml +++ b/frame/balances/Cargo.toml @@ -1,23 +1,26 @@ [package] name = "pallet-balances" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet to manage balances" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/runtime" } +frame-benchmarking = { version = "2.0.0-alpha.2", default-features = false, path = "../benchmarking", optional = true } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../system" } [dev-dependencies] -sp-io = { version = "2.0.0", path = "../../primitives/io" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -pallet-transaction-payment = { version = "2.0.0", path = "../transaction-payment" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +pallet-transaction-payment = { version = "2.0.0-alpha.2", path = "../transaction-payment" } [features] default = ["std"] @@ -27,6 +30,8 @@ std = [ "sp-std/std", "sp-io/std", "sp-runtime/std", + "frame-benchmarking/std", "frame-support/std", "frame-system/std", ] +runtime-benchmarks = ["frame-benchmarking"] diff --git a/frame/balances/src/benchmarking.rs b/frame/balances/src/benchmarking.rs index 605561d5975995d6099a6c24e7ac4cac46cb932d..2473ce292004754574a96be0d07c3d2af3c038a5 100644 --- a/frame/balances/src/benchmarking.rs +++ b/frame/balances/src/benchmarking.rs @@ -19,303 +19,101 @@ use super::*; use frame_system::RawOrigin; -use sp_io::hashing::blake2_256; -use sp_runtime::{BenchmarkResults, BenchmarkParameter}; -use sp_runtime::traits::{Bounded, Benchmarking, BenchmarkingSetup, Dispatchable}; +use frame_benchmarking::{benchmarks, account}; +use sp_runtime::traits::Bounded; use crate::Module as Balances; -// Support Functions -fn account(name: &'static str, index: u32) -> T::AccountId { - let entropy = (name, index).using_encoded(blake2_256); - T::AccountId::decode(&mut &entropy[..]).unwrap_or_default() -} +const SEED: u32 = 0; +const MAX_EXISTENTIAL_DEPOSIT: u32 = 1000; +const MAX_USER_INDEX: u32 = 1000; -// Benchmark `transfer` extrinsic with the worst possible conditions: -// * Transfer will kill the sender account. -// * Transfer will create the recipient account. -struct Transfer; -impl BenchmarkingSetup, RawOrigin> for Transfer { - fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { - vec![ - // Existential Deposit Multiplier - (BenchmarkParameter::E, 2, 1000), - // User Seed - (BenchmarkParameter::U, 1, 1000), - ] +benchmarks! { + _ { + let e in 2 .. MAX_EXISTENTIAL_DEPOSIT => (); + let u in 1 .. MAX_USER_INDEX => (); } - fn instance(&self, components: &[(BenchmarkParameter, u32)]) - -> Result<(crate::Call, RawOrigin), &'static str> - { - // Constants - let ed = T::ExistentialDeposit::get(); + // Benchmark `transfer` extrinsic with the worst possible conditions: + // * Transfer will kill the sender account. + // * Transfer will create the recipient account. + transfer { + let u in ...; + let e in ...; - // Select an account - let u = components.iter().find(|&c| c.0 == BenchmarkParameter::U).unwrap().1; - let user = account::("user", u); - let user_origin = RawOrigin::Signed(user.clone()); + let existential_deposit = T::ExistentialDeposit::get(); + let caller = account("caller", u, SEED); // Give some multiple of the existential deposit + creation fee + transfer fee - let e = components.iter().find(|&c| c.0 == BenchmarkParameter::E).unwrap().1; - let balance = ed.saturating_mul(e.into()); - let _ = as Currency<_>>::make_free_balance_be(&user, balance); + let balance = existential_deposit.saturating_mul(e.into()); + let _ = as Currency<_>>::make_free_balance_be(&caller, balance); // Transfer `e - 1` existential deposits + 1 unit, which guarantees to create one account, and reap this user. - let recipient = account::("recipient", u); + let recipient = account("recipient", u, SEED); let recipient_lookup: ::Source = T::Lookup::unlookup(recipient); - let transfer_amt = ed.saturating_mul((e - 1).into()) + 1.into(); + let transfer_amount = existential_deposit.saturating_mul((e - 1).into()) + 1.into(); + }: _(RawOrigin::Signed(caller), recipient_lookup, transfer_amount) - // Return the `transfer` call - Ok((crate::Call::::transfer(recipient_lookup, transfer_amt), user_origin)) - } -} - -// Benchmark `transfer` with the best possible condition: -// * Both accounts exist and will continue to exist. -struct TransferBestCase; -impl BenchmarkingSetup, RawOrigin> for TransferBestCase { - fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { - vec![ - // Existential Deposit Multiplier - (BenchmarkParameter::E, 2, 1000), - // User Seed - (BenchmarkParameter::U, 1, 1000), - ] - } - - fn instance(&self, components: &[(BenchmarkParameter, u32)]) - -> Result<(crate::Call, RawOrigin), &'static str> - { - // Constants - let ed = T::ExistentialDeposit::get(); - - // Select a sender - let u = components.iter().find(|&c| c.0 == BenchmarkParameter::U).unwrap().1; - let user = account::("user", u); - let user_origin = RawOrigin::Signed(user.clone()); + // Benchmark `transfer` with the best possible condition: + // * Both accounts exist and will continue to exist. + transfer_best_case { + let u in ...; + let e in ...; - // Select a recipient - let recipient = account::("recipient", u); + let caller = account("caller", u, SEED); + let recipient: T::AccountId = account("recipient", u, SEED); let recipient_lookup: ::Source = T::Lookup::unlookup(recipient.clone()); - // Get the existential deposit multiplier - let e = components.iter().find(|&c| c.0 == BenchmarkParameter::E).unwrap().1; - // Give the sender account max funds for transfer (their account will never reasonably be killed). - let _ = as Currency<_>>::make_free_balance_be(&user, T::Balance::max_value()); + let _ = as Currency<_>>::make_free_balance_be(&caller, T::Balance::max_value()); // Give the recipient account existential deposit (thus their account already exists). - let _ = as Currency<_>>::make_free_balance_be(&recipient, ed); - - // Transfer e * existential deposit. - let transfer_amt = ed.saturating_mul(e.into()); - - // Return the `transfer` call - Ok((crate::Call::::transfer(recipient_lookup, transfer_amt), user_origin)) - } -} - -// Benchmark `transfer_keep_alive` with the worst possible condition: -// * The recipient account is created. -struct TransferKeepAlive; -impl BenchmarkingSetup, RawOrigin> for TransferKeepAlive { - fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { - vec![ - // Existential Deposit Multiplier - (BenchmarkParameter::E, 2, 1000), - // User Seed - (BenchmarkParameter::U, 1, 1000), - ] - } - - fn instance(&self, components: &[(BenchmarkParameter, u32)]) - -> Result<(crate::Call, RawOrigin), &'static str> - { - // Constants - let ed = T::ExistentialDeposit::get(); - - // Select a sender - let u = components.iter().find(|&c| c.0 == BenchmarkParameter::U).unwrap().1; - let user = account::("user", u); - let user_origin = RawOrigin::Signed(user.clone()); - - // Select a recipient - let recipient = account::("recipient", u); - let recipient_lookup: ::Source = T::Lookup::unlookup(recipient.clone()); - - // Get the existential deposit multiplier - let e = components.iter().find(|&c| c.0 == BenchmarkParameter::E).unwrap().1; + let existential_deposit = T::ExistentialDeposit::get(); + let _ = as Currency<_>>::make_free_balance_be(&recipient, existential_deposit); + let transfer_amount = existential_deposit.saturating_mul(e.into()); + }: transfer(RawOrigin::Signed(caller), recipient_lookup, transfer_amount) + + // Benchmark `transfer_keep_alive` with the worst possible condition: + // * The recipient account is created. + transfer_keep_alive { + let u in ...; + let e in ...; + + let caller = account("caller", u, SEED); + let recipient = account("recipient", u, SEED); + let recipient_lookup: ::Source = T::Lookup::unlookup(recipient); // Give the sender account max funds, thus a transfer will not kill account. - let _ = as Currency<_>>::make_free_balance_be(&user, T::Balance::max_value()); + let _ = as Currency<_>>::make_free_balance_be(&caller, T::Balance::max_value()); + let existential_deposit = T::ExistentialDeposit::get(); + let transfer_amount = existential_deposit.saturating_mul(e.into()); + }: _(RawOrigin::Signed(caller), recipient_lookup, transfer_amount) - // Transfer e * existential deposit. - let transfer_amt = ed.saturating_mul(e.into()); + // Benchmark `set_balance` coming from ROOT account. This always creates an account. + set_balance { + let u in ...; + let e in ...; - // Return the `transfer_keep_alive` call - Ok((crate::Call::::transfer_keep_alive(recipient_lookup, transfer_amt), user_origin)) - } -} - -// Benchmark `set_balance` coming from ROOT account. This always creates an account. -struct SetBalance; -impl BenchmarkingSetup, RawOrigin> for SetBalance { - fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { - vec![ - // Existential Deposit Multiplier - (BenchmarkParameter::E, 2, 1000), - // User Seed - (BenchmarkParameter::U, 1, 1000), - ] - } - - fn instance(&self, components: &[(BenchmarkParameter, u32)]) - -> Result<(crate::Call, RawOrigin), &'static str> - { - // Constants - let ed = T::ExistentialDeposit::get(); - - // Select a sender - let u = components.iter().find(|&c| c.0 == BenchmarkParameter::U).unwrap().1; - let user = account::("user", u); + let user: T::AccountId = account("user", u, SEED); let user_lookup: ::Source = T::Lookup::unlookup(user.clone()); - // Get the existential deposit multiplier for free and reserved - let e = components.iter().find(|&c| c.0 == BenchmarkParameter::E).unwrap().1; - let balance_amt = ed.saturating_mul(e.into()); - - // Return the `set_balance` call - Ok((crate::Call::::set_balance(user_lookup, balance_amt, balance_amt), RawOrigin::Root)) - } -} - -// Benchmark `set_balance` coming from ROOT account. This always kills an account. -struct SetBalanceKilling; -impl BenchmarkingSetup, RawOrigin> for SetBalanceKilling { - fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { - vec![ - // Existential Deposit Multiplier - (BenchmarkParameter::E, 2, 1000), - // User Seed - (BenchmarkParameter::U, 1, 1000), - ] - } + // Give the user some initial balance. + let existential_deposit = T::ExistentialDeposit::get(); + let balance_amount = existential_deposit.saturating_mul(e.into()); + let _ = as Currency<_>>::make_free_balance_be(&user, balance_amount); + }: _(RawOrigin::Root, user_lookup, balance_amount, balance_amount) - fn instance(&self, components: &[(BenchmarkParameter, u32)]) - -> Result<(crate::Call, RawOrigin), &'static str> - { - // Constants - let ed = T::ExistentialDeposit::get(); + // Benchmark `set_balance` coming from ROOT account. This always kills an account. + set_balance_killing { + let u in ...; + let e in ...; - // Select a sender - let u = components.iter().find(|&c| c.0 == BenchmarkParameter::U).unwrap().1; - let user = account::("user", u); + let user: T::AccountId = account("user", u, SEED); let user_lookup: ::Source = T::Lookup::unlookup(user.clone()); - // Get the existential deposit multiplier for free and reserved - let e = components.iter().find(|&c| c.0 == BenchmarkParameter::E).unwrap().1; - // Give the user some initial balance - let balance_amt = ed.saturating_mul(e.into()); - let _ = as Currency<_>>::make_free_balance_be(&user, balance_amt); - - // Return the `set_balance` call that will kill the account - Ok((crate::Call::::set_balance(user_lookup, 0.into(), 0.into()), RawOrigin::Root)) - } -} - -// The list of available benchmarks for this pallet. -enum SelectedBenchmark { - Transfer, - TransferBestCase, - TransferKeepAlive, - SetBalance, - SetBalanceKilling, -} - -// Allow us to select a benchmark from the list of available benchmarks. -impl BenchmarkingSetup, RawOrigin> for SelectedBenchmark { - fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { - match self { - Self::Transfer => , RawOrigin>>::components(&Transfer), - Self::TransferBestCase => , RawOrigin>>::components(&TransferBestCase), - Self::TransferKeepAlive => , RawOrigin>>::components(&TransferKeepAlive), - Self::SetBalance => , RawOrigin>>::components(&SetBalance), - Self::SetBalanceKilling => , RawOrigin>>::components(&SetBalanceKilling), - } - } - - fn instance(&self, components: &[(BenchmarkParameter, u32)]) - -> Result<(crate::Call, RawOrigin), &'static str> - { - match self { - Self::Transfer => , RawOrigin>>::instance(&Transfer, components), - Self::TransferBestCase => , RawOrigin>>::instance(&TransferBestCase, components), - Self::TransferKeepAlive => , RawOrigin>>::instance(&TransferKeepAlive, components), - Self::SetBalance => , RawOrigin>>::instance(&SetBalance, components), - Self::SetBalanceKilling => , RawOrigin>>::instance(&SetBalanceKilling, components), - } - } -} - -impl Benchmarking for Module { - fn run_benchmark(extrinsic: Vec, steps: u32, repeat: u32) -> Result, &'static str> { - // Map the input to the selected benchmark. - let selected_benchmark = match extrinsic.as_slice() { - b"transfer" => SelectedBenchmark::Transfer, - b"transfer_best_case" => SelectedBenchmark::TransferBestCase, - b"transfer_keep_alive" => SelectedBenchmark::TransferKeepAlive, - b"set_balance" => SelectedBenchmark::SetBalance, - b"set_balance_killing" => SelectedBenchmark::SetBalanceKilling, - _ => return Err("Could not find extrinsic."), - }; - - // Warm up the DB - sp_io::benchmarking::commit_db(); - sp_io::benchmarking::wipe_db(); - - let components = , RawOrigin>>::components(&selected_benchmark); - // results go here - let mut results: Vec = Vec::new(); - // Select the component we will be benchmarking. Each component will be benchmarked. - for (name, low, high) in components.iter() { - // Create up to `STEPS` steps for that component between high and low. - let step_size = ((high - low) / steps).max(1); - let num_of_steps = (high - low) / step_size; - for s in 0..num_of_steps { - // This is the value we will be testing for component `name` - let component_value = low + step_size * s; - - // Select the mid value for all the other components. - let c: Vec<(BenchmarkParameter, u32)> = components.iter() - .map(|(n, l, h)| - (*n, if n == name { component_value } else { (h - l) / 2 + l }) - ).collect(); - - // Run the benchmark `repeat` times. - for _r in 0..repeat { - // Set up the externalities environment for the setup we want to benchmark. - let (call, caller) = , RawOrigin>>::instance(&selected_benchmark, &c)?; - // Commit the externalities to the database, flushing the DB cache. - // This will enable worst case scenario for reading from the database. - sp_io::benchmarking::commit_db(); - // Run the benchmark. - let start = sp_io::benchmarking::current_time(); - call.dispatch(caller.clone().into())?; - let finish = sp_io::benchmarking::current_time(); - let elapsed = finish - start; - sp_std::if_std!{ - if let RawOrigin::Signed(who) = caller.clone() { - let balance = Account::::get(&who).free; - println!("Free Balance {:?}", balance); - } - } - results.push((c.clone(), elapsed)); - // Wipe the DB back to the genesis state. - sp_io::benchmarking::wipe_db(); - } - } - } - return Ok(results); - } -} + // Give the user some initial balance. + let existential_deposit = T::ExistentialDeposit::get(); + let balance_amount = existential_deposit.saturating_mul(e.into()); + let _ = as Currency<_>>::make_free_balance_be(&user, balance_amount); + }: set_balance(RawOrigin::Root, user_lookup, 0.into(), 0.into()) +} \ No newline at end of file diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index 2c8ae3e18fe8e05b6937698f8e47ff17be10ccee..f91c9e8229050cfd9182d2fe55f15de9133b0db2 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -92,7 +92,7 @@ //! //! The following examples show how to use the Balances module in your custom module. //! -//! ### Examples from the SRML +//! ### Examples from the FRAME //! //! The Contract module uses the `Currency` trait to handle gas payment, and its types inherit from `Currency`: //! @@ -155,15 +155,18 @@ mod tests_composite; #[cfg(test)] #[macro_use] mod tests; +#[cfg(feature = "runtime-benchmarks")] mod benchmarking; +mod migration; use sp_std::prelude::*; use sp_std::{cmp, result, mem, fmt::Debug, ops::BitOr, convert::Infallible}; +use sp_io::hashing::twox_64; use codec::{Codec, Encode, Decode}; use frame_support::{ StorageValue, Parameter, decl_event, decl_storage, decl_module, decl_error, ensure, weights::SimpleDispatchInfo, traits::{ - Currency, OnReapAccount, OnUnbalanced, TryDrop, StoredMap, + Currency, OnKilledAccount, OnUnbalanced, TryDrop, StoredMap, WithdrawReason, WithdrawReasons, LockIdentifier, LockableCurrency, ExistenceRequirement, Imbalance, SignedImbalance, ReservableCurrency, Get, ExistenceRequirement::KeepAlive, ExistenceRequirement::AllowDeath, IsDeadAccount, BalanceStatus as Status @@ -178,7 +181,7 @@ use sp_runtime::{ }; use frame_system::{self as system, ensure_signed, ensure_root}; use frame_support::storage::migration::{ - get_storage_value, take_storage_value, put_storage_value, StorageIterator + get_storage_value, take_storage_value, put_storage_value, StorageIterator, have_storage_value }; pub use self::imbalances::{PositiveImbalance, NegativeImbalance}; @@ -346,6 +349,21 @@ impl AccountData { } } +// A value placed in storage that represents the current version of the Balances storage. +// This value is used by the `on_runtime_upgrade` logic to determine whether we run +// storage migration logic. This should match directly with the semantic versions of the Rust crate. +#[derive(Encode, Decode, Clone, Copy, PartialEq, Eq, RuntimeDebug)] +enum Releases { + V1_0_0, + V2_0_0, +} + +impl Default for Releases { + fn default() -> Self { + Releases::V1_0_0 + } +} + decl_storage! { trait Store for Module, I: Instance=DefaultInstance> as Balances { /// The total units issued in the system. @@ -365,10 +383,10 @@ decl_storage! { /// NOTE: Should only be accessed when setting, changing and freeing a lock. pub Locks get(fn locks): map hasher(blake2_256) T::AccountId => Vec>; - /// True if network has been upgraded to this version. + /// Storage version of the pallet. /// - /// True for new networks. - IsUpgraded build(|_: &GenesisConfig| true): bool; + /// This is set to v2.0.0 for new networks. + StorageVersion build(|_: &GenesisConfig| Releases::V2_0_0): Releases; } add_extra_genesis { config(balances): Vec<(T::AccountId, T::Balance)>; @@ -516,11 +534,8 @@ decl_module! { >::transfer(&transactor, &dest, value, KeepAlive)?; } - fn on_initialize() { - if !IsUpgraded::::get() { - IsUpgraded::::put(true); - Self::do_upgrade(); - } + fn on_runtime_upgrade() { + migration::on_runtime_upgrade::(); } } } @@ -546,73 +561,6 @@ impl OldBalanceLock { impl, I: Instance> Module { // PRIVATE MUTABLES - // Upgrade from the pre-#4649 balances/vesting into the new balances. - pub fn do_upgrade() { - sp_runtime::print("Upgrading account balances..."); - // First, migrate from old FreeBalance to new Account. - // We also move all locks across since only accounts with FreeBalance values have locks. - // FreeBalance: map T::AccountId => T::Balance - for (hash, free) in StorageIterator::::new(b"Balances", b"FreeBalance").drain() { - let mut account = AccountData { free, ..Default::default() }; - // Locks: map T::AccountId => Vec - let old_locks = get_storage_value::>>(b"Balances", b"Locks", &hash); - if let Some(locks) = old_locks { - let locks = locks.into_iter() - .map(|i| { - let (result, expiry) = i.upgraded(); - if expiry != T::BlockNumber::max_value() { - // Any `until`s that are not T::BlockNumber::max_value come from - // democracy and need to be migrated over there. - // Democracy: Locks get(locks): map T::AccountId => Option; - put_storage_value(b"Democracy", b"Locks", &hash, expiry); - } - result - }) - .collect::>(); - for l in locks.iter() { - if l.reasons == Reasons::All || l.reasons == Reasons::Misc { - account.misc_frozen = account.misc_frozen.max(l.amount); - } - if l.reasons == Reasons::All || l.reasons == Reasons::Fee { - account.fee_frozen = account.fee_frozen.max(l.amount); - } - } - put_storage_value(b"Balances", b"Locks", &hash, locks); - } - put_storage_value(b"Balances", b"Account", &hash, account); - } - // Second, migrate old ReservedBalance into new Account. - // ReservedBalance: map T::AccountId => T::Balance - for (hash, reserved) in StorageIterator::::new(b"Balances", b"ReservedBalance").drain() { - let mut account = get_storage_value::>(b"Balances", b"Account", &hash).unwrap_or_default(); - account.reserved = reserved; - put_storage_value(b"Balances", b"Account", &hash, account); - } - - // Finally, migrate vesting and ensure locks are in place. We will be lazy and just lock - // for the maximum amount (i.e. at genesis). Users will need to call "vest" to reduce the - // lock to something sensible. - // pub Vesting: map T::AccountId => Option; - for (hash, vesting) in StorageIterator::<(T::Balance, T::Balance, T::BlockNumber)>::new(b"Balances", b"Vesting").drain() { - let mut account = get_storage_value::>(b"Balances", b"Account", &hash).unwrap_or_default(); - let mut locks = get_storage_value::>>(b"Balances", b"Locks", &hash).unwrap_or_default(); - locks.push(BalanceLock { - id: *b"vesting ", - amount: vesting.0.clone(), - reasons: Reasons::Misc, - }); - account.misc_frozen = account.misc_frozen.max(vesting.0.clone()); - put_storage_value(b"Vesting", b"Vesting", &hash, vesting); - put_storage_value(b"Balances", b"Locks", &hash, locks); - put_storage_value(b"Balances", b"Account", &hash, account); - } - - for (hash, balances) in StorageIterator::>::new(b"Balances", b"Account").drain() { - let nonce = take_storage_value::(b"System", b"AccountNonce", &hash).unwrap_or_default(); - put_storage_value(b"System", b"Account", &hash, (nonce, balances)); - } - } - /// Get the free balance of an account. pub fn free_balance(who: impl sp_std::borrow::Borrow) -> T::Balance { Self::account(who.borrow()).free @@ -721,14 +669,21 @@ impl, I: Instance> Module { } } }); - Locks::::insert(who, locks); - } -} -impl, I: Instance> OnReapAccount for Module { - fn on_reap_account(who: &T::AccountId) { - Locks::::remove(who); - Account::::remove(who); + let existed = Locks::::contains_key(who); + if locks.is_empty() { + Locks::::remove(who); + if existed { + // TODO: use Locks::::hashed_key + // https://github.com/paritytech/substrate/issues/4969 + system::Module::::dec_ref(who); + } + } else { + Locks::::insert(who, locks); + if !existed { + system::Module::::inc_ref(who); + } + } } } @@ -896,7 +851,7 @@ mod imbalances { // This works as long as `increase_total_issuance_by` doesn't use the Imbalance // types (basically for charging fees). // This should eventually be refactored so that the type item that -// depends on the Imbalance type (DustRemoval) is placed in its own SRML module. +// depends on the Imbalance type (DustRemoval) is placed in its own pallet. struct ElevatedTrait, I: Instance>(T, I); impl, I: Instance> Clone for ElevatedTrait { fn clone(&self) -> Self { unimplemented!() } @@ -923,7 +878,7 @@ impl, I: Instance> frame_system::Trait for ElevatedTrait { type Version = T::Version; type ModuleToIndex = T::ModuleToIndex; type OnNewAccount = T::OnNewAccount; - type OnReapAccount = T::OnReapAccount; + type OnKilledAccount = T::OnKilledAccount; type AccountData = T::AccountData; } impl, I: Instance> Trait for ElevatedTrait { @@ -1040,6 +995,7 @@ impl, I: Instance> Currency for Module where )?; let allow_death = existence_requirement == ExistenceRequirement::AllowDeath; + let allow_death = allow_death && system::Module::::allow_death(transactor); ensure!(allow_death || from_account.free >= ed, Error::::KeepAlive); Ok(()) @@ -1283,6 +1239,17 @@ impl, I: Instance> ReservableCurrency for Module } } +/// Implement `OnKilledAccount` to remove the local account, if using local account storage. +/// +/// NOTE: You probably won't need to use this! This only needs to be "wired in" to System module +/// if you're using the local balance storage. **If you're using the composite system account +/// storage (which is the default in most examples and tests) then there's no need.** +impl, I: Instance> OnKilledAccount for Module { + fn on_killed_account(who: &T::AccountId) { + Account::::remove(who); + } +} + impl, I: Instance> LockableCurrency for Module where T::Balance: MaybeSerializeDeserialize + Debug diff --git a/frame/balances/src/migration.rs b/frame/balances/src/migration.rs new file mode 100644 index 0000000000000000000000000000000000000000..16c764d59f1aa4f5e73812ab028141bbcd985405 --- /dev/null +++ b/frame/balances/src/migration.rs @@ -0,0 +1,88 @@ +use super::*; + +pub fn on_runtime_upgrade, I: Instance>() { + match StorageVersion::::get() { + Releases::V2_0_0 => return, + Releases::V1_0_0 => upgrade_v1_to_v2::(), + } +} + +// Upgrade from the pre-#4649 balances/vesting into the new balances. +fn upgrade_v1_to_v2, I: Instance>() { + sp_runtime::print("Upgrading account balances..."); + // First, migrate from old FreeBalance to new Account. + // We also move all locks across since only accounts with FreeBalance values have locks. + // FreeBalance: map T::AccountId => T::Balance + for (hash, free) in StorageIterator::::new(b"Balances", b"FreeBalance").drain() { + let mut account = AccountData { free, ..Default::default() }; + // Locks: map T::AccountId => Vec + let old_locks = get_storage_value::>>(b"Balances", b"Locks", &hash); + if let Some(locks) = old_locks { + let locks = locks.into_iter() + .map(|i| { + let (result, expiry) = i.upgraded(); + if expiry != T::BlockNumber::max_value() { + // Any `until`s that are not T::BlockNumber::max_value come from + // democracy and need to be migrated over there. + // Democracy: Locks get(locks): map T::AccountId => Option; + put_storage_value(b"Democracy", b"Locks", &hash, expiry); + } + result + }) + .collect::>(); + for l in locks.iter() { + if l.reasons == Reasons::All || l.reasons == Reasons::Misc { + account.misc_frozen = account.misc_frozen.max(l.amount); + } + if l.reasons == Reasons::All || l.reasons == Reasons::Fee { + account.fee_frozen = account.fee_frozen.max(l.amount); + } + } + put_storage_value(b"Balances", b"Locks", &hash, locks); + } + put_storage_value(b"Balances", b"Account", &hash, account); + } + // Second, migrate old ReservedBalance into new Account. + // ReservedBalance: map T::AccountId => T::Balance + for (hash, reserved) in StorageIterator::::new(b"Balances", b"ReservedBalance").drain() { + let mut account = get_storage_value::>(b"Balances", b"Account", &hash).unwrap_or_default(); + account.reserved = reserved; + put_storage_value(b"Balances", b"Account", &hash, account); + } + + // Finally, migrate vesting and ensure locks are in place. We will be lazy and just lock + // for the maximum amount (i.e. at genesis). Users will need to call "vest" to reduce the + // lock to something sensible. + // pub Vesting: map T::AccountId => Option; + for (hash, vesting) in StorageIterator::<(T::Balance, T::Balance, T::BlockNumber)>::new(b"Balances", b"Vesting").drain() { + let mut account = get_storage_value::>(b"Balances", b"Account", &hash).unwrap_or_default(); + let mut locks = get_storage_value::>>(b"Balances", b"Locks", &hash).unwrap_or_default(); + locks.push(BalanceLock { + id: *b"vesting ", + amount: vesting.0.clone(), + reasons: Reasons::Misc, + }); + account.misc_frozen = account.misc_frozen.max(vesting.0.clone()); + put_storage_value(b"Vesting", b"Vesting", &hash, vesting); + put_storage_value(b"Balances", b"Locks", &hash, locks); + put_storage_value(b"Balances", b"Account", &hash, account); + } + + for (hash, balances) in StorageIterator::>::new(b"Balances", b"Account").drain() { + let nonce = take_storage_value::(b"System", b"AccountNonce", &hash).unwrap_or_default(); + let mut refs: system::RefCount = 0; + // The items in Kusama that would result in a ref count being incremented. + if have_storage_value(b"Democracy", b"Proxy", &hash) { refs += 1 } + // We skip Recovered since it's being replaced anyway. + let mut prefixed_hash = twox_64(&b":session:keys"[..]).to_vec(); + prefixed_hash.extend(&b":session:keys"[..]); + prefixed_hash.extend(&hash[..]); + if have_storage_value(b"Session", b"NextKeys", &prefixed_hash) { refs += 1 } + if have_storage_value(b"Staking", b"Bonded", &hash) { refs += 1 } + put_storage_value(b"System", b"Account", &hash, (nonce, refs, &balances)); + } + + take_storage_value::(b"Balances", b"IsUpgraded", &[]); + + StorageVersion::::put(Releases::V2_0_0); +} diff --git a/frame/balances/src/tests.rs b/frame/balances/src/tests.rs index 3d0c9e9207f5d82faf9b58ee3958fa11fdaf3b4c..98c7c856bc88a3e74c56c64e77107bc57bacd500 100644 --- a/frame/balances/src/tests.rs +++ b/frame/balances/src/tests.rs @@ -24,8 +24,10 @@ macro_rules! decl_tests { use sp_runtime::{Fixed64, traits::{SignedExtension, BadOrigin}}; use frame_support::{ assert_noop, assert_ok, assert_err, - traits::{LockableCurrency, LockIdentifier, WithdrawReason, WithdrawReasons, - Currency, ReservableCurrency, ExistenceRequirement::AllowDeath} + traits::{ + LockableCurrency, LockIdentifier, WithdrawReason, WithdrawReasons, + Currency, ReservableCurrency, ExistenceRequirement::AllowDeath, StoredMap + } }; use pallet_transaction_payment::ChargeTransactionPayment; use frame_system::RawOrigin; @@ -55,6 +57,15 @@ macro_rules! decl_tests { }); } + #[test] + fn account_should_be_reaped() { + <$ext_builder>::default().existential_deposit(1).monied(true).build().execute_with(|| { + assert_eq!(Balances::free_balance(1), 10); + assert_ok!(>::transfer(&1, &2, 10, AllowDeath)); + assert!(!<::AccountStore as StoredMap>>::is_explicit(&1)); + }); + } + #[test] fn partial_locking_should_work() { <$ext_builder>::default().existential_deposit(1).monied(true).build().execute_with(|| { diff --git a/frame/balances/src/tests_composite.rs b/frame/balances/src/tests_composite.rs index 73d63d06c6636cef4d204e13f6983c862ecb0b68..2f54e1ec780ff6fd0bef05a15981251a6cf03be1 100644 --- a/frame/balances/src/tests_composite.rs +++ b/frame/balances/src/tests_composite.rs @@ -67,7 +67,7 @@ impl frame_system::Trait for Test { type ModuleToIndex = (); type AccountData = super::AccountData; type OnNewAccount = (); - type OnReapAccount = Module; + type OnKilledAccount = (); } parameter_types! { pub const TransactionBaseFee: u64 = 0; diff --git a/frame/balances/src/tests_local.rs b/frame/balances/src/tests_local.rs index 958164a8a057d99c74be01171bfae4ffa24bfd03..3456db5f13b92eab7c972a2ee2e065983e200d2d 100644 --- a/frame/balances/src/tests_local.rs +++ b/frame/balances/src/tests_local.rs @@ -67,7 +67,7 @@ impl frame_system::Trait for Test { type ModuleToIndex = (); type AccountData = super::AccountData; type OnNewAccount = (); - type OnReapAccount = Module; + type OnKilledAccount = Module; } parameter_types! { pub const TransactionBaseFee: u64 = 0; diff --git a/frame/benchmark/Cargo.toml b/frame/benchmark/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..bf237d992cd64cecc2c771a1bec4af4ffc02002d --- /dev/null +++ b/frame/benchmark/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "pallet-benchmark" +version = "2.0.0-alpha.3" +authors = ["Parity Technologies "] +edition = "2018" +license = "GPL-3.0" + +[dependencies] +serde = { version = "1.0.101", optional = true } +codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-alpha.3", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.3", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-alpha.3", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-alpha.3", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.3", default-features = false, path = "../system" } +frame-benchmarking = { version = "2.0.0-alpha.3", default-features = false, path = "../benchmarking" } + +[features] +default = ["std"] +std = [ + "serde", + "codec/std", + "sp-std/std", + "sp-io/std", + "sp-runtime/std", + "frame-support/std", + "frame-system/std", + "frame-benchmarking/std", +] diff --git a/frame/benchmark/src/benchmarking.rs b/frame/benchmark/src/benchmarking.rs new file mode 100644 index 0000000000000000000000000000000000000000..29f9e8ee972a5bf95dd774eaa0cc6dda316e8de1 --- /dev/null +++ b/frame/benchmark/src/benchmarking.rs @@ -0,0 +1,131 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Benchmarks for common FRAME Pallet operations. + +use super::*; + +use frame_system::RawOrigin; +use sp_std::prelude::*; +use frame_benchmarking::{benchmarks, account}; + +use crate::Module as Benchmark; + +const SEED: u32 = 0; + +benchmarks! { + _ { + let m in 1 .. 1000 => { + let origin = RawOrigin::Signed(account("member", m, SEED)); + Benchmark::::add_member_list(origin.into())? + }; + let i in 1 .. 1000 => { + MyMap::insert(i, i); + }; + let d in 1 .. 1000 => { + for i in 0..d { + for j in 0..100 { + MyDoubleMap::insert(i, j, d); + } + } + }; + } + + add_member_list { + let m in ...; + }: _(RawOrigin::Signed(account("member", m + 1, SEED))) + + append_member_list { + let m in ...; + }: _(RawOrigin::Signed(account("member", m + 1, SEED))) + + read_value { + let n in 1 .. 1000; + MyValue::put(n); + }: _(RawOrigin::Signed(account("user", 0, SEED)), n) + + put_value { + let n in 1 .. 1000; + }: _(RawOrigin::Signed(account("user", 0, SEED)), n) + + exists_value { + let n in 1 .. 1000; + MyValue::put(n); + }: _(RawOrigin::Signed(account("user", 0, SEED)), n) + + remove_value { + let i in ...; + }: _(RawOrigin::Signed(account("user", 0, SEED)), i) + + read_map { + let i in ...; + }: _(RawOrigin::Signed(account("user", 0, SEED)), i) + + insert_map { + let n in 1 .. 1000; + }: _(RawOrigin::Signed(account("user", 0, SEED)), n) + + contains_key_map { + let i in ...; + }: _(RawOrigin::Signed(account("user", 0, SEED)), i) + + remove_prefix { + let d in ...; + }: _(RawOrigin::Signed(account("user", 0, SEED)), d) + + do_nothing { + let n in 1 .. 1000; + }: _(RawOrigin::Signed(account("user", 0, SEED)), n) + + encode_accounts { + let a in 1 .. 1000; + let mut accounts = Vec::new(); + for _ in 0..a { + accounts.push(account::("encode", a, SEED)); + } + }: _(RawOrigin::Signed(account("user", 0, SEED)), accounts) + + decode_accounts { + let a in 1 .. 1000; + let mut accounts = Vec::new(); + for _ in 0..a { + accounts.push(account::("encode", a, SEED)); + } + let bytes = accounts.encode(); + }: _(RawOrigin::Signed(account("user", 0, SEED)), bytes) + + // Custom implementation to handle benchmarking of storage recalculation. + // Puts `repeat` number of items into random storage keys, and then times how + // long it takes to recalculate the storage root. + storage_root { + let z in 0 .. 10000; + }: { + for index in 0 .. z { + let random = (index).using_encoded(sp_io::hashing::blake2_256); + sp_io::storage::set(&random, &random); + } + } + + // Custom implementation to handle benchmarking of calling a host function. + // Will check how long it takes to call `current_time()`. + current_time { + let z in 0 .. 1000; + }: { + for _ in 0 .. z { + let _ = frame_benchmarking::benchmarking::current_time(); + } + } +} diff --git a/frame/benchmark/src/lib.rs b/frame/benchmark/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..ef7731eea43d4610a93213843ef04b26d13db1b8 --- /dev/null +++ b/frame/benchmark/src/lib.rs @@ -0,0 +1,177 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! A pallet that contains common runtime patterns in an isolated manner. +//! This pallet is **not** meant to be used in a production blockchain, just +//! for benchmarking and testing purposes. + +#![cfg_attr(not(feature = "std"), no_std)] + +use frame_support::{decl_module, decl_storage, decl_event, decl_error}; +use frame_support::traits::Currency; +use frame_system::{self as system, ensure_signed}; +use codec::{Encode, Decode}; +use sp_std::prelude::Vec; + +pub mod benchmarking; + +/// Type alias for currency balance. +pub type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; + +/// The pallet's configuration trait. +pub trait Trait: system::Trait { + type Event: From> + Into<::Event>; + type Currency: Currency; +} + +// This pallet's storage items. +decl_storage! { + trait Store for Module as Benchmark { + MyMemberList: Vec; + MyMemberMap: map hasher(blake2_256) T::AccountId => bool; + MyValue: u32; + MyMap: map hasher(blake2_256) u32 => u32; + MyDoubleMap: double_map hasher(blake2_256) u32, hasher(blake2_256) u32 => u32; + } +} + +// The pallet's events +decl_event!( + pub enum Event where AccountId = ::AccountId { + Dummy(u32, AccountId), + } +); + +// The pallet's errors +decl_error! { + pub enum Error for Module { + } +} + +// The pallet's dispatchable functions. +decl_module! { + /// The module declaration. + pub struct Module for enum Call where origin: T::Origin { + type Error = Error; + + fn deposit_event() = default; + + /// Do nothing. + pub fn do_nothing(_origin, input: u32) { + if input > 0 { + return Ok(()); + } + } + + /// Read a value from storage value `repeat` number of times. + /// Note the first `get()` read here will pull from the underlying + /// storage database, however, the `repeat` calls will all pull from the + /// storage overlay cache. You must consider this when analyzing the + /// results of the benchmark. + pub fn read_value(_origin, repeat: u32) { + for _ in 0..repeat { + MyValue::get(); + } + } + + /// Put a value into a storage value. + pub fn put_value(_origin, repeat: u32) { + for r in 0..repeat { + MyValue::put(r); + } + } + + /// Read a value from storage `repeat` number of times. + /// Note the first `exists()` read here will pull from the underlying + /// storage database, however, the `repeat` calls will all pull from the + /// storage overlay cache. You must consider this when analyzing the + /// results of the benchmark. + pub fn exists_value(_origin, repeat: u32) { + for _ in 0..repeat { + MyValue::exists(); + } + } + + /// Remove a value from storage `repeat` number of times. + pub fn remove_value(_origin, repeat: u32) { + for r in 0..repeat { + MyMap::remove(r); + } + } + + /// Read a value from storage map `repeat` number of times. + pub fn read_map(_origin, repeat: u32) { + for r in 0..repeat { + MyMap::get(r); + } + } + + /// Insert a value into a map. + pub fn insert_map(_origin, repeat: u32) { + for r in 0..repeat { + MyMap::insert(r, r); + } + } + + /// Check is a map contains a value `repeat` number of times. + pub fn contains_key_map(_origin, repeat: u32) { + for r in 0..repeat { + MyMap::contains_key(r); + } + } + + /// Read a value from storage `repeat` number of times. + pub fn remove_prefix(_origin, repeat: u32) { + for r in 0..repeat { + MyDoubleMap::remove_prefix(r); + } + } + + // Add user to the list. + pub fn add_member_list(origin) { + let who = ensure_signed(origin)?; + MyMemberList::::mutate(|x| x.push(who)); + } + + // Append user to the list. + pub fn append_member_list(origin) { + let who = ensure_signed(origin)?; + MyMemberList::::append(&[who])?; + } + + // Encode a vector of accounts to bytes. + pub fn encode_accounts(_origin, accounts: Vec) { + let bytes = accounts.encode(); + + // In an attempt to tell the compiler not to optimize away this benchmark, we will use + // the result of encoding the accounts. + if bytes.is_empty() { + frame_support::print("You are encoding zero accounts."); + } + } + + // Decode bytes into a vector of accounts. + pub fn decode_accounts(_origin, bytes: Vec) { + let accounts: Vec = Decode::decode(&mut bytes.as_slice()).map_err(|_| "Could not decode")?; + + // In an attempt to tell the compiler not to optimize away this benchmark, we will use + // the result of decoding the bytes. + if accounts.is_empty() { + frame_support::print("You are decoding zero bytes."); + } + } + } +} diff --git a/frame/benchmarking/Cargo.toml b/frame/benchmarking/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..b39031a6b77f4422b6ea558d0644201cdf8a3b8d --- /dev/null +++ b/frame/benchmarking/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "frame-benchmarking" +version = "2.0.0-alpha.3" +authors = ["Parity Technologies "] +edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Macro for benchmarking a FRAME runtime." + +[dependencies] +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +sp-api = { version = "2.0.0-alpha.2", path = "../../primitives/api", default-features = false } +sp-runtime-interface = { version = "2.0.0-alpha.2", path = "../../primitives/runtime-interface", default-features = false } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../primitives/runtime", default-features = false } +sp-std = { version = "2.0.0-alpha.2", path = "../../primitives/std", default-features = false } +sp-io = { path = "../../primitives/io", default-features = false, version = "2.0.0-alpha.2" } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../system" } + +[features] +default = [ "std" ] +std = [ + "codec/std", + "sp-runtime-interface/std", + "sp-runtime/std", + "sp-api/std", + "sp-std/std", + "frame-support/std", + "frame-system/std", +] diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..a18048d3053370ca4f89461d83436e00d1a1af74 --- /dev/null +++ b/frame/benchmarking/src/lib.rs @@ -0,0 +1,491 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Macro for benchmarking a FRAME runtime. + +#![cfg_attr(not(feature = "std"), no_std)] + +mod tests; +mod utils; +pub use utils::*; +#[doc(hidden)] +pub use sp_io::storage::root as storage_root; +pub use sp_runtime::traits::Dispatchable; + +/// Construct pallet benchmarks for weighing dispatchables. +/// +/// Works around the idea of complexity parameters, named by a single letter (which is usually +/// upper cased in complexity notation but is lower-cased for use in this macro). +/// +/// Complexity parameters ("parameters") have a range which is a `u32` pair. Every time a benchmark +/// is prepared and run, this parameter takes a concrete value within the range. There is an +/// associated instancing block, which is a single expression that is evaluated during +/// preparation. It may use `?` (`i.e. `return Err(...)`) to bail with a string error. Here's a +/// few examples: +/// +/// ```ignore +/// // These two are equivalent: +/// let x in 0 .. 10; +/// let x in 0 .. 10 => (); +/// // This one calls a setup function and might return an error (which would be terminal). +/// let y in 0 .. 10 => setup(y)?; +/// // This one uses a code block to do lots of stuff: +/// let z in 0 .. 10 => { +/// let a = z * z / 5; +/// let b = do_something(a)?; +/// combine_into(z, b); +/// } +/// ``` +/// +/// Note that due to parsing restrictions, if the `from` expression is not a single token (i.e. a +/// literal or constant), then it must be parenthesised. +/// +/// The macro allows for a number of "arms", each representing an individual benchmark. Using the +/// simple syntax, the associated dispatchable function maps 1:1 with the benchmark and the name of +/// the benchmark is the same as that of the associated function. However, extended syntax allows +/// for arbitrary expresions to be evaluated in a benchmark (including for example, +/// `on_initialize`). +/// +/// The macro allows for common parameters whose ranges and instancing expressions may be drawn upon +/// (or not) by each arm. Syntax is available to allow for only the range to be drawn upon if +/// desired, allowing an alternative instancing expression to be given. +/// +/// Each arm may also have a block of code which is run prior to any instancing and a block of code +/// which is run afterwards. All code blocks may draw upon the specific value of each parameter +/// at any time. Local variables are shared between the two pre- and post- code blocks, but do not +/// leak from the interior of any instancing expressions. +/// +/// Any common parameters that are unused in an arm do not have their instancing expressions +/// evaluated. +/// +/// Example: +/// ```ignore +/// benchmarks! { +/// // common parameter; just one for this example. +/// _ { +/// let l in 1 .. MAX_LENGTH => initialize_l(l); +/// } +/// +/// // first dispatchable: foo; this is a user dispatchable and operates on a `u8` vector of +/// // size `l`, which we allow to be initialized as usual. +/// foo { +/// let caller = account::(b"caller", 0, benchmarks_seed); +/// let l = ...; +/// }: _(Origin::Signed(caller), vec![0u8; l]) +/// +/// // second dispatchable: bar; this is a root dispatchable and accepts a `u8` vector of size +/// // `l`. We don't want it pre-initialized like before so we override using the `=> ()` notation. +/// // In this case, we explicitly name the call using `bar` instead of `_`. +/// bar { +/// let l = _ .. _ => (); +/// }: bar(Origin::Root, vec![0u8; l]) +/// +/// // third dispatchable: baz; this is a user dispatchable. It isn't dependent on length like the +/// // other two but has its own complexity `c` that needs setting up. It uses `caller` (in the +/// // pre-instancing block) within the code block. This is only allowed in the param instancers +/// // of arms. Instancers of common params cannot optimistically draw upon hypothetical variables +/// // that the arm's pre-instancing code block might have declared. +/// baz1 { +/// let caller = account::(b"caller", 0, benchmarks_seed); +/// let c = 0 .. 10 => setup_c(&caller, c); +/// }: baz(Origin::Signed(caller)) +/// +/// // this is a second benchmark of the baz dispatchable with a different setup. +/// baz2 { +/// let caller = account::(b"caller", 0, benchmarks_seed); +/// let c = 0 .. 10 => setup_c_in_some_other_way(&caller, c); +/// }: baz(Origin::Signed(caller)) +/// +/// // this is benchmarking some code that is not a dispatchable. +/// populate_a_set { +/// let x in 0 .. 10_000; +/// let mut m = Vec::::new(); +/// for i in 0..x { +/// m.insert(i); +/// } +/// }: { m.into_iter().collect::() } +/// } +/// ``` +#[macro_export] +macro_rules! benchmarks { + ( + _ { + $( + let $common:ident in $common_from:tt .. $common_to:expr => $common_instancer:expr; + )* + } + $( $rest:tt )* + ) => { + $crate::benchmarks_iter!({ + $( { $common , $common_from , $common_to , $common_instancer } )* + } ( ) $( $rest )* ); + } +} + +#[macro_export] +#[allow(missing_docs)] +macro_rules! benchmarks_iter { + // mutation arm: + ( + { $( $common:tt )* } + ( $( $names:ident )* ) + $name:ident { $( $code:tt )* }: _ ( $origin:expr $( , $arg:expr )* ) + $( $rest:tt )* + ) => { + $crate::benchmarks_iter! { + { $( $common )* } ( $( $names )* ) $name { $( $code )* }: $name ( $origin $( , $arg )* ) $( $rest )* + } + }; + // mutation arm: + ( + { $( $common:tt )* } + ( $( $names:ident )* ) + $name:ident { $( $code:tt )* }: $dispatch:ident ( $origin:expr $( , $arg:expr )* ) + $( $rest:tt )* + ) => { + $crate::benchmarks_iter! { + { $( $common )* } ( $( $names )* ) $name { $( $code )* }: { + as $crate::Dispatchable>::dispatch(Call::::$dispatch($($arg),*), $origin.into())?; + } $( $rest )* + } + }; + // iteration arm: + ( + { $( $common:tt )* } + ( $( $names:ident )* ) + $name:ident { $( $code:tt )* }: $eval:block + $( $rest:tt )* + ) => { + $crate::benchmark_backend! { + $name { $( $common )* } { } { $eval } { $( $code )* } + } + $crate::benchmarks_iter!( { $( $common )* } ( $( $names )* $name ) $( $rest )* ); + }; + // iteration-exit arm + ( { $( $common:tt )* } ( $( $names:ident )* ) ) => { + $crate::selected_benchmark!( $( $names ),* ); + $crate::impl_benchmark!( $( $names ),* ); + } +} + +#[macro_export] +#[allow(missing_docs)] +macro_rules! benchmark_backend { + // parsing arms + ($name:ident { + $( $common:tt )* + } { + $( PRE { $( $pre_parsed:tt )* } )* + } { $eval:block } { + let $pre_id:tt : $pre_ty:ty = $pre_ex:expr; + $( $rest:tt )* + } ) => { + $crate::benchmark_backend! { + $name { $( $common )* } { + $( PRE { $( $pre_parsed )* } )* + PRE { $pre_id , $pre_ty , $pre_ex } + } { $eval } { $( $rest )* } + } + }; + ($name:ident { + $( $common:tt )* + } { + $( $parsed:tt )* + } { $eval:block } { + let $param:ident in ( $param_from:expr ) .. $param_to:expr => $param_instancer:expr; + $( $rest:tt )* + }) => { + $crate::benchmark_backend! { + $name { $( $common )* } { + $( $parsed )* + PARAM { $param , $param_from , $param_to , $param_instancer } + } { $eval } { $( $rest )* } + } + }; + // mutation arm to look after defaulting to a common param + ($name:ident { + $( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )* + } { + $( $parsed:tt )* + } { $eval:block } { + let $param:ident in ...; + $( $rest:tt )* + }) => { + $crate::benchmark_backend! { + $name { + $( { $common , $common_from , $common_to , $common_instancer } )* + } { + $( $parsed )* + } { $eval } { + let $param + in ({ $( let $common = $common_from; )* $param }) + .. ({ $( let $common = $common_to; )* $param }) + => ({ $( let $common = || -> Result<(), &'static str> { $common_instancer ; Ok(()) }; )* $param()? }); + $( $rest )* + } + } + }; + // mutation arm to look after defaulting only the range to common param + ($name:ident { + $( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )* + } { + $( $parsed:tt )* + } { $eval:block } { + let $param:ident in _ .. _ => $param_instancer:expr ; + $( $rest:tt )* + }) => { + $crate::benchmark_backend! { + $name { + $( { $common , $common_from , $common_to , $common_instancer } )* + } { + $( $parsed )* + } { $eval } { + let $param + in ({ $( let $common = $common_from; )* $param }) + .. ({ $( let $common = $common_to; )* $param }) + => $param_instancer ; + $( $rest )* + } + } + }; + // mutation arm to look after a single tt for param_from. + ($name:ident { + $( $common:tt )* + } { + $( $parsed:tt )* + } { $eval:block } { + let $param:ident in $param_from:tt .. $param_to:expr => $param_instancer:expr ; + $( $rest:tt )* + }) => { + $crate::benchmark_backend! { + $name { $( $common )* } { $( $parsed )* } { $eval } { + let $param in ( $param_from ) .. $param_to => $param_instancer; + $( $rest )* + } + } + }; + // mutation arm to look after the default tail of `=> ()` + ($name:ident { + $( $common:tt )* + } { + $( $parsed:tt )* + } { $eval:block } { + let $param:ident in $param_from:tt .. $param_to:expr; + $( $rest:tt )* + }) => { + $crate::benchmark_backend! { + $name { $( $common )* } { $( $parsed )* } { $eval } { + let $param in $param_from .. $param_to => (); + $( $rest )* + } + } + }; + // mutation arm to look after `let _ =` + ($name:ident { + $( $common:tt )* + } { + $( $parsed:tt )* + } { $eval:block } { + let $pre_id:tt = $pre_ex:expr; + $( $rest:tt )* + }) => { + $crate::benchmark_backend! { + $name { $( $common )* } { $( $parsed )* } { $eval } { + let $pre_id : _ = $pre_ex; + $( $rest )* + } + } + }; + // actioning arm + ($name:ident { + $( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )* + } { + $( PRE { $pre_id:tt , $pre_ty:ty , $pre_ex:expr } )* + $( PARAM { $param:ident , $param_from:expr , $param_to:expr , $param_instancer:expr } )* + } { $eval:block } { $( $post:tt )* } ) => { + #[allow(non_camel_case_types)] + struct $name; + #[allow(unused_variables)] + impl $crate::BenchmarkingSetup for $name { + fn components(&self) -> Vec<($crate::BenchmarkParameter, u32, u32)> { + vec! [ + $( + ($crate::BenchmarkParameter::$param, $param_from, $param_to) + ),* + ] + } + + fn instance(&self, components: &[($crate::BenchmarkParameter, u32)]) + -> Result Result<(), &'static str>>, &'static str> + { + $( + let $common = $common_from; + )* + $( + // Prepare instance + let $param = components.iter().find(|&c| c.0 == $crate::BenchmarkParameter::$param).unwrap().1; + )* + $( + let $pre_id : $pre_ty = $pre_ex; + )* + $( $param_instancer ; )* + $( $post )* + + Ok(Box::new(move || -> Result<(), &'static str> { $eval; Ok(()) })) + } + } + } +} + +/// Creates a `SelectedBenchmark` enum implementing `BenchmarkingSetup`. +/// +/// Every variant must implement [`BenchmarkingSetup`]. +/// +/// ```nocompile +/// +/// struct Transfer; +/// impl BenchmarkingSetup for Transfer { ... } +/// +/// struct SetBalance; +/// impl BenchmarkingSetup for SetBalance { ... } +/// +/// selected_benchmark!(Transfer, SetBalance); +/// ``` +#[macro_export] +macro_rules! selected_benchmark { + ( + $( $bench:ident ),* + ) => { + // The list of available benchmarks for this pallet. + #[allow(non_camel_case_types)] + enum SelectedBenchmark { + $( $bench, )* + } + + // Allow us to select a benchmark from the list of available benchmarks. + impl $crate::BenchmarkingSetup for SelectedBenchmark { + fn components(&self) -> Vec<($crate::BenchmarkParameter, u32, u32)> { + match self { + $( Self::$bench => <$bench as $crate::BenchmarkingSetup>::components(&$bench), )* + } + } + + fn instance(&self, components: &[($crate::BenchmarkParameter, u32)]) + -> Result Result<(), &'static str>>, &'static str> + { + match self { + $( Self::$bench => <$bench as $crate::BenchmarkingSetup>::instance(&$bench, components), )* + } + } + } + }; +} + +#[macro_export] +macro_rules! impl_benchmark { + ( + $( $name:ident ),* + ) => { + impl $crate::Benchmarking<$crate::BenchmarkResults> for Module { + fn run_benchmark( + extrinsic: Vec, + lowest_range_values: Vec, + highest_range_values: Vec, + steps: Vec, + repeat: u32, + ) -> Result, &'static str> { + // Map the input to the selected benchmark. + let extrinsic = sp_std::str::from_utf8(extrinsic.as_slice()) + .map_err(|_| "`extrinsic` is not a valid utf8 string!")?; + let selected_benchmark = match extrinsic { + $( stringify!($name) => SelectedBenchmark::$name, )* + _ => return Err("Could not find extrinsic."), + }; + + // Warm up the DB + $crate::benchmarking::commit_db(); + $crate::benchmarking::wipe_db(); + + let components = >::components(&selected_benchmark); + let mut results: Vec<$crate::BenchmarkResults> = Vec::new(); + + // Default number of steps for a component. + let mut prev_steps = 10; + + // Select the component we will be benchmarking. Each component will be benchmarked. + for (idx, (name, low, high)) in components.iter().enumerate() { + // Get the number of steps for this component. + let steps = steps.get(idx).cloned().unwrap_or(prev_steps); + prev_steps = steps; + + let lowest = lowest_range_values.get(idx).cloned().unwrap_or(*low); + let highest = highest_range_values.get(idx).cloned().unwrap_or(*high); + + let diff = highest - lowest; + + // Create up to `STEPS` steps for that component between high and low. + let step_size = (diff / steps).max(1); + let num_of_steps = diff / step_size + 1; + + for s in 0..num_of_steps { + // This is the value we will be testing for component `name` + let component_value = lowest + step_size * s; + + // Select the max value for all the other components. + let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() + .enumerate() + .map(|(idx, (n, _, h))| + if n == name { + (*n, component_value) + } else { + (*n, *highest_range_values.get(idx).unwrap_or(h)) + } + ) + .collect(); + + // Run the benchmark `repeat` times. + for _ in 0..repeat { + // Set up the externalities environment for the setup we want to benchmark. + let closure_to_benchmark = >::instance(&selected_benchmark, &c)?; + + // Commit the externalities to the database, flushing the DB cache. + // This will enable worst case scenario for reading from the database. + $crate::benchmarking::commit_db(); + + // Time the extrinsic logic. + let start_extrinsic = $crate::benchmarking::current_time(); + closure_to_benchmark()?; + let finish_extrinsic = $crate::benchmarking::current_time(); + let elapsed_extrinsic = finish_extrinsic - start_extrinsic; + + // Time the storage root recalculation. + let start_storage_root = $crate::benchmarking::current_time(); + $crate::storage_root(); + let finish_storage_root = $crate::benchmarking::current_time(); + let elapsed_storage_root = finish_storage_root - start_storage_root; + + results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root)); + + // Wipe the DB back to the genesis state. + $crate::benchmarking::wipe_db(); + } + } + } + return Ok(results); + } + } + } +} diff --git a/frame/benchmarking/src/tests.rs b/frame/benchmarking/src/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..4327476c4a6c7c59dae5019935ab06879d61e353 --- /dev/null +++ b/frame/benchmarking/src/tests.rs @@ -0,0 +1,166 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Tests for the module. + +#![cfg(test)] + +use super::*; +use codec::Decode; +use sp_std::prelude::*; +use sp_runtime::{traits::{BlakeTwo256, IdentityLookup}, testing::{H256, Header}}; +use frame_support::{dispatch::DispatchResult, decl_module, impl_outer_origin}; +use frame_system::{RawOrigin, ensure_signed, ensure_none}; + +decl_module! { + pub struct Module for enum Call where origin: T::Origin { + fn dummy(origin, _n: u32) -> DispatchResult { + let _sender = ensure_signed(origin)?; + Ok(()) + } + + fn other_dummy(origin, _n: u32) -> DispatchResult { + let _sender = ensure_none(origin)?; + Ok(()) + } + } +} + +impl_outer_origin! { + pub enum Origin for Test where system = frame_system {} +} + +pub trait Trait { + type Event; + type BlockNumber; + type AccountId: 'static + Default + Decode; + type Origin: From> + Into, Self::Origin>>; +} + +#[derive(Clone, Eq, PartialEq)] +pub struct Test; + +impl frame_system::Trait for Test { + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Call = (); + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = (); + type BlockHashCount = (); + type MaximumBlockWeight = (); + type MaximumBlockLength = (); + type AvailableBlockRatio = (); + type Version = (); + type ModuleToIndex = (); + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); +} + +impl Trait for Test { + type Event = (); + type BlockNumber = u32; + type Origin = Origin; + type AccountId = u64; +} + +// This function basically just builds a genesis storage key/value store according to +// our desired mockup. +fn new_test_ext() -> sp_io::TestExternalities { + frame_system::GenesisConfig::default().build_storage::().unwrap().into() +} + +benchmarks!{ + _ { + // Define a common range for `b`. + let b in 1 .. 1000 => (); + } + + dummy { + let b in ...; + let caller = account("caller", 0, 0); + }: _ (RawOrigin::Signed(caller), b.into()) + + other_name { + let b in ...; + let caller = account("caller", 0, 0); + }: other_dummy (RawOrigin::Signed(caller), b.into()) + + sort_vector { + let x in 0 .. 10000; + let mut m = Vec::::new(); + for i in 0..x { + m.push(i); + } + }: { + m.sort(); + } +} + +#[test] +fn benchmarks_macro_works() { + // Check benchmark creation for `dummy`. + let selected_benchmark = SelectedBenchmark::dummy; + + let components = >::components(&selected_benchmark); + assert_eq!(components, vec![(BenchmarkParameter::b, 1, 1000)]); + + let closure = >::instance( + &selected_benchmark, + &[(BenchmarkParameter::b, 1)], + ).expect("failed to create closure"); + + new_test_ext().execute_with(|| { + assert_eq!(closure(), Ok(())); + }); +} + +#[test] +fn benchmarks_macro_rename_works() { + // Check benchmark creation for `other_dummy`. + let selected_benchmark = SelectedBenchmark::other_name; + let components = >::components(&selected_benchmark); + assert_eq!(components, vec![(BenchmarkParameter::b, 1, 1000)]); + + let closure = >::instance( + &selected_benchmark, + &[(BenchmarkParameter::b, 1)], + ).expect("failed to create closure"); + + new_test_ext().execute_with(|| { + assert_eq!(closure(), Err("Bad origin")); + }); +} + +#[test] +fn benchmarks_macro_works_for_non_dispatchable() { + let selected_benchmark = SelectedBenchmark::sort_vector; + + let components = >::components(&selected_benchmark); + assert_eq!(components, vec![(BenchmarkParameter::x, 0, 10000)]); + + let closure = >::instance( + &selected_benchmark, + &[(BenchmarkParameter::x, 1)], + ).expect("failed to create closure"); + + assert_eq!(closure(), Ok(())); +} diff --git a/frame/benchmarking/src/utils.rs b/frame/benchmarking/src/utils.rs new file mode 100644 index 0000000000000000000000000000000000000000..bc6cfbcc86ea1e221cdceb7728fd89b2e7bb2bd9 --- /dev/null +++ b/frame/benchmarking/src/utils.rs @@ -0,0 +1,108 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Interfaces, types and utils for benchmarking a FRAME runtime. + +use codec::{Encode, Decode}; +use sp_std::{vec::Vec, prelude::Box}; +use sp_io::hashing::blake2_256; +use sp_runtime::RuntimeString; + +/// An alphabet of possible parameters to use for benchmarking. +#[derive(codec::Encode, codec::Decode, Clone, Copy, PartialEq, Debug)] +#[allow(missing_docs)] +#[allow(non_camel_case_types)] +pub enum BenchmarkParameter { + a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, +} + +/// Results from running benchmarks on a FRAME pallet. +/// Contains duration of the function call in nanoseconds along with the benchmark parameters +/// used for that benchmark result. +pub type BenchmarkResults = (Vec<(BenchmarkParameter, u32)>, u128, u128); + +sp_api::decl_runtime_apis! { + /// Runtime api for benchmarking a FRAME runtime. + pub trait Benchmark { + /// Dispatch the given benchmark. + fn dispatch_benchmark( + module: Vec, + extrinsic: Vec, + lowest_range_values: Vec, + highest_range_values: Vec, + steps: Vec, + repeat: u32, + ) -> Result, RuntimeString>; + } +} + +/// Interface that provides functions for benchmarking the runtime. +#[sp_runtime_interface::runtime_interface] +pub trait Benchmarking { + /// Get the number of nanoseconds passed since the UNIX epoch + /// + /// WARNING! This is a non-deterministic call. Do not use this within + /// consensus critical logic. + fn current_time() -> u128 { + std::time::SystemTime::now().duration_since(std::time::SystemTime::UNIX_EPOCH) + .expect("Unix time doesn't go backwards; qed") + .as_nanos() + } + + /// Reset the trie database to the genesis state. + fn wipe_db(&mut self) { + self.wipe() + } + + /// Commit pending storage changes to the trie database and clear the database cache. + fn commit_db(&mut self) { + self.commit() + } +} + +/// The pallet benchmarking trait. +pub trait Benchmarking { + /// Run the benchmarks for this pallet. + /// + /// Parameters + /// - `extrinsic`: The name of extrinsic function you want to benchmark encoded as bytes. + /// - `steps`: The number of sample points you want to take across the range of parameters. + /// - `lowest_range_values`: The lowest number for each range of parameters. + /// - `highest_range_values`: The highest number for each range of parameters. + /// - `repeat`: The number of times you want to repeat a benchmark. + fn run_benchmark( + extrinsic: Vec, + lowest_range_values: Vec, + highest_range_values: Vec, + steps: Vec, + repeat: u32, + ) -> Result, &'static str>; +} + +/// The required setup for creating a benchmark. +pub trait BenchmarkingSetup { + /// Return the components and their ranges which should be tested in this benchmark. + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)>; + + /// Set up the storage, and prepare a closure to test in a single run of the benchmark. + fn instance(&self, components: &[(BenchmarkParameter, u32)]) -> Result Result<(), &'static str>>, &'static str>; +} + +/// Grab an account, seeded by a name and index. +pub fn account(name: &'static str, index: u32, seed: u32) -> AccountId { + let entropy = (name, index, seed).using_encoded(blake2_256); + AccountId::decode(&mut &entropy[..]).unwrap_or_default() +} diff --git a/frame/collective/Cargo.toml b/frame/collective/Cargo.toml index 8f7ffa76534908320aeed2761ef09b3d9d5a789c..60899feb4fbb96aa7eb44d637c5cae43bd52612a 100644 --- a/frame/collective/Cargo.toml +++ b/frame/collective/Cargo.toml @@ -1,23 +1,26 @@ [package] name = "pallet-collective" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Collective system: Members of a set of account IDs can make their collective feelings known through dispatched calls from one of two specialized origins." [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +sp-core = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../system" } [dev-dependencies] hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0", path = "../balances" } +pallet-balances = { version = "2.0.0-alpha.2", path = "../balances" } [features] default = ["std"] diff --git a/frame/collective/src/lib.rs b/frame/collective/src/lib.rs index a3ea901a6fd248c4c775e024525d286e913ecdaa..b5620e34065f64415e93de725af1aa6cd372352c 100644 --- a/frame/collective/src/lib.rs +++ b/frame/collective/src/lib.rs @@ -18,7 +18,20 @@ //! through dispatched calls from one of two specialized origins. //! //! The membership can be provided in one of two ways: either directly, using the Root-dispatchable -//! function `set_members`, or indirectly, through implementing the `ChangeMembers` +//! function `set_members`, or indirectly, through implementing the `ChangeMembers`. +//! +//! A "prime" member may be set allowing their vote to act as the default vote in case of any +//! abstentions after the voting period. +//! +//! Voting happens through motions comprising a proposal (i.e. a curried dispatchable) plus a +//! number of approvals required for it to pass and be called. Motions are open for members to +//! vote on for a minimum period given by `MotionDuration`. As soon as the needed number of +//! approvals is given, the motion is closed and executed. If the number of approvals is not reached +//! during the voting period, then `close` may be called by any account in order to force the end +//! the motion explicitly. If a prime member is defined then their vote is used in place of any +//! abstentions and the proposal is executed if there are enough approvals counting the new votes. +//! +//! If there are not, or if no prime is set, then the motion is dropped without being executed. #![cfg_attr(not(feature = "std"), no_std)] #![recursion_limit="128"] @@ -30,7 +43,7 @@ use sp_runtime::traits::{Hash, EnsureOrigin}; use frame_support::weights::SimpleDispatchInfo; use frame_support::{ dispatch::{Dispatchable, Parameter}, codec::{Encode, Decode}, - traits::{ChangeMembers, InitializeMembers}, decl_module, decl_event, + traits::{Get, ChangeMembers, InitializeMembers}, decl_module, decl_event, decl_storage, decl_error, ensure, }; use frame_system::{self as system, ensure_signed, ensure_root}; @@ -53,6 +66,9 @@ pub trait Trait: frame_system::Trait { /// The outer event type. type Event: From> + Into<::Event>; + + /// The time-out for council motions. + type MotionDuration: Get; } /// Origin for the collective module. @@ -71,7 +87,7 @@ pub type Origin = RawOrigin<::Ac #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug)] /// Info for keeping track of a motion being voted on. -pub struct Votes { +pub struct Votes { /// The proposal's unique index. index: ProposalIndex, /// The number of approval votes that are needed to pass the motion. @@ -80,6 +96,8 @@ pub struct Votes { ayes: Vec, /// The current set of voters that rejected it. nays: Vec, + /// The hard end time of this vote. + end: BlockNumber, } decl_storage! { @@ -91,11 +109,14 @@ decl_storage! { map hasher(blake2_256) T::Hash => Option<>::Proposal>; /// Votes on a given proposal, if it is ongoing. pub Voting get(fn voting): - map hasher(blake2_256) T::Hash => Option>; + map hasher(blake2_256) T::Hash => Option>; /// Proposals so far. pub ProposalCount get(fn proposal_count): u32; /// The current members of the collective. This is stored sorted (just by value). pub Members get(fn members): Vec; + /// The member who provides the default vote for any other members that do not vote before + /// the timeout. If None, then no member has that privilege. + pub Prime get(fn prime): Option; } add_extra_genesis { config(phantom): sp_std::marker::PhantomData; @@ -123,6 +144,8 @@ decl_event! { Executed(Hash, bool), /// A single member did some action; `bool` is true if returned without error. MemberExecuted(Hash, bool), + /// A proposal was closed after its duration was up. + Closed(Hash, MemberCount, MemberCount), } } @@ -140,6 +163,8 @@ decl_error! { DuplicateVote, /// Members are already initialized! AlreadyInitialized, + /// The close call is made too early, before the end of the voting. + TooEarly, } } @@ -152,19 +177,21 @@ decl_module! { fn deposit_event() = default; - /// Set the collective's membership manually to `new_members`. Be nice to the chain and - /// provide it pre-sorted. + /// Set the collective's membership. + /// + /// - `new_members`: The new member list. Be nice to the chain and + // provide it sorted. + /// - `prime`: The prime member whose vote sets the default. /// /// Requires root origin. #[weight = SimpleDispatchInfo::FixedOperational(100_000)] - fn set_members(origin, new_members: Vec) { + fn set_members(origin, new_members: Vec, prime: Option) { ensure_root(origin)?; let mut new_members = new_members; new_members.sort(); - >::mutate(|m| { - >::set_members_sorted(&new_members[..], m); - *m = new_members; - }); + let old = Members::::get(); + >::set_members_sorted(&new_members[..], &old); + Prime::::set(prime); } /// Dispatch a proposal from a member using the `Member` origin. @@ -202,7 +229,8 @@ decl_module! { >::mutate(|i| *i += 1); >::mutate(|proposals| proposals.push(proposal_hash)); >::insert(proposal_hash, *proposal); - let votes = Votes { index, threshold, ayes: vec![who.clone()], nays: vec![] }; + let end = system::Module::::block_number() + T::MotionDuration::get(); + let votes = Votes { index, threshold, ayes: vec![who.clone()], nays: vec![], end }; >::insert(proposal_hash, votes); Self::deposit_event(RawEvent::Proposed(who, index, proposal_hash, threshold)); @@ -249,32 +277,55 @@ decl_module! { Self::deposit_event(RawEvent::Voted(who, proposal, approve, yes_votes, no_votes)); let seats = Self::members().len() as MemberCount; + let approved = yes_votes >= voting.threshold; let disapproved = seats.saturating_sub(no_votes) < voting.threshold; if approved || disapproved { - if approved { - Self::deposit_event(RawEvent::Approved(proposal)); - - // execute motion, assuming it exists. - if let Some(p) = >::take(&proposal) { - let origin = RawOrigin::Members(voting.threshold, seats).into(); - let ok = p.dispatch(origin).is_ok(); - Self::deposit_event(RawEvent::Executed(proposal, ok)); - } - } else { - // disapproved - >::remove(&proposal); - Self::deposit_event(RawEvent::Disapproved(proposal)); - } - - // remove vote - >::remove(&proposal); - >::mutate(|proposals| proposals.retain(|h| h != &proposal)); + Self::finalize_proposal(approved, seats, voting, proposal); } else { - // update voting - >::insert(&proposal, voting); + Voting::::insert(&proposal, voting); } } + + /// May be called by any signed account after the voting duration has ended in order to + /// finish voting and close the proposal. + /// + /// Abstentions are counted as rejections unless there is a prime member set and the prime + /// member cast an approval. + /// + /// - the weight of `proposal` preimage. + /// - up to three events deposited. + /// - one read, two removals, one mutation. (plus three static reads.) + /// - computation and i/o `O(P + L + M)` where: + /// - `M` is number of members, + /// - `P` is number of active proposals, + /// - `L` is the encoded length of `proposal` preimage. + #[weight = SimpleDispatchInfo::FixedOperational(200_000)] + fn close(origin, proposal: T::Hash, #[compact] index: ProposalIndex) { + let _ = ensure_signed(origin)?; + + let voting = Self::voting(&proposal).ok_or(Error::::ProposalMissing)?; + ensure!(voting.index == index, Error::::WrongIndex); + ensure!(system::Module::::block_number() >= voting.end, Error::::TooEarly); + + // default to true only if there's a prime and they voted in favour. + let default = Self::prime().map_or( + false, + |who| voting.ayes.iter().any(|a| a == &who), + ); + + let mut no_votes = voting.nays.len() as MemberCount; + let mut yes_votes = voting.ayes.len() as MemberCount; + let seats = Self::members().len() as MemberCount; + let abstentions = seats - (yes_votes + no_votes); + match default { + true => yes_votes += abstentions, + false => no_votes += abstentions, + } + + Self::deposit_event(RawEvent::Closed(proposal, yes_votes, no_votes)); + Self::finalize_proposal(yes_votes >= voting.threshold, seats, voting, proposal); + } } } @@ -282,10 +333,54 @@ impl, I: Instance> Module { pub fn is_member(who: &T::AccountId) -> bool { Self::members().contains(who) } + + /// Weight: + /// If `approved`: + /// - the weight of `proposal` preimage. + /// - two events deposited. + /// - two removals, one mutation. + /// - computation and i/o `O(P + L)` where: + /// - `P` is number of active proposals, + /// - `L` is the encoded length of `proposal` preimage. + /// + /// If not `approved`: + /// - one event deposited. + /// Two removals, one mutation. + /// Computation and i/o `O(P)` where: + /// - `P` is number of active proposals + fn finalize_proposal( + approved: bool, + seats: MemberCount, + voting: Votes, + proposal: T::Hash, + ) { + if approved { + Self::deposit_event(RawEvent::Approved(proposal)); + + // execute motion, assuming it exists. + if let Some(p) = ProposalOf::::take(&proposal) { + let origin = RawOrigin::Members(voting.threshold, seats).into(); + let ok = p.dispatch(origin).is_ok(); + Self::deposit_event(RawEvent::Executed(proposal, ok)); + } + } else { + // disapproved + ProposalOf::::remove(&proposal); + Self::deposit_event(RawEvent::Disapproved(proposal)); + } + + // remove vote + Voting::::remove(&proposal); + Proposals::::mutate(|proposals| proposals.retain(|h| h != &proposal)); + } } impl, I: Instance> ChangeMembers for Module { - fn change_members_sorted(_incoming: &[T::AccountId], outgoing: &[T::AccountId], new: &[T::AccountId]) { + fn change_members_sorted( + _incoming: &[T::AccountId], + outgoing: &[T::AccountId], + new: &[T::AccountId], + ) { // remove accounts from all current voting in motions. let mut outgoing = outgoing.to_vec(); outgoing.sort_unstable(); @@ -302,7 +397,12 @@ impl, I: Instance> ChangeMembers for Module { } ); } - >::put(new); + Members::::put(new); + Prime::::kill(); + } + + fn set_prime(prime: Option) { + Prime::::set(prime); } } @@ -415,6 +515,7 @@ mod tests { pub const MaximumBlockWeight: Weight = 1024; pub const MaximumBlockLength: u32 = 2 * 1024; pub const AvailableBlockRatio: Perbill = Perbill::one(); + pub const MotionDuration: u64 = 3; } impl frame_system::Trait for Test { type Origin = Origin; @@ -435,17 +536,19 @@ mod tests { type ModuleToIndex = (); type AccountData = (); type OnNewAccount = (); - type OnReapAccount = (); + type OnKilledAccount = (); } impl Trait for Test { type Origin = Origin; type Proposal = Call; type Event = Event; + type MotionDuration = MotionDuration; } impl Trait for Test { type Origin = Origin; type Proposal = Call; type Event = Event; + type MotionDuration = MotionDuration; } pub type Block = sp_runtime::generic::Block; @@ -486,22 +589,101 @@ mod tests { Call::System(frame_system::Call::remark(value.encode())) } + #[test] + fn close_works() { + make_ext().execute_with(|| { + System::set_block_number(1); + let proposal = make_proposal(42); + let hash = BlakeTwo256::hash_of(&proposal); + + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); + assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true)); + + System::set_block_number(3); + assert_noop!( + Collective::close(Origin::signed(4), hash.clone(), 0), + Error::::TooEarly + ); + + System::set_block_number(4); + assert_ok!(Collective::close(Origin::signed(4), hash.clone(), 0)); + + let record = |event| EventRecord { phase: Phase::Finalization, event, topics: vec![] }; + assert_eq!(System::events(), vec![ + record(Event::collective_Instance1(RawEvent::Proposed(1, 0, hash.clone(), 3))), + record(Event::collective_Instance1(RawEvent::Voted(2, hash.clone(), true, 2, 0))), + record(Event::collective_Instance1(RawEvent::Closed(hash.clone(), 2, 1))), + record(Event::collective_Instance1(RawEvent::Disapproved(hash.clone()))) + ]); + }); + } + + #[test] + fn close_with_prime_works() { + make_ext().execute_with(|| { + System::set_block_number(1); + let proposal = make_proposal(42); + let hash = BlakeTwo256::hash_of(&proposal); + assert_ok!(Collective::set_members(Origin::ROOT, vec![1, 2, 3], Some(3))); + + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); + assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true)); + + System::set_block_number(4); + assert_ok!(Collective::close(Origin::signed(4), hash.clone(), 0)); + + let record = |event| EventRecord { phase: Phase::Finalization, event, topics: vec![] }; + assert_eq!(System::events(), vec![ + record(Event::collective_Instance1(RawEvent::Proposed(1, 0, hash.clone(), 3))), + record(Event::collective_Instance1(RawEvent::Voted(2, hash.clone(), true, 2, 0))), + record(Event::collective_Instance1(RawEvent::Closed(hash.clone(), 2, 1))), + record(Event::collective_Instance1(RawEvent::Disapproved(hash.clone()))) + ]); + }); + } + + #[test] + fn close_with_voting_prime_works() { + make_ext().execute_with(|| { + System::set_block_number(1); + let proposal = make_proposal(42); + let hash = BlakeTwo256::hash_of(&proposal); + assert_ok!(Collective::set_members(Origin::ROOT, vec![1, 2, 3], Some(1))); + + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); + assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true)); + + System::set_block_number(4); + assert_ok!(Collective::close(Origin::signed(4), hash.clone(), 0)); + + let record = |event| EventRecord { phase: Phase::Finalization, event, topics: vec![] }; + assert_eq!(System::events(), vec![ + record(Event::collective_Instance1(RawEvent::Proposed(1, 0, hash.clone(), 3))), + record(Event::collective_Instance1(RawEvent::Voted(2, hash.clone(), true, 2, 0))), + record(Event::collective_Instance1(RawEvent::Closed(hash.clone(), 3, 0))), + record(Event::collective_Instance1(RawEvent::Approved(hash.clone()))), + record(Event::collective_Instance1(RawEvent::Executed(hash.clone(), false))) + ]); + }); + } + #[test] fn removal_of_old_voters_votes_works() { make_ext().execute_with(|| { System::set_block_number(1); let proposal = make_proposal(42); let hash = BlakeTwo256::hash_of(&proposal); + let end = 4; assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true)); assert_eq!( Collective::voting(&hash), - Some(Votes { index: 0, threshold: 3, ayes: vec![1, 2], nays: vec![] }) + Some(Votes { index: 0, threshold: 3, ayes: vec![1, 2], nays: vec![], end }) ); Collective::change_members_sorted(&[4], &[1], &[2, 3, 4]); assert_eq!( Collective::voting(&hash), - Some(Votes { index: 0, threshold: 3, ayes: vec![2], nays: vec![] }) + Some(Votes { index: 0, threshold: 3, ayes: vec![2], nays: vec![], end }) ); let proposal = make_proposal(69); @@ -510,12 +692,12 @@ mod tests { assert_ok!(Collective::vote(Origin::signed(3), hash.clone(), 1, false)); assert_eq!( Collective::voting(&hash), - Some(Votes { index: 1, threshold: 2, ayes: vec![2], nays: vec![3] }) + Some(Votes { index: 1, threshold: 2, ayes: vec![2], nays: vec![3], end }) ); Collective::change_members_sorted(&[], &[3], &[2, 4]); assert_eq!( Collective::voting(&hash), - Some(Votes { index: 1, threshold: 2, ayes: vec![2], nays: vec![] }) + Some(Votes { index: 1, threshold: 2, ayes: vec![2], nays: vec![], end }) ); }); } @@ -526,16 +708,17 @@ mod tests { System::set_block_number(1); let proposal = make_proposal(42); let hash = BlakeTwo256::hash_of(&proposal); + let end = 4; assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true)); assert_eq!( Collective::voting(&hash), - Some(Votes { index: 0, threshold: 3, ayes: vec![1, 2], nays: vec![] }) + Some(Votes { index: 0, threshold: 3, ayes: vec![1, 2], nays: vec![], end }) ); - assert_ok!(Collective::set_members(Origin::ROOT, vec![2, 3, 4])); + assert_ok!(Collective::set_members(Origin::ROOT, vec![2, 3, 4], None)); assert_eq!( Collective::voting(&hash), - Some(Votes { index: 0, threshold: 3, ayes: vec![2], nays: vec![] }) + Some(Votes { index: 0, threshold: 3, ayes: vec![2], nays: vec![], end }) ); let proposal = make_proposal(69); @@ -544,12 +727,12 @@ mod tests { assert_ok!(Collective::vote(Origin::signed(3), hash.clone(), 1, false)); assert_eq!( Collective::voting(&hash), - Some(Votes { index: 1, threshold: 2, ayes: vec![2], nays: vec![3] }) + Some(Votes { index: 1, threshold: 2, ayes: vec![2], nays: vec![3], end }) ); - assert_ok!(Collective::set_members(Origin::ROOT, vec![2, 4])); + assert_ok!(Collective::set_members(Origin::ROOT, vec![2, 4], None)); assert_eq!( Collective::voting(&hash), - Some(Votes { index: 1, threshold: 2, ayes: vec![2], nays: vec![] }) + Some(Votes { index: 1, threshold: 2, ayes: vec![2], nays: vec![], end }) ); }); } @@ -560,12 +743,13 @@ mod tests { System::set_block_number(1); let proposal = make_proposal(42); let hash = proposal.blake2_256().into(); + let end = 4; assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); assert_eq!(Collective::proposals(), vec![hash]); assert_eq!(Collective::proposal_of(&hash), Some(proposal)); assert_eq!( Collective::voting(&hash), - Some(Votes { index: 0, threshold: 3, ayes: vec![1], nays: vec![] }) + Some(Votes { index: 0, threshold: 3, ayes: vec![1], nays: vec![], end }) ); assert_eq!(System::events(), vec![ @@ -629,10 +813,11 @@ mod tests { System::set_block_number(1); let proposal = make_proposal(42); let hash: H256 = proposal.blake2_256().into(); + let end = 4; assert_ok!(Collective::propose(Origin::signed(1), 2, Box::new(proposal.clone()))); assert_eq!( Collective::voting(&hash), - Some(Votes { index: 0, threshold: 2, ayes: vec![1], nays: vec![] }) + Some(Votes { index: 0, threshold: 2, ayes: vec![1], nays: vec![], end }) ); assert_noop!( Collective::vote(Origin::signed(1), hash.clone(), 0, true), @@ -641,7 +826,7 @@ mod tests { assert_ok!(Collective::vote(Origin::signed(1), hash.clone(), 0, false)); assert_eq!( Collective::voting(&hash), - Some(Votes { index: 0, threshold: 2, ayes: vec![], nays: vec![1] }) + Some(Votes { index: 0, threshold: 2, ayes: vec![], nays: vec![1], end }) ); assert_noop!( Collective::vote(Origin::signed(1), hash.clone(), 0, false), diff --git a/frame/contracts/Cargo.toml b/frame/contracts/Cargo.toml index 159e9f9d0c199464d13f044b4ea92a475a38323a..86f39f8a82b94d65e2da3e8535174d2ecde4ea77 100644 --- a/frame/contracts/Cargo.toml +++ b/frame/contracts/Cargo.toml @@ -1,32 +1,35 @@ [package] name = "pallet-contracts" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet for WASM contracts" [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } pwasm-utils = { version = "0.12.0", default-features = false } -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } parity-wasm = { version = "0.41.0", default-features = false } wasmi-validation = { version = "0.3.0", default-features = false } -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-sandbox = { version = "0.8.0", default-features = false, path = "../../primitives/sandbox" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } -pallet-contracts-primitives = { version = "2.0.0", default-features = false, path = "common" } +sp-core = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/runtime" } +sp-io = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/io" } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/std" } +sp-sandbox = { version = "0.8.0-alpha.2", default-features = false, path = "../../primitives/sandbox" } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../system" } +pallet-contracts-primitives = { version = "2.0.0-alpha.2", default-features = false, path = "common" } [dev-dependencies] wabt = "0.9.2" assert_matches = "1.3.0" hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0", path = "../balances" } -pallet-timestamp = { version = "2.0.0", path = "../timestamp" } -pallet-randomness-collective-flip = { version = "2.0.0", path = "../randomness-collective-flip" } +pallet-balances = { version = "2.0.0-alpha.2", path = "../balances" } +pallet-timestamp = { version = "2.0.0-alpha.2", path = "../timestamp" } +pallet-randomness-collective-flip = { version = "2.0.0-alpha.2", path = "../randomness-collective-flip" } [features] default = ["std"] diff --git a/frame/contracts/common/Cargo.toml b/frame/contracts/common/Cargo.toml index 6e4ee050bd1978b3da839e5f2d07a68192d05390..0d009e7c82b41285fe12abdbe68ac07343c80985 100644 --- a/frame/contracts/common/Cargo.toml +++ b/frame/contracts/common/Cargo.toml @@ -1,14 +1,18 @@ [package] name = "pallet-contracts-primitives" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "A crate that hosts a common definitions that are relevant for the pallet-contracts." [dependencies] # This crate should not rely on any of the frame primitives. -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0", default-features = false, path = "../../../primitives/std" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/std" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../../primitives/runtime" } [features] default = ["std"] diff --git a/frame/contracts/rpc/Cargo.toml b/frame/contracts/rpc/Cargo.toml index d59260d11f536da807679531e4350edaf9d4e8d8..092d049c5414ba7964860732fd8f8b3e2e3022a1 100644 --- a/frame/contracts/rpc/Cargo.toml +++ b/frame/contracts/rpc/Cargo.toml @@ -1,23 +1,26 @@ [package] name = "pallet-contracts-rpc" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Node-specific RPC methods for interaction with contracts." [dependencies] -codec = { package = "parity-scale-codec", version = "1.0.0" } +codec = { package = "parity-scale-codec", version = "1.2.0" } jsonrpc-core = "14.0.3" jsonrpc-core-client = "14.0.3" jsonrpc-derive = "14.0.3" -sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } -sp-core = { version = "2.0.0", path = "../../../primitives/core" } -sp-rpc = { version = "2.0.0", path = "../../../primitives/rpc" } +sp-blockchain = { version = "2.0.0-alpha.2", path = "../../../primitives/blockchain" } +sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } +sp-rpc = { version = "2.0.0-alpha.2", path = "../../../primitives/rpc" } serde = { version = "1.0.101", features = ["derive"] } -sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0", path = "../../../primitives/api" } -pallet-contracts-primitives = { version = "2.0.0", path = "../common" } -pallet-contracts-rpc-runtime-api = { version = "0.8.0", path = "./runtime-api" } +sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-alpha.2", path = "../../../primitives/api" } +pallet-contracts-primitives = { version = "2.0.0-alpha.2", path = "../common" } +pallet-contracts-rpc-runtime-api = { version = "0.8.0-alpha.2", path = "./runtime-api" } [dev-dependencies] serde_json = "1.0.41" diff --git a/frame/contracts/rpc/runtime-api/Cargo.toml b/frame/contracts/rpc/runtime-api/Cargo.toml index dad9b92f6a30891b30f5307a3d732b87f05b9f49..8435a6c4238c9e07d60c79a49aed9ae282993a76 100644 --- a/frame/contracts/rpc/runtime-api/Cargo.toml +++ b/frame/contracts/rpc/runtime-api/Cargo.toml @@ -1,16 +1,19 @@ [package] name = "pallet-contracts-rpc-runtime-api" -version = "0.8.0" +version = "0.8.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Runtime API definition required by Contracts RPC extensions." [dependencies] -sp-api = { version = "2.0.0", default-features = false, path = "../../../../primitives/api" } -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0", default-features = false, path = "../../../../primitives/std" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../../../primitives/runtime" } -pallet-contracts-primitives = { version = "2.0.0", default-features = false, path = "../../common" } +sp-api = { version = "2.0.0-alpha.2", default-features = false, path = "../../../../primitives/api" } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../../../primitives/std" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../../../primitives/runtime" } +pallet-contracts-primitives = { version = "2.0.0-alpha.2", default-features = false, path = "../../common" } [features] default = ["std"] diff --git a/frame/contracts/src/account_db.rs b/frame/contracts/src/account_db.rs index 5204f1003a6c5920714c483a4329de9c1cfdf386..374c55c374d7bf26c0603017f3838b9c0a6a0bb7 100644 --- a/frame/contracts/src/account_db.rs +++ b/frame/contracts/src/account_db.rs @@ -151,7 +151,7 @@ impl AccountDb for DirectAccountDb { let exists = !T::Currency::total_balance(&address).is_zero(); total_imbalance = total_imbalance.merge(imbalance); if existed && !exists { - // Account killed. This will ultimately lead to calling `OnReapAccount` callback + // Account killed. This will ultimately lead to calling `OnKilledAccount` callback // which will make removal of CodeHashOf and AccountStorage for this account. // In order to avoid writing over the deleted properties we `continue` here. continue; diff --git a/frame/contracts/src/gas.rs b/frame/contracts/src/gas.rs index c8572daaa43d487efd84725b26a6492beaa63794..362f15f3aae795ef44b346285de79cd557291a23 100644 --- a/frame/contracts/src/gas.rs +++ b/frame/contracts/src/gas.rs @@ -250,7 +250,11 @@ pub fn refund_unused_gas( pub fn approx_gas_for_balance(gas_price: Balance, balance: Balance) -> Gas where Balance: AtLeast32Bit { - (balance / gas_price).saturated_into::() + if gas_price.is_zero() { + Zero::zero() + } else { + (balance / gas_price).saturated_into::() + } } /// A simple utility macro that helps to match against a @@ -294,7 +298,7 @@ macro_rules! match_tokens { #[cfg(test)] mod tests { use super::{GasMeter, Token}; - use crate::tests::Test; + use crate::{tests::Test, gas::approx_gas_for_balance}; /// A trivial token that charges the specified number of gas units. #[derive(Copy, Clone, PartialEq, Eq, Debug)] @@ -382,4 +386,22 @@ mod tests { let mut gas_meter = GasMeter::::with_limit(25, 10); assert!(!gas_meter.charge(&(), SimpleToken(25)).is_out_of_gas()); } + + // A unit test for `fn approx_gas_for_balance()`, and makes + // sure setting gas_price 0 does not cause `div by zero` error. + #[test] + fn approx_gas_for_balance_works() { + let tests = vec![ + (approx_gas_for_balance(0_u64, 123), 0), + (approx_gas_for_balance(0_u64, 456), 0), + (approx_gas_for_balance(1_u64, 123), 123), + (approx_gas_for_balance(1_u64, 456), 456), + (approx_gas_for_balance(100_u64, 900), 9), + (approx_gas_for_balance(123_u64, 900), 7), + ]; + + for (lhs, rhs) in tests { + assert_eq!(lhs, rhs); + } + } } diff --git a/frame/contracts/src/lib.rs b/frame/contracts/src/lib.rs index 42cbaa3a7c2af2e7251d885c1e55c2187fa145b8..93e470a51bb83fbccb28dba4709985c22d7362e8 100644 --- a/frame/contracts/src/lib.rs +++ b/frame/contracts/src/lib.rs @@ -125,7 +125,7 @@ use frame_support::{ parameter_types, IsSubType, weights::DispatchInfo, }; -use frame_support::traits::{OnReapAccount, OnUnbalanced, Currency, Get, Time, Randomness}; +use frame_support::traits::{OnKilledAccount, OnUnbalanced, Currency, Get, Time, Randomness}; use frame_system::{self as system, ensure_signed, RawOrigin, ensure_root}; use sp_core::storage::well_known_keys::CHILD_STORAGE_KEY_PREFIX; use pallet_contracts_primitives::{RentProjection, ContractAccessError}; @@ -254,7 +254,7 @@ where let mut buf = Vec::new(); storage_root.using_encoded(|encoded| buf.extend_from_slice(encoded)); buf.extend_from_slice(code_hash.as_ref()); - RawTombstoneContractInfo(Hasher::hash(&buf[..]), PhantomData) + RawTombstoneContractInfo(::hash(&buf[..]), PhantomData) } } @@ -941,8 +941,12 @@ decl_storage! { } } -impl OnReapAccount for Module { - fn on_reap_account(who: &T::AccountId) { +// TODO: this should be removed in favour of a self-destruct contract host function allowing the +// contract to delete all storage and the `ContractInfoOf` key and transfer remaining balance to +// some other account. As it stands, it's an economic insecurity on any smart-contract chain. +// https://github.com/paritytech/substrate/issues/4952 +impl OnKilledAccount for Module { + fn on_killed_account(who: &T::AccountId) { if let Some(ContractInfo::Alive(info)) = >::take(who) { child::kill_storage(&info.trie_id, info.child_trie_unique_id()); } diff --git a/frame/contracts/src/tests.rs b/frame/contracts/src/tests.rs index ddd532334c158181feb8d3041f4f6984d45db727..51da3e8d1ff97ee0048290d073ac53e6e10f291f 100644 --- a/frame/contracts/src/tests.rs +++ b/frame/contracts/src/tests.rs @@ -117,7 +117,7 @@ impl frame_system::Trait for Test { type ModuleToIndex = (); type AccountData = pallet_balances::AccountData; type OnNewAccount = (); - type OnReapAccount = (Balances, Contracts); + type OnKilledAccount = Contracts; } impl pallet_balances::Trait for Test { type Balance = u64; @@ -1606,7 +1606,7 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: assert_eq!(System::events(), vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: MetaEvent::system(system::RawEvent::ReapedAccount(DJANGO)), + event: MetaEvent::system(system::RawEvent::KilledAccount(DJANGO)), topics: vec![], }, EventRecord { @@ -2079,6 +2079,22 @@ fn deploy_and_call_other_contract() { }); } +#[test] +fn deploy_works_without_gas_price() { + let (wasm, code_hash) = compile_module::(CODE_GET_RUNTIME_STORAGE).unwrap(); + ExtBuilder::default().existential_deposit(50).gas_price(0).build().execute_with(|| { + Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 100, + 100_000, + code_hash.into(), + vec![], + )); + }); +} + const CODE_SELF_DESTRUCT: &str = r#" (module (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) diff --git a/frame/democracy/Cargo.toml b/frame/democracy/Cargo.toml index 2428d2c1544dcf7309294f9238b2031cb8cd3210..d2938956bd015333e28c7bbd67fc4a61a1bea282 100644 --- a/frame/democracy/Cargo.toml +++ b/frame/democracy/Cargo.toml @@ -1,23 +1,26 @@ [package] name = "pallet-democracy" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet for democracy" [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0", path = "../balances" } -sp-storage = { version = "2.0.0", path = "../../primitives/storage" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-alpha.2", path = "../balances" } +sp-storage = { version = "2.0.0-alpha.2", path = "../../primitives/storage" } hex-literal = "0.2.1" [features] diff --git a/frame/democracy/src/lib.rs b/frame/democracy/src/lib.rs index c2c44918b036cfb05974c7f748870b36417d3754..5632103ab317e684d9bf5b70e7258da2873b25ea 100644 --- a/frame/democracy/src/lib.rs +++ b/frame/democracy/src/lib.rs @@ -14,7 +14,140 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Democratic system: Handles administration of general stakeholder voting. +//! # Democracy Pallet +//! +//! - [`democracy::Trait`](./trait.Trait.html) +//! - [`Call`](./enum.Call.html) +//! +//! ## Overview +//! +//! The Democracy pallet handles the administration of general stakeholder voting. +//! +//! There are two different queues that a proposal can be added to before it +//! becomes a referendum, 1) the proposal queue consisting of all public proposals +//! and 2) the external queue consisting of a single proposal that originates +//! from one of the _external_ origins (such as a collective group). +//! +//! Every launch period - a length defined in the runtime - the Democracy pallet +//! launches a referendum from a proposal that it takes from either the proposal +//! queue or the external queue in turn. Any token holder in the system can vote +//! on referenda. The voting system +//! uses time-lock voting by allowing the token holder to set their _conviction_ +//! behind a vote. The conviction will dictate the length of time the tokens +//! will be locked, as well as the multiplier that scales the vote power. +//! +//! ### Terminology +//! +//! - **Enactment Period:** The minimum period of locking and the period between a proposal being +//! approved and enacted. +//! - **Lock Period:** A period of time after proposal enactment that the tokens of _winning_ voters +//! will be locked. +//! - **Conviction:** An indication of a voter's strength of belief in their vote. An increase +//! of one in conviction indicates that a token holder is willing to lock their tokens for twice +//! as many lock periods after enactment. +//! - **Vote:** A value that can either be in approval ("Aye") or rejection ("Nay") +//! of a particular referendum. +//! - **Proposal:** A submission to the chain that represents an action that a proposer (either an +//! account or an external origin) suggests that the system adopt. +//! - **Referendum:** A proposal that is in the process of being voted on for +//! either acceptance or rejection as a change to the system. +//! - **Proxy:** An account that votes on behalf of a separate "Stash" account +//! that holds the funds. +//! - **Delegation:** The act of granting your voting power to the decisions of another account. +//! +//! ### Adaptive Quorum Biasing +//! +//! A _referendum_ can be either simple majority-carries in which 50%+1 of the +//! votes decide the outcome or _adaptive quorum biased_. Adaptive quorum biasing +//! makes the threshold for passing or rejecting a referendum higher or lower +//! depending on how the referendum was originally proposed. There are two types of +//! adaptive quorum biasing: 1) _positive turnout bias_ makes a referendum +//! require a super-majority to pass that decreases as turnout increases and +//! 2) _negative turnout bias_ makes a referendum require a super-majority to +//! reject that decreases as turnout increases. Another way to think about the +//! quorum biasing is that _positive bias_ referendums will be rejected by +//! default and _negative bias_ referendums get passed by default. +//! +//! ## Interface +//! +//! ### Dispatchable Functions +//! +//! #### Public +//! +//! These calls can be made from any externally held account capable of creating +//! a signed extrinsic. +//! +//! - `propose` - Submits a sensitive action, represented as a hash. +//! Requires a deposit. +//! - `second` - Signals agreement with a proposal, moves it higher on the +//! proposal queue, and requires a matching deposit to the original. +//! - `vote` - Votes in a referendum, either the vote is "Aye" to enact the +//! proposal or "Nay" to keep the status quo. +//! - `proxy_vote` - Votes in a referendum on behalf of a stash account. +//! - `activate_proxy` - Activates a proxy that is already open to the sender. +//! - `close_proxy` - Clears the proxy status, called by the proxy. +//! - `deactivate_proxy` - Deactivates a proxy back to the open status, called by +//! the stash. +//! - `open_proxy` - Opens a proxy account on behalf of the sender. +//! - `delegate` - Delegates the voting power (tokens * conviction) to another +//! account. +//! - `undelegate` - Stops the delegation of voting power to another account. +//! - `note_preimage` - Registers the preimage for an upcoming proposal, requires +//! a deposit that is returned once the proposal is enacted. +//! - `note_imminent_preimage` - Registers the preimage for an upcoming proposal. +//! Does not require a deposit, but the proposal must be in the dispatch queue. +//! - `reap_preimage` - Removes the preimage for an expired proposal. Will only +//! work under the condition that it's the same account that noted it and +//! after the voting period, OR it's a different account after the enactment period. +//! - `unlock` - Unlocks tokens that have an expired lock. +//! +//! #### Cancellation Origin +//! +//! This call can only be made by the `CancellationOrigin`. +//! +//! - `emergency_cancel` - Schedules an emergency cancellation of a referendum. +//! Can only happen once to a specific referendum. +//! +//! #### ExternalOrigin +//! +//! This call can only be made by the `ExternalOrigin`. +//! +//! - `external_propose` - Schedules a proposal to become a referendum once it is is legal +//! for an externally proposed referendum. +//! +//! #### External Majority Origin +//! +//! This call can only be made by the `ExternalMajorityOrigin`. +//! +//! - `external_propose_majority` - Schedules a proposal to become a majority-carries +//! referendum once it is legal for an externally proposed referendum. +//! +//! #### External Default Origin +//! +//! This call can only be made by the `ExternalDefaultOrigin`. +//! +//! - `external_propose_default` - Schedules a proposal to become a negative-turnout-bias +//! referendum once it is legal for an externally proposed referendum. +//! +//! #### Fast Track Origin +//! +//! This call can only be made by the `FastTrackOrigin`. +//! +//! - `fast_track` - Schedules the current externally proposed proposal that +//! is "majority-carries" to become a referendum immediately. +//! +//! #### Veto Origin +//! +//! This call can only be made by the `VetoOrigin`. +//! +//! - `veto_external` - Vetoes and blacklists the external proposal hash. +//! +//! #### Root +//! +//! - `cancel_referendum` - Removes a referendum. +//! - `cancel_queued` - Cancels a proposal that is queued for enactment. +//! - `clear_public_proposal` - Removes all public proposals. + #![recursion_limit="128"] #![cfg_attr(not(feature = "std"), no_std)] @@ -30,7 +163,7 @@ use frame_support::{ weights::SimpleDispatchInfo, traits::{ Currency, ReservableCurrency, LockableCurrency, WithdrawReason, LockIdentifier, Get, - OnReapAccount, OnUnbalanced, BalanceStatus + OnUnbalanced, BalanceStatus } }; use frame_system::{self as system, ensure_signed, ensure_root}; @@ -260,6 +393,24 @@ impl ReferendumInfo } } +/// State of a proxy voting account. +#[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug)] +pub enum ProxyState { + /// Account is open to becoming a proxy but is not yet assigned. + Open(AccountId), + /// Account is actively being a proxy. + Active(AccountId), +} + +impl ProxyState { + fn as_active(self) -> Option { + match self { + ProxyState::Active(a) => Some(a), + ProxyState::Open(_) => None, + } + } +} + decl_storage! { trait Store for Module as Democracy { /// The number of (public) proposals that have been made so far. @@ -299,7 +450,7 @@ decl_storage! { /// Who is able to vote for whom. Value is the fund-holding account, key is the /// vote-transaction-sending account. - pub Proxy get(fn proxy): map hasher(blake2_256) T::AccountId => Option; + pub Proxy get(fn proxy): map hasher(blake2_256) T::AccountId => Option>; /// Get the account (and lock periods) to which another account is delegating vote. pub Delegations get(fn delegations): @@ -423,6 +574,12 @@ decl_error! { NotLocked, /// The lock on the account to be unlocked has not yet expired. NotExpired, + /// A proxy-pairing was attempted to an account that was not open. + NotOpen, + /// A proxy-pairing was attempted to an account that was open to another account. + WrongOpen, + /// A proxy-de-pairing was attempted to an account that was not active. + NotActive } } @@ -458,8 +615,16 @@ decl_module! { /// Propose a sensitive action to be taken. /// + /// The dispatch origin of this call must be _Signed_ and the sender must + /// have funds to cover the deposit. + /// + /// - `proposal_hash`: The hash of the proposal preimage. + /// - `value`: The amount of deposit (must be at least `MinimumDeposit`). + /// + /// Emits `Proposed`. + /// /// # - /// - O(1). + /// - `O(1)`. /// - Two DB changes, one DB entry. /// # #[weight = SimpleDispatchInfo::FixedNormal(5_000_000)] @@ -481,10 +646,15 @@ decl_module! { Self::deposit_event(RawEvent::Proposed(index, value)); } - /// Propose a sensitive action to be taken. + /// Signals agreement with a particular proposal. + /// + /// The dispatch origin of this call must be _Signed_ and the sender + /// must have funds to cover the deposit, equal to the original deposit. + /// + /// - `proposal`: The index of the proposal to second. /// /// # - /// - O(1). + /// - `O(1)`. /// - One DB entry. /// # #[weight = SimpleDispatchInfo::FixedNormal(5_000_000)] @@ -500,8 +670,13 @@ decl_module! { /// Vote in a referendum. If `vote.is_aye()`, the vote is to enact the proposal; /// otherwise it is a vote to keep the status quo. /// + /// The dispatch origin of this call must be _Signed_. + /// + /// - `ref_index`: The index of the referendum to vote for. + /// - `vote`: The vote configuration. + /// /// # - /// - O(1). + /// - `O(1)`. /// - One DB change, one DB entry. /// # #[weight = SimpleDispatchInfo::FixedNormal(200_000)] @@ -516,8 +691,13 @@ decl_module! { /// Vote in a referendum on behalf of a stash. If `vote.is_aye()`, the vote is to enact /// the proposal; otherwise it is a vote to keep the status quo. /// + /// The dispatch origin of this call must be _Signed_. + /// + /// - `ref_index`: The index of the referendum to proxy vote for. + /// - `vote`: The vote configuration. + /// /// # - /// - O(1). + /// - `O(1)`. /// - One DB change, one DB entry. /// # #[weight = SimpleDispatchInfo::FixedNormal(200_000)] @@ -525,12 +705,21 @@ decl_module! { #[compact] ref_index: ReferendumIndex, vote: Vote ) -> DispatchResult { - let who = Self::proxy(ensure_signed(origin)?).ok_or(Error::::NotProxy)?; - Self::do_vote(who, ref_index, vote) + let who = ensure_signed(origin)?; + let voter = Self::proxy(who).and_then(|a| a.as_active()).ok_or(Error::::NotProxy)?; + Self::do_vote(voter, ref_index, vote) } /// Schedule an emergency cancellation of a referendum. Cannot happen twice to the same /// referendum. + /// + /// The dispatch origin of this call must be `CancellationOrigin`. + /// + /// -`ref_index`: The index of the referendum to cancel. + /// + /// # + /// - Depends on size of storage vec `VotersFor` for this referendum. + /// # #[weight = SimpleDispatchInfo::FixedOperational(500_000)] fn emergency_cancel(origin, ref_index: ReferendumIndex) { T::CancellationOrigin::ensure_origin(origin)?; @@ -545,6 +734,15 @@ decl_module! { /// Schedule a referendum to be tabled once it is legal to schedule an external /// referendum. + /// + /// The dispatch origin of this call must be `ExternalOrigin`. + /// + /// - `proposal_hash`: The preimage hash of the proposal. + /// + /// # + /// - `O(1)`. + /// - One DB change. + /// # #[weight = SimpleDispatchInfo::FixedNormal(5_000_000)] fn external_propose(origin, proposal_hash: T::Hash) { T::ExternalOrigin::ensure_origin(origin)?; @@ -561,8 +759,17 @@ decl_module! { /// Schedule a majority-carries referendum to be tabled next once it is legal to schedule /// an external referendum. /// + /// The dispatch of this call must be `ExternalMajorityOrigin`. + /// + /// - `proposal_hash`: The preimage hash of the proposal. + /// /// Unlike `external_propose`, blacklisting has no effect on this and it may replace a /// pre-scheduled `external_propose` call. + /// + /// # + /// - `O(1)`. + /// - One DB change. + /// # #[weight = SimpleDispatchInfo::FixedNormal(5_000_000)] fn external_propose_majority(origin, proposal_hash: T::Hash) { T::ExternalMajorityOrigin::ensure_origin(origin)?; @@ -572,8 +779,17 @@ decl_module! { /// Schedule a negative-turnout-bias referendum to be tabled next once it is legal to /// schedule an external referendum. /// + /// The dispatch of this call must be `ExternalDefaultOrigin`. + /// + /// - `proposal_hash`: The preimage hash of the proposal. + /// /// Unlike `external_propose`, blacklisting has no effect on this and it may replace a /// pre-scheduled `external_propose` call. + /// + /// # + /// - `O(1)`. + /// - One DB change. + /// # #[weight = SimpleDispatchInfo::FixedNormal(5_000_000)] fn external_propose_default(origin, proposal_hash: T::Hash) { T::ExternalDefaultOrigin::ensure_origin(origin)?; @@ -584,11 +800,21 @@ decl_module! { /// immediately. If there is no externally-proposed referendum currently, or if there is one /// but it is not a majority-carries referendum then it fails. /// + /// The dispatch of this call must be `FastTrackOrigin`. + /// /// - `proposal_hash`: The hash of the current external proposal. /// - `voting_period`: The period that is allowed for voting on this proposal. Increased to /// `EmergencyVotingPeriod` if too low. /// - `delay`: The number of block after voting has ended in approval and this should be /// enacted. This doesn't have a minimum amount. + /// + /// Emits `Started`. + /// + /// # + /// - One DB clear. + /// - One DB change. + /// - One extra DB entry. + /// # #[weight = SimpleDispatchInfo::FixedNormal(200_000)] fn fast_track(origin, proposal_hash: T::Hash, @@ -611,6 +837,19 @@ decl_module! { } /// Veto and blacklist the external proposal hash. + /// + /// The dispatch origin of this call must be `VetoOrigin`. + /// + /// - `proposal_hash`: The preimage hash of the proposal to veto and blacklist. + /// + /// Emits `Vetoed`. + /// + /// # + /// - Two DB entries. + /// - One DB clear. + /// - Performs a binary search on `existing_vetoers` which should not + /// be very large. + /// # #[weight = SimpleDispatchInfo::FixedNormal(200_000)] fn veto_external(origin, proposal_hash: T::Hash) { let who = T::VetoOrigin::ensure_origin(origin)?; @@ -636,6 +875,14 @@ decl_module! { } /// Remove a referendum. + /// + /// The dispatch origin of this call must be _Root_. + /// + /// - `ref_index`: The index of the referendum to cancel. + /// + /// # + /// - `O(1)`. + /// # #[weight = SimpleDispatchInfo::FixedOperational(10_000)] fn cancel_referendum(origin, #[compact] ref_index: ReferendumIndex) { ensure_root(origin)?; @@ -643,6 +890,14 @@ decl_module! { } /// Cancel a proposal queued for enactment. + /// + /// The dispatch origin of this call must be _Root_. + /// + /// - `which`: The index of the referendum to cancel. + /// + /// # + /// - One DB change. + /// # #[weight = SimpleDispatchInfo::FixedOperational(10_000)] fn cancel_queued(origin, which: ReferendumIndex) { ensure_root(origin)?; @@ -659,46 +914,89 @@ decl_module! { } } - /// Specify a proxy. Called by the stash. + /// Specify a proxy that is already open to us. Called by the stash. + /// + /// NOTE: Used to be called `set_proxy`. + /// + /// The dispatch origin of this call must be _Signed_. + /// + /// - `proxy`: The account that will be activated as proxy. /// /// # /// - One extra DB entry. /// # #[weight = SimpleDispatchInfo::FixedNormal(100_000)] - fn set_proxy(origin, proxy: T::AccountId) { + fn activate_proxy(origin, proxy: T::AccountId) { let who = ensure_signed(origin)?; - ensure!(!>::contains_key(&proxy), Error::::AlreadyProxy); - >::insert(proxy, who) + Proxy::::try_mutate(&proxy, |a| match a.take() { + None => Err(Error::::NotOpen), + Some(ProxyState::Active(_)) => Err(Error::::AlreadyProxy), + Some(ProxyState::Open(x)) if &x == &who => { + *a = Some(ProxyState::Active(who)); + Ok(()) + } + Some(ProxyState::Open(_)) => Err(Error::::WrongOpen), + })?; } /// Clear the proxy. Called by the proxy. /// + /// NOTE: Used to be called `resign_proxy`. + /// + /// The dispatch origin of this call must be _Signed_. + /// /// # /// - One DB clear. /// # #[weight = SimpleDispatchInfo::FixedNormal(100_000)] - fn resign_proxy(origin) { + fn close_proxy(origin) { let who = ensure_signed(origin)?; - >::remove(who); + Proxy::::mutate(&who, |a| { + if a.is_some() { + system::Module::::dec_ref(&who); + } + *a = None; + }); } - /// Clear the proxy. Called by the stash. + /// Deactivate the proxy, but leave open to this account. Called by the stash. + /// + /// The proxy must already be active. + /// + /// NOTE: Used to be called `remove_proxy`. + /// + /// The dispatch origin of this call must be _Signed_. + /// + /// - `proxy`: The account that will be deactivated as proxy. /// /// # /// - One DB clear. /// # #[weight = SimpleDispatchInfo::FixedNormal(100_000)] - fn remove_proxy(origin, proxy: T::AccountId) { + fn deactivate_proxy(origin, proxy: T::AccountId) { let who = ensure_signed(origin)?; - ensure!( - &Self::proxy(&proxy).ok_or(Error::::NotProxy)? == &who, - Error::::WrongProxy, - ); - >::remove(proxy); + Proxy::::try_mutate(&proxy, |a| match a.take() { + None | Some(ProxyState::Open(_)) => Err(Error::::NotActive), + Some(ProxyState::Active(x)) if &x == &who => { + *a = Some(ProxyState::Open(who)); + Ok(()) + } + Some(ProxyState::Active(_)) => Err(Error::::WrongProxy), + })?; } /// Delegate vote. /// + /// Currency is locked indefinitely for as long as it's delegated. + /// + /// The dispatch origin of this call must be _Signed_. + /// + /// - `to`: The account to make a delegate of the sender. + /// - `conviction`: The conviction that will be attached to the delegated + /// votes. + /// + /// Emits `Delegated`. + /// /// # /// - One extra DB entry. /// # @@ -719,6 +1017,14 @@ decl_module! { /// Undelegate vote. /// + /// Must be sent from an account that has called delegate previously. + /// The tokens will be reduced from an indefinite lock to the maximum + /// possible according to the conviction of the prior delegation. + /// + /// The dispatch origin of this call must be _Signed_. + /// + /// Emits `Undelegated`. + /// /// # /// - O(1). /// # @@ -740,7 +1046,14 @@ decl_module! { Self::deposit_event(RawEvent::Undelegated(who)); } - /// Veto and blacklist the proposal hash. Must be from Root origin. + /// Clears all public proposals. + /// + /// The dispatch origin of this call must be _Root_. + /// + /// # + /// - `O(1)`. + /// - One DB clear. + /// # #[weight = SimpleDispatchInfo::FixedNormal(10_000)] fn clear_public_proposals(origin) { ensure_root(origin)?; @@ -750,6 +1063,17 @@ decl_module! { /// Register the preimage for an upcoming proposal. This doesn't require the proposal to be /// in the dispatch queue but does require a deposit, returned once enacted. + /// + /// The dispatch origin of this call must be _Signed_. + /// + /// - `encoded_proposal`: The preimage of a proposal. + /// + /// Emits `PreimageNoted`. + /// + /// # + /// - Dependent on the size of `encoded_proposal` but protected by a + /// required deposit. + /// # #[weight = SimpleDispatchInfo::FixedNormal(100_000)] fn note_preimage(origin, encoded_proposal: Vec) { let who = ensure_signed(origin)?; @@ -768,6 +1092,16 @@ decl_module! { /// Register the preimage for an upcoming proposal. This requires the proposal to be /// in the dispatch queue. No deposit is needed. + /// + /// The dispatch origin of this call must be _Signed_. + /// + /// - `encoded_proposal`: The preimage of a proposal. + /// + /// Emits `PreimageNoted`. + /// + /// # + /// - Dependent on the size of `encoded_proposal`. + /// # #[weight = SimpleDispatchInfo::FixedNormal(100_000)] fn note_imminent_preimage(origin, encoded_proposal: Vec) { let who = ensure_signed(origin)?; @@ -785,9 +1119,19 @@ decl_module! { /// Remove an expired proposal preimage and collect the deposit. /// + /// The dispatch origin of this call must be _Signed_. + /// + /// - `proposal_hash`: The preimage hash of a proposal. + /// /// This will only work after `VotingPeriod` blocks from the time that the preimage was /// noted, if it's the same account doing it. If it's a different account, then it'll only /// work an additional `EnactmentPeriod` later. + /// + /// Emits `PreimageReaped`. + /// + /// # + /// - One DB clear. + /// # #[weight = SimpleDispatchInfo::FixedNormal(10_000)] fn reap_preimage(origin, proposal_hash: T::Hash) { let who = ensure_signed(origin)?; @@ -807,6 +1151,17 @@ decl_module! { Self::deposit_event(RawEvent::PreimageReaped(proposal_hash, old, deposit, who)); } + /// Unlock tokens that have an expired lock. + /// + /// The dispatch origin of this call must be _Signed_. + /// + /// - `target`: The account to remove the lock on. + /// + /// Emits `Unlocked`. + /// + /// # + /// - `O(1)`. + /// # #[weight = SimpleDispatchInfo::FixedNormal(10_000)] fn unlock(origin, target: T::AccountId) { ensure_signed(origin)?; @@ -818,6 +1173,30 @@ decl_module! { Locks::::remove(&target); Self::deposit_event(RawEvent::Unlocked(target)); } + + /// Become a proxy. + /// + /// This must be called prior to a later `activate_proxy`. + /// + /// Origin must be a Signed. + /// + /// - `target`: The account whose votes will later be proxied. + /// + /// `close_proxy` must be called before the account can be destroyed. + /// + /// # + /// - One extra DB entry. + /// # + #[weight = SimpleDispatchInfo::FixedNormal(100_000)] + fn open_proxy(origin, target: T::AccountId) { + let who = ensure_signed(origin)?; + Proxy::::mutate(&who, |a| { + if a.is_none() { + system::Module::::inc_ref(&who); + } + *a = Some(ProxyState::Open(target)); + }); + } } } @@ -935,7 +1314,12 @@ impl Module { #[cfg(feature = "std")] pub fn force_proxy(stash: T::AccountId, proxy: T::AccountId) { - >::insert(proxy, stash) + Proxy::::mutate(&proxy, |o| { + if o.is_none() { + system::Module::::inc_ref(&proxy); + } + *o = Some(ProxyState::Active(stash)) + }) } /// Start a referendum. @@ -989,7 +1373,6 @@ impl Module { fn clear_referendum(ref_index: ReferendumIndex) { >::remove(ref_index); - LowestUnbaked::mutate(|i| if *i == ref_index { *i += 1; let end = ReferendumCount::get(); @@ -1162,12 +1545,6 @@ impl Module { } } -impl OnReapAccount for Module { - fn on_reap_account(who: &T::AccountId) { - >::remove(who) - } -} - #[cfg(test)] mod tests { use super::*; @@ -1178,7 +1555,7 @@ mod tests { }; use sp_core::H256; use sp_runtime::{ - traits::{BlakeTwo256, IdentityLookup, Bounded, BadOrigin, OnInitialize}, + traits::{BlakeTwo256, IdentityLookup, Bounded, BadOrigin, OnRuntimeUpgrade}, testing::Header, Perbill, }; use pallet_balances::{BalanceLock, Error as BalancesError}; @@ -1229,7 +1606,7 @@ mod tests { type ModuleToIndex = (); type AccountData = pallet_balances::AccountData; type OnNewAccount = (); - type OnReapAccount = Balances; + type OnKilledAccount = (); } parameter_types! { pub const ExistentialDeposit: u64 = 1; @@ -1336,7 +1713,7 @@ mod tests { ]; s.top = data.into_iter().collect(); sp_io::TestExternalities::new(s).execute_with(|| { - Balances::on_initialize(1); + Balances::on_runtime_upgrade(); assert_eq!(Balances::free_balance(1), 5); assert_eq!(Balances::reserved_balance(1), 5); assert_eq!(Balances::usable_balance(&1), 2); @@ -1968,29 +2345,45 @@ mod tests { fn proxy_should_work() { new_test_ext().execute_with(|| { assert_eq!(Democracy::proxy(10), None); - assert_ok!(Democracy::set_proxy(Origin::signed(1), 10)); - assert_eq!(Democracy::proxy(10), Some(1)); + assert!(System::allow_death(&10)); + + assert_noop!(Democracy::activate_proxy(Origin::signed(1), 10), Error::::NotOpen); + + assert_ok!(Democracy::open_proxy(Origin::signed(10), 1)); + assert!(!System::allow_death(&10)); + assert_eq!(Democracy::proxy(10), Some(ProxyState::Open(1))); + + assert_noop!(Democracy::activate_proxy(Origin::signed(2), 10), Error::::WrongOpen); + assert_ok!(Democracy::activate_proxy(Origin::signed(1), 10)); + assert_eq!(Democracy::proxy(10), Some(ProxyState::Active(1))); // Can't set when already set. - assert_noop!(Democracy::set_proxy(Origin::signed(2), 10), Error::::AlreadyProxy); + assert_noop!(Democracy::activate_proxy(Origin::signed(2), 10), Error::::AlreadyProxy); // But this works because 11 isn't proxying. - assert_ok!(Democracy::set_proxy(Origin::signed(2), 11)); - assert_eq!(Democracy::proxy(10), Some(1)); - assert_eq!(Democracy::proxy(11), Some(2)); + assert_ok!(Democracy::open_proxy(Origin::signed(11), 2)); + assert_ok!(Democracy::activate_proxy(Origin::signed(2), 11)); + assert_eq!(Democracy::proxy(10), Some(ProxyState::Active(1))); + assert_eq!(Democracy::proxy(11), Some(ProxyState::Active(2))); // 2 cannot fire 1's proxy: - assert_noop!(Democracy::remove_proxy(Origin::signed(2), 10), Error::::WrongProxy); + assert_noop!(Democracy::deactivate_proxy(Origin::signed(2), 10), Error::::WrongProxy); - // 1 fires his proxy: - assert_ok!(Democracy::remove_proxy(Origin::signed(1), 10)); - assert_eq!(Democracy::proxy(10), None); - assert_eq!(Democracy::proxy(11), Some(2)); + // 1 deactivates their proxy: + assert_ok!(Democracy::deactivate_proxy(Origin::signed(1), 10)); + assert_eq!(Democracy::proxy(10), Some(ProxyState::Open(1))); + // but the proxy account cannot be killed until the proxy is closed. + assert!(!System::allow_death(&10)); - // 11 resigns: - assert_ok!(Democracy::resign_proxy(Origin::signed(11))); + // and then 10 closes it completely: + assert_ok!(Democracy::close_proxy(Origin::signed(10))); assert_eq!(Democracy::proxy(10), None); + assert!(System::allow_death(&10)); + + // 11 just closes without 2's "permission". + assert_ok!(Democracy::close_proxy(Origin::signed(11))); assert_eq!(Democracy::proxy(11), None); + assert!(System::allow_death(&11)); }); } @@ -2002,7 +2395,8 @@ mod tests { fast_forward_to(2); let r = 0; - assert_ok!(Democracy::set_proxy(Origin::signed(1), 10)); + assert_ok!(Democracy::open_proxy(Origin::signed(10), 1)); + assert_ok!(Democracy::activate_proxy(Origin::signed(1), 10)); assert_ok!(Democracy::proxy_vote(Origin::signed(10), r, AYE)); assert_eq!(Democracy::voters_for(r), vec![1]); diff --git a/frame/elections-phragmen/Cargo.toml b/frame/elections-phragmen/Cargo.toml index f3cfc800f01c015738304e2281ba6180ffa4b339..62ae091e9a41dea8fa039e48d329d571659b3852 100644 --- a/frame/elections-phragmen/Cargo.toml +++ b/frame/elections-phragmen/Cargo.toml @@ -1,24 +1,27 @@ [package] name = "pallet-elections-phragmen" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME election pallet for PHRAGMEN" [dependencies] -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -sp-phragmen = { version = "2.0.0", default-features = false, path = "../../primitives/phragmen" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/runtime" } +sp-phragmen = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/phragmen" } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/std" } [dev-dependencies] -sp-io = { version = "2.0.0", path = "../../primitives/io" } +sp-io = { version = "2.0.0-alpha.2", path = "../../primitives/io" } hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0", path = "../balances" } -sp-core = { version = "2.0.0", path = "../../primitives/core" } -substrate-test-utils = { version = "2.0.0", path = "../../test-utils" } +pallet-balances = { version = "2.0.0-alpha.2", path = "../balances" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } +substrate-test-utils = { version = "2.0.0-alpha.2", path = "../../test-utils" } serde = { version = "1.0.101" } [features] diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs index f250e771216e601f4129bd8043d7e86d97791bc3..55b7b3f1280d30df19545143cb7a8972df52f71c 100644 --- a/frame/elections-phragmen/src/lib.rs +++ b/frame/elections-phragmen/src/lib.rs @@ -690,6 +690,7 @@ impl Module { // split new set into winners and runners up. let split_point = desired_seats.min(new_set_with_stake.len()); let mut new_members = (&new_set_with_stake[..split_point]).to_vec(); + let most_popular = new_members.first().map(|x| x.0.clone()); // save the runners up as-is. They are sorted based on desirability. // sort and save the members. @@ -722,6 +723,7 @@ impl Module { &outgoing.clone(), &new_members_ids, ); + T::ChangeMembers::set_prime(most_popular); // outgoing candidates lose their bond. let mut to_burn_bond = outgoing.to_vec(); @@ -816,7 +818,7 @@ mod tests { type ModuleToIndex = (); type AccountData = pallet_balances::AccountData; type OnNewAccount = (); - type OnReapAccount = Balances; + type OnKilledAccount = (); } parameter_types! { @@ -864,6 +866,7 @@ mod tests { thread_local! { pub static MEMBERS: RefCell> = RefCell::new(vec![]); + pub static PRIME: RefCell> = RefCell::new(None); } pub struct TestChangeMembers; @@ -898,6 +901,11 @@ mod tests { assert_eq!(old_plus_incoming, new_plus_outgoing); MEMBERS.with(|m| *m.borrow_mut() = new.to_vec()); + PRIME.with(|p| *p.borrow_mut() = None); + } + + fn set_prime(who: Option) { + PRIME.with(|p| *p.borrow_mut() = who); } } @@ -1250,6 +1258,30 @@ mod tests { }); } + #[test] + fn prime_works() { + ExtBuilder::default().build().execute_with(|| { + assert_ok!(Elections::submit_candidacy(Origin::signed(3))); + assert_ok!(Elections::submit_candidacy(Origin::signed(4))); + assert_ok!(Elections::submit_candidacy(Origin::signed(5))); + + assert_ok!(Elections::vote(Origin::signed(1), vec![4, 3], 10)); + assert_ok!(Elections::vote(Origin::signed(2), vec![4], 20)); + assert_ok!(Elections::vote(Origin::signed(3), vec![3], 30)); + assert_ok!(Elections::vote(Origin::signed(4), vec![4], 40)); + assert_ok!(Elections::vote(Origin::signed(5), vec![5], 50)); + + System::set_block_number(5); + assert_ok!(Elections::end_block(System::block_number())); + + assert_eq!(Elections::members_ids(), vec![4, 5]); + assert_eq!(Elections::candidates(), vec![]); + + assert_ok!(Elections::vote(Origin::signed(3), vec![4, 5], 10)); + assert_eq!(PRIME.with(|p| *p.borrow()), Some(4)); + }); + } + #[test] fn cannot_vote_for_more_than_candidates() { ExtBuilder::default().build().execute_with(|| { diff --git a/frame/elections/Cargo.toml b/frame/elections/Cargo.toml index b7c98a65e15b45aa0a52113319102497236b58b5..e65f4e5d4627827380cbb06cc20732e0de40a5dc 100644 --- a/frame/elections/Cargo.toml +++ b/frame/elections/Cargo.toml @@ -1,23 +1,26 @@ [package] name = "pallet-elections" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet for elections" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] } +sp-core = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../system" } [dev-dependencies] hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0", path = "../balances" } +pallet-balances = { version = "2.0.0-alpha.2", path = "../balances" } [features] default = ["std"] diff --git a/frame/elections/src/mock.rs b/frame/elections/src/mock.rs index c5af6ba0456b95a0b0fe23e72588adf89489d70b..b82e73d512aa6eb254a1eda22ff801b7ad768a95 100644 --- a/frame/elections/src/mock.rs +++ b/frame/elections/src/mock.rs @@ -56,7 +56,7 @@ impl frame_system::Trait for Test { type ModuleToIndex = (); type AccountData = pallet_balances::AccountData; type OnNewAccount = (); - type OnReapAccount = Balances; + type OnKilledAccount = (); } parameter_types! { diff --git a/frame/evm/Cargo.toml b/frame/evm/Cargo.toml index 66c809fa44c7ee76d148298e019fbade3aba8d60..76407d57e42846394039d3f9af9f5dfb70f2016d 100644 --- a/frame/evm/Cargo.toml +++ b/frame/evm/Cargo.toml @@ -1,21 +1,24 @@ [package] name = "pallet-evm" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME EVM contracts pallet" [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } -pallet-timestamp = { version = "2.0.0", default-features = false, path = "../timestamp" } -pallet-balances = { version = "2.0.0", default-features = false, path = "../balances" } -sp-core = { version = "2.0.0", default-features = false, path = "../../primitives/core" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../system" } +pallet-timestamp = { version = "2.0.0-alpha.2", default-features = false, path = "../timestamp" } +pallet-balances = { version = "2.0.0-alpha.2", default-features = false, path = "../balances" } +sp-core = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/io" } primitive-types = { version = "0.6.2", default-features = false, features = ["rlp"] } rlp = { version = "0.4", default-features = false } evm = { version = "0.15", default-features = false } diff --git a/frame/example-offchain-worker/Cargo.toml b/frame/example-offchain-worker/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..ff8d780c3cff326c3b1fb749270dc8ec09c7043a --- /dev/null +++ b/frame/example-offchain-worker/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "pallet-example-offchain-worker" +version = "2.0.0-alpha.3" +authors = ["Parity Technologies "] +edition = "2018" +license = "Unlicense" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME example pallet for offchain worker" + +[dependencies] +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../system" } +serde = { version = "1.0.101", optional = true } +sp-core = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/core" } +sp-io = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/std" } +serde_json = { version = "1.0.46", default-features = false, features = ["alloc"] } + +[features] +default = ["std"] +std = [ + "codec/std", + "frame-support/std", + "frame-system/std", + "serde", + "sp-core/std", + "sp-io/std", + "sp-runtime/std", + "sp-std/std", +] diff --git a/frame/example-offchain-worker/src/lib.rs b/frame/example-offchain-worker/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..1c72a8be68653974803ee7702ae3aa53accdd1b5 --- /dev/null +++ b/frame/example-offchain-worker/src/lib.rs @@ -0,0 +1,548 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! # Offchain Worker Example Module +//! +//! The Offchain Worker Example: A simple pallet demonstrating +//! concepts, APIs and structures common to most offchain workers. +//! +//! Run `cargo doc --package pallet-example-offchain-worker --open` to view this module's +//! documentation. +//! +//! - \[`pallet_example_offchain_worker::Trait`](./trait.Trait.html) +//! - \[`Call`](./enum.Call.html) +//! - \[`Module`](./struct.Module.html) +//! +//! +//! \## Overview +//! +//! In this example we are going to build a very simplistic, naive and definitely NOT +//! production-ready oracle for BTC/USD price. +//! Offchain Worker (OCW) will be triggered after every block, fetch the current price +//! and prepare either signed or unsigned transaction to feed the result back on chain. +//! The on-chain logic will simply aggregate the results and store last `64` values to compute +//! the average price. +//! Additional logic in OCW is put in place to prevent spamming the network with both signed +//! and unsigned transactions, and custom `UnsignedValidator` makes sure that there is only +//! one unsigned transaction floating in the network. +#![cfg_attr(not(feature = "std"), no_std)] + +use frame_support::{ + debug, + dispatch::DispatchResult, decl_module, decl_storage, decl_event, + traits::Get, + weights::SimpleDispatchInfo, +}; +use frame_system::{self as system, ensure_signed, ensure_none, offchain}; +use serde_json as json; +use sp_core::crypto::KeyTypeId; +use sp_runtime::{ + offchain::{http, Duration, storage::StorageValueRef}, + traits::Zero, + transaction_validity::{InvalidTransaction, ValidTransaction, TransactionValidity}, +}; + +#[cfg(test)] +mod tests; + +/// Defines application identifier for crypto keys of this module. +/// +/// Every module that deals with signatures needs to declare its unique identifier for +/// its crypto keys. +/// When offchain worker is signing transactions it's going to request keys of type +/// `KeyTypeId` from the keystore and use the ones it finds to sign the transaction. +/// The keys can be inserted manually via RPC (see `author_insertKey`). +pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"btc!"); + +/// Based on the above `KeyTypeId` we need to generate a pallet-specific crypto type wrappers. +/// We can use from supported crypto kinds (`sr25519`, `ed25519` and `ecdsa`) and augment +/// the types with this pallet-specific identifier. +pub mod crypto { + use super::KEY_TYPE; + use sp_runtime::app_crypto::{app_crypto, sr25519}; + app_crypto!(sr25519, KEY_TYPE); +} + +/// This pallet's configuration trait +pub trait Trait: frame_system::Trait { + /// The type to sign and submit transactions. + type SubmitSignedTransaction: + offchain::SubmitSignedTransaction::Call>; + /// The type to submit unsigned transactions. + type SubmitUnsignedTransaction: + offchain::SubmitUnsignedTransaction::Call>; + + /// The overarching event type. + type Event: From> + Into<::Event>; + /// The overarching dispatch call type. + type Call: From>; + + // Configuration parameters + + /// A grace period after we send transaction. + /// + /// To avoid sending too many transactions, we only attempt to send one + /// every `GRACE_PERIOD` blocks. We use Local Storage to coordinate + /// sending between distinct runs of this offchain worker. + type GracePeriod: Get; + + /// Number of blocks of cooldown after unsigned transaction is included. + /// + /// This ensures that we only accept unsigned transactions once, every `UnsignedInterval` blocks. + type UnsignedInterval: Get; +} + +decl_storage! { + trait Store for Module as Example { + /// A vector of recently submitted prices. + /// + /// This is used to calculate average price, should have bounded size. + Prices get(fn prices): Vec; + /// Defines the block when next unsigned transaction will be accepted. + /// + /// To prevent spam of unsigned (and unpayed!) transactions on the network, + /// we only allow one transaction every `T::UnsignedInterval` blocks. + /// This storage entry defines when new transaction is going to be accepted. + NextUnsignedAt get(fn next_unsigned_at): T::BlockNumber; + } +} + +decl_event!( + /// Events generated by the module. + pub enum Event where AccountId = ::AccountId { + /// Event generated when new price is accepted to contribute to the average. + NewPrice(u32, AccountId), + } +); + +decl_module! { + /// A public part of the pallet. + pub struct Module for enum Call where origin: T::Origin { + fn deposit_event() = default; + + /// Submit new price to the list. + /// + /// This method is a public function of the module and can be called from within + /// a transaction. It appends given `price` to current list of prices. + /// In our example the `offchain worker` will create, sign & submit a transaction that + /// calls this function passing the price. + /// + /// The transaction needs to be signed (see `ensure_signed`) check, so that the caller + /// pays a fee to execute it. + /// This makes sure that it's not easy (or rather cheap) to attack the chain by submitting + /// excesive transactions, but note that it doesn't ensure the price oracle is actually + /// working and receives (and provides) meaningful data. + /// This example is not focused on correctness of the oracle itself, but rather its + /// purpose is to showcase offchain worker capabilities. + #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + pub fn submit_price(origin, price: u32) -> DispatchResult { + // Retrieve sender of the transaction. + let who = ensure_signed(origin)?; + // Add the price to the on-chain list. + Self::add_price(who, price); + Ok(()) + } + + /// Submit new price to the list via unsigned transaction. + /// + /// Works exactly like the `submit_price` function, but since we allow sending the + /// transaction without a signature, and hence without paying any fees, + /// we need a way to make sure that only some transactions are accepted. + /// This function can be called only once every `T::UnsignedInterval` blocks. + /// Transactions that call that function are de-duplicated on the pool level + /// via `validate_unsigned` implementation and also are rendered invalid if + /// the function has already been called in current "session". + /// + /// It's important to specify `weight` for unsigned calls as well, because even though + /// they don't charge fees, we still don't want a single block to contain unlimited + /// number of such transactions. + /// + /// This example is not focused on correctness of the oracle itself, but rather its + /// purpose is to showcase offchain worker capabilities. + #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + pub fn submit_price_unsigned(origin, _block_number: T::BlockNumber, price: u32) + -> DispatchResult + { + // This ensures that the function can only be called via unsigned transaction. + ensure_none(origin)?; + // Add the price to the on-chain list, but mark it as coming from an empty address. + Self::add_price(Default::default(), price); + // now increment the block number at which we expect next unsigned transaction. + let current_block = >::block_number(); + >::put(current_block + T::UnsignedInterval::get()); + Ok(()) + } + + /// Offchain Worker entry point. + /// + /// By implementing `fn offchain_worker` within `decl_module!` you declare a new offchain + /// worker. + /// This function will be called when the node is fully synced and a new best block is + /// succesfuly imported. + /// Note that it's not guaranteed for offchain workers to run on EVERY block, there might + /// be cases where some blocks are skipped, or for some the worker runs twice (re-orgs), + /// so the code should be able to handle that. + /// You can use `Local Storage` API to coordinate runs of the worker. + fn offchain_worker(block_number: T::BlockNumber) { + // It's a good idea to add logs to your offchain workers. + // Using the `frame_support::debug` module you have access to the same API exposed by + // the `log` crate. + // Note that having logs compiled to WASM may cause the size of the blob to increase + // significantly. You can use `RuntimeDebug` custom derive to hide details of the types + // in WASM or use `debug::native` namespace to produce logs only when the worker is + // running natively. + debug::native::info!("Hello World from offchain workers!"); + + // Since off-chain workers are just part of the runtime code, they have direct access + // to the storage and other included pallets. + // + // We can easily import `frame_system` and retrieve a block hash of the parent block. + let parent_hash = >::block_hash(block_number - 1.into()); + debug::debug!("Current block: {:?} (parent hash: {:?})", block_number, parent_hash); + + // It's a good practice to keep `fn offchain_worker()` function minimal, and move most + // of the code to separate `impl` block. + // Here we call a helper function to calculate current average price. + // This function reads storage entries of the current state. + let average: Option = Self::average_price(); + debug::debug!("Current price: {:?}", average); + + // For this example we are going to send both signed and unsigned transactions + // depending on the block number. + // Usually it's enough to choose one or the other. + let should_send = Self::choose_transaction_type(block_number); + let res = match should_send { + TransactionType::Signed => Self::fetch_price_and_send_signed(), + TransactionType::Unsigned => Self::fetch_price_and_send_unsigned(block_number), + TransactionType::None => Ok(()), + }; + if let Err(e) = res { + debug::error!("Error: {}", e); + } + } + } +} + +enum TransactionType { + Signed, + Unsigned, + None, +} + +/// Most of the functions are moved outside of the `decl_module!` macro. +/// +/// This greatly helps with error messages, as the ones inside the macro +/// can sometimes be hard to debug. +impl Module { + /// Chooses which transaction type to send. + /// + /// This function serves mostly to showcase `StorageValue` helper + /// and local storage usage. + /// + /// Returns a type of transaction that should be produced in current run. + fn choose_transaction_type(block_number: T::BlockNumber) -> TransactionType { + /// A friendlier name for the error that is going to be returned in case we are in the grace + /// period. + const RECENTLY_SENT: () = (); + + // Start off by creating a reference to Local Storage value. + // Since the local storage is common for all offchain workers, it's a good practice + // to prepend your entry with the module name. + let val = StorageValueRef::persistent(b"example_ocw::last_send"); + // The Local Storage is persisted and shared between runs of the offchain workers, + // and offchain workers may run concurrently. We can use the `mutate` function, to + // write a storage entry in an atomic fashion. Under the hood it uses `compare_and_set` + // low-level method of local storage API, which means that only one worker + // will be able to "acquire a lock" and send a transaction if multiple workers + // happen to be executed concurrently. + let res = val.mutate(|last_send: Option>| { + // We match on the value decoded from the storage. The first `Option` + // indicates if the value was present in the storage at all, + // the second (inner) `Option` indicates if the value was succesfuly + // decoded to expected type (`T::BlockNumber` in our case). + match last_send { + // If we already have a value in storage and the block number is recent enough + // we avoid sending another transaction at this time. + Some(Some(block)) if block + T::GracePeriod::get() < block_number => { + Err(RECENTLY_SENT) + }, + // In every other case we attempt to acquire the lock and send a transaction. + _ => Ok(block_number) + } + }); + + // The result of `mutate` call will give us a nested `Result` type. + // The first one matches the return of the closure passed to `mutate`, i.e. + // if we return `Err` from the closure, we get an `Err` here. + // In case we return `Ok`, here we will have another (inner) `Result` that indicates + // if the value has been set to the storage correctly - i.e. if it wasn't + // written to in the meantime. + match res { + // The value has been set correctly, which means we can safely send a transaction now. + Ok(Ok(block_number)) => { + // Depending if the block is even or odd we will send a `Signed` or `Unsigned` + // transaction. + // Note that this logic doesn't really guarantee that the transactions will be sent + // in an alternating fashion (i.e. fairly distributed). Depending on the execution + // order and lock acquisition, we may end up for instance sending two `Signed` + // transactions in a row. If a strict order is desired, it's better to use + // the storage entry for that. (for instance store both block number and a flag + // indicating the type of next transaction to send). + let send_signed = block_number % 2.into() == Zero::zero(); + if send_signed { + TransactionType::Signed + } else { + TransactionType::Unsigned + } + }, + // We are in the grace period, we should not send a transaction this time. + Err(RECENTLY_SENT) => TransactionType::None, + // We wanted to send a transaction, but failed to write the block number (acquire a + // lock). This indicates that another offchain worker that was running concurrently + // most likely executed the same logic and succeeded at writing to storage. + // Thus we don't really want to send the transaction, knowing that the other run + // already did. + Ok(Err(_)) => TransactionType::None, + } + } + + /// A helper function to fetch the price and send signed transaction. + fn fetch_price_and_send_signed() -> Result<(), String> { + use system::offchain::SubmitSignedTransaction; + // Firstly we check if there are any accounts in the local keystore that are capable of + // signing the transaction. + // If not it doesn't even make sense to make external HTTP requests, since we won't be able + // to put the results back on-chain. + if !T::SubmitSignedTransaction::can_sign() { + return Err( + "No local accounts available. Consider adding one via `author_insertKey` RPC." + )? + } + + // Make an external HTTP request to fetch the current price. + // Note this call will block until response is received. + let price = Self::fetch_price().map_err(|e| format!("{:?}", e))?; + + // Received price is wrapped into a call to `submit_price` public function of this pallet. + // This means that the transaction, when executed, will simply call that function passing + // `price` as an argument. + let call = Call::submit_price(price); + + // Using `SubmitSignedTransaction` associated type we create and submit a transaction + // representing the call, we've just created. + // Submit signed will return a vector of results for all accounts that were found in the + // local keystore with expected `KEY_TYPE`. + let results = T::SubmitSignedTransaction::submit_signed(call); + for (acc, res) in &results { + match res { + Ok(()) => debug::info!("[{:?}] Submitted price of {} cents", acc, price), + Err(e) => debug::error!("[{:?}] Failed to submit transaction: {:?}", acc, e), + } + } + + Ok(()) + } + + /// A helper function to fetch the price and send unsigned transaction. + fn fetch_price_and_send_unsigned(block_number: T::BlockNumber) -> Result<(), String> { + use system::offchain::SubmitUnsignedTransaction; + // Make sure we don't fetch the price if unsigned transaction is going to be rejected + // anyway. + let next_unsigned_at = >::get(); + if next_unsigned_at > block_number { + return Err( + format!("Too early to send unsigned transaction. Next at: {:?}", next_unsigned_at) + )? + } + + // Make an external HTTP request to fetch the current price. + // Note this call will block until response is received. + let price = Self::fetch_price().map_err(|e| format!("{:?}", e))?; + + // Received price is wrapped into a call to `submit_price_unsigned` public function of this + // pallet. This means that the transaction, when executed, will simply call that function + // passing `price` as an argument. + let call = Call::submit_price_unsigned(block_number, price); + + // Now let's create an unsigned transaction out of this call and submit it to the pool. + // By default unsigned transactions are disallowed, so we need to whitelist this case + // by writing `UnsignedValidator`. Note that it's EXTREMELY important to carefuly + // implement unsigned validation logic, as any mistakes can lead to opening DoS or spam + // attack vectors. See validation logic docs for more details. + T::SubmitUnsignedTransaction::submit_unsigned(call) + .map_err(|()| "Unable to submit unsigned transaction.".into()) + + } + + /// Fetch current price and return the result in cents. + fn fetch_price() -> Result { + // We want to keep the offchain worker execution time reasonable, so we set a hard-coded + // deadline to 2s to complete the external call. + // You can also wait idefinitely for the response, however you may still get a timeout + // coming from the host machine. + let deadline = sp_io::offchain::timestamp().add(Duration::from_millis(2_000)); + // Initiate an external HTTP GET request. + // This is using high-level wrappers from `sp_runtime`, for the low-level calls that + // you can find in `sp_io`. The API is trying to be similar to `reqwest`, but + // since we are running in a custom WASM execution environment we can't simply + // import the library here. + let request = http::Request::get( + "https://min-api.cryptocompare.com/data/price?fsym=BTC&tsyms=USD" + ); + // We set the deadline for sending of the request, note that awaiting response can + // have a separate deadline. Next we send the request, before that it's also possible + // to alter request headers or stream body content in case of non-GET requests. + let pending = request + .deadline(deadline) + .send() + .map_err(|_| http::Error::IoError)?; + + // The request is already being processed by the host, we are free to do anything + // else in the worker (we can send multiple concurrent requests too). + // At some point however we probably want to check the response though, + // so we can block current thread and wait for it to finish. + // Note that since the request is being driven by the host, we don't have to wait + // for the request to have it complete, we will just not read the response. + let response = pending.try_wait(deadline) + .map_err(|_| http::Error::DeadlineReached)??; + // Let's check the status code before we proceed to reading the response. + if response.code != 200 { + debug::warn!("Unexpected status code: {}", response.code); + return Err(http::Error::Unknown); + } + + // Next we want to fully read the response body and collect it to a vector of bytes. + // Note that the return object allows you to read the body in chunks as well + // with a way to control the deadline. + let body = response.body().collect::>(); + // Next we parse the response using `serde_json`. Even though it's possible to use + // `serde_derive` and deserialize to a struct it's not recommended due to blob size + // overhead introduced by such code. Deserializing to `json::Value` is much more + // lightweight and should be preferred, especially if we only care about a small number + // of properties from the response. + let val: Result = json::from_slice(&body); + // Let's parse the price as float value. Note that you should avoid using floats in the + // runtime, it's fine to do that in the offchain worker, but we do convert it to an integer + // before submitting on-chain. + let price = val.ok().and_then(|v| v.get("USD").and_then(|v| v.as_f64())); + let price = match price { + Some(pricef) => Ok((pricef * 100.) as u32), + None => { + let s = core::str::from_utf8(&body); + debug::warn!("Unable to extract price from the response: {:?}", s); + Err(http::Error::Unknown) + } + }?; + + debug::warn!("Got price: {} cents", price); + + Ok(price) + } + + /// Add new price to the list. + fn add_price(who: T::AccountId, price: u32) { + debug::info!("Adding to the average: {}", price); + Prices::mutate(|prices| { + const MAX_LEN: usize = 64; + + if prices.len() < MAX_LEN { + prices.push(price); + } else { + prices[price as usize % MAX_LEN] = price; + } + }); + + let average = Self::average_price() + .expect("The average is not empty, because it was just mutated; qed"); + debug::info!("Current average price is: {}", average); + // here we are raising the NewPrice event + Self::deposit_event(RawEvent::NewPrice(price, who)); + } + + /// Calculate current average price. + fn average_price() -> Option { + let prices = Prices::get(); + if prices.is_empty() { + None + } else { + Some(prices.iter().fold(0_u32, |a, b| a.saturating_add(*b)) / prices.len() as u32) + } + } +} + +#[allow(deprecated)] // ValidateUnsigned +impl frame_support::unsigned::ValidateUnsigned for Module { + type Call = Call; + + /// Validate unsigned call to this module. + /// + /// By default unsigned transactions are disallowed, but implementing the validator + /// here we make sure that some particular calls (the ones produced by offchain worker) + /// are being whitelisted and marked as valid. + fn validate_unsigned(call: &Self::Call) -> TransactionValidity { + // Firstly let's check that we call the right function. + if let Call::submit_price_unsigned(block_number, new_price) = call { + // Now let's check if the transaction has any chance to succeed. + let next_unsigned_at = >::get(); + if &next_unsigned_at > block_number { + return InvalidTransaction::Stale.into(); + } + // Let's make sure to reject transactions from the future. + let current_block = >::block_number(); + if ¤t_block < block_number { + return InvalidTransaction::Future.into(); + } + + // We prioritize transactions that are more far away from current average. + // + // Note this doesn't make much sense when building an actual oracle, but this example + // is here mostly to show off offchain workers capabilities, not about building an + // oracle. + let avg_price = Self::average_price() + .map(|price| if &price > new_price { price - new_price } else { new_price - price }) + .unwrap_or(0); + + Ok(ValidTransaction { + // We set base priority to 2**20 to make sure it's included before any other + // transactions in the pool. Next we tweak the priority depending on how much + // it differs from the current average. (the more it differs the more priority it + // has). + priority: (1 << 20) + avg_price as u64, + // This transaction does not require anything else to go before into the pool. + // In theory we could require `previous_unsigned_at` transaction to go first, + // but it's not necessary in our case. + requires: vec![], + // We set the `provides` tag to be the same as `next_unsigned_at`. This makes + // sure only one transaction produced after `next_unsigned_at` will ever + // get to the transaction pool and will end up in the block. + // We can still have multiple transactions compete for the same "spot", + // and the one with higher priority will replace other one in the pool. + provides: vec![codec::Encode::encode(&(KEY_TYPE.0, next_unsigned_at))], + // The transaction is only valid for next 5 blocks. After that it's + // going to be revalidated by the pool. + longevity: 5, + // It's fine to propagate that transaction to other peers, which means it can be + // created even by nodes that don't produce blocks. + // Note that sometimes it's better to keep it for yourself (if you are the block + // producer), since for instance in some schemes others may copy your solution and + // claim a reward. + propagate: true, + }) + } else { + InvalidTransaction::Call.into() + } + } +} diff --git a/frame/example-offchain-worker/src/tests.rs b/frame/example-offchain-worker/src/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..9b6a567a178403d9cb1b717705fa520820358bef --- /dev/null +++ b/frame/example-offchain-worker/src/tests.rs @@ -0,0 +1,210 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::*; + +use codec::Decode; +use frame_support::{ + assert_ok, impl_outer_origin, parameter_types, + weights::{GetDispatchInfo, Weight}, +}; +use sp_core::{ + H256, + offchain::{OffchainExt, TransactionPoolExt, testing}, + testing::KeyStore, + traits::KeystoreExt, +}; +use sp_runtime::{ + Perbill, RuntimeAppPublic, + testing::{Header, TestXt}, + traits::{BlakeTwo256, IdentityLookup, Extrinsic as ExtrinsicsT}, +}; + +impl_outer_origin! { + pub enum Origin for Test where system = frame_system {} +} + +// For testing the module, we construct most of a mock runtime. This means +// first constructing a configuration type (`Test`) which `impl`s each of the +// configuration traits of modules we want to use. +#[derive(Clone, Eq, PartialEq)] +pub struct Test; +parameter_types! { + pub const BlockHashCount: u64 = 250; + pub const MaximumBlockWeight: Weight = 1024; + pub const MaximumBlockLength: u32 = 2 * 1024; + pub const AvailableBlockRatio: Perbill = Perbill::one(); +} +impl frame_system::Trait for Test { + type Origin = Origin; + type Call = (); + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = sp_core::sr25519::Public; + type Lookup = IdentityLookup; + type Header = Header; + type Event = (); + type BlockHashCount = BlockHashCount; + type MaximumBlockWeight = MaximumBlockWeight; + type MaximumBlockLength = MaximumBlockLength; + type AvailableBlockRatio = AvailableBlockRatio; + type Version = (); + type ModuleToIndex = (); + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); +} + +type Extrinsic = TestXt, ()>; +type SubmitTransaction = frame_system::offchain::TransactionSubmitter< + crypto::Public, + Test, + Extrinsic +>; + +impl frame_system::offchain::CreateTransaction for Test { + type Public = sp_core::sr25519::Public; + type Signature = sp_core::sr25519::Signature; + + fn create_transaction>( + call: ::Call, + _public: Self::Public, + _account: ::AccountId, + nonce: ::Index, + ) -> Option<(::Call, ::SignaturePayload)> { + Some((call, (nonce, ()))) + } +} + +parameter_types! { + pub const GracePeriod: u64 = 5; + pub const UnsignedInterval: u64 = 128; +} + +impl Trait for Test { + type Event = (); + type Call = Call; + type SubmitSignedTransaction = SubmitTransaction; + type SubmitUnsignedTransaction = SubmitTransaction; + type GracePeriod = GracePeriod; + type UnsignedInterval = UnsignedInterval; +} + +type Example = Module; + +#[test] +fn it_aggregates_the_price() { + sp_io::TestExternalities::default().execute_with(|| { + assert_eq!(Example::average_price(), None); + + assert_ok!(Example::submit_price(Origin::signed(Default::default()), 27)); + assert_eq!(Example::average_price(), Some(27)); + + assert_ok!(Example::submit_price(Origin::signed(Default::default()), 43)); + assert_eq!(Example::average_price(), Some(35)); + }); +} + +#[test] +fn should_make_http_call_and_parse_result() { + let (offchain, state) = testing::TestOffchainExt::new(); + let mut t = sp_io::TestExternalities::default(); + t.register_extension(OffchainExt::new(offchain)); + + price_oracle_response(&mut state.write()); + + t.execute_with(|| { + // when + let price = Example::fetch_price().unwrap(); + // then + assert_eq!(price, 15522); + }); +} + +#[test] +fn should_submit_signed_transaction_on_chain() { + const PHRASE: &str = "news slush supreme milk chapter athlete soap sausage put clutch what kitten"; + + let (offchain, offchain_state) = testing::TestOffchainExt::new(); + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + let keystore = KeyStore::new(); + keystore.write().sr25519_generate_new( + crate::crypto::Public::ID, + Some(&format!("{}/hunter1", PHRASE)) + ).unwrap(); + + + let mut t = sp_io::TestExternalities::default(); + t.register_extension(OffchainExt::new(offchain)); + t.register_extension(TransactionPoolExt::new(pool)); + t.register_extension(KeystoreExt(keystore)); + + price_oracle_response(&mut offchain_state.write()); + + t.execute_with(|| { + // when + Example::fetch_price_and_send_signed().unwrap(); + // then + let tx = pool_state.write().transactions.pop().unwrap(); + assert!(pool_state.read().transactions.is_empty()); + let tx = Extrinsic::decode(&mut &*tx).unwrap(); + assert_eq!(tx.signature.unwrap().0, 0); + assert_eq!(tx.call, Call::submit_price(15522)); + }); +} + +#[test] +fn should_submit_unsigned_transaction_on_chain() { + let (offchain, offchain_state) = testing::TestOffchainExt::new(); + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + let mut t = sp_io::TestExternalities::default(); + t.register_extension(OffchainExt::new(offchain)); + t.register_extension(TransactionPoolExt::new(pool)); + + price_oracle_response(&mut offchain_state.write()); + + t.execute_with(|| { + // when + Example::fetch_price_and_send_unsigned(1).unwrap(); + // then + let tx = pool_state.write().transactions.pop().unwrap(); + assert!(pool_state.read().transactions.is_empty()); + let tx = Extrinsic::decode(&mut &*tx).unwrap(); + assert_eq!(tx.signature, None); + assert_eq!(tx.call, Call::submit_price_unsigned(1, 15522)); + }); +} + +#[test] +fn weights_work() { + // must have a default weight. + let default_call = >::submit_price(10); + let info = default_call.get_dispatch_info(); + // aka. `let info = as GetDispatchInfo>::get_dispatch_info(&default_call);` + assert_eq!(info.weight, 10_000); +} + +fn price_oracle_response(state: &mut testing::OffchainState) { + state.expect_request(0, testing::PendingRequest { + method: "GET".into(), + uri: "https://min-api.cryptocompare.com/data/price?fsym=BTC&tsyms=USD".into(), + response: Some(br#"{"USD": 155.23}"#.to_vec()), + sent: true, + ..Default::default() + }); +} diff --git a/frame/example/Cargo.toml b/frame/example/Cargo.toml index 0d37940fc73170feaae1eace009531ea9fb692d7..515d1fd8d7c7a10b805b43cba89793f7f1fb67af 100644 --- a/frame/example/Cargo.toml +++ b/frame/example/Cargo.toml @@ -1,22 +1,26 @@ [package] name = "pallet-example" -version = "2.0.0" +version = "2.0.0-alpha.3" authors = ["Parity Technologies "] edition = "2018" -license = "GPL-3.0" +license = "Unlicense" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME example pallet" [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false } -frame-support = { version = "2.0.0", default-features = false, path = "../support" } -frame-system = { version = "2.0.0", default-features = false, path = "../system" } -pallet-balances = { version = "2.0.0", default-features = false, path = "../balances" } -sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } +codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } +frame-benchmarking = { version = "2.0.0-alpha.2", default-features = false, path = "../benchmarking" } +frame-support = { version = "2.0.0-alpha.2", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-alpha.2", default-features = false, path = "../system" } +pallet-balances = { version = "2.0.0-alpha.2", default-features = false, path = "../balances" } +sp-runtime = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-alpha.2", default-features = false, path = "../../primitives/io" } [dev-dependencies] -sp-core = { version = "2.0.0", path = "../../primitives/core" } +sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } [features] default = ["std"] @@ -24,6 +28,7 @@ std = [ "serde", "codec/std", "sp-runtime/std", + "frame-benchmarking/std", "frame-support/std", "frame-system/std", "pallet-balances/std", diff --git a/frame/example/src/lib.rs b/frame/example/src/lib.rs index dbdc2efcf80b514bbc7686d79ea16cdb70030256..826538ae582fe7e7b3073408cd1fd4042eccdf9c 100644 --- a/frame/example/src/lib.rs +++ b/frame/example/src/lib.rs @@ -14,22 +14,22 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! # Example Module +//! # Example Pallet //! //! -//! The Example: A simple example of a runtime module demonstrating -//! concepts, APIs and structures common to most runtime modules. +//! The Example: A simple example of a FRAME pallet demonstrating +//! concepts, APIs and structures common to most FRAME runtimes. //! -//! Run `cargo doc --package pallet-example --open` to view this module's documentation. +//! Run `cargo doc --package pallet-example --open` to view this pallet's documentation. //! //! ### Documentation Guidelines: //! //! -//! +//! //!