diff --git a/.github/workflows/auto-label-issues.yml b/.github/workflows/auto-label-issues.yml new file mode 100644 index 0000000000000000000000000000000000000000..cd889b5941989a62e0784a73e974b25c166e6f4c --- /dev/null +++ b/.github/workflows/auto-label-issues.yml @@ -0,0 +1,17 @@ +# If the author of the issues is not a contributor to the project, label +# the issue with 'Z0-unconfirmed' + +name: Label New Issues +on: + issues: + types: [opened] + +jobs: + label-new-issues: + runs-on: ubuntu-latest + steps: + - name: Label drafts + uses: andymckay/labeler@master + if: github.event.issue.author_association == 'NONE' + with: + add-labels: 'Z0-unconfirmed' diff --git a/.github/workflows/auto-label-prs.yml b/.github/workflows/auto-label-prs.yml new file mode 100644 index 0000000000000000000000000000000000000000..f0b8e9b343e29ce7eb25bfb41a3f56277b4d7558 --- /dev/null +++ b/.github/workflows/auto-label-prs.yml @@ -0,0 +1,21 @@ +name: Label PRs +on: + pull_request: + types: [opened,ready_for_review] + +jobs: + label-new-prs: + runs-on: ubuntu-latest + steps: + - name: Label drafts + uses: andymckay/labeler@master + if: github.event.pull_request.draft == true + with: + add-labels: 'A3-inprogress' + remove-labels: 'A0-pleasereview' + - name: Label PRs + uses: andymckay/labeler@master + if: github.event.pull_request.draft == false && ! contains(github.event.pull_request.labels.*.name, 'A2-insubstantial') + with: + add-labels: 'A0-pleasereview' + remove-labels: 'A3-inprogress' diff --git a/.github/workflows/burnin-label-notification.yml b/.github/workflows/burnin-label-notification.yml new file mode 100644 index 0000000000000000000000000000000000000000..da422a659ee08600885c4241bbd39ddb5e62f25d --- /dev/null +++ b/.github/workflows/burnin-label-notification.yml @@ -0,0 +1,17 @@ +name: Notify devops when burn-in label applied +on: + pull_request: + types: [labeled] + +jobs: + notify-devops: + runs-on: ubuntu-latest + steps: + - name: Notify devops + if: github.event.label.name == 'A1-needsburnin' + uses: s3krit/matrix-message-action@v0.0.2 + with: + room_id: ${{ secrets.POLKADOT_DEVOPS_MATRIX_ROOM_ID }} + access_token: ${{ secrets.POLKADOT_DEVOPS_MATRIX_ACCESS_TOKEN }} + message: "@room Burn-in request received for [${{ github.event.pull_request.title }}](${{ github.event.pull_request.html_url }})" + server: "matrix.parity.io" diff --git a/.github/workflows/polkadot-companion-labels.yml b/.github/workflows/polkadot-companion-labels.yml new file mode 100644 index 0000000000000000000000000000000000000000..27f743e1bd4529dc83ee7dfd5b96c6c5bb4ac3f8 --- /dev/null +++ b/.github/workflows/polkadot-companion-labels.yml @@ -0,0 +1,31 @@ +name: Check Polkadot Companion and Label + +on: + pull_request: + types: [opened, synchronize] + +jobs: + check_status: + runs-on: ubuntu-latest + steps: + - name: Monitor the status of the gitlab-check-companion-build job + uses: s3krit/await-status-action@v1.0.1 + id: 'check-companion-status' + with: + authToken: ${{ secrets.GITHUB_TOKEN }} + ref: ${{ github.event.pull_request.head.sha }} + contexts: 'continuous-integration/gitlab-check-polkadot-companion-build' + timeout: 1800 + notPresentTimeout: 3600 # It can take quite a while before the job starts... + failureStates: failure + interruptedStates: error # Error = job was probably cancelled. We don't want to label the PR in that case + - name: Label success + uses: andymckay/labeler@master + if: steps.check-companion-status.outputs.result == 'success' + with: + remove-labels: 'A7-needspolkadotpr' + - name: Label failure + uses: andymckay/labeler@master + if: steps.check-companion-status.outputs.result == 'failure' + with: + add-labels: 'A7-needspolkadotpr' diff --git a/.gitignore b/.gitignore index 6398c09fe796278130978c2a8598ee9270399388..353d49df28f34b896567b0aa91bb08e056a5832c 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ rls*.log .local **/hfuzz_target/ **/hfuzz_workspace/ +.cargo/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0c06ace76cf8e494fb08c1ba03eebeeb7ac5a4d6..d3a7f36980063cb6719eee6494f909d360c42c5c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ # # my-example-job: # stage: test # One of the stages listed below this job (required) -# image: parity/tools:latest # Any docker image (required) +# image: paritytech/tools:latest # Any docker image (required) # allow_failure: true # Allow the pipeline to continue if this job fails (default: false) # dependencies: # - build-rust-doc-release # Any jobs that are required to run before this job (optional) @@ -22,6 +22,7 @@ # - ./.maintain/gitlab/my_amazing_script.sh stages: + - check - test - build - post-build-test @@ -29,21 +30,18 @@ stages: - kubernetes - flaming-fir -variables: +variables: &default-vars GIT_STRATEGY: fetch GIT_DEPTH: 100 - SCCACHE_DIR: "/ci-cache/${CI_PROJECT_NAME}/sccache" CARGO_INCREMENTAL: 0 - CI_SERVER_NAME: "GitLab CI" DOCKER_OS: "debian:stretch" ARCH: "x86_64" # FIXME set to release CARGO_UNLEASH_INSTALL_PARAMS: "--version 1.0.0-alpha.10" CARGO_UNLEASH_PKG_DEF: "--skip node node-* pallet-template pallet-example pallet-example-* subkey chain-spec-builder" - CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: "wasm-bindgen-test-runner" - WASM_BINDGEN_TEST_TIMEOUT: 120 - CHROMEDRIVER_ARGS: "--log-level=INFO --whitelisted-ips=127.0.0.1" +default: + cache: {} .collect-artifacts: &collect-artifacts artifacts: @@ -58,9 +56,10 @@ variables: - kubernetes-parity-build environment: name: parity-build + interruptible: true .docker-env: &docker-env - image: parity/rust-builder:latest + image: paritytech/ci-linux:production before_script: - rustup show - cargo --version @@ -77,7 +76,6 @@ variables: - runner_system_failure - unknown_failure - api_failure - dependencies: [] interruptible: true tags: - linux-docker @@ -92,7 +90,7 @@ variables: #### stage: .pre skip-if-draft: - image: parity/tools:latest + image: paritytech/tools:latest <<: *kubernetes-build stage: .pre only: @@ -100,51 +98,53 @@ skip-if-draft: script: - ./.maintain/gitlab/skip_if_draft.sh -#### stage: test +#### stage: check check-runtime: - stage: test - image: parity/tools:latest + stage: check + image: paritytech/tools:latest <<: *kubernetes-build only: - /^[0-9]+$/ variables: + <<: *default-vars GITLAB_API: "https://gitlab.parity.io/api/v4" GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" script: - ./.maintain/gitlab/check_runtime.sh - interruptible: true allow_failure: true check-signed-tag: - stage: test - image: parity/tools:latest + stage: check + image: paritytech/tools:latest <<: *kubernetes-build only: - /^ci-release-.*$/ - /^v[0-9]+\.[0-9]+\.[0-9]+.*$/ script: - ./.maintain/gitlab/check_signed.sh - allow_failure: false check-line-width: - stage: test - image: parity/tools:latest + stage: check + image: paritytech/tools:latest <<: *kubernetes-build only: - /^[0-9]+$/ script: - ./.maintain/gitlab/check_line_width.sh - interruptible: true allow_failure: true -check-polkadot-companion-build: - stage: build - <<: *docker-env +test-dependency-rules: + stage: check + image: paritytech/tools:latest + <<: *kubernetes-build + except: + variables: + - $DEPLOY_TAG script: - - ./.maintain/gitlab/check_polkadot_companion_build.sh - interruptible: true - allow_failure: true + - .maintain/ensure-deps.sh + +#### stage: test cargo-audit: stage: test @@ -158,6 +158,10 @@ cargo-audit: cargo-deny: stage: test <<: *docker-env + only: + - schedules + - tags + - web script: - cargo deny check --hide-inclusion-graph -c .maintain/deny.toml after_script: @@ -175,6 +179,8 @@ cargo-check-benches: <<: *docker-env script: - BUILD_DUMMY_WASM_BINARY=1 time cargo +nightly check --benches --all + - cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small + - cargo run --release -p node-bench -- ::trie::read::small - sccache -s cargo-check-subkey: @@ -191,33 +197,17 @@ test-linux-stable: &test-linux stage: test <<: *docker-env variables: + <<: *default-vars # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. - RUSTFLAGS: -Cdebug-assertions=y + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + RUST_BACKTRACE: 1 except: variables: - $DEPLOY_TAG script: - - WASM_BUILD_NO_COLOR=1 time cargo test --all --release --verbose --locked |& - tee output.log + - WASM_BUILD_NO_COLOR=1 time cargo test --all --release --verbose --locked - sccache -s - - echo "____Test job successful, checking for warnings____" - - awk '/^warning:/,/^$/ { print }' output.log > ${CI_COMMIT_SHORT_SHA}_warnings.log - - if [ -s ${CI_COMMIT_SHORT_SHA}_warnings.log ]; then - cat ${CI_COMMIT_SHORT_SHA}_warnings.log; - exit 1; - else - echo "___No warnings___"; - fi - -test-dependency-rules: - stage: test - <<: *docker-env - except: - variables: - - $DEPLOY_TAG - script: - - .maintain/ensure-deps.sh unleash-check: stage: test @@ -230,9 +220,11 @@ unleash-check: - cargo unleash check ${CARGO_UNLEASH_PKG_DEF} test-frame-staking: + # into one job stage: test <<: *docker-env variables: + <<: *default-vars # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. RUSTFLAGS: -Cdebug-assertions=y @@ -242,13 +234,15 @@ test-frame-staking: - $DEPLOY_TAG script: - cd frame/staking/ - - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --no-default-features --features "std testing-utils" + - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --no-default-features --features "std" - sccache -s test-frame-examples-compile-to-wasm: + # into one job stage: test <<: *docker-env variables: + <<: *default-vars # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. RUSTFLAGS: -Cdebug-assertions=y @@ -267,8 +261,9 @@ test-wasmtime: stage: test <<: *docker-env variables: + <<: *default-vars # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. + # but still want to have debug assertions. RUSTFLAGS: -Cdebug-assertions=y RUST_BACKTRACE: 1 except: @@ -280,9 +275,11 @@ test-wasmtime: - sccache -s test-runtime-benchmarks: + # into one job stage: test <<: *docker-env variables: + <<: *default-vars # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. RUSTFLAGS: -Cdebug-assertions=y @@ -292,7 +289,7 @@ test-runtime-benchmarks: - $DEPLOY_TAG script: - cd bin/node/cli - - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --features runtime-benchmarks + - WASM_BUILD_NO_COLOR=1 time cargo test --workspace --release --verbose --features runtime-benchmarks - sccache -s test-linux-stable-int: @@ -305,7 +302,8 @@ test-linux-stable-int: script: - echo "___Logs will be partly shown at the end in case of failure.___" - echo "___Full log will be saved to the job artifacts only in case of failure.___" - - WASM_BUILD_NO_COLOR=1 RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace + - WASM_BUILD_NO_COLOR=1 + RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace time cargo test -p node-cli --release --verbose --locked -- --ignored &> ${CI_COMMIT_SHORT_SHA}_int_failure.log - sccache -s @@ -341,6 +339,7 @@ test-full-crypto-feature: stage: test <<: *docker-env variables: + <<: *default-vars # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. RUSTFLAGS: -Cdebug-assertions=y @@ -357,6 +356,7 @@ test-full-crypto-feature: cargo-check-macos: stage: test + # shell runner on mac ignores the image set in *docker-env <<: *docker-env script: - BUILD_DUMMY_WASM_BINARY=1 time cargo check --release @@ -364,14 +364,46 @@ cargo-check-macos: tags: - osx +test-prometheus-alerting-rules: + stage: test + image: paritytech/tools:latest + <<: *kubernetes-build + script: + - promtool check rules .maintain/monitoring/alerting-rules/alerting-rules.yaml + - cat .maintain/monitoring/alerting-rules/alerting-rules.yaml | promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml + #### stage: build +check-polkadot-companion-status: + stage: build + image: paritytech/tools:latest + <<: *kubernetes-build + only: + - /^[0-9]+$/ # PRs + script: + - ./.maintain/gitlab/check_polkadot_companion_status.sh + +check-polkadot-companion-build: + stage: build + <<: *docker-env + needs: + - job: test-linux-stable-int + artifacts: false + script: + - ./.maintain/gitlab/check_polkadot_companion_build.sh + allow_failure: true + test-browser-node: stage: build <<: *docker-env needs: - job: check-web-wasm artifacts: false + variables: + <<: *default-vars + CHROMEDRIVER_ARGS: "--log-level=INFO --whitelisted-ips=127.0.0.1" + CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: "wasm-bindgen-test-runner" + WASM_BINDGEN_TEST_TIMEOUT: 120 script: - cargo +nightly test --target wasm32-unknown-unknown -p node-browser-testing -Z features=itarget @@ -380,6 +412,9 @@ build-linux-substrate: &build-binary <<: *collect-artifacts <<: *docker-env <<: *build-only + needs: + - job: test-linux-stable + artifacts: false before_script: - mkdir -p ./artifacts/substrate/ except: @@ -394,7 +429,7 @@ build-linux-substrate: &build-binary else ./artifacts/substrate/substrate --version | sed -n -E 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' | - tee ./artifacts/substrate/VERSION; + tee ./artifacts/substrate/VERSION; fi - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256 - printf '\n# building node-template\n\n' @@ -405,6 +440,9 @@ build-linux-substrate: &build-binary build-linux-subkey: &build-subkey <<: *build-binary + needs: + - job: cargo-check-subkey + artifacts: false before_script: - mkdir -p ./artifacts/subkey script: @@ -441,23 +479,16 @@ build-rust-doc-release: <<: *build-only script: - rm -f ./crate-docs/index.html # use it as an indicator if the job succeeds - - BUILD_DUMMY_WASM_BINARY=1 RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html" time cargo +nightly doc --release --all --verbose + - BUILD_DUMMY_WASM_BINARY=1 RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html" + time cargo +nightly doc --release --all --verbose - cp -R ./target/doc ./crate-docs - echo "" > ./crate-docs/index.html - sccache -s -check-polkadot-companion-status: - stage: post-build-test - image: parity/tools:latest - <<: *kubernetes-build - only: - - /^[0-9]+$/ - script: - - ./.maintain/gitlab/check_polkadot_companion_status.sh - +#### stage: post-build-test trigger-contracts-ci: - stage: publish + stage: post-build-test needs: - job: build-linux-substrate artifacts: false @@ -473,12 +504,19 @@ trigger-contracts-ci: #### stage: publish -.publish-docker-release: &publish-docker-release +.build-push-docker-image: &build-push-docker-image <<: *build-only <<: *kubernetes-build image: docker:stable services: - docker:dind + variables: &docker-build-vars + <<: *default-vars + DOCKER_HOST: tcp://localhost:2375 + DOCKER_DRIVER: overlay2 + GIT_STRATEGY: none + DOCKERFILE: $PRODUCT.Dockerfile + CONTAINER_IMAGE: parity/$PRODUCT before_script: - test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" || ( echo "no docker credentials provided"; exit 1 ) @@ -500,18 +538,15 @@ trigger-contracts-ci: publish-docker-substrate: stage: publish - <<: *publish-docker-release + <<: *build-push-docker-image # collect VERSION artifact here to pass it on to kubernetes <<: *collect-artifacts - dependencies: - - build-linux-substrate + needs: + - job: build-linux-substrate + artifacts: true variables: - DOCKER_HOST: tcp://localhost:2375 - DOCKER_DRIVER: overlay2 - GIT_STRATEGY: none + <<: *docker-build-vars PRODUCT: substrate - DOCKERFILE: $PRODUCT.Dockerfile - CONTAINER_IMAGE: parity/$PRODUCT after_script: - docker logout # only VERSION information is needed for the deployment @@ -519,16 +554,13 @@ publish-docker-substrate: publish-docker-subkey: stage: publish - <<: *publish-docker-release - dependencies: - - build-linux-subkey + <<: *build-push-docker-image + needs: + - job: build-linux-subkey + artifacts: true variables: - DOCKER_HOST: tcp://localhost:2375 - DOCKER_DRIVER: overlay2 - GIT_STRATEGY: none + <<: *docker-build-vars PRODUCT: subkey - DOCKERFILE: $PRODUCT.Dockerfile - CONTAINER_IMAGE: parity/$PRODUCT after_script: - docker logout @@ -536,10 +568,12 @@ publish-s3-release: stage: publish <<: *build-only <<: *kubernetes-build - dependencies: - - build-linux-substrate - - build-linux-subkey - image: parity/awscli:latest + needs: + - job: build-linux-substrate + artifacts: true + - job: build-linux-subkey + artifacts: true + image: paritytech/awscli:latest variables: GIT_STRATEGY: none BUCKET: "releases.parity.io" @@ -555,11 +589,11 @@ publish-s3-release: publish-s3-doc: stage: publish - image: parity/awscli:latest + image: paritytech/awscli:latest allow_failure: true - dependencies: - - build-rust-doc-release - cache: {} + needs: + - job: build-rust-doc-release + artifacts: true <<: *build-only <<: *kubernetes-build variables: @@ -579,13 +613,12 @@ publish-s3-doc: publish-draft-release: stage: publish - image: parity/tools:latest + image: paritytech/tools:latest only: - /^ci-release-.*$/ - /^v[0-9]+\.[0-9]+\.[0-9]+.*$/ script: - ./.maintain/gitlab/publish_draft_release.sh - interruptible: true allow_failure: true publish-to-crates-io: @@ -597,14 +630,13 @@ publish-to-crates-io: script: - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} - cargo unleash em-dragons --no-check ${CARGO_UNLEASH_PKG_DEF} - interruptible: true allow_failure: true .deploy-template: &deploy stage: kubernetes when: manual retry: 1 - image: parity/kubetools:latest + image: paritytech/kubetools:latest <<: *build-only tags: # this is the runner that is used to deploy it @@ -618,18 +650,18 @@ publish-to-crates-io: - echo "Substrate version = ${DEPLOY_TAG}" # or use helm to render the template - helm template - --values ./.maintain/kubernetes/values.yaml - --set image.tag=${DEPLOY_TAG} - --set validator.keys=${VALIDATOR_KEYS} - ./.maintain/kubernetes | kubectl apply -f - --dry-run=false + --values ./.maintain/kubernetes/values.yaml + --set image.tag=${DEPLOY_TAG} + --set validator.keys=${VALIDATOR_KEYS} + ./.maintain/kubernetes | kubectl apply -f - --dry-run=false - echo "# substrate namespace ${KUBE_NAMESPACE}" - kubectl -n ${KUBE_NAMESPACE} get all - echo "# substrate's nodes' external ip addresses:" - kubectl get nodes -l node=substrate - -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range @.status.addresses[?(@.type=="ExternalIP")]}{.address}{"\n"}{end}' + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range @.status.addresses[?(@.type=="ExternalIP")]}{.address}{"\n"}{end}' - echo "# substrate' nodes" - kubectl -n ${KUBE_NAMESPACE} get pods - -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.nodeName}{"\n"}{end}' + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.nodeName}{"\n"}{end}' - echo "# wait for the rollout to complete" - kubectl -n ${KUBE_NAMESPACE} rollout status statefulset/substrate @@ -637,8 +669,9 @@ publish-to-crates-io: .deploy-cibuild: &deploy-cibuild <<: *deploy - dependencies: - - publish-docker-substrate + needs: + - job: publish-docker-substrate + artifacts: false .deploy-tag: &deploy-tag <<: *deploy @@ -669,14 +702,16 @@ deploy-ue1-tag: name: parity-prod-ue1 .validator-deploy: &validator-deploy - # script will fail if there is no artifacts/substrate/VERSION <<: *build-only stage: flaming-fir - dependencies: - - build-linux-substrate + needs: + # script will fail if there is no artifacts/substrate/VERSION + - job: publish-docker-substrate + artifacts: true image: parity/azure-ansible:v1 allow_failure: true when: manual + interruptible: true tags: - linux-docker @@ -684,15 +719,29 @@ validator 1 4: <<: *validator-deploy script: - ./.maintain/flamingfir-deploy.sh flamingfir-validator1 + validator 2 4: <<: *validator-deploy script: - ./.maintain/flamingfir-deploy.sh flamingfir-validator2 + validator 3 4: <<: *validator-deploy script: - ./.maintain/flamingfir-deploy.sh flamingfir-validator3 + validator 4 4: <<: *validator-deploy script: - ./.maintain/flamingfir-deploy.sh flamingfir-validator4 + +#### stage: .post + +check-labels: + stage: .post + image: paritytech/tools:latest + <<: *kubernetes-build + only: + - /^[0-9]+$/ + script: + - ./.maintain/gitlab/check_labels.sh diff --git a/.maintain/flamingfir-deploy.sh b/.maintain/flamingfir-deploy.sh index 596bb04ece091cd03f3c993d655f5d45770362ea..8f0fb3a2bc016067d0bf11ae993a3bfa1636280f 100755 --- a/.maintain/flamingfir-deploy.sh +++ b/.maintain/flamingfir-deploy.sh @@ -5,7 +5,7 @@ RETRY_ATTEMPT=0 SLEEP_TIME=15 TARGET_HOST="$1" COMMIT=$(cat artifacts/substrate/VERSION) -DOWNLOAD_URL="https://releases.parity.io/substrate/x86_64-debian:stretch/${COMMIT}/substrate" +DOWNLOAD_URL="https://releases.parity.io/substrate/x86_64-debian:stretch/${COMMIT}/substrate/substrate" POST_DATA='{"extra_vars":{"artifact_path":"'${DOWNLOAD_URL}'","target_host":"'${TARGET_HOST}'"}}' JOB_ID=$(wget -O - --header "Authorization: Bearer ${AWX_TOKEN}" --header "Content-type: application/json" --post-data "${POST_DATA}" https://ansible-awx.parity.io/api/v2/job_templates/32/launch/ | jq .job) diff --git a/.maintain/gitlab/check_labels.sh b/.maintain/gitlab/check_labels.sh new file mode 100755 index 0000000000000000000000000000000000000000..5ab099b38291cc66b6f370430b8f06e6b2f5c24f --- /dev/null +++ b/.maintain/gitlab/check_labels.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +#shellcheck source=lib.sh +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh" + +ensure_labels() { + for label in "$@"; do + if has_label 'paritytech/substrate' "$CI_COMMIT_BRANCH" "$label"; then + return 0 + fi + done + return 1 +} + +# Must have one of the following labels +releasenotes_labels=( + 'B0-silent' + 'B3-apinoteworthy' + 'B5-clientnoteworthy' + 'B7-runtimenoteworthy' +) + +criticality_labels=( + 'C1-low' + 'C3-medium' + 'C7-high' + 'C9-critical' +) + +echo "[+] Checking release notes (B) labels for $CI_COMMIT_BRANCH" +if ensure_labels "${releasenotes_labels[@]}"; then + echo "[+] Release notes label detected. All is well." +else + echo "[!] Release notes label not detected. Please add one of: ${releasenotes_labels[*]}" + exit 1 +fi + +echo "[+] Checking release criticality (C) labels for $CI_COMMIT_BRANCH" +if ensure_labels "${criticality_labels[@]}"; then + echo "[+] Release criticality label detected. All is well." +else + echo "[!] Release criticality label not detected. Please add one of: ${criticality_labels[*]}" + exit 1 +fi + +exit 0 diff --git a/.maintain/gitlab/check_line_width.sh b/.maintain/gitlab/check_line_width.sh index 85092260b6a649187de6d03c0936cb0d54fc751e..611d3ae2681e2ef145753a0f774c5183fc9df7d9 100755 --- a/.maintain/gitlab/check_line_width.sh +++ b/.maintain/gitlab/check_line_width.sh @@ -10,14 +10,15 @@ BASE_BRANCH_NAME="master" LINE_WIDTH="120" GOOD_LINE_WIDTH="100" BASE_BRANCH="${BASE_ORIGIN}/${BASE_BRANCH_NAME}" +git fetch ${BASE_ORIGIN} ${BASE_BRANCH_NAME} --depth 100 +BASE_HASH=$(git merge-base ${BASE_BRANCH} HEAD) -git fetch ${BASE_ORIGIN} ${BASE_BRANCH_NAME} --depth 1 -git diff --name-only ${BASE_BRANCH} -- \*.rs | ( while read file +git diff --name-only ${BASE_HASH} -- \*.rs | ( while read file do if [ ! -f ${file} ]; then echo "Skipping removed file." - elif git diff ${BASE_BRANCH} -- ${file} | grep -q "^+.\{$(( $LINE_WIDTH + 1 ))\}" + elif git diff ${BASE_HASH} -- ${file} | grep -q "^+.\{$(( $LINE_WIDTH + 1 ))\}" then if [ -z "${FAIL}" ] then @@ -29,11 +30,11 @@ do FAIL="true" fi echo "| file: ${file}" - git diff ${BASE_BRANCH} -- ${file} \ + git diff ${BASE_HASH} -- ${file} \ | grep -n "^+.\{$(( $LINE_WIDTH + 1))\}" echo "|" else - if git diff ${BASE_BRANCH} -- ${file} | grep -q "^+.\{$(( $GOOD_LINE_WIDTH + 1 ))\}" + if git diff ${BASE_HASH} -- ${file} | grep -q "^+.\{$(( $GOOD_LINE_WIDTH + 1 ))\}" then if [ -z "${FAIL}" ] then @@ -44,7 +45,7 @@ do echo "|" fi echo "| file: ${file}" - git diff ${BASE_BRANCH} -- ${file} | grep -n "^+.\{$(( $GOOD_LINE_WIDTH + 1 ))\}" + git diff ${BASE_HASH} -- ${file} | grep -n "^+.\{$(( $GOOD_LINE_WIDTH + 1 ))\}" echo "|" fi fi diff --git a/.maintain/gitlab/check_runtime.sh b/.maintain/gitlab/check_runtime.sh index 5b7e25e3afc4eabef01d984047a98d675ab09757..6d009c5aafc6a8907fb9d8e598e8870854af8db7 100755 --- a/.maintain/gitlab/check_runtime.sh +++ b/.maintain/gitlab/check_runtime.sh @@ -67,7 +67,7 @@ sub_spec_version="$(git diff tags/release...${CI_COMMIT_SHA} ${VERSIONS_FILE} \ if [ "${add_spec_version}" != "${sub_spec_version}" ] then - github_label "B2-breaksapi" + github_label "D2-breaksapi" boldcat <<-EOT diff --git a/.maintain/gitlab/generate_changelog.sh b/.maintain/gitlab/generate_changelog.sh index ba2a507e4cac694f918aa2c88cf1585991faa373..b872d324438d64bd2b64cdf9745bf76a983675ed 100755 --- a/.maintain/gitlab/generate_changelog.sh +++ b/.maintain/gitlab/generate_changelog.sh @@ -19,18 +19,17 @@ while IFS= read -r line; do if has_label 'paritytech/substrate' "$pr_id" 'B0-silent'; then continue fi - if has_label 'paritytech/substrate' "$pr_id" 'B1-runtimenoteworthy'; then - runtime_changes="$runtime_changes + if has_label 'paritytech/substrate' "$pr_id" 'B3-apinoteworthy' ; then + api_changes="$api_changes $line" fi - if has_label 'paritytech/substrate' "$pr_id" 'B1-clientnoteworthy'; then + if has_label 'paritytech/substrate' "$pr_id" 'B5-clientnoteworthy'; then client_changes="$client_changes $line" fi - if has_label 'paritytech/substrate' "$pr_id" 'B1-apinoteworthy' ; then - api_changes="$api_changes + if has_label 'paritytech/substrate' "$pr_id" 'B7-runtimenoteworthy'; then + runtime_changes="$runtime_changes $line" - continue fi done <<< "$all_changes" diff --git a/.maintain/gitlab/lib.sh b/.maintain/gitlab/lib.sh index ecc9a5f54288cd4636a05cb7f2a2d0535e40ebe9..33477b52f5891a6855f15aec62cd3c41454dc85c 100755 --- a/.maintain/gitlab/lib.sh +++ b/.maintain/gitlab/lib.sh @@ -5,7 +5,7 @@ api_base="https://api.github.com/repos" # Function to take 2 git tags/commits and get any lines from commit messages # that contain something that looks like a PR reference: e.g., (#1234) sanitised_git_logs(){ - git --no-pager log --pretty=format:"%s" "$1..$2" | + git --no-pager log --pretty=format:"%s" "$1...$2" | # Only find messages referencing a PR grep -E '\(#[0-9]+\)' | # Strip any asterisks @@ -66,8 +66,12 @@ has_label(){ repo="$1" pr_id="$2" label="$3" - out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/pulls/$pr_id") - [ -n "$(echo "$out" | jq ".labels | .[] | select(.name==\"$label\")")" ] + if [ -n "$GITHUB_RELEASE_TOKEN" ]; then + out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/pulls/$pr_id") + else + out=$(curl -H "Authorization: token $GITHUB_PR_TOKEN" -s "$api_base/$repo/pulls/$pr_id") + fi + [ -n "$(echo "$out" | tr -d '\r\n' | jq ".labels | .[] | select(.name==\"$label\")")" ] } # Formats a message into a JSON string for posting to Matrix diff --git a/.maintain/monitoring/alerting-rules/alerting-rule-tests.yaml b/.maintain/monitoring/alerting-rules/alerting-rule-tests.yaml new file mode 100644 index 0000000000000000000000000000000000000000..288750be3c108f3f9b820e038d81997626897806 --- /dev/null +++ b/.maintain/monitoring/alerting-rules/alerting-rule-tests.yaml @@ -0,0 +1,262 @@ +rule_files: + - /dev/stdin + +evaluation_interval: 1m + +tests: + - interval: 1m + input_series: + - series: 'polkadot_sub_libp2p_peers_count{ + job="polkadot", + pod="polkadot-abcdef01234-abcdef", + instance="polkadot-abcdef01234-abcdef", + }' + values: '3 2+0x4 1+0x9' # 3 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 + + - series: 'polkadot_sub_txpool_validations_scheduled{ + job="polkadot", + pod="polkadot-abcdef01234-abcdef", + instance="polkadot-abcdef01234-abcdef", + }' + values: '11+1x10 22+2x30 10043x5' + + - series: 'polkadot_sub_txpool_validations_finished{ + job="polkadot", + pod="polkadot-abcdef01234-abcdef", + instance="polkadot-abcdef01234-abcdef", + }' + values: '0+1x42 42x5' + + - series: 'polkadot_block_height{ + status="best", job="polkadot", + pod="polkadot-abcdef01234-abcdef", + instance="polkadot-abcdef01234-abcdef", + }' + values: '1+1x3 4+0x13' # 1 2 3 4 4 4 4 4 4 4 4 4 ... + + - series: 'polkadot_block_height{ + status="finalized", + job="polkadot", + pod="polkadot-abcdef01234-abcdef", + instance="polkadot-abcdef01234-abcdef", + }' + values: '1+1x3 4+0x13' # 1 2 3 4 4 4 4 4 4 4 4 4 ... + + - series: 'polkadot_cpu_usage_percentage{ + job="polkadot", + pod="polkadot-abcdef01234-abcdef", + instance="polkadot-abcdef01234-abcdef", + }' + values: '0+20x5 100+0x5' # 0 20 40 60 80 100 100 100 100 100 100 + + alert_rule_test: + + ###################################################################### + # Resource usage + ###################################################################### + + - eval_time: 9m + alertname: HighCPUUsage + exp_alerts: + - eval_time: 10m + alertname: HighCPUUsage + exp_alerts: + - exp_labels: + severity: warning + pod: polkadot-abcdef01234-abcdef + instance: polkadot-abcdef01234-abcdef + job: polkadot + exp_annotations: + message: "The node polkadot-abcdef01234-abcdef has a CPU + usage higher than 100% for more than 5 minutes" + + ###################################################################### + # Block production + ###################################################################### + + - eval_time: 6m + alertname: LowNumberOfNewBlocks + exp_alerts: + - eval_time: 7m + alertname: LowNumberOfNewBlocks + exp_alerts: + - exp_labels: + severity: warning + pod: polkadot-abcdef01234-abcdef + instance: polkadot-abcdef01234-abcdef + job: polkadot + status: best + exp_annotations: + message: "Less than one new block per minute on instance + polkadot-abcdef01234-abcdef." + + - eval_time: 14m + alertname: LowNumberOfNewBlocks + exp_alerts: + - exp_labels: + severity: warning + pod: polkadot-abcdef01234-abcdef + instance: polkadot-abcdef01234-abcdef + job: polkadot + status: best + exp_annotations: + message: "Less than one new block per minute on instance + polkadot-abcdef01234-abcdef." + - exp_labels: + severity: critical + pod: polkadot-abcdef01234-abcdef + instance: polkadot-abcdef01234-abcdef + job: polkadot + status: best + exp_annotations: + message: "Less than one new block per minute on instance + polkadot-abcdef01234-abcdef." + + ###################################################################### + # Block finalization + ###################################################################### + + - eval_time: 6m + alertname: BlockFinalizationSlow + exp_alerts: + - eval_time: 7m + alertname: BlockFinalizationSlow + exp_alerts: + - exp_labels: + severity: warning + pod: polkadot-abcdef01234-abcdef + instance: polkadot-abcdef01234-abcdef + job: polkadot + status: finalized + exp_annotations: + message: "Finalized block on instance + polkadot-abcdef01234-abcdef increases by less than 1 per + minute." + + - eval_time: 14m + alertname: BlockFinalizationSlow + exp_alerts: + - exp_labels: + severity: warning + pod: polkadot-abcdef01234-abcdef + instance: polkadot-abcdef01234-abcdef + job: polkadot + status: finalized + exp_annotations: + message: "Finalized block on instance + polkadot-abcdef01234-abcdef increases by less than 1 per + minute." + - exp_labels: + severity: critical + pod: polkadot-abcdef01234-abcdef + instance: polkadot-abcdef01234-abcdef + job: polkadot + status: finalized + exp_annotations: + message: "Finalized block on instance + polkadot-abcdef01234-abcdef increases by less than 1 per + minute." + + ###################################################################### + # Transaction queue + ###################################################################### + + - eval_time: 11m + alertname: TransactionQueueSizeIncreasing + # Number of validations scheduled and finished both grow at a rate + # of 1 in the first 10 minutes, thereby the queue is not increasing + # in size, thus don't expect an alert. + exp_alerts: + - eval_time: 22m + alertname: TransactionQueueSizeIncreasing + # Number of validations scheduled is growing twice as fast as the + # number of validations finished after minute 10. Thus expect + # warning alert after 20 minutes. + exp_alerts: + - exp_labels: + severity: warning + pod: polkadot-abcdef01234-abcdef + instance: polkadot-abcdef01234-abcdef + job: polkadot + exp_annotations: + message: "The transaction pool size on node + polkadot-abcdef01234-abcdef has been monotonically + increasing for the last 10 minutes." + - eval_time: 43m + alertname: TransactionQueueSizeIncreasing + # Number of validations scheduled is growing twice as fast as the + # number of validations finished after minute 10. Thus expect + # both warning and critical alert after 40 minutes. + exp_alerts: + - exp_labels: + severity: warning + pod: polkadot-abcdef01234-abcdef + instance: polkadot-abcdef01234-abcdef + job: polkadot + exp_annotations: + message: "The transaction pool size on node + polkadot-abcdef01234-abcdef has been monotonically + increasing for the last 10 minutes." + - exp_labels: + severity: critical + pod: polkadot-abcdef01234-abcdef + instance: polkadot-abcdef01234-abcdef + job: polkadot + exp_annotations: + message: "The transaction pool size on node + polkadot-abcdef01234-abcdef has been monotonically + increasing for the last 30 minutes." + - eval_time: 49m + alertname: TransactionQueueSizeHigh + # After minute 43 the number of validations scheduled jumps up + # drastically while the number of validations finished stays the + # same. Thus expect an alert. + exp_alerts: + - exp_labels: + severity: critical + pod: polkadot-abcdef01234-abcdef + instance: polkadot-abcdef01234-abcdef + job: polkadot + exp_annotations: + message: "The transaction pool size on node + polkadot-abcdef01234-abcdef has been above 10_000 for the + last 5 minutes." + + ###################################################################### + # Networking + ###################################################################### + + - eval_time: 3m # Values: 3 2 2 + alertname: LowNumberOfPeers + exp_alerts: + - eval_time: 4m # Values: 2 2 2 + alertname: LowNumberOfPeers + exp_alerts: + - exp_labels: + severity: warning + pod: polkadot-abcdef01234-abcdef + instance: polkadot-abcdef01234-abcdef + job: polkadot + exp_annotations: + message: "The node polkadot-abcdef01234-abcdef has less + than 3 peers for more than 3 minutes" + + - eval_time: 16m # Values: 3 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 + alertname: LowNumberOfPeers + exp_alerts: + - exp_labels: + severity: warning + pod: polkadot-abcdef01234-abcdef + instance: polkadot-abcdef01234-abcdef + job: polkadot + exp_annotations: + message: "The node polkadot-abcdef01234-abcdef has less + than 3 peers for more than 3 minutes" + - exp_labels: + severity: critical + pod: polkadot-abcdef01234-abcdef + instance: polkadot-abcdef01234-abcdef + job: polkadot + exp_annotations: + message: "The node polkadot-abcdef01234-abcdef has less + than 3 peers for more than 15 minutes" diff --git a/.maintain/monitoring/alerting-rules/alerting-rules.yaml b/.maintain/monitoring/alerting-rules/alerting-rules.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ed3889a2c2f94d80b79e9376201432d65cf89bc --- /dev/null +++ b/.maintain/monitoring/alerting-rules/alerting-rules.yaml @@ -0,0 +1,138 @@ +groups: +- name: polkadot.rules + rules: + + ############################################################################## + # Resource usage + ############################################################################## + + - alert: HighCPUUsage + expr: polkadot_cpu_usage_percentage >= 100 + for: 5m + labels: + severity: warning + annotations: + message: 'The node {{ $labels.instance }} has a CPU usage higher than 100% + for more than 5 minutes' + + ############################################################################## + # Block production + ############################################################################## + + - alert: LowNumberOfNewBlocks + annotations: + message: 'Less than one new block per minute on instance {{ + $labels.instance }}.' + expr: increase(polkadot_block_height{status="best"}[1m]) < 1 + for: 3m + labels: + severity: warning + - alert: LowNumberOfNewBlocks + annotations: + message: 'Less than one new block per minute on instance {{ + $labels.instance }}.' + expr: increase(polkadot_block_height{status="best"}[1m]) < 1 + for: 10m + labels: + severity: critical + + ############################################################################## + # Block finalization + ############################################################################## + + - alert: BlockFinalizationSlow + expr: increase(polkadot_block_height{status="finalized"}[1m]) < 1 + for: 3m + labels: + severity: warning + annotations: + message: 'Finalized block on instance {{ $labels.instance }} increases by + less than 1 per minute.' + - alert: BlockFinalizationSlow + expr: increase(polkadot_block_height{status="finalized"}[1m]) < 1 + for: 10m + labels: + severity: critical + annotations: + message: 'Finalized block on instance {{ $labels.instance }} increases by + less than 1 per minute.' + - alert: BlockFinalizationLaggingBehind + # Under the assumption of an average block production of 6 seconds, + # "best" and "finalized" being more than 10 blocks apart would imply + # more than a 1 minute delay between block production and finalization. + expr: '(polkadot_block_height_number{status="best"} - ignoring(status) + polkadot_block_height_number{status="finalized"}) > 10' + for: 8m + labels: + severity: critical + annotations: + message: "Block finalization on instance {{ $labels.instance }} is behind + block production by {{ $value }} for more than 8m" + + ############################################################################## + # Transaction queue + ############################################################################## + + - alert: TransactionQueueSizeIncreasing + expr: 'increase(polkadot_sub_txpool_validations_scheduled[5m]) - + increase(polkadot_sub_txpool_validations_finished[5m]) > 0' + for: 10m + labels: + severity: warning + annotations: + message: 'The transaction pool size on node {{ $labels.instance }} has + been monotonically increasing for the last 10 minutes.' + - alert: TransactionQueueSizeIncreasing + expr: 'increase(polkadot_sub_txpool_validations_scheduled[5m]) - + increase(polkadot_sub_txpool_validations_finished[5m]) > 0' + for: 30m + labels: + severity: critical + annotations: + message: 'The transaction pool size on node {{ $labels.instance }} has + been monotonically increasing for the last 30 minutes.' + - alert: TransactionQueueSizeHigh + expr: 'polkadot_sub_txpool_validations_scheduled - + polkadot_sub_txpool_validations_finished > 10000' + for: 5m + labels: + severity: critical + annotations: + message: 'The transaction pool size on node {{ $labels.instance }} has + been above 10_000 for the last 5 minutes.' + + ############################################################################## + # Networking + ############################################################################## + + - alert: LowNumberOfPeers + expr: polkadot_sub_libp2p_peers_count < 3 + for: 3m + labels: + severity: warning + annotations: + message: 'The node {{ $labels.instance }} has less than 3 peers for more + than 3 minutes' + - alert: LowNumberOfPeers + expr: polkadot_sub_libp2p_peers_count < 3 + for: 15m + labels: + severity: critical + annotations: + message: 'The node {{ $labels.instance }} has less than 3 peers for more + than 15 minutes' + + ############################################################################## + # Others + ############################################################################## + + - alert: AuthorityDiscoveryHighDiscoveryFailure + expr: 'polkadot_authority_discovery_handle_value_found_event_failure / + ignoring(name) + polkadot_authority_discovery_dht_event_received{name="value_found"} > 0.5' + for: 2h + labels: + severity: warning + annotations: + message: "Authority discovery on node {{ $labels.instance }} fails to + process more than 50 % of the values found on the DHT." diff --git a/.maintain/monitoring/grafana-dashboards/substrate-networking.json b/.maintain/monitoring/grafana-dashboards/substrate-networking.json new file mode 100644 index 0000000000000000000000000000000000000000..f18ca66c13a0774eca86ceef42795cf4b36a35d3 --- /dev/null +++ b/.maintain/monitoring/grafana-dashboards/substrate-networking.json @@ -0,0 +1,2722 @@ +{ + "__inputs": [ + { + "name": "VAR_METRIC_NAMESPACE", + "type": "constant", + "label": "Prefix of the metrics", + "value": "polkadot", + "description": "" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "6.7.3" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "$$hashKey": "object:821", + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "limit": 100, + "name": "Annotations & Alerts", + "showIn": 0, + "type": "dashboard" + }, + { + "$$hashKey": "object:822", + "datasource": "$data_source", + "enable": true, + "expr": "count(count(${metric_namespace}_sub_libp2p_connections / max_over_time(${metric_namespace}_sub_libp2p_connections[1h]) < 0.1) >= count(${metric_namespace}_sub_libp2p_connections) / 10)", + "hide": false, + "iconColor": "#C4162A", + "limit": 100, + "name": "Connection losses events", + "showIn": 0, + "step": "5m", + "tags": [], + "titleFormat": "Network-wide connectivity loss", + "type": "tags" + } + ] + }, + "description": "Information related to the networking layer of Substrate", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1590742405831, + "links": [], + "panels": [ + { + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 167, + "title": "Sync", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 101, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "1 - (${metric_namespace}_sub_libp2p_peerset_num_requested{instance=~\"${nodename}\"} - ${metric_namespace}_sub_libp2p_peers_count{instance=~\"${nodename}\"}) / ${metric_namespace}_sub_libp2p_peerset_num_requested{instance=~\"${nodename}\"}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of peer slots filled", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1044", + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1.0", + "min": null, + "show": true + }, + { + "$$hashKey": "object:1045", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 29, + "panels": [], + "repeat": "request_protocol", + "title": "Requests (${request_protocol})", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 148, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(${metric_namespace}_sub_libp2p_requests_out_started_total{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests emitted per second", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 151, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(${metric_namespace}_sub_libp2p_requests_in_total_count{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests served per second", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 11 + }, + "hiddenSeries": false, + "id": 146, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum(rate(${metric_namespace}_sub_libp2p_requests_out_finished_bucket{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])) by (instance, le)) > 0", + "instant": false, + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Median request answer time", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1230", + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1231", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 11 + }, + "hiddenSeries": false, + "id": 145, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum(rate(${metric_namespace}_sub_libp2p_requests_in_total_bucket{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])) by (instance, le))", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Median request serving time", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 150, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(${metric_namespace}_sub_libp2p_requests_out_finished_bucket{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])) by (instance, le)) > 0", + "instant": false, + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "99th percentile request answer time", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 15 + }, + "hiddenSeries": false, + "id": 149, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(${metric_namespace}_sub_libp2p_requests_in_total_bucket{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])) by (instance, le))", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "99th percentile request serving time", + "tooltip": { + "shared": false, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 185 + }, + "id": 23, + "panels": [], + "repeat": "notif_protocol", + "title": "Notifications (${notif_protocol})", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 186 + }, + "hiddenSeries": false, + "id": 31, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "maxPerRow": 2, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "seriesOverrides": [ + { + "$$hashKey": "object:850", + "alias": "/(in)/", + "color": "#73BF69" + }, + { + "$$hashKey": "object:851", + "alias": "/(out)/", + "color": "#F2495C" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg by (direction) (irate(${metric_namespace}_sub_libp2p_notifications_sizes_count{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[$__interval]))", + "interval": "", + "legendFormat": "{{direction}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average network notifications per second", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:874", + "format": "cps", + "label": "Notifs/sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:875", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 186 + }, + "hiddenSeries": false, + "id": 37, + "interval": "1m", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "maxPerRow": 2, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "seriesOverrides": [ + { + "$$hashKey": "object:942", + "alias": "/(in)/", + "color": "#73BF69" + }, + { + "$$hashKey": "object:943", + "alias": "/(out)/", + "color": "#F2495C" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(${metric_namespace}_sub_libp2p_notifications_sizes_sum{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[$__interval])) by (direction)", + "instant": false, + "interval": "", + "legendFormat": "{{direction}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average bandwidth used by notifications", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:966", + "format": "Bps", + "label": "Bandwidth", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:967", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 193 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(${metric_namespace}_sub_libp2p_out_events_notifications_sizes{instance=~\"${nodename}\", protocol=\"${notif_protocol}\", action=\"sent\"} - ignoring(action) ${metric_namespace}_sub_libp2p_out_events_notifications_sizes{instance=~\"${nodename}\", protocol=\"${notif_protocol}\", action=\"received\"}) by (instance) > 0", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total sizes of notifications waiting to be delivered to the rest of Substrate", + "tooltip": { + "shared": false, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 1, + "fillGradient": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 193 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "6.4.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(${metric_namespace}_sub_libp2p_notifications_sizes_sum{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[5m])) by (direction, protocol) / sum(rate(${metric_namespace}_sub_libp2p_notifications_sizes_count{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[5m])) by (direction, protocol)", + "format": "time_series", + "interval": "", + "legendFormat": "{{direction}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average size of sent and received notifications in the past 5 minutes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2050", + "format": "bytes", + "label": "Max. notification size", + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:2051", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "description": "99.9% of the time, the output queue size for this protocol is below the given value", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 201 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fill": 1, + "linewidth": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(${metric_namespace}_sub_libp2p_notifications_queues_size_bucket{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[2m])) by (le, instance))", + "hide": false, + "interval": "", + "legendFormat": "{{protocol}}", + "refId": "A" + }, + { + "expr": "max(histogram_quantile(0.99, sum(rate(${metric_namespace}_sub_libp2p_notifications_queues_size_bucket{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[2m])) by (le, instance)))", + "interval": "", + "legendFormat": "max", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "99th percentile of queues sizes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": "300", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 1, + "fillGradient": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 201 + }, + "hiddenSeries": false, + "id": 134, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "6.4.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1.0, sum(rate(${metric_namespace}_sub_libp2p_notifications_sizes_bucket{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[5m])) by (direction, le))", + "format": "time_series", + "interval": "", + "legendFormat": "{{direction}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Maximum size of sent and received notifications in the past 5 minutes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1524", + "format": "bytes", + "label": "Max. notification size", + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:1525", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1853 + }, + "id": 27, + "panels": [], + "title": "Transport", + "type": "row" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 1854 + }, + "hiddenSeries": false, + "id": 19, + "interval": "1m", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "maxPerRow": 2, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "seriesOverrides": [ + { + "alias": "established (in)", + "color": "#37872D" + }, + { + "alias": "established (out)", + "color": "#C4162A" + }, + { + "alias": "pending (out)", + "color": "#FF7383" + }, + { + "alias": "closed-recently", + "color": "#FADE2A", + "steppedLine": true + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(sum(${metric_namespace}_sub_libp2p_connections_opened_total{direction=\"in\", instance=~\"${nodename}\"}) by (instance) - sum(${metric_namespace}_sub_libp2p_connections_closed_total{direction=\"in\", instance=~\"${nodename}\"}) by (instance))", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "established (in)", + "refId": "A" + }, + { + "expr": "avg(sum(${metric_namespace}_sub_libp2p_connections_opened_total{direction=\"out\", instance=~\"${nodename}\"}) by (instance) - sum(${metric_namespace}_sub_libp2p_connections_closed_total{direction=\"out\", instance=~\"${nodename}\"}) by (instance))", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "established (out)", + "refId": "C" + }, + { + "expr": "avg(sum by (instance) (${metric_namespace}_sub_libp2p_pending_connections{instance=~\"${nodename}\"}))", + "hide": false, + "interval": "", + "legendFormat": "pending (out)", + "refId": "B" + }, + { + "expr": "avg(sum by(instance) (increase(${metric_namespace}_sub_libp2p_connections_closed_total{instance=~\"${nodename}\"}[$__interval])))", + "hide": false, + "interval": "", + "legendFormat": "closed-recently", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average transport-level (TCP, QUIC, ...) connections per node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Connections", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 1860 + }, + "hiddenSeries": false, + "id": 39, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*/", + "color": "#FF780A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(increase(${metric_namespace}_sub_libp2p_incoming_connections_handshake_errors_total{instance=~\"${nodename}\"}[$__interval])) by (reason)", + "interval": "", + "legendFormat": "{{reason}}", + "refId": "A" + }, + { + "expr": "avg(increase(${metric_namespace}_sub_libp2p_listeners_errors_total{instance=~\"${nodename}\"}[$__interval]))", + "interval": "", + "legendFormat": "pre-handshake", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of incoming connection errors, averaged by node", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Errors", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "max": 100, + "min": 0, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "$data_source", + "description": "Each bucket represent a certain number of nodes using a certain bandwidth range.", + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1860 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 4, + "legend": { + "show": false + }, + "reverseYBuckets": false, + "targets": [ + { + "expr": "${metric_namespace}_network_per_sec_bytes{instance=~\"${nodename}\"}", + "format": "time_series", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Heatmap of network bandwidth", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": "2.5m", + "yAxis": { + "decimals": null, + "format": "Bps", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1866 + }, + "hiddenSeries": false, + "id": 81, + "interval": "1m", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(increase(${metric_namespace}_sub_libp2p_pending_connections_errors_total{instance=~\"${nodename}\"}[$__interval])) by (reason)", + "interval": "", + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Dialing attempt errors, averaged per node", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1866 + }, + "hiddenSeries": false, + "id": 46, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "maxPerRow": 2, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(increase(${metric_namespace}_sub_libp2p_connections_closed_total{instance=~\"${nodename}\"}[$__interval])) by (reason)", + "interval": "", + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disconnects, averaged per node", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "Disconnects", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1873 + }, + "id": 52, + "panels": [], + "title": "GrandPa", + "type": "row" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 1874 + }, + "hiddenSeries": false, + "id": 54, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "seriesOverrides": [ + { + "alias": "/discard/", + "color": "#FA6400", + "zindex": -2 + }, + { + "alias": "/keep/", + "color": "#73BF69", + "zindex": 2 + }, + { + "alias": "/process_and_discard/", + "color": "#5794F2" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(increase(${metric_namespace}_finality_grandpa_communication_gossip_validator_messages{instance=~\"${nodename}\"}[$__interval])) by (action, message)", + "interval": "", + "legendFormat": "{{message}} => {{action}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GrandPa messages received from the network, and action", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1880 + }, + "id": 25, + "panels": [], + "repeat": null, + "title": "Kademlia", + "type": "row" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 1881 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*/", + "color": "#B877D9" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "${metric_namespace}_sub_libp2p_kbuckets_num_nodes{instance=~\"${nodename}\"}", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "Number nodes in all kbuckets", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Distribution over number of entries in k-buckets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "max": 0, + "min": null, + "mode": "histogram", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 7, + "fillGradient": 7, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1881 + }, + "hiddenSeries": false, + "id": 35, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "60 * sum(irate(${metric_namespace}_sub_libp2p_random_kademalia_queries_total{instance=~\"${nodename}\"}[$__interval]))", + "interval": "", + "legendFormat": "Number of Kademlia random queries started per minute on all nodes", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Kademlia discovery queries per minute on all nodes combined", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "cpm", + "label": "Queries per minute", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 1887 + }, + "hiddenSeries": false, + "id": 111, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(${metric_namespace}_sub_libp2p_kademlia_records_count{instance=~\"${nodename}\"})", + "interval": "", + "legendFormat": "avg", + "refId": "A" + }, + { + "expr": "min(${metric_namespace}_sub_libp2p_kademlia_records_count{instance=~\"${nodename}\"})", + "interval": "", + "legendFormat": "min", + "refId": "B" + }, + { + "expr": "max(${metric_namespace}_sub_libp2p_kademlia_records_count{instance=~\"${nodename}\"})", + "interval": "", + "legendFormat": "max", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Kademlia records (average/min/max per node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 1887 + }, + "hiddenSeries": false, + "id": 112, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "max", + "fillBelowTo": "min", + "lines": false + }, + { + "alias": "min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(${metric_namespace}_sub_libp2p_kademlia_records_sizes_total{instance=~\"${nodename}\"})", + "interval": "", + "legendFormat": "avg", + "refId": "A" + }, + { + "expr": "min(${metric_namespace}_sub_libp2p_kademlia_records_sizes_total{instance=~\"${nodename}\"})", + "interval": "", + "legendFormat": "min", + "refId": "B" + }, + { + "expr": "max(${metric_namespace}_sub_libp2p_kademlia_records_sizes_total{instance=~\"${nodename}\"})", + "interval": "", + "legendFormat": "max", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total size of Kademlia records (average/min/max per node)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 1891 + }, + "hiddenSeries": false, + "id": 68, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(\n # The amount of inflight Kademlia queries per node.\r\n sum by (instance) (\r\n # The total amount of Kademlia GET_VALUE queries started.\r\n ${metric_namespace}_authority_discovery_authority_addresses_requested_total{instance=~\"${nodename}\"}\r\n \r\n # The total amount of Kademlia PUT_VALUE queries started.\r\n + ${metric_namespace}_authority_discovery_times_published_total{instance=~\"${nodename}\"}\r\n )\r\n - sum by (instance) (\r\n # The total amount of Kademlia queries (both GET_VALUE and PUT_VALUE) finished.\r\n ${metric_namespace}_authority_discovery_dht_event_received{instance=~\"${nodename}\"}\r\n )\n)", + "interval": "", + "legendFormat": "in-progress", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Authority discovery Kademlia queries in progress, averaged per node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 1896 + }, + "hiddenSeries": false, + "id": 72, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "v", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(sum without(instance) (delta(${metric_namespace}_authority_discovery_times_published_total{instance=~\"${nodename}\"}[$__interval])))", + "interval": "", + "legendFormat": "publications", + "refId": "A" + }, + { + "expr": "avg(sum without(instance) (delta(${metric_namespace}_authority_discovery_dht_event_received{instance=~\"${nodename}\", name=\"value_put\"}[$__interval])))", + "interval": "", + "legendFormat": "successes", + "refId": "B" + }, + { + "expr": "avg(sum without(instance) (delta(${metric_namespace}_authority_discovery_dht_event_received{instance=~\"${nodename}\", name=\"value_put_failed\"}[$__interval])))", + "interval": "", + "legendFormat": "failures", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Authority discovery publications, averaged per node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "1m", + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "$data_source", + "definition": "${metric_namespace}_cpu_usage_percentage", + "hide": 0, + "includeAll": true, + "index": -1, + "label": "Instance name filter", + "multi": true, + "name": "nodename", + "options": [], + "query": "${metric_namespace}_cpu_usage_percentage", + "refresh": 1, + "regex": "/instance=\"(.*?)\"/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "$data_source", + "definition": "${metric_namespace}_sub_libp2p_notifications_sizes_count", + "hide": 2, + "includeAll": true, + "index": -1, + "label": null, + "multi": false, + "name": "notif_protocol", + "options": [], + "query": "${metric_namespace}_sub_libp2p_notifications_sizes_count", + "refresh": 1, + "regex": "/protocol=\"(.*?)\"/", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "$data_source", + "definition": "${metric_namespace}_sub_libp2p_requests_out_started_total", + "hide": 2, + "includeAll": true, + "index": -1, + "label": null, + "multi": false, + "name": "request_protocol", + "options": [], + "query": "${metric_namespace}_sub_libp2p_requests_out_started_total", + "refresh": 1, + "regex": "/protocol=\"(.*?)\"/", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "tags": [], + "text": "prometheus.parity-mgmt", + "value": "prometheus.parity-mgmt" + }, + "hide": 0, + "includeAll": false, + "label": "Source of data", + "multi": false, + "name": "data_source", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "value": "${VAR_METRIC_NAMESPACE}", + "text": "${VAR_METRIC_NAMESPACE}" + }, + "hide": 2, + "label": "Prefix of the metrics", + "name": "metric_namespace", + "options": [ + { + "value": "${VAR_METRIC_NAMESPACE}", + "text": "${VAR_METRIC_NAMESPACE}" + } + ], + "query": "${VAR_METRIC_NAMESPACE}", + "skipUrlSync": false, + "type": "constant" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Substrate Networking", + "uid": "vKVuiD9Zk", + "variables": { + "list": [] + }, + "version": 103 +} \ No newline at end of file diff --git a/.maintain/rename-crates-for-2.0.sh b/.maintain/rename-crates-for-2.0.sh index 5d873d26cdfca69db389ae569b6be534d142af1d..01d559448438b3cb857d1b46af2f8bc544232066 100755 --- a/.maintain/rename-crates-for-2.0.sh +++ b/.maintain/rename-crates-for-2.0.sh @@ -64,7 +64,7 @@ TO_RENAME=( "substrate-keyring sp-keyring" "substrate-offchain-primitives sp-offchain" "substrate-panic-handler sp-panic-handler" - "substrate-phragmen sp-phragmen" + "substrate-phragmen sp-npos-elections" "substrate-rpc-primitives sp-rpc" "substrate-runtime-interface sp-runtime-interface" "substrate-runtime-interface-proc-macro sp-runtime-interface-proc-macro" diff --git a/.maintain/sentry-node/docker-compose.yml b/.maintain/sentry-node/docker-compose.yml index 376538dde5786e15d1524dc8f75072aa4c1cb59d..2af9449853c771be95d34c8a25aef8adc867d72c 100644 --- a/.maintain/sentry-node/docker-compose.yml +++ b/.maintain/sentry-node/docker-compose.yml @@ -47,9 +47,9 @@ services: - "--validator" - "--alice" - "--sentry-nodes" - - "/dns4/sentry-a/tcp/30333/p2p/QmV7EhW6J6KgmNdr558RH1mPx2xGGznW7At4BhXzntRFsi" + - "/dns/sentry-a/tcp/30333/p2p/QmV7EhW6J6KgmNdr558RH1mPx2xGGznW7At4BhXzntRFsi" - "--reserved-nodes" - - "/dns4/sentry-a/tcp/30333/p2p/QmV7EhW6J6KgmNdr558RH1mPx2xGGznW7At4BhXzntRFsi" + - "/dns/sentry-a/tcp/30333/p2p/QmV7EhW6J6KgmNdr558RH1mPx2xGGznW7At4BhXzntRFsi" # Not only bind to localhost. - "--unsafe-ws-external" - "--unsafe-rpc-external" @@ -83,11 +83,11 @@ services: - "--port" - "30333" - "--sentry" - - "/dns4/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR" + - "/dns/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR" - "--reserved-nodes" - - "/dns4/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR" + - "/dns/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR" - "--bootnodes" - - "/dns4/validator-b/tcp/30333/p2p/QmSVnNf9HwVMT1Y4cK1P6aoJcEZjmoTXpjKBmAABLMnZEk" + - "/dns/validator-b/tcp/30333/p2p/QmSVnNf9HwVMT1Y4cK1P6aoJcEZjmoTXpjKBmAABLMnZEk" - "--no-telemetry" - "--rpc-cors" - "all" @@ -118,9 +118,9 @@ services: - "--validator" - "--bob" - "--bootnodes" - - "/dns4/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR" + - "/dns/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR" - "--bootnodes" - - "/dns4/sentry-a/tcp/30333/p2p/QmV7EhW6J6KgmNdr558RH1mPx2xGGznW7At4BhXzntRFsi" + - "/dns/sentry-a/tcp/30333/p2p/QmV7EhW6J6KgmNdr558RH1mPx2xGGznW7At4BhXzntRFsi" - "--no-telemetry" - "--rpc-cors" - "all" @@ -131,11 +131,6 @@ services: - "sub-authority-discovery=trace" - "--prometheus-external" - ui: - image: polkadot-js/apps - ports: - - "3000:80" - prometheus: image: prom/prometheus networks: diff --git a/.maintain/sentry-node/prometheus/prometheus.yml b/.maintain/sentry-node/prometheus/prometheus.yml index 831b84ba0b7018948ae235619eb600b51e75c75d..547d4bea57ae5c55f210743165b347dd732ec0de 100644 --- a/.maintain/sentry-node/prometheus/prometheus.yml +++ b/.maintain/sentry-node/prometheus/prometheus.yml @@ -2,7 +2,7 @@ global: scrape_interval: 15s scrape_configs: - - job_name: 'substrate_validator-a' + - job_name: 'substrate-nodes' static_configs: - targets: ['validator-a:9615'] labels: diff --git a/Cargo.lock b/Cargo.lock index eac3ec05621e22e30be6fb08dbb86b704f42e159..afdfb5e81cc9af20f2ce0bf8e43eaeb3f6b2e36d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,6 +16,26 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2" +[[package]] +name = "aead" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf01b9b56e767bb57b94ebf91a58b338002963785cdd7013e21c0d4679471e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "aes" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54eb1d8fe354e5fc611daf4f2ea97dd45a765f4f1e4512306ec183ae2e8f20c9" +dependencies = [ + "aes-soft", + "aesni", + "block-cipher-trait", +] + [[package]] name = "aes-ctr" version = "0.3.0" @@ -28,6 +48,20 @@ dependencies = [ "stream-cipher", ] +[[package]] +name = "aes-gcm" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "834a6bda386024dbb7c8fc51322856c10ffe69559f972261c868485f5759c638" +dependencies = [ + "aead", + "aes", + "block-cipher-trait", + "ghash", + "subtle 2.2.2", + "zeroize", +] + [[package]] name = "aes-soft" version = "0.3.3" @@ -35,7 +69,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfd7e7ae3f9a1fb5c03b389fc6bb9a51400d0c13053f0dca698c832bfd893a0d" dependencies = [ "block-cipher-trait", - "byteorder 1.3.4", + "byteorder", "opaque-debug", ] @@ -103,18 +137,6 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9a60d744a80c30fcb657dfe2c1b22bcb3e814c1a1e3674f32bf5820b570fbff" -[[package]] -name = "app_dirs" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e73a24bad9bd6a94d6395382a6c69fe071708ae4409f763c5475e14ee896313d" -dependencies = [ - "ole32-sys", - "shell32-sys", - "winapi 0.2.8", - "xdg", -] - [[package]] name = "approx" version = "0.3.2" @@ -172,7 +194,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d0864d84b8e07b145449be9a8537db86bf9de5ce03b913214694643b4743502" dependencies = [ - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -237,7 +259,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95fd83426b89b034bf4e9ceb9c533c2f2386b813fd3dcae0a425ec6f1837d78a" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "rustls", "webpki", "webpki-roots 0.19.0", @@ -312,7 +334,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf" dependencies = [ - "byteorder 1.3.4", + "byteorder", "serde", ] @@ -333,7 +355,7 @@ dependencies = [ "log", "peeking_take_while", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "regex", "rustc-hash", "shlex", @@ -414,7 +436,7 @@ checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" dependencies = [ "block-padding", "byte-tools", - "byteorder 1.3.4", + "byteorder", "generic-array", ] @@ -452,9 +474,9 @@ dependencies = [ [[package]] name = "bs58" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b170cd256a3f9fa6b9edae3e44a7dfdfc77e8124dbc3e2612d75f9c3e2396dae" +checksum = "476e9cd489f9e121e02ffa6014a8ef220ecb15c05ed23fc34cca13925dc283fb" [[package]] name = "bstr" @@ -495,12 +517,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" -[[package]] -name = "byteorder" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc10e8cc6b2580fda3f36eb6dc5316657f812a3df879a44a66fc9f0fdbc4855" - [[package]] name = "byteorder" version = "1.3.4" @@ -513,7 +529,7 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" dependencies = [ - "byteorder 1.3.4", + "byteorder", "either", "iovec", ] @@ -532,9 +548,9 @@ checksum = "4964518bd3b4a8190e832886cdc0da9794f12e8e6c1613a9e90ff331c4c8724b" [[package]] name = "cargo_metadata" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46e3374c604fb39d1a2f35ed5e4a4e30e60d01fab49446e08f1b3e9a90aef202" +checksum = "b8de60b887edf6d74370fc8eb177040da4847d971d6234c7b13a6da324ef0caf" dependencies = [ "semver 0.9.0", "serde", @@ -576,17 +592,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] -name = "chacha20-poly1305-aead" -version = "0.1.2" +name = "chacha20" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77d2058ba29594f69c75e8a9018e0485e3914ca5084e3613cd64529042f5423b" +checksum = "f6a7ae4c498f8447d86baef0fa0831909333f558866fabcb21600625ac5a31c7" dependencies = [ - "constant_time_eq", + "stream-cipher", + "zeroize", +] + +[[package]] +name = "chacha20poly1305" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48901293601228db2131606f741db33561f7576b5d19c99cd66222380a7dc863" +dependencies = [ + "aead", + "chacha20", + "poly1305", + "stream-cipher", + "zeroize", ] [[package]] name = "chain-spec-builder" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "ansi_term 0.12.1", "node-cli", @@ -740,7 +770,7 @@ version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d166b289fd30062ee6de86284750fc3fe5d037c6b864b3326ce153239b0626e1" dependencies = [ - "byteorder 1.3.4", + "byteorder", "cranelift-bforest", "cranelift-codegen-meta", "cranelift-codegen-shared", @@ -749,7 +779,7 @@ dependencies = [ "log", "regalloc", "serde", - "smallvec 1.3.0", + "smallvec 1.4.0", "target-lexicon", "thiserror", ] @@ -787,7 +817,7 @@ checksum = "e45f82e3446dd1ebb8c2c2f6a6b0e6cd6cd52965c7e5f7b1b35e9a9ace31ccde" dependencies = [ "cranelift-codegen", "log", - "smallvec 1.3.0", + "smallvec 1.4.0", "target-lexicon", ] @@ -837,7 +867,7 @@ dependencies = [ "clap", "criterion-plot 0.3.1", "csv", - "itertools", + "itertools 0.8.2", "lazy_static", "libc", "num-traits 0.2.11", @@ -864,7 +894,7 @@ dependencies = [ "clap", "criterion-plot 0.4.1", "csv", - "itertools", + "itertools 0.8.2", "lazy_static", "num-traits 0.2.11", "oorandom", @@ -884,9 +914,9 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76f9212ddf2f4a9eb2d401635190600656a1f88a932ef53d06e7fa4c7e02fb8e" dependencies = [ - "byteorder 1.3.4", + "byteorder", "cast", - "itertools", + "itertools 0.8.2", ] [[package]] @@ -896,7 +926,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a01e15e0ea58e8234f96146b1f91fa9d0e4dd7a38da93ff7a75d42c0b9d3a545" dependencies = [ "cast", - "itertools", + "itertools 0.8.2", ] [[package]] @@ -1009,7 +1039,7 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47c5e5ac752e18207b12e16b10631ae5f7f68f8805f335f9b817ead83d9ffce1" dependencies = [ - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -1023,23 +1053,13 @@ dependencies = [ "stream-cipher", ] -[[package]] -name = "cuckoofilter" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dd43f7cfaffe0a386636a10baea2ee05cc50df3b77bea4a456c9572a939bf1f" -dependencies = [ - "byteorder 0.5.3", - "rand 0.3.23", -] - [[package]] name = "curve25519-dalek" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26778518a7f6cffa1d25a44b602b62b979bd88adb9e99ffec546998cf3404839" dependencies = [ - "byteorder 1.3.4", + "byteorder", "digest", "rand_core 0.5.1", "subtle 2.2.2", @@ -1059,7 +1079,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2323f3f47db9a0e77ce7a300605d8d2098597fc451ed1a97bb1f6411bb550a7" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -1106,7 +1126,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4d33be9473d06f75f58220f71f7a9317aca647dc061dbd3c361b0bef505fbea" dependencies = [ - "byteorder 1.3.4", + "byteorder", "quick-error", ] @@ -1161,7 +1181,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ed9afacaea0301eefb738c9deea725e6d53938004597cdc518a8cf9a7aa2f03" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -1279,7 +1299,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e43f2f1833d64e33f15592464d6fdd70f349dda7b1a53088eb83cd94014008c5" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", ] [[package]] @@ -1314,7 +1334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", "synstructure", ] @@ -1357,7 +1377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8feb87a63249689640ac9c011742c33139204e3c134293d3054022276869133b" dependencies = [ "either", - "futures 0.3.4", + "futures 0.3.5", "futures-timer 2.0.2", "log", "num-traits 0.2.11", @@ -1372,7 +1392,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32529fc42e86ec06e5047092082aab9ad459b070c5d2a76b14f4f5ce70bf2e84" dependencies = [ - "byteorder 1.3.4", + "byteorder", "rand 0.7.3", "rustc-hex", "static_assertions", @@ -1405,14 +1425,14 @@ checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" [[package]] name = "fork-tree" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "parity-scale-codec", ] [[package]] name = "frame-benchmarking" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -1428,7 +1448,7 @@ dependencies = [ [[package]] name = "frame-benchmarking-cli" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "parity-scale-codec", @@ -1445,7 +1465,7 @@ dependencies = [ [[package]] name = "frame-executive" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -1465,7 +1485,7 @@ dependencies = [ [[package]] name = "frame-metadata" -version = "11.0.0-alpha.8" +version = "11.0.0-rc4" dependencies = [ "parity-scale-codec", "serde", @@ -1475,7 +1495,7 @@ dependencies = [ [[package]] name = "frame-support" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "bitmask", "frame-metadata", @@ -1488,6 +1508,7 @@ dependencies = [ "paste", "pretty_assertions", "serde", + "smallvec 1.4.0", "sp-arithmetic", "sp-core", "sp-inherents", @@ -1500,37 +1521,37 @@ dependencies = [ [[package]] name = "frame-support-procedural" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support-procedural-tools", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] [[package]] name = "frame-support-procedural-tools" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support-procedural-tools-derive", "proc-macro-crate", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] [[package]] name = "frame-support-procedural-tools-derive" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] [[package]] name = "frame-support-test" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "parity-scale-codec", @@ -1542,12 +1563,13 @@ dependencies = [ "sp-io", "sp-runtime", "sp-state-machine", + "sp-std", "trybuild", ] [[package]] name = "frame-system" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "criterion 0.2.11", "frame-support", @@ -1565,7 +1587,7 @@ dependencies = [ [[package]] name = "frame-system-benchmarking" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "frame-support", @@ -1580,7 +1602,7 @@ dependencies = [ [[package]] name = "frame-system-rpc-runtime-api" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "parity-scale-codec", "sp-api", @@ -1644,9 +1666,9 @@ checksum = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef" [[package]] name = "futures" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c329ae8753502fb44ae4fc2b622fa2a94652c41e795143765ba0927f92ab780" +checksum = "1e05b85ec287aac0dc34db7d4a569323df697f9c55b99b15d6b4ef8cde49f613" dependencies = [ "futures-channel", "futures-core", @@ -1659,9 +1681,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c77d04ce8edd9cb903932b608268b3fffec4163dc053b3b402bf47eac1f1a8" +checksum = "f366ad74c28cca6ba456d95e6422883cfb4b252a83bed929c83abfdbbf2967d5" dependencies = [ "futures-core", "futures-sink", @@ -1678,9 +1700,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f25592f769825e89b92358db00d26f965761e094951ac44d3663ef25b7ac464a" +checksum = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399" [[package]] name = "futures-core-preview" @@ -1705,7 +1727,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdcef58a173af8148b182684c9f2d5250875adbcaff7b5794073894f9d8634a9" dependencies = [ "futures 0.1.29", - "futures 0.3.4", + "futures 0.3.5", "lazy_static", "log", "parking_lot 0.9.0", @@ -1716,9 +1738,9 @@ dependencies = [ [[package]] name = "futures-executor" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f674f3e1bcb15b37284a90cedf55afdba482ab061c407a9c0ebbd0f3109741ba" +checksum = "10d6bb888be1153d3abeb9006b11b02cf5e9b209fda28693c31ae1e4e012e314" dependencies = [ "futures-core", "futures-task", @@ -1728,33 +1750,36 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a638959aa96152c7a4cddf50fcb1e3fede0583b27157c26e67d6f99904090dc6" +checksum = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789" [[package]] name = "futures-macro" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a5081aa3de1f7542a794a397cde100ed903b0630152d0973479018fd85423a7" +checksum = "d0b5a30a4328ab5473878237c447333c093297bded83a4983d10f4deea240d39" dependencies = [ "proc-macro-hack", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] [[package]] name = "futures-sink" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3466821b4bc114d95b087b850a724c6f83115e929bc88f1fa98a3304a944c8a6" +checksum = "3f2032893cb734c7a05d85ce0cc8b8c4075278e93b24b66f9de99d6eb0fa8acc" [[package]] name = "futures-task" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0a34e53cf6cdcd0178aa573aed466b646eb3db769570841fda0c7ede375a27" +checksum = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626" +dependencies = [ + "once_cell", +] [[package]] name = "futures-timer" @@ -1774,9 +1799,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22766cf25d64306bedf0384da004d05c9974ab104fcc4528f1236181c18004c5" +checksum = "8764574ff08b701a084482c3c7031349104b07ac897393010494beaa18ce32c6" dependencies = [ "futures 0.1.29", "futures-channel", @@ -1786,6 +1811,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", + "pin-project", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -1811,7 +1837,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0a73299e4718f5452e45980fc1d6957a070abe308d3700b63b8673f47e1c2b3" dependencies = [ "bytes 0.5.4", - "futures 0.3.4", + "futures 0.3.5", + "memchr", + "pin-project", +] + +[[package]] +name = "futures_codec" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce54d63f8b0c75023ed920d46fd71d0cbbb830b0ee012726b5b4f506fb6dea5b" +dependencies = [ + "bytes 0.5.4", + "futures 0.3.5", "memchr", "pin-project", ] @@ -1865,6 +1903,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f0930ed19a7184089ea46d2fedead2f6dc2b674c5db4276b7da336c7cd83252" +dependencies = [ + "polyval", +] + [[package]] name = "gimli" version = "0.20.0" @@ -1872,10 +1919,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81dd6190aad0f05ddbbf3245c54ed14ca4aa6dd32f22312b70d8f168c3e3e633" dependencies = [ "arrayvec 0.5.1", - "byteorder 1.3.4", + "byteorder", "fallible-iterator", "indexmap", - "smallvec 1.3.0", + "smallvec 1.4.0", "stable_deref_trait", ] @@ -1934,7 +1981,7 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462" dependencies = [ - "byteorder 1.3.4", + "byteorder", "bytes 0.4.12", "fnv", "futures 0.1.29", @@ -2056,9 +2103,9 @@ dependencies = [ [[package]] name = "honggfuzz" -version = "0.5.47" +version = "0.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3de2c3273ef7735df1c5a72128ca85b1d20105b9aac643cdfd7a6e581311150" +checksum = "832bac18a82ec7d6c21887daa8616b238fe90d5d5e762d0d4b9372cdaa9e097f" dependencies = [ "arbitrary", "lazy_static", @@ -2261,7 +2308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef5550a42e3740a0e71f909d4c861056a284060af885ae7aa6242820f920d9d" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -2286,7 +2333,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64fa110ec7b8f493f416eed552740d10e7030ad5f63b2308f82c9608ec2df275" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "futures-timer 2.0.2", ] @@ -2320,6 +2367,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.5" @@ -2346,9 +2402,9 @@ dependencies = [ [[package]] name = "jsonrpc-client-transports" -version = "14.1.0" +version = "14.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2307a7e78cf969759e390a8a2151ea12e783849a45bb00aa871b468ba58ea79e" +checksum = "ecbdaacc17243168d9d1fa6b2bd7556a27e1e60a621d8a2a6e590ae2b145d158" dependencies = [ "failure", "futures 0.1.29", @@ -2363,9 +2419,9 @@ dependencies = [ [[package]] name = "jsonrpc-core" -version = "14.1.0" +version = "14.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25525f6002338fb4debb5167a89a0b47f727a5a48418417545ad3429758b7fec" +checksum = "a0747307121ffb9703afd93afbd0fb4f854c38fb873f2c8b90e0e902f27c7b62" dependencies = [ "futures 0.1.29", "log", @@ -2376,30 +2432,30 @@ dependencies = [ [[package]] name = "jsonrpc-core-client" -version = "14.1.0" +version = "14.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f9382e831a6d630c658df103aac3f971da096deb57c136ea2b760d3b4e3f9f" +checksum = "34221123bc79b66279a3fde2d3363553835b43092d629b34f2e760c44dc94713" dependencies = [ "jsonrpc-client-transports", ] [[package]] name = "jsonrpc-derive" -version = "14.0.5" +version = "14.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8609af8f63b626e8e211f52441fcdb6ec54f1a446606b10d5c89ae9bf8a20058" +checksum = "0fadf6945e227246825a583514534d864554e9f23d80b3c77d034b10983db5ef" dependencies = [ "proc-macro-crate", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] [[package]] name = "jsonrpc-http-server" -version = "14.1.0" +version = "14.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52860f0549694aa4abb12766856f56952ab46d3fb9f0815131b2db3d9cc2f29" +checksum = "0da906d682799df05754480dac1b9e70ec92e12c19ebafd2662a5ea1c9fd6522" dependencies = [ "hyper 0.12.35", "jsonrpc-core", @@ -2410,23 +2466,38 @@ dependencies = [ "unicase", ] +[[package]] +name = "jsonrpc-ipc-server" +version = "14.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dedccd693325d833963b549e959137f30a7a0ea650cde92feda81dc0c1393cb5" +dependencies = [ + "jsonrpc-core", + "jsonrpc-server-utils", + "log", + "parity-tokio-ipc", + "parking_lot 0.10.2", + "tokio-service", +] + [[package]] name = "jsonrpc-pubsub" -version = "14.1.0" +version = "14.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4ca5e391d6c6a2261d4adca029f427fe63ea546ad6cef2957c654c08495ec16" +checksum = "2d44f5602a11d657946aac09357956d2841299ed422035edf140c552cb057986" dependencies = [ "jsonrpc-core", "log", "parking_lot 0.10.2", + "rand 0.7.3", "serde", ] [[package]] name = "jsonrpc-server-utils" -version = "14.1.0" +version = "14.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f06add502b48351e05dd95814835327fb115e4e9f834ca42fd522d3b769d4d2" +checksum = "56cbfb462e7f902e21121d9f0d1c2b77b2c5b642e1a4e8f4ebfa2e15b94402bb" dependencies = [ "bytes 0.4.12", "globset", @@ -2440,9 +2511,9 @@ dependencies = [ [[package]] name = "jsonrpc-ws-server" -version = "14.1.0" +version = "14.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "017a7dd5083d9ed62c5e1dd3e317975c33c3115dac5447f4480fe05a8c354754" +checksum = "903d3109fe7c4acb932b567e1e607e0f524ed04741b09fb0e61841bc40a022fc" dependencies = [ "jsonrpc-core", "jsonrpc-server-utils", @@ -2495,7 +2566,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e763b2a9b500ba47948061d1e8bc3b5f03a8a1f067dbcf822a4d2c84d2b54a3a" dependencies = [ "parity-util-mem", - "smallvec 1.3.0", + "smallvec 1.4.0", ] [[package]] @@ -2524,7 +2595,7 @@ dependencies = [ "parking_lot 0.10.2", "regex", "rocksdb", - "smallvec 1.3.0", + "smallvec 1.4.0", ] [[package]] @@ -2533,7 +2604,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c7f36acb1841d4c701d30ae1f2cfd242e805991443f75f6935479ed3de64903" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "js-sys", "kvdb", "kvdb-memorydb", @@ -2598,61 +2669,55 @@ checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" [[package]] name = "libp2p" -version = "0.18.1" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ea742c86405b659c358223a8f0f9f5a9eb27bb6083894c6340959b05269662" +checksum = "db81113df355dea9dddfcb01cd867555298dca29d915f25d1b1a0aad2e29338b" dependencies = [ "bytes 0.5.4", - "futures 0.3.4", + "futures 0.3.5", "lazy_static", "libp2p-core", "libp2p-core-derive", - "libp2p-deflate", "libp2p-dns", - "libp2p-floodsub", - "libp2p-gossipsub", "libp2p-identify", "libp2p-kad", "libp2p-mdns", "libp2p-mplex", "libp2p-noise", "libp2p-ping", - "libp2p-plaintext", - "libp2p-pnet", "libp2p-secio", "libp2p-swarm", "libp2p-tcp", - "libp2p-uds", "libp2p-wasm-ext", "libp2p-websocket", "libp2p-yamux", "multihash", - "parity-multiaddr 0.8.0", + "parity-multiaddr 0.9.1", "parking_lot 0.10.2", "pin-project", - "smallvec 1.3.0", + "smallvec 1.4.0", "wasm-timer", ] [[package]] name = "libp2p-core" -version = "0.18.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d2c17158c4dca984a77a5927aac6f0862d7f50c013470a415f93be498b5739" +checksum = "3a0387b930c3d4c2533dc4893c1e0394185ddcc019846121b1b27491e45a2c08" dependencies = [ "asn1_der", "bs58", "ed25519-dalek", "either", "fnv", - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "lazy_static", "libsecp256k1", "log", "multihash", "multistream-select", - "parity-multiaddr 0.8.0", + "parity-multiaddr 0.9.1", "parking_lot 0.10.2", "pin-project", "prost", @@ -2661,115 +2726,62 @@ dependencies = [ "ring", "rw-stream-sink", "sha2", - "smallvec 1.3.0", + "smallvec 1.4.0", "thiserror", - "unsigned-varint", + "unsigned-varint 0.4.0", "void", "zeroize", ] [[package]] name = "libp2p-core-derive" -version = "0.18.0" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "329127858e4728db5ab60c33d5ae352a999325fdf190ed022ec7d3a4685ae2e6" +checksum = "f09548626b737ed64080fde595e06ce1117795b8b9fc4d2629fa36561c583171" dependencies = [ - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] -[[package]] -name = "libp2p-deflate" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad32b006ea922da8cc66e537cf2df4b0fad8ebaa467d2a8c63d7784ac252ec6" -dependencies = [ - "flate2", - "futures 0.3.4", - "libp2p-core", -] - [[package]] name = "libp2p-dns" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0d0993481203d68e5ce2f787d033fb0cac6b850659ed6c784612db678977c71" +checksum = "3cc186d9a941fd0207cf8f08ef225a735e2d7296258f570155e525f6ee732f87" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "libp2p-core", "log", ] -[[package]] -name = "libp2p-floodsub" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3673153ca967c179d745fadf047d069355d6669ecf7f261b450fbaebf1bffd3d" -dependencies = [ - "cuckoofilter", - "fnv", - "futures 0.3.4", - "libp2p-core", - "libp2p-swarm", - "prost", - "prost-build", - "rand 0.7.3", - "smallvec 1.3.0", -] - -[[package]] -name = "libp2p-gossipsub" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f7f3f79f060864db0317cc47641b7d35276dee52a0ffa91553fbd0c153863a3" -dependencies = [ - "base64 0.11.0", - "byteorder 1.3.4", - "bytes 0.5.4", - "fnv", - "futures 0.3.4", - "futures_codec", - "libp2p-core", - "libp2p-swarm", - "log", - "lru", - "prost", - "prost-build", - "rand 0.7.3", - "sha2", - "smallvec 1.3.0", - "unsigned-varint", - "wasm-timer", -] - [[package]] name = "libp2p-identify" -version = "0.18.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a38ca3eb807789e26f41c82ca7cd2b3843c66c5587b8b5f709a2f421f3061414" +checksum = "62f76075b170d908bae616f550ade410d9d27c013fa69042551dbfc757c7c094" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "libp2p-core", "libp2p-swarm", "log", "prost", "prost-build", - "smallvec 1.3.0", + "smallvec 1.4.0", "wasm-timer", ] [[package]] name = "libp2p-kad" -version = "0.18.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92cda1fb8149ea64d092a2b99d2bd7a2c309eee38ea322d02e4480bd6ee1759" +checksum = "f7c819a5425b2eb3416d67e9c868c5c1e922b6658655e06b9eeafaa41304b876" dependencies = [ "arrayvec 0.5.1", "bytes 0.5.4", "either", "fnv", - "futures 0.3.4", - "futures_codec", + "futures 0.3.5", + "futures_codec 0.4.1", "libp2p-core", "libp2p-swarm", "log", @@ -2778,59 +2790,59 @@ dependencies = [ "prost-build", "rand 0.7.3", "sha2", - "smallvec 1.3.0", + "smallvec 1.4.0", "uint", - "unsigned-varint", + "unsigned-varint 0.4.0", "void", "wasm-timer", ] [[package]] name = "libp2p-mdns" -version = "0.18.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41e908d2aaf8ff0ec6ad1f02fe1844fd777fb0b03a68a226423630750ab99471" +checksum = "7f55b2d4b80986e5bf158270ab23268ec0e7f644ece5436fbaabc5155472f357" dependencies = [ "async-std", "data-encoding", "dns-parser", "either", - "futures 0.3.4", + "futures 0.3.5", "lazy_static", "libp2p-core", "libp2p-swarm", "log", "net2", "rand 0.7.3", - "smallvec 1.3.0", + "smallvec 1.4.0", "void", "wasm-timer", ] [[package]] name = "libp2p-mplex" -version = "0.18.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0832882b06619b2e81d74e71447753ea3c068164a0bca67847d272e856a04a02" +checksum = "be7d913a4cd57de2013257ec73f07d77bfce390b370023e2d59083e5ca079864" dependencies = [ "bytes 0.5.4", "fnv", - "futures 0.3.4", - "futures_codec", + "futures 0.3.5", + "futures_codec 0.4.1", "libp2p-core", "log", "parking_lot 0.10.2", - "unsigned-varint", + "unsigned-varint 0.4.0", ] [[package]] name = "libp2p-noise" -version = "0.18.0" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "918e94a649e1139c24ee9f1f8c1f2adaba6d157b9471af787f2d9beac8c29c77" +checksum = "a03db664653369f46ee03fcec483a378c20195089bb43a26cb9fb0058009ac88" dependencies = [ "curve25519-dalek", - "futures 0.3.4", + "futures 0.3.5", "lazy_static", "libp2p-core", "log", @@ -2846,11 +2858,11 @@ dependencies = [ [[package]] name = "libp2p-ping" -version = "0.18.0" +version = "0.19.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9bfbf87eebb492d040f9899c5c81c9738730465ac5e78d9b7a7d086d0f07230" +checksum = "b8dedd34e35a9728d52d59ef36a218e411359a353f9011b2574b86ee790978f6" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "libp2p-core", "libp2p-swarm", "log", @@ -2859,47 +2871,15 @@ dependencies = [ "wasm-timer", ] -[[package]] -name = "libp2p-plaintext" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabb00553a49bf6d4a8ce362f6eefac410227a14d03c3acffbb8cc3f022ea019" -dependencies = [ - "bytes 0.5.4", - "futures 0.3.4", - "futures_codec", - "libp2p-core", - "log", - "prost", - "prost-build", - "rw-stream-sink", - "unsigned-varint", - "void", -] - -[[package]] -name = "libp2p-pnet" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f81b8b37ff529e1f51c20c396dac657def2108da174c1d27e57e72c9fe2d411" -dependencies = [ - "futures 0.3.4", - "log", - "pin-project", - "rand 0.7.3", - "salsa20", - "sha3", -] - [[package]] name = "libp2p-secio" -version = "0.18.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a0509a7e47245259954fef58b85b81bf4d29ae33a4365e38d718a866698774" +checksum = "c99b3c33e96bb402486d5b4f7cbeab14e66e6a2ed010abbb5bb032a05460bfda" dependencies = [ "aes-ctr", "ctr", - "futures 0.3.4", + "futures 0.3.5", "hmac", "js-sys", "lazy_static", @@ -2923,53 +2903,42 @@ dependencies = [ [[package]] name = "libp2p-swarm" -version = "0.18.1" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44ab289ae44cc691da0a6fe96aefa43f26c86c6c7813998e203f6d80f1860f18" +checksum = "ce53ff4d127cf8b39adf84dbd381ca32d49bd85788cee08e6669da2495993930" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "libp2p-core", "log", "rand 0.7.3", - "smallvec 1.3.0", + "smallvec 1.4.0", "void", "wasm-timer", ] [[package]] name = "libp2p-tcp" -version = "0.18.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b37ea44823d3ed223e4605da94b50177bc520f05ae2452286700549a32d81669" +checksum = "9481500c5774c62e8c413e9535b3f33a0e3dbacf2da63b8d3056c686a9df4146" dependencies = [ "async-std", - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "get_if_addrs", "ipnet", "libp2p-core", "log", -] - -[[package]] -name = "libp2p-uds" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "281c18ea2faacb9c8a6ff76c4405df5918d9a263770e3847bf03f099abadc010" -dependencies = [ - "async-std", - "futures 0.3.4", - "libp2p-core", - "log", + "socket2", ] [[package]] name = "libp2p-wasm-ext" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ac7dbde0f88cad191dcdfd073b8bae28d01823e8ca313f117b6ecb914160c3" +checksum = "f59fdbb5706f2723ca108c088b1c7a37f735a8c328021f0508007162627e9885" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "js-sys", "libp2p-core", "parity-send-wrapper", @@ -2979,14 +2948,13 @@ dependencies = [ [[package]] name = "libp2p-websocket" -version = "0.18.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6874c9069ce93d899df9dc7b29f129c706b2a0fdc048f11d878935352b580190" +checksum = "7e4440551bf6519e0a684cd859ea809aec6d798f686e0d6ed03a28c3e76849b8" dependencies = [ "async-tls", - "bytes 0.5.4", "either", - "futures 0.3.4", + "futures 0.3.5", "libp2p-core", "log", "quicksink", @@ -3000,11 +2968,11 @@ dependencies = [ [[package]] name = "libp2p-yamux" -version = "0.18.0" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02f91aea50f6571e0bc6c058dc0e9b270afd41ec28dd94e9e4bf607e78b9ab87" +checksum = "8da33e7b5f49c75c6a8afb0b8d1e229f5fa48be9f39bd14cdbc21459a02ac6fc" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "libp2p-core", "parking_lot 0.10.2", "thiserror", @@ -3170,9 +3138,9 @@ dependencies = [ [[package]] name = "memory-db" -version = "0.20.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be512cb2ccb4ecbdca937fdd4a62ea5f09f8e7195466a85e4632b3d5bcce82e6" +checksum = "fb2999ff7a65d5a1d72172f6d51fa2ea03024b51aee709ba5ff81c3c629a2410" dependencies = [ "ahash", "hash-db", @@ -3192,7 +3160,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6feca46f4fa3443a01769d768727f10c10a20fdb65e52dc16a81f0c8269bb78" dependencies = [ - "byteorder 1.3.4", + "byteorder", "keccak", "rand_core 0.5.1", "zeroize", @@ -3220,7 +3188,7 @@ dependencies = [ "kernel32-sys", "libc", "log", - "miow", + "miow 0.2.1", "net2", "slab", "winapi 0.2.8", @@ -3238,6 +3206,18 @@ dependencies = [ "slab", ] +[[package]] +name = "mio-named-pipes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5e374eff525ce1c5b7687c4cef63943e7686524a387933ad27ca7ec43779cb3" +dependencies = [ + "log", + "mio", + "miow 0.3.5", + "winapi 0.3.8", +] + [[package]] name = "mio-uds" version = "0.6.7" @@ -3261,6 +3241,16 @@ dependencies = [ "ws2_32-sys", ] +[[package]] +name = "miow" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07b88fb9795d4d36d62a012dfbf49a8f5cf12751f36d31a9dbe66d528e58979e" +dependencies = [ + "socket2", + "winapi 0.3.8", +] + [[package]] name = "more-asserts" version = "0.2.1" @@ -3269,9 +3259,9 @@ checksum = "0debeb9fcf88823ea64d64e4a815ab1643f33127d995978e099942ce38f25238" [[package]] name = "multihash" -version = "0.10.1" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47fbc227f7e2b1cb701f95404579ecb2668abbdd3c7ef7a6cbb3cc0d3b236869" +checksum = "ae32179a9904ccc6e063de8beee7f5dd55fae85ecb851ca923d55722bc28cf5d" dependencies = [ "blake2b_simd", "blake2s_simd", @@ -3279,7 +3269,7 @@ dependencies = [ "sha-1", "sha2", "sha3", - "unsigned-varint", + "unsigned-varint 0.3.3", ] [[package]] @@ -3290,16 +3280,16 @@ checksum = "d8883adfde9756c1d30b0f519c9b8c502a94b41ac62f696453c37c7fc0a958ce" [[package]] name = "multistream-select" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74cdcf7cfb3402881e15a1f95116cb033d69b33c83d481e1234777f5ef0c3d2c" +checksum = "c9157e87afbc2ef0d84cc0345423d715f445edde00141c93721c162de35a05e5" dependencies = [ "bytes 0.5.4", - "futures 0.3.4", + "futures 0.3.5", "log", "pin-project", - "smallvec 1.3.0", - "unsigned-varint", + "smallvec 1.4.0", + "unsigned-varint 0.4.0", ] [[package]] @@ -3346,7 +3336,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29449d242064c48d3057a194b049a2bdcccadda16faa18a91468677b44e8d422" dependencies = [ "bitflags", - "byteorder 1.3.4", + "byteorder", "enum-primitive-derive", "libc", "num-traits 0.2.11", @@ -3368,10 +3358,11 @@ dependencies = [ [[package]] name = "node-bench" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "derive_more", "fs_extra", + "futures 0.3.5", "hash-db", "hex", "kvdb", @@ -3379,17 +3370,24 @@ dependencies = [ "lazy_static", "log", "node-primitives", + "node-runtime", "node-testing", "parity-db", "parity-util-mem", "rand 0.7.3", + "sc-basic-authorship", "sc-cli", "sc-client-api", "serde", "serde_json", + "sp-consensus", "sp-core", + "sp-finality-tracker", + "sp-inherents", "sp-runtime", "sp-state-machine", + "sp-timestamp", + "sp-transaction-pool", "sp-trie", "structopt", "tempfile", @@ -3397,9 +3395,9 @@ dependencies = [ [[package]] name = "node-browser-testing" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "jsonrpc-core", "libp2p", @@ -3414,13 +3412,13 @@ dependencies = [ [[package]] name = "node-cli" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "assert_cmd", "frame-benchmarking-cli", "frame-support", "frame-system", - "futures 0.3.4", + "futures 0.3.5", "hex-literal", "jsonrpc-core", "log", @@ -3488,7 +3486,7 @@ dependencies = [ [[package]] name = "node-executor" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "criterion 0.3.1", "frame-benchmarking", @@ -3522,7 +3520,7 @@ dependencies = [ [[package]] name = "node-inspect" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "derive_more", "log", @@ -3538,7 +3536,7 @@ dependencies = [ [[package]] name = "node-primitives" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-system", "parity-scale-codec", @@ -3551,7 +3549,7 @@ dependencies = [ [[package]] name = "node-rpc" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "jsonrpc-core", "node-primitives", @@ -3565,7 +3563,9 @@ dependencies = [ "sc-finality-grandpa", "sc-finality-grandpa-rpc", "sc-keystore", + "sc-rpc-api", "sp-api", + "sp-block-builder", "sp-blockchain", "sp-consensus", "sp-consensus-babe", @@ -3576,7 +3576,7 @@ dependencies = [ [[package]] name = "node-rpc-client" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "env_logger 0.7.1", "futures 0.1.29", @@ -3589,7 +3589,7 @@ dependencies = [ [[package]] name = "node-runtime" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "frame-executive", @@ -3597,6 +3597,7 @@ dependencies = [ "frame-system", "frame-system-benchmarking", "frame-system-rpc-runtime-api", + "hex-literal", "integer-sqrt", "node-primitives", "pallet-authority-discovery", @@ -3615,8 +3616,10 @@ dependencies = [ "pallet-im-online", "pallet-indices", "pallet-membership", + "pallet-multisig", "pallet-offences", "pallet-offences-benchmarking", + "pallet-proxy", "pallet-randomness-collective-flip", "pallet-recovery", "pallet-scheduler", @@ -3655,9 +3658,9 @@ dependencies = [ [[package]] name = "node-template" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "log", "node-template-runtime", "parking_lot 0.10.2", @@ -3684,7 +3687,7 @@ dependencies = [ [[package]] name = "node-template-runtime" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-executive", "frame-support", @@ -3716,13 +3719,13 @@ dependencies = [ [[package]] name = "node-testing" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "criterion 0.3.1", "frame-support", "frame-system", "fs_extra", - "futures 0.3.4", + "futures 0.3.5", "log", "node-executor", "node-primitives", @@ -3872,16 +3875,6 @@ dependencies = [ "target-lexicon", ] -[[package]] -name = "ole32-sys" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2c49021782e5233cd243168edfa8037574afed4eba4bbaf538b3d8d1789d8c" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "once_cell" version = "1.3.1" @@ -3929,7 +3922,7 @@ dependencies = [ [[package]] name = "pallet-assets" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -3941,9 +3934,24 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-atomic-swap" +version = "2.0.0-rc4" +dependencies = [ + "frame-support", + "frame-system", + "pallet-balances", + "parity-scale-codec", + "serde", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", +] + [[package]] name = "pallet-aura" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -3965,7 +3973,7 @@ dependencies = [ [[package]] name = "pallet-authority-discovery" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -3983,7 +3991,7 @@ dependencies = [ [[package]] name = "pallet-authorship" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -3999,7 +4007,7 @@ dependencies = [ [[package]] name = "pallet-babe" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -4021,7 +4029,7 @@ dependencies = [ [[package]] name = "pallet-balances" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "frame-support", @@ -4037,7 +4045,7 @@ dependencies = [ [[package]] name = "pallet-benchmark" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "frame-support", @@ -4051,7 +4059,7 @@ dependencies = [ [[package]] name = "pallet-collective" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "frame-support", @@ -4068,7 +4076,7 @@ dependencies = [ [[package]] name = "pallet-contracts" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "assert_matches", "frame-support", @@ -4078,9 +4086,9 @@ dependencies = [ "pallet-contracts-primitives", "pallet-randomness-collective-flip", "pallet-timestamp", - "pallet-transaction-payment", "parity-scale-codec", "parity-wasm 0.41.0", + "pretty_assertions", "pwasm-utils", "serde", "sp-core", @@ -4094,7 +4102,7 @@ dependencies = [ [[package]] name = "pallet-contracts-primitives" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "parity-scale-codec", "sp-runtime", @@ -4103,7 +4111,7 @@ dependencies = [ [[package]] name = "pallet-contracts-rpc" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -4122,7 +4130,7 @@ dependencies = [ [[package]] name = "pallet-contracts-rpc-runtime-api" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "pallet-contracts-primitives", "parity-scale-codec", @@ -4133,7 +4141,7 @@ dependencies = [ [[package]] name = "pallet-democracy" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "frame-support", @@ -4153,7 +4161,7 @@ dependencies = [ [[package]] name = "pallet-elections" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -4169,7 +4177,7 @@ dependencies = [ [[package]] name = "pallet-elections-phragmen" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "frame-support", @@ -4180,7 +4188,7 @@ dependencies = [ "serde", "sp-core", "sp-io", - "sp-phragmen", + "sp-npos-elections", "sp-runtime", "sp-std", "substrate-test-utils", @@ -4188,7 +4196,7 @@ dependencies = [ [[package]] name = "pallet-evm" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "evm", "frame-support", @@ -4208,7 +4216,7 @@ dependencies = [ [[package]] name = "pallet-example" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "frame-support", @@ -4224,7 +4232,7 @@ dependencies = [ [[package]] name = "pallet-example-offchain-worker" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -4239,7 +4247,7 @@ dependencies = [ [[package]] name = "pallet-finality-tracker" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -4256,7 +4264,7 @@ dependencies = [ [[package]] name = "pallet-generic-asset" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -4270,7 +4278,7 @@ dependencies = [ [[package]] name = "pallet-grandpa" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "finality-grandpa", "frame-support", @@ -4297,7 +4305,7 @@ dependencies = [ [[package]] name = "pallet-identity" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "enumflags2", "frame-benchmarking", @@ -4314,7 +4322,7 @@ dependencies = [ [[package]] name = "pallet-im-online" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "frame-support", @@ -4333,8 +4341,9 @@ dependencies = [ [[package]] name = "pallet-indices" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ + "frame-benchmarking", "frame-support", "frame-system", "pallet-balances", @@ -4349,7 +4358,7 @@ dependencies = [ [[package]] name = "pallet-membership" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -4361,9 +4370,25 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-multisig" +version = "2.0.0-rc4" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "pallet-balances", + "parity-scale-codec", + "serde", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", +] + [[package]] name = "pallet-nicks" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -4378,7 +4403,7 @@ dependencies = [ [[package]] name = "pallet-offences" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -4393,33 +4418,50 @@ dependencies = [ ] [[package]] -name = "pallet-offences-benchmarking" -version = "2.0.0-alpha.8" +name = "pallet-offences-benchmarking" +version = "2.0.0-rc4" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "pallet-babe", + "pallet-balances", + "pallet-grandpa", + "pallet-im-online", + "pallet-offences", + "pallet-session", + "pallet-staking", + "pallet-staking-reward-curve", + "pallet-timestamp", + "parity-scale-codec", + "serde", + "sp-core", + "sp-io", + "sp-runtime", + "sp-staking", + "sp-std", +] + +[[package]] +name = "pallet-proxy" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "pallet-babe", "pallet-balances", - "pallet-grandpa", - "pallet-im-online", - "pallet-offences", - "pallet-session", - "pallet-staking", - "pallet-staking-reward-curve", - "pallet-timestamp", + "pallet-utility", "parity-scale-codec", "serde", "sp-core", "sp-io", "sp-runtime", - "sp-staking", "sp-std", ] [[package]] name = "pallet-randomness-collective-flip" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -4433,7 +4475,7 @@ dependencies = [ [[package]] name = "pallet-recovery" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "enumflags2", "frame-support", @@ -4449,7 +4491,7 @@ dependencies = [ [[package]] name = "pallet-scheduler" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "frame-support", @@ -4464,7 +4506,7 @@ dependencies = [ [[package]] name = "pallet-scored-pool" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -4479,7 +4521,7 @@ dependencies = [ [[package]] name = "pallet-session" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -4500,7 +4542,7 @@ dependencies = [ [[package]] name = "pallet-session-benchmarking" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "frame-support", @@ -4520,7 +4562,7 @@ dependencies = [ [[package]] name = "pallet-society" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -4536,7 +4578,7 @@ dependencies = [ [[package]] name = "pallet-staking" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "env_logger 0.7.1", "frame-benchmarking", @@ -4545,19 +4587,17 @@ dependencies = [ "hex", "pallet-authorship", "pallet-balances", - "pallet-indices", "pallet-session", "pallet-staking-reward-curve", "pallet-timestamp", "parity-scale-codec", "parking_lot 0.10.2", - "rand 0.7.3", "rand_chacha 0.2.2", "serde", "sp-application-crypto", "sp-core", "sp-io", - "sp-phragmen", + "sp-npos-elections", "sp-runtime", "sp-staking", "sp-std", @@ -4582,25 +4622,25 @@ dependencies = [ "parity-scale-codec", "sp-core", "sp-io", - "sp-phragmen", + "sp-npos-elections", "sp-runtime", "sp-std", ] [[package]] name = "pallet-staking-reward-curve" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "proc-macro-crate", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "sp-runtime", "syn 1.0.17", ] [[package]] name = "pallet-sudo" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -4614,7 +4654,7 @@ dependencies = [ [[package]] name = "pallet-template" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", @@ -4626,7 +4666,7 @@ dependencies = [ [[package]] name = "pallet-timestamp" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "frame-support", @@ -4644,13 +4684,15 @@ dependencies = [ [[package]] name = "pallet-transaction-payment" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", "pallet-balances", "pallet-transaction-payment-rpc-runtime-api", "parity-scale-codec", + "serde", + "smallvec 1.4.0", "sp-core", "sp-io", "sp-runtime", @@ -4660,7 +4702,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -4677,7 +4719,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc-runtime-api" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "parity-scale-codec", @@ -4690,7 +4732,7 @@ dependencies = [ [[package]] name = "pallet-treasury" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "frame-support", @@ -4702,11 +4744,12 @@ dependencies = [ "sp-io", "sp-runtime", "sp-std", + "sp-storage", ] [[package]] name = "pallet-utility" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-benchmarking", "frame-support", @@ -4722,7 +4765,7 @@ dependencies = [ [[package]] name = "pallet-vesting" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "enumflags2", "frame-benchmarking", @@ -4761,31 +4804,31 @@ checksum = "f77055f9e81921a8cc7bebeb6cded3d128931d51f1e3dd6251f0770a6d431477" dependencies = [ "arrayref", "bs58", - "byteorder 1.3.4", + "byteorder", "data-encoding", "parity-multihash", "percent-encoding 2.1.0", "serde", "static_assertions", - "unsigned-varint", + "unsigned-varint 0.3.3", "url 2.1.1", ] [[package]] name = "parity-multiaddr" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4db35e222f783ef4e6661873f6c165c4eb7b65e0c408349818517d5705c2d7d3" +checksum = "cc20af3143a62c16e7c9e92ea5c6ae49f7d271d97d4d8fe73afc28f0514a3d0f" dependencies = [ "arrayref", "bs58", - "byteorder 1.3.4", + "byteorder", "data-encoding", "multihash", "percent-encoding 2.1.0", "serde", "static_assertions", - "unsigned-varint", + "unsigned-varint 0.4.0", "url 2.1.1", ] @@ -4801,14 +4844,14 @@ dependencies = [ "sha-1", "sha2", "sha3", - "unsigned-varint", + "unsigned-varint 0.3.3", ] [[package]] name = "parity-scale-codec" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "329c8f7f4244ddb5c37c103641027a76c530e65e8e4b8240b29f81ea40508b17" +checksum = "a74f02beb35d47e0706155c9eac554b50c671e0d868fe8296bcdf44a9a4847bf" dependencies = [ "arrayvec 0.5.1", "bitvec", @@ -4825,7 +4868,7 @@ checksum = "5a0ec292e92e8ec7c58e576adacc1e3f399c597c8f263c42f18420abe58e7245" dependencies = [ "proc-macro-crate", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -4835,6 +4878,25 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa9777aa91b8ad9dd5aaa04a9b6bcb02c7f1deb952fca5a66034d5e63afc5c6f" +[[package]] +name = "parity-tokio-ipc" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e57fea504fea33f9fbb5f49f378359030e7e026a6ab849bb9e8f0787376f1bf" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.29", + "libc", + "log", + "mio-named-pipes", + "miow 0.3.5", + "rand 0.7.3", + "tokio 0.1.22", + "tokio-named-pipes", + "tokio-uds", + "winapi 0.3.8", +] + [[package]] name = "parity-util-mem" version = "0.6.1" @@ -4846,7 +4908,7 @@ dependencies = [ "parity-util-mem-derive", "parking_lot 0.10.2", "primitive-types", - "smallvec 1.3.0", + "smallvec 1.4.0", "winapi 0.3.8", ] @@ -4867,7 +4929,7 @@ version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16ad52817c4d343339b3bc2e26861bd21478eda0b7509acf83505727000512ac" dependencies = [ - "byteorder 1.3.4", + "byteorder", ] [[package]] @@ -4922,7 +4984,7 @@ dependencies = [ "cloudabi", "libc", "redox_syscall", - "smallvec 1.3.0", + "smallvec 1.4.0", "winapi 0.3.8", ] @@ -4944,7 +5006,7 @@ checksum = "a62486e111e571b1e93b710b61e8f493c0013be39629b714cb166bdb06aa5a8a" dependencies = [ "proc-macro-hack", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -4954,7 +5016,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" dependencies = [ - "byteorder 1.3.4", + "byteorder", "crypto-mac", ] @@ -4994,21 +5056,21 @@ dependencies = [ [[package]] name = "pin-project" -version = "0.4.9" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f6a7f5eee6292c559c793430c55c00aea9d3b3d1905e855806ca4d7253426a2" +checksum = "12e3a6cdbfe94a5e4572812a0201f8c0ed98c1c452c7b8563ce2276988ef9c17" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.9" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8988430ce790d8682672117bc06dda364c0be32d3abd738234f19f3240bad99a" +checksum = "6a0ffd45cf79d88737d7cc85bfd5d2894bee1139b356e616fe85dc389c61aaf7" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -5020,9 +5082,9 @@ checksum = "237844750cfbb86f67afe27eee600dfbbcb6188d734139b534cbfbf4f96792ae" [[package]] name = "pin-utils" -version = "0.1.0-alpha.4" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" @@ -5054,6 +5116,25 @@ dependencies = [ "web-sys", ] +[[package]] +name = "poly1305" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5829f50f48e9ddb79f3f7c3097029d0caee30f8286accb241416df603b080b8" +dependencies = [ + "universal-hash", +] + +[[package]] +name = "polyval" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ec3341498978de3bfd12d1b22f1af1de22818f5473a11e8a6ef997989e3a212" +dependencies = [ + "cfg-if", + "universal-hash", +] + [[package]] name = "ppv-lite86" version = "0.2.6" @@ -5128,7 +5209,7 @@ checksum = "98e9e4b82e0ef281812565ea4751049f1bdcdfccda7d3f459f2e138a40c08678" dependencies = [ "proc-macro-error-attr", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", "version_check", ] @@ -5140,7 +5221,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f5444ead4e9935abd7f27dc51f7e852a0569ac888096d5ec2499470794e2e53" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", "syn-mid", "version_check", @@ -5174,7 +5255,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe50036aa1b71e553a4a0c48ab7baabf8aa8c7a5a61aae06bf38c2eab7430475" dependencies = [ "bitflags", - "byteorder 1.3.4", + "byteorder", "chrono", "hex", "lazy_static", @@ -5214,7 +5295,7 @@ checksum = "02b10678c913ecbd69350e8535c3aef91a8676c0773fc1d7b95cdd196d7f2f26" dependencies = [ "bytes 0.5.4", "heck", - "itertools", + "itertools 0.8.2", "log", "multimap", "petgraph", @@ -5231,9 +5312,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "537aa19b95acde10a12fec4301466386f757403de4cd4e5b4fa78fb5ecb18f72" dependencies = [ "anyhow", - "itertools", + "itertools 0.8.2", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -5259,7 +5340,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f7a12f176deee919f4ba55326ee17491c8b707d0987aed822682c821b660192" dependencies = [ - "byteorder 1.3.4", + "byteorder", "log", "parity-wasm 0.41.0", ] @@ -5301,9 +5382,9 @@ checksum = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" [[package]] name = "quote" -version = "1.0.3" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" +checksum = "54a21852a652ad6f610c9510194f398ff6f8692e334fd1145fed931f7fbe44ea" dependencies = [ "proc-macro2", ] @@ -5514,7 +5595,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03b418169fb9c46533f326efd6eed2576699c44ca92d3052a066214a8d828929" dependencies = [ - "byteorder 1.3.4", + "byteorder", "rand_core 0.3.1", ] @@ -5601,7 +5682,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "602eb59cda66fcb9aec25841fb76bc01d2b34282dcdd705028da297db6f3eec8" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -5613,7 +5694,7 @@ checksum = "b27b256b41986ac5141b37b8bbba85d314fbf546c182eb255af6720e07e4f804" dependencies = [ "log", "rustc-hash", - "smallvec 1.3.0", + "smallvec 1.4.0", ] [[package]] @@ -5634,7 +5715,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" dependencies = [ - "byteorder 1.3.4", + "byteorder", ] [[package]] @@ -5664,6 +5745,27 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "rental" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8545debe98b2b139fb04cad8618b530e9b07c152d99a5de83c860b877d67847f" +dependencies = [ + "rental-impl", + "stable_deref_trait", +] + +[[package]] +name = "rental-impl" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "475e68978dc5b743f2f40d8e0a8fdc83f1c5e78cbf4b8fa5e74e73beebc340de" +dependencies = [ + "proc-macro2", + "quote 1.0.6", + "syn 1.0.17", +] + [[package]] name = "ring" version = "0.16.12" @@ -5785,7 +5887,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3bba175698996010c4f6dce5e7f173b6eb781fce25d2cfc45e27091ce0b79f6" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -5795,7 +5897,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4da5fcb054c46f5a5dff833b129285a93d3f0179531735e6c866e8cc307d2020" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "pin-project", "static_assertions", ] @@ -5815,26 +5917,6 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "salsa20" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2324b0e8c3bb9a586a571fdb3136f70e7e2c748de00a78043f86e0cff91f91fe" -dependencies = [ - "byteorder 1.3.4", - "salsa20-core", - "stream-cipher", -] - -[[package]] -name = "salsa20-core" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fe6cc1b9f5a5867853ade63099de70f042f7679e408d1ffe52821c9248e6e69" -dependencies = [ - "stream-cipher", -] - [[package]] name = "same-file" version = "1.0.6" @@ -5846,12 +5928,12 @@ dependencies = [ [[package]] name = "sc-authority-discovery" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "bytes 0.5.4", "derive_more", "env_logger 0.7.1", - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "libp2p", "log", @@ -5876,15 +5958,16 @@ dependencies = [ [[package]] name = "sc-basic-authorship" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "log", "parity-scale-codec", "parking_lot 0.10.2", "sc-block-builder", "sc-client-api", + "sc-proposer-metrics", "sc-telemetry", "sc-transaction-pool", "sp-api", @@ -5894,13 +5977,14 @@ dependencies = [ "sp-inherents", "sp-runtime", "sp-transaction-pool", + "substrate-prometheus-endpoint", "substrate-test-runtime-client", "tokio-executor 0.2.0-alpha.6", ] [[package]] name = "sc-block-builder" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "parity-scale-codec", "sc-client-api", @@ -5917,7 +6001,7 @@ dependencies = [ [[package]] name = "sc-chain-spec" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "impl-trait-for-tuples", "sc-chain-spec-derive", @@ -5932,27 +6016,25 @@ dependencies = [ [[package]] name = "sc-chain-spec-derive" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "proc-macro-crate", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] [[package]] name = "sc-cli" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "ansi_term 0.12.1", - "app_dirs", "atty", "chrono", - "clap", "derive_more", "env_logger 0.7.1", "fdlimit", - "futures 0.3.4", + "futures 0.3.5", "lazy_static", "log", "names", @@ -5984,11 +6066,11 @@ dependencies = [ [[package]] name = "sc-client-api" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "derive_more", "fnv", - "futures 0.3.4", + "futures 0.3.5", "hash-db", "hex-literal", "kvdb", @@ -6022,7 +6104,7 @@ dependencies = [ [[package]] name = "sc-client-db" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "blake2-rfc", "env_logger 0.7.1", @@ -6055,7 +6137,7 @@ dependencies = [ [[package]] name = "sc-consensus" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "sc-client-api", "sp-blockchain", @@ -6065,11 +6147,11 @@ dependencies = [ [[package]] name = "sc-consensus-aura" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "derive_more", "env_logger 0.7.1", - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "log", "parity-scale-codec", @@ -6103,12 +6185,12 @@ dependencies = [ [[package]] name = "sc-consensus-babe" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "derive_more", "env_logger 0.7.1", "fork-tree", - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "log", "merlin", @@ -6119,6 +6201,7 @@ dependencies = [ "parking_lot 0.10.2", "pdqselect", "rand 0.7.3", + "rand_chacha 0.2.2", "sc-block-builder", "sc-client-api", "sc-consensus-epochs", @@ -6153,17 +6236,20 @@ dependencies = [ [[package]] name = "sc-consensus-babe-rpc" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "derive_more", - "futures 0.3.4", + "futures 0.3.5", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", + "sc-consensus", "sc-consensus-babe", "sc-consensus-epochs", "sc-keystore", + "sc-rpc-api", "serde", + "serde_json", "sp-api", "sp-application-crypto", "sp-blockchain", @@ -6178,7 +6264,7 @@ dependencies = [ [[package]] name = "sc-consensus-epochs" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "fork-tree", "parity-scale-codec", @@ -6190,12 +6276,12 @@ dependencies = [ [[package]] name = "sc-consensus-manual-seal" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "assert_matches", "derive_more", "env_logger 0.7.1", - "futures 0.3.4", + "futures 0.3.5", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", @@ -6220,10 +6306,10 @@ dependencies = [ [[package]] name = "sc-consensus-pow" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "derive_more", - "futures 0.3.4", + "futures 0.3.5", "log", "parity-scale-codec", "sc-client-api", @@ -6241,9 +6327,9 @@ dependencies = [ [[package]] name = "sc-consensus-slots" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "log", "parity-scale-codec", @@ -6263,7 +6349,7 @@ dependencies = [ [[package]] name = "sc-consensus-uncles" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "log", "sc-client-api", @@ -6276,7 +6362,7 @@ dependencies = [ [[package]] name = "sc-executor" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "assert_matches", "derive_more", @@ -6291,6 +6377,7 @@ dependencies = [ "sc-executor-wasmi", "sc-executor-wasmtime", "sc-runtime-test", + "sc-tracing", "sp-api", "sp-core", "sp-externalities", @@ -6300,18 +6387,20 @@ dependencies = [ "sp-runtime-interface", "sp-serializer", "sp-state-machine", + "sp-tracing", "sp-trie", "sp-version", "sp-wasm-interface", "substrate-test-runtime", "test-case", + "tracing", "wabt", "wasmi", ] [[package]] name = "sc-executor-common" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "derive_more", "log", @@ -6327,7 +6416,7 @@ dependencies = [ [[package]] name = "sc-executor-wasmi" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "log", "parity-scale-codec", @@ -6341,7 +6430,7 @@ dependencies = [ [[package]] name = "sc-executor-wasmtime" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "assert_matches", "cranelift-codegen", @@ -6362,14 +6451,14 @@ dependencies = [ [[package]] name = "sc-finality-grandpa" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "assert_matches", "derive_more", "env_logger 0.7.1", "finality-grandpa", "fork-tree", - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "log", "parity-scale-codec", @@ -6386,6 +6475,7 @@ dependencies = [ "sc-telemetry", "serde_json", "sp-api", + "sp-application-crypto", "sp-arithmetic", "sp-blockchain", "sp-consensus", @@ -6406,11 +6496,11 @@ dependencies = [ [[package]] name = "sc-finality-grandpa-rpc" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "derive_more", "finality-grandpa", - "futures 0.3.4", + "futures 0.3.5", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", @@ -6423,26 +6513,29 @@ dependencies = [ [[package]] name = "sc-informant" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "ansi_term 0.12.1", - "futures 0.3.4", + "futures 0.3.5", "log", "parity-util-mem", + "parking_lot 0.10.2", "sc-client-api", "sc-network", - "sc-service", "sp-blockchain", "sp-runtime", + "sp-transaction-pool", + "sp-utils", "wasm-timer", ] [[package]] name = "sc-keystore" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "derive_more", "hex", + "merlin", "parking_lot 0.10.2", "rand 0.7.3", "serde_json", @@ -6452,13 +6545,32 @@ dependencies = [ "tempfile", ] +[[package]] +name = "sc-light" +version = "2.0.0-rc4" +dependencies = [ + "hash-db", + "lazy_static", + "parity-scale-codec", + "parking_lot 0.10.2", + "sc-client-api", + "sc-executor", + "sp-api", + "sp-blockchain", + "sp-core", + "sp-externalities", + "sp-runtime", + "sp-state-machine", +] + [[package]] name = "sc-network" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "assert_matches", "async-std", "bitflags", + "bs58", "bytes 0.5.4", "derive_more", "either", @@ -6466,9 +6578,9 @@ dependencies = [ "erased-serde", "fnv", "fork-tree", - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", - "futures_codec", + "futures_codec 0.3.4", "hex", "ip_network", "libp2p", @@ -6505,7 +6617,7 @@ dependencies = [ "substrate-test-runtime-client", "tempfile", "thiserror", - "unsigned-varint", + "unsigned-varint 0.3.3", "void", "wasm-timer", "zeroize", @@ -6513,10 +6625,10 @@ dependencies = [ [[package]] name = "sc-network-gossip" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "async-std", - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "libp2p", "log", @@ -6531,10 +6643,10 @@ dependencies = [ [[package]] name = "sc-network-test" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "env_logger 0.7.1", - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "libp2p", "log", @@ -6557,16 +6669,16 @@ dependencies = [ [[package]] name = "sc-offchain" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "bytes 0.5.4", "env_logger 0.7.1", - "fdlimit", "fnv", - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "hyper 0.13.4", "hyper-rustls", + "lazy_static", "log", "num_cpus", "parity-scale-codec", @@ -6590,9 +6702,9 @@ dependencies = [ [[package]] name = "sc-peerset" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "libp2p", "log", "rand 0.7.3", @@ -6601,13 +6713,21 @@ dependencies = [ "wasm-timer", ] +[[package]] +name = "sc-proposer-metrics" +version = "0.8.0-rc4" +dependencies = [ + "log", + "substrate-prometheus-endpoint", +] + [[package]] name = "sc-rpc" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "assert_matches", "futures 0.1.29", - "futures 0.3.4", + "futures 0.3.5", "hash-db", "jsonrpc-core", "jsonrpc-pubsub", @@ -6642,10 +6762,10 @@ dependencies = [ [[package]] name = "sc-rpc-api" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "derive_more", - "futures 0.3.4", + "futures 0.3.5", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", @@ -6665,10 +6785,11 @@ dependencies = [ [[package]] name = "sc-rpc-server" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "jsonrpc-core", "jsonrpc-http-server", + "jsonrpc-ipc-server", "jsonrpc-pubsub", "jsonrpc-ws-server", "log", @@ -6679,7 +6800,7 @@ dependencies = [ [[package]] name = "sc-runtime-test" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "sp-allocator", "sp-core", @@ -6692,14 +6813,16 @@ dependencies = [ [[package]] name = "sc-service" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "derive_more", + "directories", "exit-future", "futures 0.1.29", - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "hash-db", + "jsonrpc-pubsub", "lazy_static", "log", "netstat2", @@ -6716,7 +6839,9 @@ dependencies = [ "sc-client-db", "sc-executor", "sc-finality-grandpa", + "sc-informant", "sc-keystore", + "sc-light", "sc-network", "sc-offchain", "sc-rpc", @@ -6747,18 +6872,19 @@ dependencies = [ "substrate-prometheus-endpoint", "substrate-test-runtime-client", "sysinfo", + "tempfile", "tracing", "wasm-timer", ] [[package]] name = "sc-service-test" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "env_logger 0.7.1", "fdlimit", "futures 0.1.29", - "futures 0.3.4", + "futures 0.3.5", "hex-literal", "log", "parity-scale-codec", @@ -6767,6 +6893,7 @@ dependencies = [ "sc-client-api", "sc-client-db", "sc-executor", + "sc-light", "sc-network", "sc-service", "sp-api", @@ -6788,7 +6915,7 @@ dependencies = [ [[package]] name = "sc-state-db" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "env_logger 0.7.1", "log", @@ -6802,10 +6929,9 @@ dependencies = [ [[package]] name = "sc-telemetry" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ - "bytes 0.5.4", - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "libp2p", "log", @@ -6823,27 +6949,29 @@ dependencies = [ [[package]] name = "sc-tracing" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "erased-serde", "log", "parking_lot 0.10.2", + "rustc-hash", "sc-telemetry", "serde", "serde_json", "slog", + "sp-tracing", "tracing", "tracing-core", ] [[package]] name = "sc-transaction-graph" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "assert_matches", "criterion 0.3.1", "derive_more", - "futures 0.3.4", + "futures 0.3.5", "linked-hash-map", "log", "parity-scale-codec", @@ -6861,11 +6989,11 @@ dependencies = [ [[package]] name = "sc-transaction-pool" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "assert_matches", "derive_more", - "futures 0.3.4", + "futures 0.3.5", "futures-diagnose", "hex", "intervalier", @@ -6873,10 +7001,12 @@ dependencies = [ "parity-scale-codec", "parity-util-mem", "parking_lot 0.10.2", + "sc-block-builder", "sc-client-api", "sc-transaction-graph", "sp-api", "sp-blockchain", + "sp-consensus", "sp-core", "sp-keyring", "sp-runtime", @@ -6945,7 +7075,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8584eea9b9ff42825b46faf46a8c24d2cff13ec152fa2a50df788b87c07ee28" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -6959,6 +7089,15 @@ dependencies = [ "untrusted", ] +[[package]] +name = "secrecy" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9182278ed645df3477a9c27bfee0621c621aa16f6972635f7f795dae3d81070f" +dependencies = [ + "zeroize", +] + [[package]] name = "security-framework" version = "0.4.2" @@ -7027,21 +7166,21 @@ checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" [[package]] name = "serde" -version = "1.0.106" +version = "1.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399" +checksum = "99e7b308464d16b56eba9964e4972a3eee817760ab60d88c3f86e1fecb08204c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.106" +version = "1.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e549e3abf4fb8621bd1609f11dfc9f5e50320802273b12f3811a67e6716ea6c" +checksum = "818fbf6bfa9a42d3bfcaca148547aa00c7b915bec71d1757aa2d44ca68771984" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -7068,12 +7207,6 @@ dependencies = [ "opaque-debug", ] -[[package]] -name = "sha1" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" - [[package]] name = "sha2" version = "0.8.1" @@ -7099,16 +7232,6 @@ dependencies = [ "opaque-debug", ] -[[package]] -name = "shell32-sys" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ee04b46101f57121c9da2b151988283b6beb79b34f5bb29a58ee48cb695122c" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "shlex" version = "0.1.1" @@ -7171,7 +7294,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a945ec7f7ce853e89ffa36be1e27dce9a43e82ff9093bf3461c30d5da74ed11b" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -7186,19 +7309,19 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05720e22615919e4734f6a99ceae50d00226c3c5aca406e102ebc33298214e0a" +checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" [[package]] name = "snow" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb767eee7d257ba202f0b9b08673bc13b22281632ef45267b19f13100accd2f" +checksum = "ce0f91be479494dd92e69d9971bd23ed27037dd1c94fcf558f6c6e74e6afa654" dependencies = [ - "arrayref", - "blake2-rfc", - "chacha20-poly1305-aead", + "aes-gcm", + "blake2", + "chacha20poly1305", "rand 0.7.3", "rand_core 0.5.1", "ring", @@ -7208,29 +7331,37 @@ dependencies = [ "x25519-dalek", ] +[[package]] +name = "socket2" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "winapi 0.3.8", +] + [[package]] name = "soketto" -version = "0.3.2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c9dab3f95c9ebdf3a88268c19af668f637a3c5039c2c56ff2d40b1b2d64a25b" +checksum = "85457366ae0c6ce56bf05a958aef14cd38513c236568618edbcd9a8c52cb80b0" dependencies = [ - "base64 0.11.0", + "base64 0.12.0", "bytes 0.5.4", "flate2", - "futures 0.3.4", - "http 0.2.1", + "futures 0.3.5", "httparse", "log", "rand 0.7.3", - "sha1", - "smallvec 1.3.0", - "static_assertions", - "thiserror", + "sha-1", ] [[package]] name = "sp-allocator" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "derive_more", "log", @@ -7241,7 +7372,7 @@ dependencies = [ [[package]] name = "sp-api" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "hash-db", "parity-scale-codec", @@ -7256,18 +7387,18 @@ dependencies = [ [[package]] name = "sp-api-proc-macro" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "blake2-rfc", "proc-macro-crate", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] [[package]] name = "sp-api-test" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "criterion 0.3.1", "parity-scale-codec", @@ -7286,7 +7417,7 @@ dependencies = [ [[package]] name = "sp-application-crypto" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "parity-scale-codec", "serde", @@ -7297,7 +7428,7 @@ dependencies = [ [[package]] name = "sp-application-crypto-test" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "sp-api", "sp-application-crypto", @@ -7308,7 +7439,7 @@ dependencies = [ [[package]] name = "sp-arithmetic" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "criterion 0.3.1", "integer-sqrt", @@ -7324,7 +7455,7 @@ dependencies = [ [[package]] name = "sp-arithmetic-fuzzer" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "honggfuzz", "num-bigint", @@ -7335,7 +7466,7 @@ dependencies = [ [[package]] name = "sp-authority-discovery" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "parity-scale-codec", "sp-api", @@ -7346,7 +7477,7 @@ dependencies = [ [[package]] name = "sp-authorship" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "parity-scale-codec", "sp-inherents", @@ -7356,7 +7487,7 @@ dependencies = [ [[package]] name = "sp-block-builder" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "parity-scale-codec", "sp-api", @@ -7367,7 +7498,7 @@ dependencies = [ [[package]] name = "sp-blockchain" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "derive_more", "log", @@ -7382,7 +7513,7 @@ dependencies = [ [[package]] name = "sp-chain-spec" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "serde", "serde_json", @@ -7390,10 +7521,10 @@ dependencies = [ [[package]] name = "sp-consensus" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "derive_more", - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "libp2p", "log", @@ -7409,11 +7540,12 @@ dependencies = [ "sp-utils", "sp-version", "substrate-prometheus-endpoint", + "wasm-timer", ] [[package]] name = "sp-consensus-aura" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "parity-scale-codec", "sp-api", @@ -7426,7 +7558,7 @@ dependencies = [ [[package]] name = "sp-consensus-babe" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "merlin", "parity-scale-codec", @@ -7434,6 +7566,7 @@ dependencies = [ "sp-application-crypto", "sp-consensus", "sp-consensus-vrf", + "sp-core", "sp-inherents", "sp-runtime", "sp-std", @@ -7442,7 +7575,7 @@ dependencies = [ [[package]] name = "sp-consensus-pow" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "parity-scale-codec", "sp-api", @@ -7453,7 +7586,7 @@ dependencies = [ [[package]] name = "sp-consensus-vrf" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "parity-scale-codec", "schnorrkel", @@ -7464,15 +7597,15 @@ dependencies = [ [[package]] name = "sp-core" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "base58", "blake2-rfc", - "byteorder 1.3.4", + "byteorder", "criterion 0.2.11", "derive_more", "ed25519-dalek", - "futures 0.3.4", + "futures 0.3.5", "hash-db", "hash256-std-hasher", "hex", @@ -7489,8 +7622,10 @@ dependencies = [ "pretty_assertions", "primitive-types", "rand 0.7.3", + "rand_chacha 0.2.2", "regex", "schnorrkel", + "secrecy", "serde", "serde_json", "sha2", @@ -7510,7 +7645,7 @@ dependencies = [ [[package]] name = "sp-database" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "kvdb", "parking_lot 0.10.2", @@ -7518,16 +7653,16 @@ dependencies = [ [[package]] name = "sp-debug-derive" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] [[package]] name = "sp-externalities" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "environmental", "parity-scale-codec", @@ -7537,7 +7672,7 @@ dependencies = [ [[package]] name = "sp-finality-grandpa" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "finality-grandpa", "log", @@ -7552,7 +7687,7 @@ dependencies = [ [[package]] name = "sp-finality-tracker" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "parity-scale-codec", "sp-inherents", @@ -7561,7 +7696,7 @@ dependencies = [ [[package]] name = "sp-inherents" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "derive_more", "parity-scale-codec", @@ -7572,9 +7707,9 @@ dependencies = [ [[package]] name = "sp-io" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "hash-db", "libsecp256k1", "log", @@ -7585,13 +7720,14 @@ dependencies = [ "sp-runtime-interface", "sp-state-machine", "sp-std", + "sp-tracing", "sp-trie", "sp-wasm-interface", ] [[package]] name = "sp-keyring" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "lazy_static", "sp-core", @@ -7600,62 +7736,61 @@ dependencies = [ ] [[package]] -name = "sp-offchain" -version = "2.0.0-alpha.8" -dependencies = [ - "sp-api", - "sp-core", - "sp-runtime", - "sp-state-machine", -] - -[[package]] -name = "sp-panic-handler" -version = "2.0.0-alpha.8" -dependencies = [ - "backtrace", - "log", -] - -[[package]] -name = "sp-phragmen" -version = "2.0.0-alpha.8" +name = "sp-npos-elections" +version = "2.0.0-rc4" dependencies = [ "parity-scale-codec", "rand 0.7.3", "serde", "sp-arithmetic", - "sp-phragmen", - "sp-phragmen-compact", + "sp-npos-elections-compact", "sp-runtime", "sp-std", "substrate-test-utils", ] [[package]] -name = "sp-phragmen-compact" -version = "2.0.0-alpha.8" +name = "sp-npos-elections-compact" +version = "2.0.0-rc4" dependencies = [ "proc-macro-crate", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] [[package]] -name = "sp-phragmen-fuzzer" +name = "sp-npos-elections-fuzzer" version = "2.0.0-alpha.5" dependencies = [ "honggfuzz", "rand 0.7.3", - "sp-phragmen", + "sp-npos-elections", "sp-runtime", "sp-std", ] +[[package]] +name = "sp-offchain" +version = "2.0.0-rc4" +dependencies = [ + "sp-api", + "sp-core", + "sp-runtime", + "sp-state-machine", +] + +[[package]] +name = "sp-panic-handler" +version = "2.0.0-rc4" +dependencies = [ + "backtrace", + "log", +] + [[package]] name = "sp-rpc" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "serde", "serde_json", @@ -7664,8 +7799,9 @@ dependencies = [ [[package]] name = "sp-runtime" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ + "either", "hash256-std-hasher", "impl-trait-for-tuples", "log", @@ -7686,7 +7822,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "parity-scale-codec", "primitive-types", @@ -7706,18 +7842,18 @@ dependencies = [ [[package]] name = "sp-runtime-interface-proc-macro" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "Inflector", "proc-macro-crate", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] [[package]] name = "sp-runtime-interface-test" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "sc-executor", "sp-core", @@ -7732,7 +7868,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface-test-wasm" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "sp-core", "sp-io", @@ -7743,7 +7879,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface-test-wasm-deprecated" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "sp-core", "sp-io", @@ -7754,7 +7890,7 @@ dependencies = [ [[package]] name = "sp-sandbox" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "assert_matches", "parity-scale-codec", @@ -7768,7 +7904,7 @@ dependencies = [ [[package]] name = "sp-serializer" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "serde", "serde_json", @@ -7776,7 +7912,7 @@ dependencies = [ [[package]] name = "sp-session" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "parity-scale-codec", "sp-api", @@ -7788,7 +7924,7 @@ dependencies = [ [[package]] name = "sp-staking" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "parity-scale-codec", "sp-runtime", @@ -7797,15 +7933,18 @@ dependencies = [ [[package]] name = "sp-state-machine" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "hash-db", "hex-literal", + "itertools 0.9.0", "log", "num-traits 0.2.11", "parity-scale-codec", "parking_lot 0.10.2", + "pretty_assertions", "rand 0.7.3", + "smallvec 1.4.0", "sp-core", "sp-externalities", "sp-panic-handler", @@ -7817,11 +7956,11 @@ dependencies = [ [[package]] name = "sp-std" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" [[package]] name = "sp-storage" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "impl-serde 0.2.3", "ref-cast", @@ -7832,7 +7971,7 @@ dependencies = [ [[package]] name = "sp-test-primitives" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "parity-scale-codec", "parity-util-mem", @@ -7844,7 +7983,7 @@ dependencies = [ [[package]] name = "sp-timestamp" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -7857,28 +7996,31 @@ dependencies = [ [[package]] name = "sp-tracing" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ + "log", + "rental", "tracing", ] [[package]] name = "sp-transaction-pool" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "derive_more", - "futures 0.3.4", + "futures 0.3.5", "log", "parity-scale-codec", "serde", "sp-api", + "sp-blockchain", "sp-runtime", "sp-utils", ] [[package]] name = "sp-trie" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "criterion 0.2.11", "hash-db", @@ -7896,17 +8038,18 @@ dependencies = [ [[package]] name = "sp-utils" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "futures-core", + "futures-timer 3.0.2", "lazy_static", "prometheus", ] [[package]] name = "sp-version" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "impl-serde 0.2.3", "parity-scale-codec", @@ -7917,7 +8060,7 @@ dependencies = [ [[package]] name = "sp-wasm-interface" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -8005,7 +8148,7 @@ dependencies = [ "heck", "proc-macro-error", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -8026,13 +8169,13 @@ checksum = "0054a7df764039a6cd8592b9de84be4bec368ff081d203a7d5371cbfa8e65c81" dependencies = [ "heck", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] [[package]] name = "subkey" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "clap", "derive_more", @@ -8041,7 +8184,7 @@ dependencies = [ "hex", "hex-literal", "hyper 0.12.35", - "itertools", + "itertools 0.8.2", "jsonrpc-core-client", "libp2p", "node-primitives", @@ -8074,14 +8217,14 @@ dependencies = [ [[package]] name = "substrate-browser-utils" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "chrono", "clear_on_drop", "console_error_panic_hook", "console_log", "futures 0.1.29", - "futures 0.3.4", + "futures 0.3.5", "futures-timer 3.0.2", "js-sys", "kvdb-web", @@ -8100,18 +8243,18 @@ dependencies = [ [[package]] name = "substrate-build-script-utils" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "platforms", ] [[package]] name = "substrate-frame-rpc-support" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "frame-support", "frame-system", - "futures 0.3.4", + "futures 0.3.5", "jsonrpc-client-transports", "jsonrpc-core", "parity-scale-codec", @@ -8123,20 +8266,22 @@ dependencies = [ [[package]] name = "substrate-frame-rpc-system" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "env_logger 0.7.1", "frame-system-rpc-runtime-api", - "futures 0.3.4", + "futures 0.3.5", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", "log", "parity-scale-codec", "sc-client-api", + "sc-rpc-api", "sc-transaction-pool", "serde", "sp-api", + "sp-block-builder", "sp-blockchain", "sp-core", "sp-runtime", @@ -8146,7 +8291,7 @@ dependencies = [ [[package]] name = "substrate-prometheus-endpoint" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" dependencies = [ "async-std", "derive_more", @@ -8159,15 +8304,16 @@ dependencies = [ [[package]] name = "substrate-test-client" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "hash-db", "parity-scale-codec", "sc-client-api", "sc-client-db", "sc-consensus", "sc-executor", + "sc-light", "sc-service", "sp-blockchain", "sp-consensus", @@ -8179,7 +8325,7 @@ dependencies = [ [[package]] name = "substrate-test-runtime" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "cfg-if", "frame-executive", @@ -8222,13 +8368,14 @@ dependencies = [ [[package]] name = "substrate-test-runtime-client" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "parity-scale-codec", "sc-block-builder", "sc-client-api", "sc-consensus", + "sc-light", "sc-service", "sp-api", "sp-blockchain", @@ -8241,10 +8388,10 @@ dependencies = [ [[package]] name = "substrate-test-runtime-transaction-pool" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" dependencies = [ "derive_more", - "futures 0.3.4", + "futures 0.3.5", "parity-scale-codec", "parking_lot 0.10.2", "sc-transaction-graph", @@ -8256,17 +8403,17 @@ dependencies = [ [[package]] name = "substrate-test-utils" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" [[package]] name = "substrate-wasm-builder" -version = "1.0.10" +version = "1.0.11" dependencies = [ "atty", "build-helper", "cargo_metadata", "fs2", - "itertools", + "itertools 0.8.2", "tempfile", "toml", "walkdir", @@ -8279,9 +8426,9 @@ version = "1.0.6" [[package]] name = "substrate-wasmtime" -version = "0.16.0-threadsafe.2" +version = "0.16.0-threadsafe.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b40a6f3d5d3c00754e348863fead4f37763c32eedf950f5b23df87769882311" +checksum = "6bd62264edc1a5f3ef44d86fb0c11c9fb142894b9a2da034f34afae482080d7a" dependencies = [ "anyhow", "backtrace", @@ -8302,9 +8449,9 @@ dependencies = [ [[package]] name = "substrate-wasmtime-jit" -version = "0.16.0-threadsafe.2" +version = "0.16.0-threadsafe.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09712de4f56a2c2912bee7763b0877d17d72cfb2237987d63ab78956907e7692" +checksum = "4ce43c159d4f3ef6b19641e1ae045847fd202d8e2cc74df7ccb2b6475e069d4a" dependencies = [ "anyhow", "cfg-if", @@ -8329,9 +8476,9 @@ dependencies = [ [[package]] name = "substrate-wasmtime-profiling" -version = "0.16.0-threadsafe.2" +version = "0.16.0-threadsafe.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31505dd221f001634a54ea51472bc0058bcbde9186eaf8dd31d0859638121385" +checksum = "c77f0ce539b5a09a54dc80a1cf0c7cd7e694df11029354fe50a2d5fe889bdb97" dependencies = [ "anyhow", "cfg-if", @@ -8348,9 +8495,9 @@ dependencies = [ [[package]] name = "substrate-wasmtime-runtime" -version = "0.16.0-threadsafe.2" +version = "0.16.0-threadsafe.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3708081f04d9216d4dee487abf94872065f930cf82e287bd0c5bdb57895460ba" +checksum = "46516af0a64a7d9b652c5aa7436b6ce13edfa54435a66ef177fc02d2283e2dc2" dependencies = [ "backtrace", "cc", @@ -8396,7 +8543,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0df0eb663f387145cab623dea85b09c2c5b4b0aef44e945d928e682fce71bb03" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "unicode-xid 0.2.0", ] @@ -8407,7 +8554,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -8427,7 +8574,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", "unicode-xid 0.2.0", ] @@ -8490,7 +8637,7 @@ checksum = "a605baa797821796a751f4a959e1206079b24a4b7e1ed302b7d785d81a9276c9" dependencies = [ "lazy_static", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", "version_check", ] @@ -8520,7 +8667,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca972988113b7715266f91250ddb98070d033c62a011fa0fcc57434a649310dd" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] @@ -8726,10 +8873,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", ] +[[package]] +name = "tokio-named-pipes" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d282d483052288b2308ba5ee795f5673b159c9bdf63c385a05609da782a5eae" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.29", + "mio", + "mio-named-pipes", + "tokio 0.1.22", +] + [[package]] name = "tokio-reactor" version = "0.1.12" @@ -8751,9 +8911,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4adb8b3e5f86b707f1b54e7c15b6de52617a823608ccda98a15d3a24222f265a" +checksum = "15cb62a0d2770787abc96e99c1cd98fcf17f94959f3af63ca85bdfb203f051b4" dependencies = [ "futures-core", "rustls", @@ -8761,6 +8921,15 @@ dependencies = [ "webpki", ] +[[package]] +name = "tokio-service" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24da22d077e0f15f55162bdbdc661228c1581892f52074fb242678d015b45162" +dependencies = [ + "futures 0.1.29", +] + [[package]] name = "tokio-sync" version = "0.1.8" @@ -8889,9 +9058,9 @@ checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" [[package]] name = "tracing" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1721cc8cf7d770cc4257872507180f35a4797272f5962f24c806af9e7faf52ab" +checksum = "a7c6b59d116d218cb2d990eb06b77b64043e0268ef7323aae63d8b30ae462923" dependencies = [ "cfg-if", "tracing-attributes", @@ -8900,11 +9069,12 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fbad39da2f9af1cae3016339ad7f2c7a9e870f12e8fd04c4fd7ef35b30c0d2b" +checksum = "99bbad0de3fd923c9c3232ead88510b783e5a4d16a6154adffa3d53308de984c" dependencies = [ - "quote 1.0.3", + "proc-macro2", + "quote 1.0.6", "syn 1.0.17", ] @@ -8925,9 +9095,9 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" -version = "0.21.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c48b309cdda1abbdada28424bdc46f8b85362b3e66d6786d91223e83874429c7" +checksum = "ed8419971832eb3333dc26066e50943a20e0934efeb451b3df5ee94f7f7323ab" dependencies = [ "criterion 0.2.11", "hash-db", @@ -8941,15 +9111,15 @@ dependencies = [ [[package]] name = "trie-db" -version = "0.20.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc309f34008563989045a4c4dbcc5770467f3a3785ee80a9b5cc0d83362475f" +checksum = "cb230c24c741993b04cfccbabb45acff6f6480c5f00d3ed8794ea43db3a9d727" dependencies = [ "hash-db", "hashbrown", "log", "rustc-hex", - "smallvec 1.3.0", + "smallvec 1.4.0", ] [[package]] @@ -8998,7 +9168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712d261e83e727c8e2dbb75dacac67c36e35db36a958ee504f2164fc052434e1" dependencies = [ "block-cipher-trait", - "byteorder 1.3.4", + "byteorder", "opaque-debug", ] @@ -9023,7 +9193,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e75a4cdd7b87b28840dba13c483b9a88ee6bbf16ba5c951ee1ecfcf723078e0d" dependencies = [ - "byteorder 1.3.4", + "byteorder", "crunchy", "rustc-hex", "static_assertions", @@ -9053,7 +9223,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" dependencies = [ - "smallvec 1.3.0", + "smallvec 1.4.0", ] [[package]] @@ -9080,6 +9250,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +[[package]] +name = "universal-hash" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0c900f2f9b4116803415878ff48b63da9edb268668e08cf9292d7503114a01" +dependencies = [ + "generic-array", + "subtle 2.2.2", +] + [[package]] name = "unsigned-varint" version = "0.3.3" @@ -9089,7 +9269,17 @@ dependencies = [ "bytes 0.5.4", "futures-io", "futures-util", - "futures_codec", + "futures_codec 0.3.4", +] + +[[package]] +name = "unsigned-varint" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "669d776983b692a906c881fcd0cfb34271a48e197e4d6cb8df32b05bfc3d3fa5" +dependencies = [ + "bytes 0.5.4", + "futures_codec 0.4.1", ] [[package]] @@ -9236,7 +9426,7 @@ dependencies = [ "lazy_static", "log", "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", "wasm-bindgen-shared", ] @@ -9259,7 +9449,7 @@ version = "0.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cd85aa2c579e8892442954685f0d801f9129de24fa2136b2c6a539c76b65776" dependencies = [ - "quote 1.0.3", + "quote 1.0.6", "wasm-bindgen-macro-support", ] @@ -9270,7 +9460,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eb197bd3a47553334907ffd2f16507b4f4f01bbec3ac921a7719e0decdfe72a" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -9303,7 +9493,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf2f86cd78a2aa7b1fb4bb6ed854eccb7f9263089c79542dca1576a1518a8467" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", ] [[package]] @@ -9323,7 +9513,7 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "324c5e65a08699c9c4334ba136597ab22b85dccd4b65dd1e36ccf8f723a95b54" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "js-sys", "parking_lot 0.9.0", "pin-utils", @@ -9528,7 +9718,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c51a2c47b5798ccc774ffb93ff536aec7c4275d722fd9c740c83cdd1af1f2d94" dependencies = [ - "byteorder 1.3.4", + "byteorder", "bytes 0.4.12", "httparse", "log", @@ -9561,19 +9751,13 @@ dependencies = [ "zeroize", ] -[[package]] -name = "xdg" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57" - [[package]] name = "yamux" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84300bb493cc878f3638b981c62b4632ec1a5c52daaa3036651e8c106d3b55ea" +checksum = "cd37e58a1256a0b328ce9c67d8b62ecdd02f4803ba443df478835cb1a41a637c" dependencies = [ - "futures 0.3.4", + "futures 0.3.5", "log", "nohash-hasher", "parking_lot 0.10.2", @@ -9597,7 +9781,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de251eec69fc7c1bc3923403d18ececb929380e016afe103da75f396704f8ca2" dependencies = [ "proc-macro2", - "quote 1.0.3", + "quote 1.0.6", "syn 1.0.17", "synstructure", ] diff --git a/Cargo.toml b/Cargo.toml index d9ee8709d1334f7ffbab48f02028a94f79b5e295..ba146e55bca3fb5d5b0c2b0e5e1f5bbc37b1c4ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ members = [ "client/executor/runtime-test", "client/finality-grandpa", "client/informant", + "client/light", "client/tracing", "client/keystore", "client/network", @@ -45,6 +46,7 @@ members = [ "client/network-gossip", "client/offchain", "client/peerset", + "client/proposer-metrics", "client/rpc-servers", "client/rpc", "client/rpc-api", @@ -58,6 +60,7 @@ members = [ "utils/wasm-builder-runner", "frame/assets", "frame/aura", + "frame/atomic-swap", "frame/authority-discovery", "frame/authorship", "frame/babe", @@ -83,8 +86,10 @@ members = [ "frame/indices", "frame/membership", "frame/metadata", + "frame/multisig", "frame/nicks", "frame/offences", + "frame/proxy", "frame/randomness-collective-flip", "frame/recovery", "frame/scheduler", @@ -135,9 +140,9 @@ members = [ "primitives/keyring", "primitives/offchain", "primitives/panic-handler", - "primitives/phragmen", - "primitives/phragmen/fuzzer", - "primitives/phragmen/compact", + "primitives/npos-elections", + "primitives/npos-elections/fuzzer", + "primitives/npos-elections/compact", "primitives/rpc", "primitives/runtime-interface", "primitives/runtime-interface/proc-macro", @@ -178,6 +183,74 @@ members = [ "utils/wasm-builder", ] +# The list of dependencies below (which can be both direct and indirect dependencies) are crates +# that are suspected to be CPU-intensive, and that are unlikely to require debugging (as some of +# their debug info might be missing) or to require to be frequently recompiled. We compile these +# dependencies with `opt-level=3` even in "dev" mode in order to make "dev" mode more usable. +# The majority of these crates are cryptographic libraries. +# +# Note that this does **not** affect crates that depend on Substrate. In other words, if you add +# a dependency on Substrate, you have to copy-paste this list in your own `Cargo.toml` (assuming +# that you want the same list). This list is only relevant when running `cargo build` from within +# the Substrate workspace. +# +# If you see an error mentioning "profile package spec ... did not match any packages", it +# probably concerns this list. +# +# This list is ordered alphabetically. +[profile.dev.package] +aes-ctr = { opt-level = 3 } +aes-soft = { opt-level = 3 } +aesni = { opt-level = 3 } +blake2 = { opt-level = 3 } +blake2-rfc = { opt-level = 3 } +blake2b_simd = { opt-level = 3 } +blake2s_simd = { opt-level = 3 } +chacha20poly1305 = { opt-level = 3 } +cranelift-codegen = { opt-level = 3 } +cranelift-wasm = { opt-level = 3 } +crc32fast = { opt-level = 3 } +crossbeam-deque = { opt-level = 3 } +crossbeam-queue = { opt-level = 3 } +crypto-mac = { opt-level = 3 } +ctr = { opt-level = 3 } +curve25519-dalek = { opt-level = 3 } +ed25519-dalek = { opt-level = 3 } +evm-core = { opt-level = 3 } +evm-runtime = { opt-level = 3 } +flate2 = { opt-level = 3 } +futures-channel = { opt-level = 3 } +hashbrown = { opt-level = 3 } +h2 = { opt-level = 3 } +hash-db = { opt-level = 3 } +hmac = { opt-level = 3 } +httparse = { opt-level = 3 } +integer-sqrt = { opt-level = 3 } +keccak = { opt-level = 3 } +libm = { opt-level = 3 } +librocksdb-sys = { opt-level = 3 } +libsecp256k1 = { opt-level = 3 } +libz-sys = { opt-level = 3 } +mio = { opt-level = 3 } +nalgebra = { opt-level = 3 } +num-bigint = { opt-level = 3 } +parking_lot = { opt-level = 3 } +parking_lot_core = { opt-level = 3 } +percent-encoding = { opt-level = 3 } +primitive-types = { opt-level = 3 } +ring = { opt-level = 3 } +rustls = { opt-level = 3 } +sha2 = { opt-level = 3 } +sha3 = { opt-level = 3 } +smallvec = { opt-level = 3 } +snow = { opt-level = 3 } +twox-hash = { opt-level = 3 } +uint = { opt-level = 3 } +wasmi = { opt-level = 3 } +x25519-dalek = { opt-level = 3 } +yamux = { opt-level = 3 } +zeroize = { opt-level = 3 } + [profile.release] # Substrate runtime requires unwinding. panic = "unwind" diff --git a/README.md b/README.md index 874ec99e69f89a8947095b5d62e9aa4ffea033cb..5764722373d434289d2c0c29d42144869af87b11 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Substrate · [![GitHub license](https://img.shields.io/github/license/paritytech/substrate)](LICENSE) [![GitLab Status](https://gitlab.parity.io/parity/substrate/badges/master/pipeline.svg)](https://gitlab.parity.io/parity/substrate/pipelines) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](docs/CONTRIBUTING.adoc) +# Substrate · [![GitHub license](https://img.shields.io/badge/license-GPL3%2FApache2-blue)](LICENSE) [![GitLab Status](https://gitlab.parity.io/parity/substrate/badges/master/pipeline.svg)](https://gitlab.parity.io/parity/substrate/pipelines) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](docs/CONTRIBUTING.adoc)

@@ -9,7 +9,9 @@ Substrate is a next-generation framework for blockchain innovation 🚀. ## Trying it out -Simply go to [substrate.dev](https://substrate.dev) and follow the [getting started](https://substrate.dev/docs/en/overview/getting-started/) instructions. +Simply go to [substrate.dev](https://substrate.dev) and follow the +[installation](https://substrate.dev/docs/en/knowledgebase/getting-started/) instructions. You can +also try out one of the [tutorials](https://substrate.dev/en/tutorials). ## Contributions & Code of Conduct @@ -21,4 +23,9 @@ The security policy and procedures can be found in [`docs/SECURITY.md`](docs/SEC ## License -Substrate Client (`/client/*` / `sc-*`) is licensed under [GPL v3.0 with a classpath linking exception](LICENSE-GPL3), primitives (`sp-*`), FRAME (`frame-*`) and pallets (`pallets-*`), binaries (`/bin`) and all other utilities are licensed under [Apache 2.0](LICENSE-APACHE2). \ No newline at end of file +- Substrate Primitives (`sp-*`), Frame (`frame-*`) and the pallets (`pallets-*`), binaries (`/bin`) and all other utilities are licensed under [Apache 2.0](LICENSE-APACHE2). +- Substrate Client (`/client/*` / `sc-*`) is licensed under [GPL v3.0 with a classpath linking exception](LICENSE-GPL3). + +The reason for the split-licensing is to ensure that for the vast majority of teams using Substrate to create feature-chains, then all changes can be made entirely in Apache2-licensed code, allowing teams full freedom over what and how they release and giving licensing clarity to commercial teams. + +In the interests of the community, we require any deeper improvements made to Substrate's core logic (e.g. Substrate's internal consensus, crypto or database code) to be contributed back so everyone can benefit. diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index c4df4f18a251e034071a6faac15de91274cb870f..6689062390b0bc8135a1b3f15284f3b932bd5fb1 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-template" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Anonymous"] description = "Substrate Node template" edition = "2018" @@ -21,25 +21,25 @@ log = "0.4.8" structopt = "0.3.8" parking_lot = "0.10.0" -sc-cli = { version = "0.8.0-alpha.8", path = "../../../client/cli" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sc-executor = { version = "0.8.0-alpha.8", path = "../../../client/executor" } -sc-service = { version = "0.8.0-alpha.8", path = "../../../client/service" } -sp-inherents = { version = "2.0.0-alpha.8", path = "../../../primitives/inherents" } -sc-transaction-pool = { version = "2.0.0-alpha.8", path = "../../../client/transaction-pool" } -sp-transaction-pool = { version = "2.0.0-alpha.8", path = "../../../primitives/transaction-pool" } -sc-network = { version = "0.8.0-alpha.8", path = "../../../client/network" } -sc-consensus-aura = { version = "0.8.0-alpha.8", path = "../../../client/consensus/aura" } -sp-consensus-aura = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/aura" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/common" } -sc-consensus = { version = "0.8.0-alpha.8", path = "../../../client/consensus/common" } -sc-finality-grandpa = { version = "0.8.0-alpha.8", path = "../../../client/finality-grandpa" } -sp-finality-grandpa = { version = "2.0.0-alpha.8", path = "../../../primitives/finality-grandpa" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../../../client/api" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sc-basic-authorship = { path = "../../../client/basic-authorship", version = "0.8.0-alpha.8"} +sc-cli = { version = "0.8.0-rc4", path = "../../../client/cli", features = ["wasmtime"] } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sc-executor = { version = "0.8.0-rc4", path = "../../../client/executor", features = ["wasmtime"] } +sc-service = { version = "0.8.0-rc4", path = "../../../client/service", features = ["wasmtime"] } +sp-inherents = { version = "2.0.0-rc4", path = "../../../primitives/inherents" } +sc-transaction-pool = { version = "2.0.0-rc4", path = "../../../client/transaction-pool" } +sp-transaction-pool = { version = "2.0.0-rc4", path = "../../../primitives/transaction-pool" } +sc-network = { version = "0.8.0-rc4", path = "../../../client/network" } +sc-consensus-aura = { version = "0.8.0-rc4", path = "../../../client/consensus/aura" } +sp-consensus-aura = { version = "0.8.0-rc4", path = "../../../primitives/consensus/aura" } +sp-consensus = { version = "0.8.0-rc4", path = "../../../primitives/consensus/common" } +sc-consensus = { version = "0.8.0-rc4", path = "../../../client/consensus/common" } +sc-finality-grandpa = { version = "0.8.0-rc4", path = "../../../client/finality-grandpa" } +sp-finality-grandpa = { version = "2.0.0-rc4", path = "../../../primitives/finality-grandpa" } +sc-client-api = { version = "2.0.0-rc4", path = "../../../client/api" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sc-basic-authorship = { path = "../../../client/basic-authorship", version = "0.8.0-rc4"} -node-template-runtime = { version = "2.0.0-alpha.8", path = "../runtime" } +node-template-runtime = { version = "2.0.0-rc4", path = "../runtime" } [build-dependencies] -substrate-build-script-utils = { version = "2.0.0-alpha.8", path = "../../../utils/build-script-utils" } +substrate-build-script-utils = { version = "2.0.0-rc4", path = "../../../utils/build-script-utils" } diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index 18e1b22a53f8ef1eac2f441d00b356fae6fe3e69..4f2fd3aad6fd3f30e3af2739705c023c2eb53dd4 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -18,7 +18,7 @@ use crate::chain_spec; use crate::cli::Cli; use crate::service; -use sc_cli::SubstrateCli; +use sc_cli::{SubstrateCli, RuntimeVersion, Role, ChainSpec}; impl SubstrateCli for Cli { fn impl_name() -> &'static str { @@ -58,6 +58,10 @@ impl SubstrateCli for Cli { )?), }) } + + fn native_runtime_version(_: &Box) -> &'static RuntimeVersion { + &node_template_runtime::VERSION + } } /// Parse and run command line arguments @@ -71,11 +75,10 @@ pub fn run() -> sc_cli::Result<()> { } None => { let runner = cli.create_runner(&cli.run)?; - runner.run_node( - service::new_light, - service::new_full, - node_template_runtime::VERSION - ) + runner.run_node_until_exit(|config| match config.role { + Role::Light => service::new_light(config), + _ => service::new_full(config), + }) } } } diff --git a/bin/node-template/node/src/lib.rs b/bin/node-template/node/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..38e43372ca3ff45d40015e83fc45a221f599f16d --- /dev/null +++ b/bin/node-template/node/src/lib.rs @@ -0,0 +1,2 @@ +pub mod chain_spec; +pub mod service; diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index d02e9ea95e4943ccc14a6b206a0c22efea0716ca..89bf159927fc648dc965e1a0800f82cfdb2ca927 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -5,7 +5,10 @@ use std::time::Duration; use sc_client_api::ExecutorProvider; use sc_consensus::LongestChain; use node_template_runtime::{self, opaque::Block, RuntimeApi}; -use sc_service::{error::{Error as ServiceError}, AbstractService, Configuration, ServiceBuilder}; +use sc_service::{ + error::{Error as ServiceError}, Configuration, ServiceBuilder, ServiceComponents, + TaskManager, +}; use sp_inherents::InherentDataProviders; use sc_executor::native_executor_instance; pub use sc_executor::NativeExecutor; @@ -34,14 +37,22 @@ macro_rules! new_full_start { let inherent_data_providers = sp_inherents::InherentDataProviders::new(); let builder = sc_service::ServiceBuilder::new_full::< - node_template_runtime::opaque::Block, node_template_runtime::RuntimeApi, crate::service::Executor + node_template_runtime::opaque::Block, + node_template_runtime::RuntimeApi, + crate::service::Executor >($config)? .with_select_chain(|_config, backend| { Ok(sc_consensus::LongestChain::new(backend.clone())) })? - .with_transaction_pool(|config, client, _fetcher, prometheus_registry| { - let pool_api = sc_transaction_pool::FullChainApi::new(client.clone()); - Ok(sc_transaction_pool::BasicPool::new(config, std::sync::Arc::new(pool_api), prometheus_registry)) + .with_transaction_pool(|builder| { + let pool_api = sc_transaction_pool::FullChainApi::new( + builder.client().clone(), + ); + Ok(sc_transaction_pool::BasicPool::new( + builder.config().transaction_pool.clone(), + std::sync::Arc::new(pool_api), + builder.prometheus_registry(), + )) })? .with_import_queue(| _config, @@ -85,7 +96,7 @@ macro_rules! new_full_start { } /// Builds a new service for a full client. -pub fn new_full(config: Configuration) -> Result { +pub fn new_full(config: Configuration) -> Result { let role = config.role.clone(); let force_authoring = config.force_authoring; let name = config.network.node_name.clone(); @@ -97,20 +108,25 @@ pub fn new_full(config: Configuration) -> Result>; Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, provider)) as _) })? - .build()?; + .build_full()?; if role.is_authority() { - let proposer = - sc_basic_authorship::ProposerFactory::new(service.client(), service.transaction_pool()); + let proposer = sc_basic_authorship::ProposerFactory::new( + client.clone(), + transaction_pool, + prometheus_registry.as_ref(), + ); - let client = service.client(); - let select_chain = service.select_chain() + let select_chain = select_chain .ok_or(ServiceError::SelectChainRequired)?; let can_author_with = @@ -118,26 +134,26 @@ pub fn new_full(config: Configuration) -> Result( sc_consensus_aura::slot_duration(&*client)?, - client, + client.clone(), select_chain, block_import, proposer, - service.network(), + network.clone(), inherent_data_providers.clone(), force_authoring, - service.keystore(), + keystore.clone(), can_author_with, )?; // the AURA authoring task is considered essential, i.e. if it // fails we take down the service with it. - service.spawn_essential_task("aura", aura); + task_manager.spawn_essential_handle().spawn_blocking("aura", aura); } // if the node isn't actively participating in consensus then it doesn't // need a keystore, regardless of which protocol we use below. let keystore = if role.is_authority() { - Some(service.keystore()) + Some(keystore.clone() as sp_core::traits::BareCryptoStorePtr) } else { None }; @@ -163,46 +179,52 @@ pub fn new_full(config: Configuration) -> Result Result { +pub fn new_light(config: Configuration) -> Result { let inherent_data_providers = InherentDataProviders::new(); ServiceBuilder::new_light::(config)? .with_select_chain(|_config, backend| { Ok(LongestChain::new(backend.clone())) })? - .with_transaction_pool(|config, client, fetcher, prometheus_registry| { - let fetcher = fetcher + .with_transaction_pool(|builder| { + let fetcher = builder.fetcher() .ok_or_else(|| "Trying to start light transaction pool without active fetcher")?; - let pool_api = sc_transaction_pool::LightChainApi::new(client.clone(), fetcher.clone()); + let pool_api = sc_transaction_pool::LightChainApi::new( + builder.client().clone(), + fetcher.clone(), + ); let pool = sc_transaction_pool::BasicPool::with_revalidation_type( - config, Arc::new(pool_api), prometheus_registry, sc_transaction_pool::RevalidationType::Light, + builder.config().transaction_pool.clone(), + Arc::new(pool_api), + builder.prometheus_registry(), + sc_transaction_pool::RevalidationType::Light, ); Ok(pool) })? @@ -247,5 +269,6 @@ pub fn new_light(config: Configuration) -> Result>; Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, provider)) as _) })? - .build() + .build_light() + .map(|ServiceComponents { task_manager, .. }| task_manager) } diff --git a/bin/node-template/pallets/template/Cargo.toml b/bin/node-template/pallets/template/Cargo.toml index 24f40f41269281f90a0fdea47cfe405a69792bcf..442fb7203098cde8c071e889b6628bfa11e3d175 100644 --- a/bin/node-template/pallets/template/Cargo.toml +++ b/bin/node-template/pallets/template/Cargo.toml @@ -2,7 +2,7 @@ authors = ['Anonymous'] edition = '2018' name = 'pallet-template' -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" license = "Unlicense" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" @@ -12,30 +12,31 @@ description = "FRAME pallet template" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } [dependencies.frame-support] default-features = false -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" path = "../../../../frame/support" [dependencies.frame-system] default-features = false -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" path = "../../../../frame/system" + [dev-dependencies.sp-core] default-features = false -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" path = "../../../../primitives/core" [dev-dependencies.sp-io] default-features = false -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" path = "../../../../primitives/io" [dev-dependencies.sp-runtime] default-features = false -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" path = "../../../../primitives/runtime" diff --git a/bin/node-template/pallets/template/src/mock.rs b/bin/node-template/pallets/template/src/mock.rs index 33c66e2a4e80d616a64255701279bdc9170a9240..0d9ae7cff775322853152d78db610b38ab891415 100644 --- a/bin/node-template/pallets/template/src/mock.rs +++ b/bin/node-template/pallets/template/src/mock.rs @@ -24,6 +24,7 @@ parameter_types! { pub const AvailableBlockRatio: Perbill = Perbill::from_percent(75); } impl system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Call = (); type Index = u64; @@ -39,6 +40,7 @@ impl system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/bin/node-template/runtime/Cargo.toml b/bin/node-template/runtime/Cargo.toml index b1151c64d9e6d89f3371624320c0c0b8fc2f40b2..ea44c805d0b1beee0691836ed5826de75bee7885 100644 --- a/bin/node-template/runtime/Cargo.toml +++ b/bin/node-template/runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-template-runtime" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Anonymous"] edition = "2018" license = "Unlicense" @@ -11,33 +11,33 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } -aura = { version = "2.0.0-alpha.8", default-features = false, package = "pallet-aura", path = "../../../frame/aura" } -balances = { version = "2.0.0-alpha.8", default-features = false, package = "pallet-balances", path = "../../../frame/balances" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/support" } -grandpa = { version = "2.0.0-alpha.8", default-features = false, package = "pallet-grandpa", path = "../../../frame/grandpa" } -randomness-collective-flip = { version = "2.0.0-alpha.8", default-features = false, package = "pallet-randomness-collective-flip", path = "../../../frame/randomness-collective-flip" } -sudo = { version = "2.0.0-alpha.8", default-features = false, package = "pallet-sudo", path = "../../../frame/sudo" } -system = { version = "2.0.0-alpha.8", default-features = false, package = "frame-system", path = "../../../frame/system" } -timestamp = { version = "2.0.0-alpha.8", default-features = false, package = "pallet-timestamp", path = "../../../frame/timestamp" } -transaction-payment = { version = "2.0.0-alpha.8", default-features = false, package = "pallet-transaction-payment", path = "../../../frame/transaction-payment" } -frame-executive = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/executive" } +aura = { version = "2.0.0-rc4", default-features = false, package = "pallet-aura", path = "../../../frame/aura" } +balances = { version = "2.0.0-rc4", default-features = false, package = "pallet-balances", path = "../../../frame/balances" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/support" } +grandpa = { version = "2.0.0-rc4", default-features = false, package = "pallet-grandpa", path = "../../../frame/grandpa" } +randomness-collective-flip = { version = "2.0.0-rc4", default-features = false, package = "pallet-randomness-collective-flip", path = "../../../frame/randomness-collective-flip" } +sudo = { version = "2.0.0-rc4", default-features = false, package = "pallet-sudo", path = "../../../frame/sudo" } +system = { version = "2.0.0-rc4", default-features = false, package = "frame-system", path = "../../../frame/system" } +timestamp = { version = "2.0.0-rc4", default-features = false, package = "pallet-timestamp", path = "../../../frame/timestamp" } +transaction-payment = { version = "2.0.0-rc4", default-features = false, package = "pallet-transaction-payment", path = "../../../frame/transaction-payment" } +frame-executive = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/executive" } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-api = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/api" } -sp-block-builder = { path = "../../../primitives/block-builder", default-features = false, version = "2.0.0-alpha.8"} -sp-consensus-aura = { version = "0.8.0-alpha.8", default-features = false, path = "../../../primitives/consensus/aura" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/core" } -sp-inherents = { path = "../../../primitives/inherents", default-features = false, version = "2.0.0-alpha.8"} -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/io" } -sp-offchain = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/offchain" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/runtime" } -sp-session = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/session" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/std" } -sp-transaction-pool = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/transaction-pool" } -sp-version = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/version" } +sp-api = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/api" } +sp-block-builder = { path = "../../../primitives/block-builder", default-features = false, version = "2.0.0-rc4"} +sp-consensus-aura = { version = "0.8.0-rc4", default-features = false, path = "../../../primitives/consensus/aura" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/core" } +sp-inherents = { path = "../../../primitives/inherents", default-features = false, version = "2.0.0-rc4"} +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/io" } +sp-offchain = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/offchain" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/runtime" } +sp-session = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/session" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/std" } +sp-transaction-pool = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/transaction-pool" } +sp-version = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/version" } -template = { version = "2.0.0-alpha.8", default-features = false, path = "../pallets/template", package = "pallet-template" } +template = { version = "2.0.0-rc4", default-features = false, path = "../pallets/template", package = "pallet-template" } [build-dependencies] wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } diff --git a/bin/node-template/runtime/build.rs b/bin/node-template/runtime/build.rs index 39f7f56feb0b1730d7f2c9a7daa898fe54bfd052..1f40a41ff8ee62a560785bc426c80787028a048f 100644 --- a/bin/node-template/runtime/build.rs +++ b/bin/node-template/runtime/build.rs @@ -3,7 +3,7 @@ use wasm_builder_runner::WasmBuilder; fn main() { WasmBuilder::new() .with_current_project() - .with_wasm_builder_from_crates("1.0.9") + .with_wasm_builder_from_crates("1.0.11") .export_heap_base() .import_memory() .build() diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index 0a35b329e340e4849b1657109224024a37ce75ce..30571b7e0b36ffd1d45551f692422b71fc7ba5c4 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -15,7 +15,7 @@ use sp_runtime::{ transaction_validity::{TransactionValidity, TransactionSource}, }; use sp_runtime::traits::{ - BlakeTwo256, Block as BlockT, IdentityLookup, Verify, ConvertInto, IdentifyAccount, NumberFor, + BlakeTwo256, Block as BlockT, IdentityLookup, Verify, IdentifyAccount, NumberFor, Saturating, }; use sp_api::impl_runtime_apis; use sp_consensus_aura::sr25519::AuthorityId as AuraId; @@ -35,7 +35,7 @@ pub use frame_support::{ construct_runtime, parameter_types, StorageValue, traits::{KeyOwnerProofSystem, Randomness}, weights::{ - Weight, + Weight, IdentityFee, constants::{BlockExecutionWeight, ExtrinsicBaseWeight, RocksDbWeight, WEIGHT_PER_SECOND}, }, }; @@ -127,11 +127,16 @@ parameter_types! { /// We allow for 2 seconds of compute with a 6 second average block time. pub const MaximumBlockWeight: Weight = 2 * WEIGHT_PER_SECOND; pub const AvailableBlockRatio: Perbill = Perbill::from_percent(75); + /// Assume 10% of weight for average on_initialize calls. + pub MaximumExtrinsicWeight: Weight = AvailableBlockRatio::get() + .saturating_sub(Perbill::from_percent(10)) * MaximumBlockWeight::get(); pub const MaximumBlockLength: u32 = 5 * 1024 * 1024; pub const Version: RuntimeVersion = VERSION; } impl system::Trait for Runtime { + /// The basic call filter to use in dispatchable. + type BaseCallFilter = (); /// The identifier used to distinguish between accounts. type AccountId = AccountId; /// The aggregated dispatch type that is available for extrinsics. @@ -164,6 +169,10 @@ impl system::Trait for Runtime { /// The base weight of any extrinsic processed by the runtime, independent of the /// logic of that extrinsic. (Signature verification, nonce increment, fee, etc...) type ExtrinsicBaseWeight = ExtrinsicBaseWeight; + /// The maximum weight that a single extrinsic of `Normal` dispatch class can have, + /// idependent of the logic of that extrinsics. (Roughly max block weight - average on + /// initialize cost). + type MaximumExtrinsicWeight = MaximumExtrinsicWeight; /// Maximum size of all encoded transactions (in bytes) that are allowed in one block. type MaximumBlockLength = MaximumBlockLength; /// Portion of the block weight that is available to all normal transactions. @@ -236,7 +245,7 @@ impl transaction_payment::Trait for Runtime { type Currency = balances::Module; type OnTransactionPayment = (); type TransactionByteFee = TransactionByteFee; - type WeightToFee = ConvertInto; + type WeightToFee = IdentityFee; type FeeMultiplierUpdate = (); } diff --git a/bin/node/bench/Cargo.toml b/bin/node/bench/Cargo.toml index 443bfaeede6eb1e3427d540122b68e51e28c7eb5..07db27a1f18099dbfcbb4b5a7551cdc7564b52aa 100644 --- a/bin/node/bench/Cargo.toml +++ b/bin/node/bench/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-bench" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "Substrate node integration benchmarks." edition = "2018" @@ -10,20 +10,27 @@ license = "GPL-3.0-or-later WITH Classpath-exception-2.0" [dependencies] log = "0.4.8" -node-primitives = { version = "2.0.0-alpha.8", path = "../primitives" } -node-testing = { version = "2.0.0-alpha.8", path = "../testing" } -sc-cli = { version = "0.8.0-alpha.8", path = "../../../client/cli" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../../../client/api/" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../../primitives/state-machine" } +node-primitives = { version = "2.0.0-rc4", path = "../primitives" } +node-testing = { version = "2.0.0-rc4", path = "../testing" } +node-runtime = { version = "2.0.0-rc4", path = "../runtime" } +sc-cli = { version = "0.8.0-rc4", path = "../../../client/cli" } +sc-client-api = { version = "2.0.0-rc4", path = "../../../client/api/" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../../primitives/state-machine" } serde = "1.0.101" serde_json = "1.0.41" structopt = "0.3" derive_more = "0.99.2" kvdb = "0.6" kvdb-rocksdb = "0.8" -sp-trie = { version = "2.0.0-alpha.8", path = "../../../primitives/trie" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } +sp-trie = { version = "2.0.0-rc4", path = "../../../primitives/trie" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-consensus = { version = "0.8.0-rc4", path = "../../../primitives/consensus/common" } +sp-transaction-pool = { version = "2.0.0-rc4", path = "../../../primitives/transaction-pool" } +sc-basic-authorship = { version = "0.8.0-rc4", path = "../../../client/basic-authorship" } +sp-inherents = { version = "2.0.0-rc4", path = "../../../primitives/inherents" } +sp-finality-tracker = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/finality-tracker" } +sp-timestamp = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/timestamp" } hash-db = "0.15.2" tempfile = "3.1.0" fs_extra = "1" @@ -32,3 +39,4 @@ rand = { version = "0.7.2", features = ["small_rng"] } lazy_static = "1.4.0" parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } parity-db = { version = "0.1.2" } +futures = "0.3.1" diff --git a/bin/node/bench/src/common.rs b/bin/node/bench/src/common.rs new file mode 100644 index 0000000000000000000000000000000000000000..2637d6e9bd04d809af5b374790adcc6e5207993f --- /dev/null +++ b/bin/node/bench/src/common.rs @@ -0,0 +1,48 @@ + +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#[derive(Clone, Copy, Debug, derive_more::Display)] +pub enum SizeType { + #[display(fmt = "empty")] + Empty, + #[display(fmt = "small")] + Small, + #[display(fmt = "medium")] + Medium, + #[display(fmt = "large")] + Large, + #[display(fmt = "full")] + Full, + #[display(fmt = "custom")] + Custom(usize), +} + +impl SizeType { + pub fn transactions(&self) -> Option { + match self { + SizeType::Empty => Some(0), + SizeType::Small => Some(10), + SizeType::Medium => Some(100), + SizeType::Large => Some(500), + SizeType::Full => None, + // Custom SizeType will use the `--transactions` input parameter + SizeType::Custom(val) => Some(*val), + } + } +} \ No newline at end of file diff --git a/bin/node/bench/src/construct.rs b/bin/node/bench/src/construct.rs new file mode 100644 index 0000000000000000000000000000000000000000..e23594dd4364a1de30d65096fe69ab1627599a36 --- /dev/null +++ b/bin/node/bench/src/construct.rs @@ -0,0 +1,296 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Block construction benchmark. +//! +//! This benchmark is expected to measure block construction. +//! We want to protect against cold-cache attacks, and so this +//! benchmark should not rely on any caching (except those entries that +//! DO NOT depend on user input). Thus transaction generation should be +//! based on randomized data. + +use std::{ + borrow::Cow, + collections::HashMap, + pin::Pin, + sync::Arc, +}; +use futures::Future; + +use node_primitives::Block; +use node_testing::bench::{BenchDb, Profile, BlockType, KeyTypes, DatabaseType}; +use sp_runtime::{ + generic::BlockId, + traits::NumberFor, + OpaqueExtrinsic, +}; +use sp_transaction_pool::{ + ImportNotificationStream, + PoolFuture, + PoolStatus, + TransactionFor, + TransactionSource, + TransactionStatusStreamFor, + TxHash, +}; +use sp_consensus::{Environment, Proposer, RecordProof}; + +use crate::{ + common::SizeType, + core::{self, Path, Mode}, +}; + +pub struct ConstructionBenchmarkDescription { + pub profile: Profile, + pub key_types: KeyTypes, + pub block_type: BlockType, + pub size: SizeType, + pub database_type: DatabaseType, +} + +pub struct ConstructionBenchmark { + profile: Profile, + database: BenchDb, + transactions: Transactions, +} + +impl core::BenchmarkDescription for ConstructionBenchmarkDescription { + fn path(&self) -> Path { + + let mut path = Path::new(&["node", "proposer"]); + + match self.profile { + Profile::Wasm => path.push("wasm"), + Profile::Native => path.push("native"), + } + + match self.key_types { + KeyTypes::Sr25519 => path.push("sr25519"), + KeyTypes::Ed25519 => path.push("ed25519"), + } + + match self.block_type { + BlockType::RandomTransfersKeepAlive => path.push("transfer"), + BlockType::RandomTransfersReaping => path.push("transfer_reaping"), + BlockType::Noop => path.push("noop"), + } + + match self.database_type { + DatabaseType::RocksDb => path.push("rocksdb"), + DatabaseType::ParityDb => path.push("paritydb"), + } + + path.push(&format!("{}", self.size)); + + path + } + + fn setup(self: Box) -> Box { + let mut extrinsics: Vec> = Vec::new(); + + let mut bench_db = BenchDb::with_key_types( + self.database_type, + 50_000, + self.key_types + ); + + let client = bench_db.client(); + + let content_type = self.block_type.to_content(self.size.transactions()); + for transaction in bench_db.block_content(content_type, &client) { + extrinsics.push(Arc::new(transaction.into())); + } + + Box::new(ConstructionBenchmark { + profile: self.profile, + database: bench_db, + transactions: Transactions(extrinsics), + }) + } + + fn name(&self) -> Cow<'static, str> { + format!( + "Block construction ({:?}/{}, {:?}, {:?} backend)", + self.block_type, + self.size, + self.profile, + self.database_type, + ).into() + } +} + +impl core::Benchmark for ConstructionBenchmark { + fn run(&mut self, mode: Mode) -> std::time::Duration { + let context = self.database.create_context(self.profile); + + let _ = context.client.runtime_version_at(&BlockId::Number(0)) + .expect("Failed to get runtime version") + .spec_version; + + if mode == Mode::Profile { + std::thread::park_timeout(std::time::Duration::from_secs(3)); + } + + let mut proposer_factory = sc_basic_authorship::ProposerFactory::new( + context.client.clone(), + self.transactions.clone().into(), + None, + ); + let inherent_data_providers = sp_inherents::InherentDataProviders::new(); + inherent_data_providers + .register_provider(sp_timestamp::InherentDataProvider) + .expect("Failed to register timestamp data provider"); + + let start = std::time::Instant::now(); + + let proposer = futures::executor::block_on(proposer_factory.init( + &context.client.header(&BlockId::number(0)) + .expect("Database error querying block #0") + .expect("Block #0 should exist"), + )).expect("Proposer initialization failed"); + + let _block = futures::executor::block_on( + proposer.propose( + inherent_data_providers.create_inherent_data().expect("Create inherent data failed"), + Default::default(), + std::time::Duration::from_secs(20), + RecordProof::Yes, + ), + ).map(|r| r.block).expect("Proposing failed"); + + let elapsed = start.elapsed(); + + if mode == Mode::Profile { + std::thread::park_timeout(std::time::Duration::from_secs(1)); + } + + elapsed + } +} + +#[derive(Clone, Debug)] +pub struct PoolTransaction { + data: OpaqueExtrinsic, + hash: node_primitives::Hash, +} + +impl From for PoolTransaction { + fn from(e: OpaqueExtrinsic) -> Self { + PoolTransaction { + data: e, + hash: node_primitives::Hash::zero(), + } + } +} + +impl sp_transaction_pool::InPoolTransaction for PoolTransaction { + type Transaction = OpaqueExtrinsic; + type Hash = node_primitives::Hash; + + fn data(&self) -> &Self::Transaction { + &self.data + } + + fn hash(&self) -> &Self::Hash { + &self.hash + } + + fn priority(&self) -> &u64 { unimplemented!() } + + fn longevity(&self) -> &u64 { unimplemented!() } + + fn requires(&self) -> &[Vec] { unimplemented!() } + + fn provides(&self) -> &[Vec] { unimplemented!() } + + fn is_propagable(&self) -> bool { unimplemented!() } +} + +#[derive(Clone, Debug)] +pub struct Transactions(Vec>); + +impl sp_transaction_pool::TransactionPool for Transactions { + type Block = Block; + type Hash = node_primitives::Hash; + type InPoolTransaction = PoolTransaction; + type Error = sp_transaction_pool::error::Error; + + /// Returns a future that imports a bunch of unverified transactions to the pool. + fn submit_at( + &self, + _at: &BlockId, + _source: TransactionSource, + _xts: Vec>, + ) -> PoolFuture>, Self::Error> { + unimplemented!() + } + + /// Returns a future that imports one unverified transaction to the pool. + fn submit_one( + &self, + _at: &BlockId, + _source: TransactionSource, + _xt: TransactionFor, + ) -> PoolFuture, Self::Error> { + unimplemented!() + } + + fn submit_and_watch( + &self, + _at: &BlockId, + _source: TransactionSource, + _xt: TransactionFor, + ) -> PoolFuture>, Self::Error> { + unimplemented!() + } + + fn ready_at(&self, _at: NumberFor) + -> Pin> + Send>> + Send>> + { + let iter: Box> + Send> = Box::new(self.0.clone().into_iter()); + Box::pin(futures::future::ready(iter)) + } + + fn ready(&self) -> Box> + Send> { + unimplemented!() + } + + fn remove_invalid(&self, _hashes: &[TxHash]) -> Vec> { + Default::default() + } + + fn status(&self) -> PoolStatus { + unimplemented!() + } + + fn import_notification_stream(&self) -> ImportNotificationStream> { + unimplemented!() + } + + fn on_broadcasted(&self, _propagations: HashMap, Vec>) { + unimplemented!() + } + + fn hash_of(&self, _xt: &TransactionFor) -> TxHash { + unimplemented!() + } + + fn ready_transaction(&self, _hash: &TxHash) -> Option> { + unimplemented!() + } +} \ No newline at end of file diff --git a/bin/node/bench/src/core.rs b/bin/node/bench/src/core.rs index ca297b1a90cb694f336b2d343f80d511bf9b231b..c1b1711549be17daf43024caae16ee356d0c889c 100644 --- a/bin/node/bench/src/core.rs +++ b/bin/node/bench/src/core.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -75,14 +75,14 @@ impl fmt::Display for NsFormatter { } if self.0 < 1_000_000 { - return write!(f, "{:.2} ms", v as f64 / 1_000_000.0) + return write!(f, "{:.4} ms", v as f64 / 1_000_000.0) } if self.0 < 100_000_000 { - return write!(f, "{} ms", v as f64 / 1_000_000.0) + return write!(f, "{:.1} ms", v as f64 / 1_000_000.0) } - write!(f, "{:.2} s", v as f64 / 1_000_000_000.0) + write!(f, "{:.4} s", v as f64 / 1_000_000_000.0) } } diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs index a42dbd748f3ef6dd05e7019008dc31dee8d25d37..759a4299c72758f540e92349de6a25591c506d39 100644 --- a/bin/node/bench/src/generator.rs +++ b/bin/node/bench/src/generator.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/node/bench/src/import.rs b/bin/node/bench/src/import.rs index d98b54ce702aa20fffb841843fce7607fb1ab07b..e49a359fb6af16fc26a7f252ca759442c35bbe1a 100644 --- a/bin/node/bench/src/import.rs +++ b/bin/node/bench/src/import.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -36,38 +36,12 @@ use node_testing::bench::{BenchDb, Profile, BlockType, KeyTypes, DatabaseType}; use node_primitives::Block; use sc_client_api::backend::Backend; use sp_runtime::generic::BlockId; +use sp_state_machine::InspectState; -use crate::core::{self, Path, Mode}; - -#[derive(Clone, Copy, Debug, derive_more::Display)] -pub enum SizeType { - #[display(fmt = "empty")] - Empty, - #[display(fmt = "small")] - Small, - #[display(fmt = "medium")] - Medium, - #[display(fmt = "large")] - Large, - #[display(fmt = "full")] - Full, - #[display(fmt = "custom")] - Custom(usize), -} - -impl SizeType { - pub fn transactions(&self) -> Option { - match self { - SizeType::Empty => Some(0), - SizeType::Small => Some(10), - SizeType::Medium => Some(100), - SizeType::Large => Some(500), - SizeType::Full => None, - // Custom SizeType will use the `--transactions` input parameter - SizeType::Custom(val) => Some(*val), - } - } -} +use crate::{ + common::SizeType, + core::{self, Path, Mode}, +}; pub struct ImportBenchmarkDescription { pub profile: Profile, @@ -81,6 +55,7 @@ pub struct ImportBenchmark { profile: Profile, database: BenchDb, block: Block, + block_type: BlockType, } impl core::BenchmarkDescription for ImportBenchmarkDescription { @@ -124,6 +99,7 @@ impl core::BenchmarkDescription for ImportBenchmarkDescription { let block = bench_db.generate_block(self.block_type.to_content(self.size.transactions())); Box::new(ImportBenchmark { database: bench_db, + block_type: self.block_type, block, profile, }) @@ -131,8 +107,9 @@ impl core::BenchmarkDescription for ImportBenchmarkDescription { fn name(&self) -> Cow<'static, str> { format!( - "Import benchmark ({:?}, {:?}, {:?} backend)", + "Block import ({:?}/{}, {:?}, {:?} backend)", self.block_type, + self.size, self.profile, self.database_type, ).into() @@ -155,6 +132,41 @@ impl core::Benchmark for ImportBenchmark { context.import_block(self.block.clone()); let elapsed = start.elapsed(); + // Sanity checks. + context.client.state_at(&BlockId::number(1)).expect("state_at failed for block#1") + .inspect_with(|| { + match self.block_type { + BlockType::RandomTransfersKeepAlive => { + // should be 5 per signed extrinsic + 1 per unsigned + // we have 1 unsigned and the rest are signed in the block + // those 5 events per signed are: + // - new account (RawEvent::NewAccount) as we always transfer fund to non-existant account + // - endowed (RawEvent::Endowed) for this new account + // - successful transfer (RawEvent::Transfer) for this transfer operation + // - deposit event for charging transaction fee + // - extrinsic success + assert_eq!( + node_runtime::System::events().len(), + (self.block.extrinsics.len() - 1) * 5 + 1, + ); + }, + BlockType::Noop => { + assert_eq!( + node_runtime::System::events().len(), + + // should be 2 per signed extrinsic + 1 per unsigned + // we have 1 unsigned and the rest are signed in the block + // those 2 events per signed are: + // - deposit event for charging transaction fee + // - extrinsic success + (self.block.extrinsics.len() - 1) * 2 + 1, + ); + }, + _ => {}, + } + } + ); + if mode == Mode::Profile { std::thread::park_timeout(std::time::Duration::from_secs(1)); } diff --git a/bin/node/bench/src/main.rs b/bin/node/bench/src/main.rs index fbf203d29e31ab41324a9cef7d193a0f387e62be..11820247112f8c99891c54b7ea4cb60fcf22c87f 100644 --- a/bin/node/bench/src/main.rs +++ b/bin/node/bench/src/main.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -16,21 +16,29 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +mod common; +mod construct; #[macro_use] mod core; mod import; -mod trie; -mod simple_trie; mod generator; -mod tempdb; +mod simple_trie; mod state_sizes; +mod tempdb; +mod trie; -use crate::core::{run_benchmark, Mode as BenchmarkMode}; -use crate::tempdb::DatabaseType; -use import::{ImportBenchmarkDescription, SizeType}; -use trie::{TrieReadBenchmarkDescription, TrieWriteBenchmarkDescription, DatabaseSize}; -use node_testing::bench::{Profile, KeyTypes, BlockType, DatabaseType as BenchDataBaseType}; use structopt::StructOpt; +use node_testing::bench::{Profile, KeyTypes, BlockType, DatabaseType as BenchDataBaseType}; + +use crate::{ + common::SizeType, + core::{run_benchmark, Mode as BenchmarkMode}, + tempdb::DatabaseType, + import::ImportBenchmarkDescription, + trie::{TrieReadBenchmarkDescription, TrieWriteBenchmarkDescription, DatabaseSize}, + construct::ConstructionBenchmarkDescription, +}; + #[derive(Debug, StructOpt)] #[structopt(name = "node-bench", about = "Node integration benchmarks")] struct Opt { @@ -126,6 +134,20 @@ fn main() { ] .iter().map(move |db_type| (size, db_type))) => TrieWriteBenchmarkDescription { database_size: *size, database_type: *db_type }, + ConstructionBenchmarkDescription { + profile: Profile::Wasm, + key_types: KeyTypes::Sr25519, + block_type: BlockType::RandomTransfersKeepAlive, + size: SizeType::Medium, + database_type: BenchDataBaseType::RocksDb, + }, + ConstructionBenchmarkDescription { + profile: Profile::Wasm, + key_types: KeyTypes::Sr25519, + block_type: BlockType::RandomTransfersKeepAlive, + size: SizeType::Large, + database_type: BenchDataBaseType::RocksDb, + }, ); if opt.list { @@ -152,6 +174,11 @@ fn main() { } } + if results.is_empty() { + eprintln!("No benchmark was found for query"); + std::process::exit(1); + } + if opt.json { let json_result: String = serde_json::to_string(&results).expect("Failed to construct json"); println!("{}", json_result); diff --git a/bin/node/bench/src/simple_trie.rs b/bin/node/bench/src/simple_trie.rs index 01ad1bb9b4f19f1f5ae02656c1f44c3e168b66c1..3cfd7ddb300a9101b22d55a2d460526a283fa0e2 100644 --- a/bin/node/bench/src/simple_trie.rs +++ b/bin/node/bench/src/simple_trie.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/node/bench/src/tempdb.rs b/bin/node/bench/src/tempdb.rs index 19f9b9099ce91ce414031a074f78ea39e93634f3..770bafec6f38dc092ac3a905152e2c0861f54e5f 100644 --- a/bin/node/bench/src/tempdb.rs +++ b/bin/node/bench/src/tempdb.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs index ada18c1016d71fab68697a06ef45a584968112c9..886dc6011492f4564aa679f6f5f483025c057b58 100644 --- a/bin/node/bench/src/trie.rs +++ b/bin/node/bench/src/trie.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/node/browser-testing/Cargo.toml b/bin/node/browser-testing/Cargo.toml index 2178fc631f649bc296482566ef353ffae75ccba7..0fa2c4d51aba0256ad56fe90c8935c7ab8a179dc 100644 --- a/bin/node/browser-testing/Cargo.toml +++ b/bin/node/browser-testing/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-browser-testing" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] description = "Tests for the in-browser light client." edition = "2018" @@ -8,8 +8,8 @@ license = "Apache-2.0" [dependencies] futures-timer = "3.0.2" -libp2p = { version = "0.18.0", default-features = false } -jsonrpc-core = "14.0.5" +libp2p = { version = "0.20.1", default-features = false } +jsonrpc-core = "14.2.0" serde = "1.0.106" serde_json = "1.0.48" wasm-bindgen = { version = "=0.2.62", features = ["serde-serialize"] } @@ -17,5 +17,5 @@ wasm-bindgen-futures = "0.4.10" wasm-bindgen-test = "0.3.10" futures = "0.3.4" -node-cli = { path = "../cli", default-features = false, features = ["browser"] , version = "2.0.0-alpha.8"} -sc-rpc-api = { path = "../../../client/rpc-api" , version = "0.8.0-alpha.8"} +node-cli = { path = "../cli", default-features = false, features = ["browser"] , version = "2.0.0-rc4"} +sc-rpc-api = { path = "../../../client/rpc-api" , version = "0.8.0-rc4"} diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 1e4c9b6be37479c5957c30ecaec449b59a9360f7..6202c1af69e8d62ba82b4bb43b58b22e9698ea4c 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-cli" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] description = "Generic Substrate node implementation in Rust." build = "build.rs" @@ -20,7 +20,7 @@ targets = ["x86_64-unknown-linux-gnu"] [badges] travis-ci = { repository = "paritytech/substrate" } -maintenance = { status = "actively-alpha.8eloped" } +maintenance = { status = "actively-developed" } is-it-maintained-issue-resolution = { repository = "paritytech/substrate" } is-it-maintained-open-issues = { repository = "paritytech/substrate" } @@ -34,11 +34,11 @@ crate-type = ["cdylib", "rlib"] [dependencies] # third-party dependencies -codec = { package = "parity-scale-codec", version = "1.3.0" } +codec = { package = "parity-scale-codec", version = "1.3.1" } serde = { version = "1.0.102", features = ["derive"] } futures = { version = "0.3.1", features = ["compat"] } hex-literal = "0.2.1" -jsonrpc-core = "14.0.3" +jsonrpc-core = "14.2.0" log = "0.4.8" rand = "0.7.2" structopt = { version = "0.3.8", optional = true } @@ -46,76 +46,76 @@ tracing = "0.1.10" parking_lot = "0.10.0" # primitives -sp-authority-discovery = { version = "2.0.0-alpha.8", path = "../../../primitives/authority-discovery" } -sp-consensus-babe = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/babe" } -grandpa-primitives = { version = "2.0.0-alpha.8", package = "sp-finality-grandpa", path = "../../../primitives/finality-grandpa" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-timestamp = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/timestamp" } -sp-finality-tracker = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/finality-tracker" } -sp-inherents = { version = "2.0.0-alpha.8", path = "../../../primitives/inherents" } -sp-keyring = { version = "2.0.0-alpha.8", path = "../../../primitives/keyring" } -sp-io = { version = "2.0.0-alpha.8", path = "../../../primitives/io" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/common" } -sp-transaction-pool = { version = "2.0.0-alpha.8", path = "../../../primitives/transaction-pool" } +sp-authority-discovery = { version = "2.0.0-rc4", path = "../../../primitives/authority-discovery" } +sp-consensus-babe = { version = "0.8.0-rc4", path = "../../../primitives/consensus/babe" } +grandpa-primitives = { version = "2.0.0-rc4", package = "sp-finality-grandpa", path = "../../../primitives/finality-grandpa" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-timestamp = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/timestamp" } +sp-finality-tracker = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/finality-tracker" } +sp-inherents = { version = "2.0.0-rc4", path = "../../../primitives/inherents" } +sp-keyring = { version = "2.0.0-rc4", path = "../../../primitives/keyring" } +sp-io = { version = "2.0.0-rc4", path = "../../../primitives/io" } +sp-consensus = { version = "0.8.0-rc4", path = "../../../primitives/consensus/common" } +sp-transaction-pool = { version = "2.0.0-rc4", path = "../../../primitives/transaction-pool" } # client dependencies -sc-client-api = { version = "2.0.0-alpha.8", path = "../../../client/api" } -sc-chain-spec = { version = "2.0.0-alpha.8", path = "../../../client/chain-spec" } -sc-consensus = { version = "0.8.0-alpha.8", path = "../../../client/consensus/common" } -sc-transaction-pool = { version = "2.0.0-alpha.8", path = "../../../client/transaction-pool" } -sc-network = { version = "0.8.0-alpha.8", path = "../../../client/network" } -sc-consensus-babe = { version = "0.8.0-alpha.8", path = "../../../client/consensus/babe" } -grandpa = { version = "0.8.0-alpha.8", package = "sc-finality-grandpa", path = "../../../client/finality-grandpa" } -sc-client-db = { version = "0.8.0-alpha.8", default-features = false, path = "../../../client/db" } -sc-offchain = { version = "2.0.0-alpha.8", path = "../../../client/offchain" } -sc-rpc = { version = "2.0.0-alpha.8", path = "../../../client/rpc" } -sc-basic-authorship = { version = "0.8.0-alpha.8", path = "../../../client/basic-authorship" } -sc-service = { version = "0.8.0-alpha.8", default-features = false, path = "../../../client/service" } -sc-tracing = { version = "2.0.0-alpha.8", path = "../../../client/tracing" } -sc-telemetry = { version = "2.0.0-alpha.8", path = "../../../client/telemetry" } -sc-authority-discovery = { version = "0.8.0-alpha.8", path = "../../../client/authority-discovery" } +sc-client-api = { version = "2.0.0-rc4", path = "../../../client/api" } +sc-chain-spec = { version = "2.0.0-rc4", path = "../../../client/chain-spec" } +sc-consensus = { version = "0.8.0-rc4", path = "../../../client/consensus/common" } +sc-transaction-pool = { version = "2.0.0-rc4", path = "../../../client/transaction-pool" } +sc-network = { version = "0.8.0-rc4", path = "../../../client/network" } +sc-consensus-babe = { version = "0.8.0-rc4", path = "../../../client/consensus/babe" } +grandpa = { version = "0.8.0-rc4", package = "sc-finality-grandpa", path = "../../../client/finality-grandpa" } +sc-client-db = { version = "0.8.0-rc4", default-features = false, path = "../../../client/db" } +sc-offchain = { version = "2.0.0-rc4", path = "../../../client/offchain" } +sc-rpc = { version = "2.0.0-rc4", path = "../../../client/rpc" } +sc-basic-authorship = { version = "0.8.0-rc4", path = "../../../client/basic-authorship" } +sc-service = { version = "0.8.0-rc4", default-features = false, path = "../../../client/service" } +sc-tracing = { version = "2.0.0-rc4", path = "../../../client/tracing" } +sc-telemetry = { version = "2.0.0-rc4", path = "../../../client/telemetry" } +sc-authority-discovery = { version = "0.8.0-rc4", path = "../../../client/authority-discovery" } # frame dependencies -pallet-indices = { version = "2.0.0-alpha.8", path = "../../../frame/indices" } -pallet-timestamp = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/timestamp" } -pallet-contracts = { version = "2.0.0-alpha.8", path = "../../../frame/contracts" } -frame-system = { version = "2.0.0-alpha.8", path = "../../../frame/system" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../../../frame/balances" } -pallet-transaction-payment = { version = "2.0.0-alpha.8", path = "../../../frame/transaction-payment" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/support" } -pallet-im-online = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/im-online" } -pallet-authority-discovery = { version = "2.0.0-alpha.8", path = "../../../frame/authority-discovery" } -pallet-staking = { version = "2.0.0-alpha.8", path = "../../../frame/staking" } -pallet-grandpa = { version = "2.0.0-alpha.8", path = "../../../frame/grandpa" } +pallet-indices = { version = "2.0.0-rc4", path = "../../../frame/indices" } +pallet-timestamp = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/timestamp" } +pallet-contracts = { version = "2.0.0-rc4", path = "../../../frame/contracts" } +frame-system = { version = "2.0.0-rc4", path = "../../../frame/system" } +pallet-balances = { version = "2.0.0-rc4", path = "../../../frame/balances" } +pallet-transaction-payment = { version = "2.0.0-rc4", path = "../../../frame/transaction-payment" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/support" } +pallet-im-online = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/im-online" } +pallet-authority-discovery = { version = "2.0.0-rc4", path = "../../../frame/authority-discovery" } +pallet-staking = { version = "2.0.0-rc4", path = "../../../frame/staking" } +pallet-grandpa = { version = "2.0.0-rc4", path = "../../../frame/grandpa" } # node-specific dependencies -node-runtime = { version = "2.0.0-alpha.8", path = "../runtime" } -node-rpc = { version = "2.0.0-alpha.8", path = "../rpc" } -node-primitives = { version = "2.0.0-alpha.8", path = "../primitives" } -node-executor = { version = "2.0.0-alpha.8", path = "../executor" } +node-runtime = { version = "2.0.0-rc4", path = "../runtime" } +node-rpc = { version = "2.0.0-rc4", path = "../rpc" } +node-primitives = { version = "2.0.0-rc4", path = "../primitives" } +node-executor = { version = "2.0.0-rc4", path = "../executor" } # CLI-specific dependencies -sc-cli = { version = "0.8.0-alpha.8", optional = true, path = "../../../client/cli" } -frame-benchmarking-cli = { version = "2.0.0-alpha.8", optional = true, path = "../../../utils/frame/benchmarking-cli" } -node-inspect = { version = "0.8.0-alpha.8", optional = true, path = "../inspect" } +sc-cli = { version = "0.8.0-rc4", optional = true, path = "../../../client/cli" } +frame-benchmarking-cli = { version = "2.0.0-rc4", optional = true, path = "../../../utils/frame/benchmarking-cli" } +node-inspect = { version = "0.8.0-rc4", optional = true, path = "../inspect" } # WASM-specific dependencies wasm-bindgen = { version = "0.2.57", optional = true } wasm-bindgen-futures = { version = "0.4.7", optional = true } -browser-utils = { package = "substrate-browser-utils", path = "../../../utils/browser", optional = true, version = "0.8.0-alpha.8"} +browser-utils = { package = "substrate-browser-utils", path = "../../../utils/browser", optional = true, version = "0.8.0-rc4"} [target.'cfg(target_arch="x86_64")'.dependencies] -node-executor = { version = "2.0.0-alpha.8", path = "../executor", features = [ "wasmtime" ] } -sc-cli = { version = "0.8.0-alpha.8", optional = true, path = "../../../client/cli", features = [ "wasmtime" ] } -sc-service = { version = "0.8.0-alpha.8", default-features = false, path = "../../../client/service", features = [ "wasmtime" ] } +node-executor = { version = "2.0.0-rc4", path = "../executor", features = [ "wasmtime" ] } +sc-cli = { version = "0.8.0-rc4", optional = true, path = "../../../client/cli", features = [ "wasmtime" ] } +sc-service = { version = "0.8.0-rc4", default-features = false, path = "../../../client/service", features = [ "wasmtime" ] } [dev-dependencies] -sc-keystore = { version = "2.0.0-alpha.8", path = "../../../client/keystore" } -sc-consensus = { version = "0.8.0-alpha.8", path = "../../../client/consensus/common" } -sc-consensus-babe = { version = "0.8.0-alpha.8", features = ["test-helpers"], path = "../../../client/consensus/babe" } -sc-consensus-epochs = { version = "0.8.0-alpha.8", path = "../../../client/consensus/epochs" } -sc-service-test = { version = "2.0.0-alpha.8", path = "../../../client/service/test" } +sc-keystore = { version = "2.0.0-rc4", path = "../../../client/keystore" } +sc-consensus = { version = "0.8.0-rc4", path = "../../../client/consensus/common" } +sc-consensus-babe = { version = "0.8.0-rc4", features = ["test-helpers"], path = "../../../client/consensus/babe" } +sc-consensus-epochs = { version = "0.8.0-rc4", path = "../../../client/consensus/epochs" } +sc-service-test = { version = "2.0.0-rc4", path = "../../../client/service/test" } futures = "0.3.4" tempfile = "3.1.0" assert_cmd = "1.0" @@ -126,12 +126,12 @@ platforms = "0.2.1" [build-dependencies] structopt = { version = "0.3.8", optional = true } -node-inspect = { version = "0.8.0-alpha.8", optional = true, path = "../inspect" } -frame-benchmarking-cli = { version = "2.0.0-alpha.8", optional = true, path = "../../../utils/frame/benchmarking-cli" } -substrate-build-script-utils = { version = "2.0.0-alpha.8", optional = true, path = "../../../utils/build-script-utils" } +node-inspect = { version = "0.8.0-rc4", optional = true, path = "../inspect" } +frame-benchmarking-cli = { version = "2.0.0-rc4", optional = true, path = "../../../utils/frame/benchmarking-cli" } +substrate-build-script-utils = { version = "2.0.0-rc4", optional = true, path = "../../../utils/build-script-utils" } [build-dependencies.sc-cli] -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" package = "sc-cli" path = "../../../client/cli" optional = true diff --git a/bin/node/cli/bin/main.rs b/bin/node/cli/bin/main.rs index 1d0fa639c06146d6e470e3b014c4189c77c7550f..299b760c82e36b2b6540f6ae4efd8111cf430640 100644 --- a/bin/node/cli/bin/main.rs +++ b/bin/node/cli/bin/main.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/node/cli/build.rs b/bin/node/cli/build.rs index 1c47dadc8dcf57ba8e2ede0fe7a132e0bd01ff13..a36f0d01a0a034a10686d3565e6abf538e6a5886 100644 --- a/bin/node/cli/build.rs +++ b/bin/node/cli/build.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/node/cli/res/flaming-fir.json b/bin/node/cli/res/flaming-fir.json index 7cc2c11c327e158842332faf6405d91983b6886c..5f2eb265880efa1853e095486b3fe18d58fa233c 100644 --- a/bin/node/cli/res/flaming-fir.json +++ b/bin/node/cli/res/flaming-fir.json @@ -14,7 +14,7 @@ ], "telemetryEndpoints": [ [ - "/dns4/telemetry.polkadot.io/tcp/443/x-parity-wss/%2Fsubmit%2F", + "/dns/telemetry.polkadot.io/tcp/443/x-parity-wss/%2Fsubmit%2F", 0 ] ], diff --git a/bin/node/cli/src/browser.rs b/bin/node/cli/src/browser.rs index df74ef3f75455dfc6aa2a28cbdfe7cd3ee65c8c5..3aec486c103700f9e7fd9e9c9a8218b78cf555ab 100644 --- a/bin/node/cli/src/browser.rs +++ b/bin/node/cli/src/browser.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -52,8 +52,10 @@ async fn start_inner(chain_spec: Option, log_level: String) -> Result ChainSpec { #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::service::{new_full, new_light}; + use crate::service::{new_full_base, new_light_base}; use sc_service_test; use sp_runtime::BuildStorage; @@ -430,8 +430,14 @@ pub(crate) mod tests { fn test_connectivity() { sc_service_test::connectivity( integration_test_config_with_two_authorities(), - |config| new_full(config), - |config| new_light(config), + |config| { + let (keep_alive, _, client, network, transaction_pool) = new_full_base(config,|_, _| ())?; + Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool)) + }, + |config| { + let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?; + Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool)) + } ); } diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index a6532790641b2b4e50536255ed8113c8b6249cef..29e916fe0180ec2c8ffde4cacdfba361b24cf86c 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -20,7 +20,7 @@ use sc_cli::RunCmd; use structopt::StructOpt; /// An overarching CLI command definition. -#[derive(Clone, Debug, StructOpt)] +#[derive(Debug, StructOpt)] pub struct Cli { /// Possible subcommand with parameters. #[structopt(subcommand)] @@ -31,7 +31,7 @@ pub struct Cli { } /// Possible subcommands of the main binary. -#[derive(Clone, Debug, StructOpt)] +#[derive(Debug, StructOpt)] pub enum Subcommand { /// A set of base subcommands handled by `sc_cli`. #[structopt(flatten)] diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 3034c81bac15373ddaa8c22c5bb20f2a12b700e7..b07e0cdc907e03ff1e5c56dc6daea7ee75ed88f3 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -19,7 +19,7 @@ use crate::{chain_spec, service, Cli, Subcommand}; use node_executor::Executor; use node_runtime::{Block, RuntimeApi}; -use sc_cli::{Result, SubstrateCli}; +use sc_cli::{Result, SubstrateCli, RuntimeVersion, Role, ChainSpec}; impl SubstrateCli for Cli { fn impl_name() -> &'static str { @@ -61,6 +61,10 @@ impl SubstrateCli for Cli { )?), }) } + + fn native_runtime_version(_: &Box) -> &'static RuntimeVersion { + &node_runtime::VERSION + } } /// Parse command line arguments into service configuration. @@ -70,11 +74,10 @@ pub fn run() -> Result<()> { match &cli.subcommand { None => { let runner = cli.create_runner(&cli.run)?; - runner.run_node( - service::new_light, - service::new_full, - node_runtime::VERSION - ) + runner.run_node_until_exit(|config| match config.role { + Role::Light => service::new_light(config), + _ => service::new_full(config), + }) } Some(Subcommand::Inspect(cmd)) => { let runner = cli.create_runner(cmd)?; diff --git a/bin/node/cli/src/lib.rs b/bin/node/cli/src/lib.rs index 8edfb5e3fb0b2a1534109fedf305618307624b88..bd2298514a7a2f1ed57fa008615b11ac0947d20b 100644 --- a/bin/node/cli/src/lib.rs +++ b/bin/node/cli/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index ee6d11de57640c1980c34b8b2dadc0526a652315..9707e3d8caf08cc79b683f1b50fa399ddd5561f0 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -21,7 +21,6 @@ //! Service implementation. Specialized wrapper over substrate service. use std::sync::Arc; - use sc_consensus_babe; use grandpa::{ self, FinalityProofProvider as GrandpaFinalityProofProvider, StorageAndProofProvider, @@ -30,10 +29,16 @@ use node_executor; use node_primitives::Block; use node_runtime::RuntimeApi; use sc_service::{ - AbstractService, ServiceBuilder, config::Configuration, error::{Error as ServiceError}, + ServiceBuilder, config::{Role, Configuration}, error::{Error as ServiceError}, + RpcHandlers, ServiceComponents, TaskManager, }; use sp_inherents::InherentDataProviders; use sc_consensus::LongestChain; +use sc_network::{Event, NetworkService}; +use sp_runtime::traits::Block as BlockT; +use futures::prelude::*; +use sc_client_api::ExecutorProvider; +use sp_core::traits::BareCryptoStorePtr; /// Starts a `ServiceBuilder` for a full service. /// @@ -43,7 +48,6 @@ macro_rules! new_full_start { ($config:expr) => {{ use std::sync::Arc; - type RpcExtension = jsonrpc_core::IoHandler; let mut import_setup = None; let mut rpc_setup = None; let inherent_data_providers = sp_inherents::InherentDataProviders::new(); @@ -54,12 +58,16 @@ macro_rules! new_full_start { .with_select_chain(|_config, backend| { Ok(sc_consensus::LongestChain::new(backend.clone())) })? - .with_transaction_pool(|config, client, _fetcher, prometheus_registry| { - let pool_api = sc_transaction_pool::FullChainApi::new(client.clone()); + .with_transaction_pool(|builder| { + let pool_api = sc_transaction_pool::FullChainApi::new( + builder.client().clone(), + ); + let config = builder.config(); + Ok(sc_transaction_pool::BasicPool::new( - config, + config.transaction_pool.clone(), std::sync::Arc::new(pool_api), - prometheus_registry, + builder.prometheus_registry(), )) })? .with_import_queue(| @@ -99,221 +107,258 @@ macro_rules! new_full_start { import_setup = Some((block_import, grandpa_link, babe_link)); Ok(import_queue) })? - .with_rpc_extensions(|builder| -> std::result::Result { - let babe_link = import_setup.as_ref().map(|s| &s.2) - .expect("BabeLink is present for full services or set up failed; qed."); + .with_rpc_extensions_builder(|builder| { let grandpa_link = import_setup.as_ref().map(|s| &s.1) .expect("GRANDPA LinkHalf is present for full services or set up failed; qed."); - let shared_authority_set = grandpa_link.shared_authority_set(); + + let shared_authority_set = grandpa_link.shared_authority_set().clone(); let shared_voter_state = grandpa::SharedVoterState::empty(); - let deps = node_rpc::FullDeps { - client: builder.client().clone(), - pool: builder.pool(), - select_chain: builder.select_chain().cloned() - .expect("SelectChain is present for full services or set up failed; qed."), - babe: node_rpc::BabeDeps { - keystore: builder.keystore(), - babe_config: sc_consensus_babe::BabeLink::config(babe_link).clone(), - shared_epoch_changes: sc_consensus_babe::BabeLink::epoch_changes(babe_link).clone() - }, - grandpa: node_rpc::GrandpaDeps { - shared_voter_state: shared_voter_state.clone(), - shared_authority_set: shared_authority_set.clone(), - }, - }; - rpc_setup = Some((shared_voter_state)); - Ok(node_rpc::create_full(deps)) + + rpc_setup = Some((shared_voter_state.clone())); + + let babe_link = import_setup.as_ref().map(|s| &s.2) + .expect("BabeLink is present for full services or set up failed; qed."); + + let babe_config = babe_link.config().clone(); + let shared_epoch_changes = babe_link.epoch_changes().clone(); + + let client = builder.client().clone(); + let pool = builder.pool().clone(); + let select_chain = builder.select_chain().cloned() + .expect("SelectChain is present for full services or set up failed; qed."); + let keystore = builder.keystore().clone(); + + Ok(move |deny_unsafe| { + let deps = node_rpc::FullDeps { + client: client.clone(), + pool: pool.clone(), + select_chain: select_chain.clone(), + deny_unsafe, + babe: node_rpc::BabeDeps { + babe_config: babe_config.clone(), + shared_epoch_changes: shared_epoch_changes.clone(), + keystore: keystore.clone(), + }, + grandpa: node_rpc::GrandpaDeps { + shared_voter_state: shared_voter_state.clone(), + shared_authority_set: shared_authority_set.clone(), + }, + }; + + node_rpc::create_full(deps) + }) })?; (builder, import_setup, inherent_data_providers, rpc_setup) }} } -/// Creates a full service from the configuration. -/// -/// We need to use a macro because the test suit doesn't work with an opaque service. It expects -/// concrete types instead. -macro_rules! new_full { - ($config:expr, $with_startup_data: expr) => {{ - use futures::prelude::*; - use sc_network::Event; - use sc_client_api::ExecutorProvider; - - let ( - role, - force_authoring, - name, - disable_grandpa, - ) = ( - $config.role.clone(), - $config.force_authoring, - $config.network.node_name.clone(), - $config.disable_grandpa, - ); +type FullClient = sc_service::TFullClient; +type FullBackend = sc_service::TFullBackend; +type GrandpaBlockImport = grandpa::GrandpaBlockImport< + FullBackend, Block, FullClient, sc_consensus::LongestChain +>; +type BabeBlockImport = sc_consensus_babe::BabeBlockImport; - let (builder, mut import_setup, inherent_data_providers, mut rpc_setup) = - new_full_start!($config); +/// Creates a full service from the configuration. +pub fn new_full_base( + config: Configuration, + with_startup_data: impl FnOnce(&BabeBlockImport, &sc_consensus_babe::BabeLink) +) -> Result<( + TaskManager, + InherentDataProviders, + Arc, Arc::Hash>>, + Arc, Block>> +), ServiceError> { + let ( + role, + force_authoring, + name, + disable_grandpa, + ) = ( + config.role.clone(), + config.force_authoring, + config.network.node_name.clone(), + config.disable_grandpa, + ); + + let (builder, mut import_setup, inherent_data_providers, mut rpc_setup) = + new_full_start!(config); + + let ServiceComponents { + client, transaction_pool, task_manager, keystore, network, select_chain, + prometheus_registry, telemetry_on_connect_sinks, .. + } = builder + .with_finality_proof_provider(|client, backend| { + // GenesisAuthoritySetProvider is implemented for StorageAndProofProvider + let provider = client as Arc>; + Ok(Arc::new(grandpa::FinalityProofProvider::new(backend, provider)) as _) + })? + .build_full()?; - let service = builder - .with_finality_proof_provider(|client, backend| { - // GenesisAuthoritySetProvider is implemented for StorageAndProofProvider - let provider = client as Arc>; - Ok(Arc::new(grandpa::FinalityProofProvider::new(backend, provider)) as _) - })? - .build()?; + let (block_import, grandpa_link, babe_link) = import_setup.take() + .expect("Link Half and Block Import are present for Full Services or setup failed before. qed"); - let (block_import, grandpa_link, babe_link) = import_setup.take() - .expect("Link Half and Block Import are present for Full Services or setup failed before. qed"); + let shared_voter_state = rpc_setup.take() + .expect("The SharedVoterState is present for Full Services or setup failed before. qed"); - let shared_voter_state = rpc_setup.take() - .expect("The SharedVoterState is present for Full Services or setup failed before. qed"); + (with_startup_data)(&block_import, &babe_link); - ($with_startup_data)(&block_import, &babe_link); + if let sc_service::config::Role::Authority { .. } = &role { + let proposer = sc_basic_authorship::ProposerFactory::new( + client.clone(), + transaction_pool.clone(), + prometheus_registry.as_ref(), + ); - if let sc_service::config::Role::Authority { .. } = &role { - let proposer = sc_basic_authorship::ProposerFactory::new( - service.client(), - service.transaction_pool() - ); + let select_chain = select_chain + .ok_or(sc_service::Error::SelectChainRequired)?; - let client = service.client(); - let select_chain = service.select_chain() - .ok_or(sc_service::Error::SelectChainRequired)?; + let can_author_with = + sp_consensus::CanAuthorWithNativeVersion::new(client.executor().clone()); - let can_author_with = - sp_consensus::CanAuthorWithNativeVersion::new(client.executor().clone()); + let babe_config = sc_consensus_babe::BabeParams { + keystore: keystore.clone(), + client: client.clone(), + select_chain, + env: proposer, + block_import, + sync_oracle: network.clone(), + inherent_data_providers: inherent_data_providers.clone(), + force_authoring, + babe_link, + can_author_with, + }; - let babe_config = sc_consensus_babe::BabeParams { - keystore: service.keystore(), - client, - select_chain, - env: proposer, - block_import, - sync_oracle: service.network(), - inherent_data_providers: inherent_data_providers.clone(), - force_authoring, - babe_link, - can_author_with, - }; + let babe = sc_consensus_babe::start_babe(babe_config)?; + task_manager.spawn_essential_handle().spawn_blocking("babe-proposer", babe); + } - let babe = sc_consensus_babe::start_babe(babe_config)?; - service.spawn_essential_task("babe-proposer", babe); - } - - // Spawn authority discovery module. - if matches!(role, sc_service::config::Role::Authority{..} | sc_service::config::Role::Sentry {..}) { - let (sentries, authority_discovery_role) = match role { - sc_service::config::Role::Authority { ref sentry_nodes } => ( - sentry_nodes.clone(), - sc_authority_discovery::Role::Authority ( - service.keystore(), - ), - ), - sc_service::config::Role::Sentry {..} => ( - vec![], - sc_authority_discovery::Role::Sentry, + // Spawn authority discovery module. + if matches!(role, Role::Authority{..} | Role::Sentry {..}) { + let (sentries, authority_discovery_role) = match role { + sc_service::config::Role::Authority { ref sentry_nodes } => ( + sentry_nodes.clone(), + sc_authority_discovery::Role::Authority ( + keystore.clone(), ), - _ => unreachable!("Due to outer matches! constraint; qed.") - }; + ), + sc_service::config::Role::Sentry {..} => ( + vec![], + sc_authority_discovery::Role::Sentry, + ), + _ => unreachable!("Due to outer matches! constraint; qed.") + }; - let network = service.network(); - let dht_event_stream = network.event_stream("authority-discovery").filter_map(|e| async move { match e { + let dht_event_stream = network.event_stream("authority-discovery") + .filter_map(|e| async move { match e { Event::Dht(e) => Some(e), _ => None, }}).boxed(); - let authority_discovery = sc_authority_discovery::AuthorityDiscovery::new( - service.client(), - network, - sentries, - dht_event_stream, - authority_discovery_role, - service.prometheus_registry(), - ); + let authority_discovery = sc_authority_discovery::AuthorityDiscovery::new( + client.clone(), + network.clone(), + sentries, + dht_event_stream, + authority_discovery_role, + prometheus_registry.clone(), + ); - service.spawn_task("authority-discovery", authority_discovery); - } + task_manager.spawn_handle().spawn("authority-discovery", authority_discovery); + } - // if the node isn't actively participating in consensus then it doesn't - // need a keystore, regardless of which protocol we use below. - let keystore = if role.is_authority() { - Some(service.keystore()) - } else { - None - }; + // if the node isn't actively participating in consensus then it doesn't + // need a keystore, regardless of which protocol we use below. + let keystore = if role.is_authority() { + Some(keystore.clone() as BareCryptoStorePtr) + } else { + None + }; - let config = grandpa::Config { - // FIXME #1578 make this available through chainspec - gossip_duration: std::time::Duration::from_millis(333), - justification_period: 512, - name: Some(name), - observer_enabled: false, - keystore, - is_authority: role.is_network_authority(), - }; + let config = grandpa::Config { + // FIXME #1578 make this available through chainspec + gossip_duration: std::time::Duration::from_millis(333), + justification_period: 512, + name: Some(name), + observer_enabled: false, + keystore, + is_authority: role.is_network_authority(), + }; - let enable_grandpa = !disable_grandpa; - if enable_grandpa { - // start the full GRANDPA voter - // NOTE: non-authorities could run the GRANDPA observer protocol, but at - // this point the full voter should provide better guarantees of block - // and vote data availability than the observer. The observer has not - // been tested extensively yet and having most nodes in a network run it - // could lead to finality stalls. - let grandpa_config = grandpa::GrandpaParams { - config, - link: grandpa_link, - network: service.network(), - inherent_data_providers: inherent_data_providers.clone(), - telemetry_on_connect: Some(service.telemetry_on_connect_stream()), - voting_rule: grandpa::VotingRulesBuilder::default().build(), - prometheus_registry: service.prometheus_registry(), - shared_voter_state, - }; + let enable_grandpa = !disable_grandpa; + if enable_grandpa { + // start the full GRANDPA voter + // NOTE: non-authorities could run the GRANDPA observer protocol, but at + // this point the full voter should provide better guarantees of block + // and vote data availability than the observer. The observer has not + // been tested extensively yet and having most nodes in a network run it + // could lead to finality stalls. + let grandpa_config = grandpa::GrandpaParams { + config, + link: grandpa_link, + network: network.clone(), + inherent_data_providers: inherent_data_providers.clone(), + telemetry_on_connect: Some(telemetry_on_connect_sinks.on_connect_stream()), + voting_rule: grandpa::VotingRulesBuilder::default().build(), + prometheus_registry: prometheus_registry.clone(), + shared_voter_state, + }; - // the GRANDPA voter task is considered infallible, i.e. - // if it fails we take down the service with it. - service.spawn_essential_task( - "grandpa-voter", - grandpa::run_grandpa_voter(grandpa_config)? - ); - } else { - grandpa::setup_disabled_grandpa( - service.client(), - &inherent_data_providers, - service.network(), - )?; - } + // the GRANDPA voter task is considered infallible, i.e. + // if it fails we take down the service with it. + task_manager.spawn_essential_handle().spawn_blocking( + "grandpa-voter", + grandpa::run_grandpa_voter(grandpa_config)? + ); + } else { + grandpa::setup_disabled_grandpa( + client.clone(), + &inherent_data_providers, + network.clone(), + )?; + } - Ok((service, inherent_data_providers)) - }}; - ($config:expr) => {{ - new_full!($config, |_, _| {}) - }} + Ok((task_manager, inherent_data_providers, client, network, transaction_pool)) } /// Builds a new service for a full client. pub fn new_full(config: Configuration) --> Result -{ - new_full!(config).map(|(service, _)| service) +-> Result { + new_full_base(config, |_, _| ()).map(|(task_manager, _, _, _, _)| { + task_manager + }) } -/// Builds a new service for a light client. -pub fn new_light(config: Configuration) --> Result { - type RpcExtension = jsonrpc_core::IoHandler; +type LightClient = sc_service::TLightClient; +type LightFetcher = sc_network::config::OnDemand; + +pub fn new_light_base(config: Configuration) -> Result<( + TaskManager, Arc, Arc, + Arc::Hash>>, + Arc, Block + >> +), ServiceError> { let inherent_data_providers = InherentDataProviders::new(); - let service = ServiceBuilder::new_light::(config)? + let ServiceComponents { + task_manager, rpc_handlers, client, network, transaction_pool, .. + } = ServiceBuilder::new_light::(config)? .with_select_chain(|_config, backend| { Ok(LongestChain::new(backend.clone())) })? - .with_transaction_pool(|config, client, fetcher, prometheus_registry| { - let fetcher = fetcher + .with_transaction_pool(|builder| { + let fetcher = builder.fetcher() .ok_or_else(|| "Trying to start light transaction pool without active fetcher")?; - let pool_api = sc_transaction_pool::LightChainApi::new(client.clone(), fetcher.clone()); + let pool_api = sc_transaction_pool::LightChainApi::new( + builder.client().clone(), + fetcher, + ); let pool = sc_transaction_pool::BasicPool::with_revalidation_type( - config, Arc::new(pool_api), prometheus_registry, sc_transaction_pool::RevalidationType::Light, + builder.config().transaction_pool.clone(), + Arc::new(pool_api), + builder.prometheus_registry(), + sc_transaction_pool::RevalidationType::Light, ); Ok(pool) })? @@ -365,9 +410,7 @@ pub fn new_light(config: Configuration) let provider = client as Arc>; Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, provider)) as _) })? - .with_rpc_extensions(|builder,| -> - Result - { + .with_rpc_extensions(|builder| { let fetcher = builder.fetcher() .ok_or_else(|| "Trying to start node RPC without active fetcher")?; let remote_blockchain = builder.remote_backend() @@ -379,19 +422,25 @@ pub fn new_light(config: Configuration) client: builder.client().clone(), pool: builder.pool(), }; + Ok(node_rpc::create_light(light_deps)) })? - .build()?; + .build_light()?; + + Ok((task_manager, rpc_handlers, client, network, transaction_pool)) +} - Ok(service) +/// Builds a new service for a light client. +pub fn new_light(config: Configuration) -> Result { + new_light_base(config).map(|(task_manager, _, _, _, _)| { + task_manager + }) } #[cfg(test)] mod tests { use std::{sync::Arc, borrow::Cow, any::Any}; - use sc_consensus_babe::{ - CompatibleDigestItem, BabeIntermediate, INTERMEDIATE_KEY - }; + use sc_consensus_babe::{CompatibleDigestItem, BabeIntermediate, INTERMEDIATE_KEY}; use sc_consensus_epochs::descendent_query; use sp_consensus::{ Environment, Proposer, BlockImportParams, BlockOrigin, ForkChoiceStrategy, BlockImport, @@ -400,101 +449,24 @@ mod tests { use node_primitives::{Block, DigestItem, Signature}; use node_runtime::{BalancesCall, Call, UncheckedExtrinsic, Address}; use node_runtime::constants::{currency::CENTS, time::SLOT_DURATION}; - use codec::{Encode, Decode}; + use codec::Encode; use sp_core::{crypto::Pair as CryptoPair, H256}; use sp_runtime::{ generic::{BlockId, Era, Digest, SignedPayload}, traits::{Block as BlockT, Header as HeaderT}, traits::Verify, - OpaqueExtrinsic, }; use sp_timestamp; use sp_finality_tracker; use sp_keyring::AccountKeyring; - use sc_service::AbstractService; - use crate::service::{new_full, new_light}; + use sc_service_test::TestNetNode; + use crate::service::{new_full_base, new_light_base}; use sp_runtime::traits::IdentifyAccount; use sp_transaction_pool::{MaintainedTransactionPool, ChainEvent}; + use sc_client_api::BlockBackend; type AccountPublic = ::Signer; - #[cfg(feature = "rhd")] - fn test_sync() { - use sp_core::ed25519::Pair; - - use {service_test, Factory}; - use sp_consensus::{BlockImportParams, BlockOrigin}; - - let alice: Arc = Arc::new(Keyring::Alice.into()); - let bob: Arc = Arc::new(Keyring::Bob.into()); - let validators = vec![alice.public().0.into(), bob.public().0.into()]; - let keys: Vec<&ed25519::Pair> = vec![&*alice, &*bob]; - let dummy_runtime = ::tokio::runtime::Runtime::new().unwrap(); - let block_factory = |service: &::FullService| { - let block_id = BlockId::number(service.client().chain_info().best_number); - let parent_header = service.client().best_header(&block_id) - .expect("db error") - .expect("best block should exist"); - - futures::executor::block_on( - service.transaction_pool().maintain( - ChainEvent::NewBlock { - is_new_best: true, - id: block_id.clone(), - retracted: vec![], - header: parent_header, - }, - ) - ); - - let consensus_net = ConsensusNetwork::new(service.network(), service.client().clone()); - let proposer_factory = consensus::ProposerFactory { - client: service.client().clone(), - transaction_pool: service.transaction_pool().clone(), - network: consensus_net, - force_delay: 0, - handle: dummy_runtime.executor(), - }; - let (proposer, _, _) = proposer_factory.init(&parent_header, &validators, alice.clone()).unwrap(); - let block = proposer.propose().expect("Error making test block"); - BlockImportParams { - origin: BlockOrigin::File, - justification: Vec::new(), - internal_justification: Vec::new(), - finalized: false, - body: Some(block.extrinsics), - storage_changes: None, - header: block.header, - auxiliary: Vec::new(), - } - }; - let extrinsic_factory = - |service: &SyncService<::FullService>| - { - let payload = ( - 0, - Call::Balances(BalancesCall::transfer(RawAddress::Id(bob.public().0.into()), 69.into())), - Era::immortal(), - service.client().genesis_hash() - ); - let signature = alice.sign(&payload.encode()).into(); - let id = alice.public().0.into(); - let xt = UncheckedExtrinsic { - signature: Some((RawAddress::Id(id), signature, payload.0, Era::immortal())), - function: payload.1, - }.encode(); - let v: Vec = Decode::decode(&mut xt.as_slice()).unwrap(); - OpaqueExtrinsic(v) - }; - sc_service_test::sync( - sc_chain_spec::integration_test_config(), - |config| new_full(config), - |mut config| new_light(config), - block_factory, - extrinsic_factory, - ); - } - #[test] // It is "ignored", but the node-cli ignored tests are running on the CI. // This can be run locally with `cargo test --release -p node-cli test_sync -- --ignored`. @@ -520,14 +492,25 @@ mod tests { chain_spec, |config| { let mut setup_handles = None; - new_full!(config, | - block_import: &sc_consensus_babe::BabeBlockImport, - babe_link: &sc_consensus_babe::BabeLink, - | { - setup_handles = Some((block_import.clone(), babe_link.clone())); - }).map(move |(node, x)| (node, (x, setup_handles.unwrap()))) + let (keep_alive, inherent_data_providers, client, network, transaction_pool) = + new_full_base(config, + | + block_import: &sc_consensus_babe::BabeBlockImport, + babe_link: &sc_consensus_babe::BabeLink, + | { + setup_handles = Some((block_import.clone(), babe_link.clone())); + } + )?; + + let node = sc_service_test::TestNetComponents::new( + keep_alive, client, network, transaction_pool + ); + Ok((node, (inherent_data_providers, setup_handles.unwrap()))) + }, + |config| { + let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?; + Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool)) }, - |config| new_light(config), |service, &mut (ref inherent_data_providers, (ref mut block_import, ref babe_link))| { let mut inherent_data = inherent_data_providers .create_inherent_data() @@ -543,8 +526,8 @@ mod tests { service.transaction_pool().maintain( ChainEvent::NewBlock { is_new_best: true, - id: parent_id.clone(), - retracted: vec![], + hash: parent_header.hash(), + tree_route: None, header: parent_header.clone(), }, ) @@ -552,7 +535,8 @@ mod tests { let mut proposer_factory = sc_basic_authorship::ProposerFactory::new( service.client(), - service.transaction_pool() + service.transaction_pool(), + None, ); let epoch_descriptor = babe_link.epoch_changes().lock().epoch_descriptor_for_child_of( @@ -657,16 +641,13 @@ mod tests { signer.sign(payload) }); let (function, extra, _) = raw_payload.deconstruct(); - let xt = UncheckedExtrinsic::new_signed( + index += 1; + UncheckedExtrinsic::new_signed( function, from.into(), signature.into(), extra, - ).encode(); - let v: Vec = Decode::decode(&mut xt.as_slice()).unwrap(); - - index += 1; - OpaqueExtrinsic(v) + ).into() }, ); } @@ -676,8 +657,14 @@ mod tests { fn test_consensus() { sc_service_test::consensus( crate::chain_spec::tests::integration_test_config_with_two_authorities(), - |config| new_full(config), - |config| new_light(config), + |config| { + let (keep_alive, _, client, network, transaction_pool) = new_full_base(config, |_, _| ())?; + Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool)) + }, + |config| { + let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?; + Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool)) + }, vec![ "//Alice".into(), "//Bob".into(), diff --git a/bin/node/cli/tests/build_spec_works.rs b/bin/node/cli/tests/build_spec_works.rs index a6fa2ad4e297139acede01547ad0c045e90bf85e..800a4a8c51e6175e2eb8fbfb1e5ddf375d30c264 100644 --- a/bin/node/cli/tests/build_spec_works.rs +++ b/bin/node/cli/tests/build_spec_works.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/node/cli/tests/check_block_works.rs b/bin/node/cli/tests/check_block_works.rs index 209b1122aa06221a8990bce83a45d8e01c0224cd..34078b08cf074f218865df548ee20b3b94f19d74 100644 --- a/bin/node/cli/tests/check_block_works.rs +++ b/bin/node/cli/tests/check_block_works.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -22,7 +22,7 @@ use assert_cmd::cargo::cargo_bin; use std::process::Command; use tempfile::tempdir; -mod common; +pub mod common; #[test] fn check_block_works() { diff --git a/bin/node/cli/tests/common.rs b/bin/node/cli/tests/common.rs index a9df32d792ecfccca0b81add2147e624db2079d2..61a07dd1ca877c8814169f2babdfccd3a506ec76 100644 --- a/bin/node/cli/tests/common.rs +++ b/bin/node/cli/tests/common.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -17,7 +17,6 @@ // along with this program. If not, see . #![cfg(unix)] -#![allow(dead_code)] use std::{process::{Child, ExitStatus}, thread, time::Duration, path::Path}; use assert_cmd::cargo::cargo_bin; diff --git a/bin/node/cli/tests/export_import_flow.rs b/bin/node/cli/tests/export_import_flow.rs new file mode 100644 index 0000000000000000000000000000000000000000..557e722ddb7b505a13fa8daa8361838dcfd93c45 --- /dev/null +++ b/bin/node/cli/tests/export_import_flow.rs @@ -0,0 +1,212 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#![cfg(unix)] + +use assert_cmd::cargo::cargo_bin; +use std::{process::Command, fs, path::PathBuf}; +use tempfile::{tempdir, TempDir}; +use regex::Regex; + +pub mod common; + +fn contains_error(logged_output: &str) -> bool { + logged_output.contains("Error") +} + +/// Helper struct to execute the export/import/revert tests. +/// The fields are paths to a temporary directory +struct ExportImportRevertExecutor<'a> { + base_path: &'a TempDir, + exported_blocks_file: &'a PathBuf, + db_path: &'a PathBuf, + num_exported_blocks: Option, +} + +/// Format options for export / import commands. +enum FormatOpt { + Json, + Binary, +} + +/// Command corresponding to the different commands we would like to run. +enum SubCommand { + ExportBlocks, + ImportBlocks, +} + +impl ToString for SubCommand { + fn to_string(&self) -> String { + match self { + SubCommand::ExportBlocks => String::from("export-blocks"), + SubCommand::ImportBlocks => String::from("import-blocks"), + } + } +} + +impl<'a> ExportImportRevertExecutor<'a> { + fn new( + base_path: &'a TempDir, + exported_blocks_file: &'a PathBuf, + db_path: &'a PathBuf + ) -> Self { + Self { + base_path, + exported_blocks_file, + db_path, + num_exported_blocks: None, + } + } + + /// Helper method to run a command. Returns a string corresponding to what has been logged. + fn run_block_command(&self, + sub_command: SubCommand, + format_opt: FormatOpt, + expected_to_fail: bool + ) -> String { + let sub_command_str = sub_command.to_string(); + // Adding "--binary" if need be. + let arguments: Vec<&str> = match format_opt { + FormatOpt::Binary => vec![&sub_command_str, "--dev", "--pruning", "archive", "--binary", "-d"], + FormatOpt::Json => vec![&sub_command_str, "--dev", "--pruning", "archive", "-d"], + }; + + let tmp: TempDir; + // Setting base_path to be a temporary folder if we are importing blocks. + // This allows us to make sure we are importing from scratch. + let base_path = match sub_command { + SubCommand::ExportBlocks => &self.base_path.path(), + SubCommand::ImportBlocks => { + tmp = tempdir().unwrap(); + tmp.path() + } + }; + + // Running the command and capturing the output. + let output = Command::new(cargo_bin("substrate")) + .args(&arguments) + .arg(&base_path) + .arg(&self.exported_blocks_file) + .output() + .unwrap(); + + let logged_output = String::from_utf8_lossy(&output.stderr).to_string(); + + if expected_to_fail { + // Checking that we did indeed find an error. + assert!(contains_error(&logged_output), "expected to error but did not error!"); + assert!(!output.status.success()); + } else { + // Making sure no error were logged. + assert!(!contains_error(&logged_output), "expected not to error but error'd!"); + assert!(output.status.success()); + } + + logged_output + } + + /// Runs the `export-blocks` command. + fn run_export(&mut self, fmt_opt: FormatOpt) { + let log = self.run_block_command(SubCommand::ExportBlocks, fmt_opt, false); + + // Using regex to find out how many block we exported. + let re = Regex::new(r"Exporting blocks from #\d* to #(?P\d*)").unwrap(); + let caps = re.captures(&log).unwrap(); + // Saving the number of blocks we've exported for further use. + self.num_exported_blocks = Some(caps["exported_blocks"].parse::().unwrap()); + + let metadata = fs::metadata(&self.exported_blocks_file).unwrap(); + assert!(metadata.len() > 0, "file exported_blocks should not be empty"); + + let _ = fs::remove_dir_all(&self.db_path); + } + + /// Runs the `import-blocks` command, asserting that an error was found or + /// not depending on `expected_to_fail`. + fn run_import(&mut self, fmt_opt: FormatOpt, expected_to_fail: bool) { + let log = self.run_block_command(SubCommand::ImportBlocks, fmt_opt, expected_to_fail); + + if !expected_to_fail { + // Using regex to find out how much block we imported, + // and what's the best current block. + let re = Regex::new(r"Imported (?P\d*) blocks. Best: #(?P\d*)").unwrap(); + let caps = re.captures(&log).expect("capture should have succeeded"); + let imported = caps["imported"].parse::().unwrap(); + let best = caps["best"].parse::().unwrap(); + + assert_eq!( + imported, + best, + "numbers of blocks imported and best number differs" + ); + assert_eq!( + best, + self.num_exported_blocks.expect("number of exported blocks cannot be None; qed"), + "best block number and number of expected blocks should not differ" + ); + } + self.num_exported_blocks = None; + } + + /// Runs the `revert` command. + fn run_revert(&self) { + let output = Command::new(cargo_bin("substrate")) + .args(&["revert", "--dev", "--pruning", "archive", "-d"]) + .arg(&self.base_path.path()) + .output() + .unwrap(); + + let logged_output = String::from_utf8_lossy(&output.stderr).to_string(); + + // Reverting should not log any error. + assert!(!contains_error(&logged_output)); + // Command should never fail. + assert!(output.status.success()); + } + + /// Helper function that runs the whole export / import / revert flow and checks for errors. + fn run(&mut self, export_fmt: FormatOpt, import_fmt: FormatOpt, expected_to_fail: bool) { + self.run_export(export_fmt); + self.run_import(import_fmt, expected_to_fail); + self.run_revert(); + } +} + +#[test] +fn export_import_revert() { + let base_path = tempdir().expect("could not create a temp dir"); + let exported_blocks_file = base_path.path().join("exported_blocks"); + let db_path = base_path.path().join("db"); + + common::run_dev_node_for_a_while(base_path.path()); + + let mut executor = ExportImportRevertExecutor::new( + &base_path, + &exported_blocks_file, + &db_path, + ); + + // Binary and binary should work. + executor.run(FormatOpt::Binary, FormatOpt::Binary, false); + // Binary and JSON should fail. + executor.run(FormatOpt::Binary, FormatOpt::Json, true); + // JSON and JSON should work. + executor.run(FormatOpt::Json, FormatOpt::Json, false); + // JSON and binary should fail. + executor.run(FormatOpt::Json, FormatOpt::Binary, true); +} diff --git a/bin/node/cli/tests/import_export_and_revert_work.rs b/bin/node/cli/tests/import_export_and_revert_work.rs deleted file mode 100644 index 26556456171ba34aa446f37348515669ecfc10e6..0000000000000000000000000000000000000000 --- a/bin/node/cli/tests/import_export_and_revert_work.rs +++ /dev/null @@ -1,61 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2020 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -#![cfg(unix)] - -use assert_cmd::cargo::cargo_bin; -use std::{process::Command, fs}; -use tempfile::tempdir; - -mod common; - -#[test] -fn import_export_and_revert_work() { - let base_path = tempdir().expect("could not create a temp dir"); - let exported_blocks = base_path.path().join("exported_blocks"); - - common::run_dev_node_for_a_while(base_path.path()); - - let status = Command::new(cargo_bin("substrate")) - .args(&["export-blocks", "--dev", "--pruning", "archive", "-d"]) - .arg(base_path.path()) - .arg(&exported_blocks) - .status() - .unwrap(); - assert!(status.success()); - - let metadata = fs::metadata(&exported_blocks).unwrap(); - assert!(metadata.len() > 0, "file exported_blocks should not be empty"); - - let _ = fs::remove_dir_all(base_path.path().join("db")); - - let status = Command::new(cargo_bin("substrate")) - .args(&["import-blocks", "--dev", "--pruning", "archive", "-d"]) - .arg(base_path.path()) - .arg(&exported_blocks) - .status() - .unwrap(); - assert!(status.success()); - - let status = Command::new(cargo_bin("substrate")) - .args(&["revert", "--dev", "--pruning", "archive", "-d"]) - .arg(base_path.path()) - .status() - .unwrap(); - assert!(status.success()); -} diff --git a/bin/node/cli/tests/inspect_works.rs b/bin/node/cli/tests/inspect_works.rs index 67872448762fe008dc5c57cf2c950e4c9be191c6..aa9653acadba5a1db8ccd295da9d802967604a77 100644 --- a/bin/node/cli/tests/inspect_works.rs +++ b/bin/node/cli/tests/inspect_works.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -22,7 +22,7 @@ use assert_cmd::cargo::cargo_bin; use std::process::Command; use tempfile::tempdir; -mod common; +pub mod common; #[test] fn inspect_works() { diff --git a/bin/node/cli/tests/purge_chain_works.rs b/bin/node/cli/tests/purge_chain_works.rs index b57084092daeac23d02ed22e06ac2bcdb536f1af..001bed8b136f5d2349f83fe86c557466f0b7a7ca 100644 --- a/bin/node/cli/tests/purge_chain_works.rs +++ b/bin/node/cli/tests/purge_chain_works.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -20,7 +20,7 @@ use assert_cmd::cargo::cargo_bin; use std::process::Command; use tempfile::tempdir; -mod common; +pub mod common; #[test] #[cfg(unix)] diff --git a/bin/node/cli/tests/running_the_node_and_interrupt.rs b/bin/node/cli/tests/running_the_node_and_interrupt.rs index fb241e177ed538123f5b06741433fc3b7b316b48..bd79dcd77a49a9e6a81b28ae360ef90b3174b565 100644 --- a/bin/node/cli/tests/running_the_node_and_interrupt.rs +++ b/bin/node/cli/tests/running_the_node_and_interrupt.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -20,7 +20,7 @@ use assert_cmd::cargo::cargo_bin; use std::{convert::TryInto, process::Command, thread, time::Duration}; use tempfile::tempdir; -mod common; +pub mod common; #[test] #[cfg(unix)] diff --git a/bin/node/cli/tests/temp_base_path_works.rs b/bin/node/cli/tests/temp_base_path_works.rs new file mode 100644 index 0000000000000000000000000000000000000000..9351568d87955b130bdde3d9f82f5ce977bc2b20 --- /dev/null +++ b/bin/node/cli/tests/temp_base_path_works.rs @@ -0,0 +1,72 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#![cfg(unix)] + +use assert_cmd::cargo::cargo_bin; +use nix::sys::signal::{kill, Signal::SIGINT}; +use nix::unistd::Pid; +use regex::Regex; +use std::convert::TryInto; +use std::io::Read; +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::Duration; + +pub mod common; + +#[test] +fn temp_base_path_works() { + let mut cmd = Command::new(cargo_bin("substrate")); + + let mut cmd = cmd + .args(&["--dev", "--tmp"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + + // Let it produce some blocks. + thread::sleep(Duration::from_secs(30)); + assert!( + cmd.try_wait().unwrap().is_none(), + "the process should still be running" + ); + + // Stop the process + kill(Pid::from_raw(cmd.id().try_into().unwrap()), SIGINT).unwrap(); + assert!(common::wait_for(&mut cmd, 40) + .map(|x| x.success()) + .unwrap_or_default()); + + // Ensure the database has been deleted + let mut stderr = String::new(); + cmd.stderr.unwrap().read_to_string(&mut stderr).unwrap(); + let re = Regex::new(r"Database: .+ at (\S+)").unwrap(); + let db_path = PathBuf::from( + re.captures(stderr.as_str()) + .unwrap() + .get(1) + .unwrap() + .as_str() + .to_string(), + ); + + assert!(!db_path.exists()); +} diff --git a/bin/node/cli/tests/version.rs b/bin/node/cli/tests/version.rs index 8d8b3a5ce0374f77f275acbd55c104cb4a9f3f63..c5240257f16dea779fa78f75fafd77df75cce08e 100644 --- a/bin/node/cli/tests/version.rs +++ b/bin/node/cli/tests/version.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/node/executor/Cargo.toml b/bin/node/executor/Cargo.toml index 78cfb3e5b4e78569d5f61e65f47fc667848b77d8..900f0cad4327ee8cd015b581b8bcba600dfba28e 100644 --- a/bin/node/executor/Cargo.toml +++ b/bin/node/executor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-executor" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] description = "Substrate node implementation in Rust." edition = "2018" @@ -12,35 +12,35 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0" } -node-primitives = { version = "2.0.0-alpha.8", path = "../primitives" } -node-runtime = { version = "2.0.0-alpha.8", path = "../runtime" } -sc-executor = { version = "0.8.0-alpha.8", path = "../../../client/executor" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-io = { version = "2.0.0-alpha.8", path = "../../../primitives/io" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../../primitives/state-machine" } -sp-trie = { version = "2.0.0-alpha.8", path = "../../../primitives/trie" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +node-primitives = { version = "2.0.0-rc4", path = "../primitives" } +node-runtime = { version = "2.0.0-rc4", path = "../runtime" } +sc-executor = { version = "0.8.0-rc4", path = "../../../client/executor" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-io = { version = "2.0.0-rc4", path = "../../../primitives/io" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../../primitives/state-machine" } +sp-trie = { version = "2.0.0-rc4", path = "../../../primitives/trie" } trie-root = "0.16.0" -frame-benchmarking = { version = "2.0.0-alpha.8", path = "../../../frame/benchmarking" } +frame-benchmarking = { version = "2.0.0-rc4", path = "../../../frame/benchmarking" } [dev-dependencies] criterion = "0.3.0" -frame-support = { version = "2.0.0-alpha.8", path = "../../../frame/support" } -frame-system = { version = "2.0.0-alpha.8", path = "../../../frame/system" } -node-testing = { version = "2.0.0-alpha.8", path = "../testing" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../../../frame/balances" } -pallet-contracts = { version = "2.0.0-alpha.8", path = "../../../frame/contracts" } -pallet-grandpa = { version = "2.0.0-alpha.8", path = "../../../frame/grandpa" } -pallet-im-online = { version = "2.0.0-alpha.8", path = "../../../frame/im-online" } -pallet-indices = { version = "2.0.0-alpha.8", path = "../../../frame/indices" } -pallet-session = { version = "2.0.0-alpha.8", path = "../../../frame/session" } -pallet-timestamp = { version = "2.0.0-alpha.8", path = "../../../frame/timestamp" } -pallet-transaction-payment = { version = "2.0.0-alpha.8", path = "../../../frame/transaction-payment" } -pallet-treasury = { version = "2.0.0-alpha.8", path = "../../../frame/treasury" } -sp-application-crypto = { version = "2.0.0-alpha.8", path = "../../../primitives/application-crypto" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-externalities = { version = "0.8.0-alpha.8", path = "../../../primitives/externalities" } -substrate-test-client = { version = "2.0.0-alpha.8", path = "../../../test-utils/client" } +frame-support = { version = "2.0.0-rc4", path = "../../../frame/support" } +frame-system = { version = "2.0.0-rc4", path = "../../../frame/system" } +node-testing = { version = "2.0.0-rc4", path = "../testing" } +pallet-balances = { version = "2.0.0-rc4", path = "../../../frame/balances" } +pallet-contracts = { version = "2.0.0-rc4", path = "../../../frame/contracts" } +pallet-grandpa = { version = "2.0.0-rc4", path = "../../../frame/grandpa" } +pallet-im-online = { version = "2.0.0-rc4", path = "../../../frame/im-online" } +pallet-indices = { version = "2.0.0-rc4", path = "../../../frame/indices" } +pallet-session = { version = "2.0.0-rc4", path = "../../../frame/session" } +pallet-timestamp = { version = "2.0.0-rc4", path = "../../../frame/timestamp" } +pallet-transaction-payment = { version = "2.0.0-rc4", path = "../../../frame/transaction-payment" } +pallet-treasury = { version = "2.0.0-rc4", path = "../../../frame/treasury" } +sp-application-crypto = { version = "2.0.0-rc4", path = "../../../primitives/application-crypto" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-externalities = { version = "0.8.0-rc4", path = "../../../primitives/externalities" } +substrate-test-client = { version = "2.0.0-rc4", path = "../../../test-utils/client" } wabt = "0.9.2" [features] diff --git a/bin/node/executor/tests/basic.rs b/bin/node/executor/tests/basic.rs index 9350c3546ff8576d6716e9ba455455bbb6486cf1..e4de98d90e94d7bd961102cea0f868926bc2f9ba 100644 --- a/bin/node/executor/tests/basic.rs +++ b/bin/node/executor/tests/basic.rs @@ -19,12 +19,12 @@ use codec::{Encode, Decode, Joiner}; use frame_support::{ StorageValue, StorageMap, traits::Currency, - weights::{GetDispatchInfo, DispatchInfo, DispatchClass, constants::ExtrinsicBaseWeight}, + weights::{GetDispatchInfo, DispatchInfo, DispatchClass}, }; use sp_core::{NeverNativeValue, traits::Externalities, storage::well_known_keys}; use sp_runtime::{ - ApplyExtrinsicResult, Fixed128, - traits::{Hash as HashT, Convert}, + ApplyExtrinsicResult, + traits::Hash as HashT, transaction_validity::InvalidTransaction, }; use pallet_contracts::ContractAddressFor; @@ -32,7 +32,7 @@ use frame_system::{self, EventRecord, Phase}; use node_runtime::{ Header, Block, UncheckedExtrinsic, CheckedExtrinsic, Call, Runtime, Balances, - System, TransactionPayment, Event, TransactionByteFee, + System, TransactionPayment, Event, constants::currency::*, }; use node_primitives::{Balance, Hash}; @@ -49,16 +49,17 @@ use self::common::{*, sign}; /// test code paths that differ between native and wasm versions. pub const BLOATY_CODE: &[u8] = node_runtime::WASM_BINARY_BLOATY; -/// Default transfer fee -fn transfer_fee(extrinsic: &E, fee_multiplier: Fixed128) -> Balance { - let length_fee = TransactionByteFee::get() * (extrinsic.encode().len() as Balance); - - let base_weight = ExtrinsicBaseWeight::get(); - let base_fee = ::WeightToFee::convert(base_weight); - let weight = default_transfer_call().get_dispatch_info().weight; - let weight_fee = ::WeightToFee::convert(weight); - - base_fee + fee_multiplier.saturated_multiply_accumulate(length_fee + weight_fee) +/// Default transfer fee. This will use the same logic that is implemented in transaction-payment module. +/// +/// Note that reads the multiplier from storage directly, hence to get the fee of `extrinsic` +/// at block `n`, it must be called prior to executing block `n` to do the calculation with the +/// correct multiplier. +fn transfer_fee(extrinsic: &E) -> Balance { + TransactionPayment::compute_fee( + extrinsic.encode().len() as u32, + &default_transfer_call().get_dispatch_info(), + 0, + ) } fn xt() -> UncheckedExtrinsic { @@ -239,7 +240,7 @@ fn successful_execution_with_native_equivalent_code_gives_ok() { ).0; assert!(r.is_ok()); - let fm = t.execute_with(TransactionPayment::next_fee_multiplier); + let fees = t.execute_with(|| transfer_fee(&xt())); let r = executor_call:: _>( &mut t, @@ -251,7 +252,6 @@ fn successful_execution_with_native_equivalent_code_gives_ok() { assert!(r.is_ok()); t.execute_with(|| { - let fees = transfer_fee(&xt(), fm); assert_eq!(Balances::total_balance(&alice()), 42 * DOLLARS - fees); assert_eq!(Balances::total_balance(&bob()), 69 * DOLLARS); }); @@ -283,7 +283,7 @@ fn successful_execution_with_foreign_code_gives_ok() { ).0; assert!(r.is_ok()); - let fm = t.execute_with(TransactionPayment::next_fee_multiplier); + let fees = t.execute_with(|| transfer_fee(&xt())); let r = executor_call:: _>( &mut t, @@ -295,7 +295,6 @@ fn successful_execution_with_foreign_code_gives_ok() { assert!(r.is_ok()); t.execute_with(|| { - let fees = transfer_fee(&xt(), fm); assert_eq!(Balances::total_balance(&alice()), 42 * DOLLARS - fees); assert_eq!(Balances::total_balance(&bob()), 69 * DOLLARS); }); @@ -308,7 +307,7 @@ fn full_native_block_import_works() { let (block1, block2) = blocks(); let mut alice_last_known_balance: Balance = Default::default(); - let mut fm = t.execute_with(TransactionPayment::next_fee_multiplier); + let mut fees = t.execute_with(|| transfer_fee(&xt())); executor_call:: _>( &mut t, @@ -319,7 +318,6 @@ fn full_native_block_import_works() { ).0.unwrap(); t.execute_with(|| { - let fees = transfer_fee(&xt(), fm); assert_eq!(Balances::total_balance(&alice()), 42 * DOLLARS - fees); assert_eq!(Balances::total_balance(&bob()), 169 * DOLLARS); alice_last_known_balance = Balances::total_balance(&alice()); @@ -358,7 +356,7 @@ fn full_native_block_import_works() { assert_eq!(System::events(), events); }); - fm = t.execute_with(TransactionPayment::next_fee_multiplier); + fees = t.execute_with(|| transfer_fee(&xt())); executor_call:: _>( &mut t, @@ -369,7 +367,6 @@ fn full_native_block_import_works() { ).0.unwrap(); t.execute_with(|| { - let fees = transfer_fee(&xt(), fm); assert_eq!( Balances::total_balance(&alice()), alice_last_known_balance - 10 * DOLLARS - fees, @@ -447,7 +444,7 @@ fn full_wasm_block_import_works() { let (block1, block2) = blocks(); let mut alice_last_known_balance: Balance = Default::default(); - let mut fm = t.execute_with(TransactionPayment::next_fee_multiplier); + let mut fees = t.execute_with(|| transfer_fee(&xt())); executor_call:: _>( &mut t, @@ -458,12 +455,12 @@ fn full_wasm_block_import_works() { ).0.unwrap(); t.execute_with(|| { - assert_eq!(Balances::total_balance(&alice()), 42 * DOLLARS - transfer_fee(&xt(), fm)); + assert_eq!(Balances::total_balance(&alice()), 42 * DOLLARS - fees); assert_eq!(Balances::total_balance(&bob()), 169 * DOLLARS); alice_last_known_balance = Balances::total_balance(&alice()); }); - fm = t.execute_with(TransactionPayment::next_fee_multiplier); + fees = t.execute_with(|| transfer_fee(&xt())); executor_call:: _>( &mut t, @@ -476,11 +473,11 @@ fn full_wasm_block_import_works() { t.execute_with(|| { assert_eq!( Balances::total_balance(&alice()), - alice_last_known_balance - 10 * DOLLARS - transfer_fee(&xt(), fm), + alice_last_known_balance - 10 * DOLLARS - fees, ); assert_eq!( Balances::total_balance(&bob()), - 179 * DOLLARS - 1 * transfer_fee(&xt(), fm), + 179 * DOLLARS - 1 * fees, ); }); } @@ -752,7 +749,7 @@ fn successful_execution_gives_ok() { assert_eq!(Balances::total_balance(&alice()), 111 * DOLLARS); }); - let fm = t.execute_with(TransactionPayment::next_fee_multiplier); + let fees = t.execute_with(|| transfer_fee(&xt())); let r = executor_call:: _>( &mut t, @@ -767,7 +764,6 @@ fn successful_execution_gives_ok() { .expect("Extrinsic failed"); t.execute_with(|| { - let fees = transfer_fee(&xt(), fm); assert_eq!(Balances::total_balance(&alice()), 42 * DOLLARS - fees); assert_eq!(Balances::total_balance(&bob()), 69 * DOLLARS); }); diff --git a/bin/node/executor/tests/fees.rs b/bin/node/executor/tests/fees.rs index 0e55f781e78c20423df0ca1372ed70448c9c7c17..8f828263c5bdb5b17532caca29cebd935c8a7b3e 100644 --- a/bin/node/executor/tests/fees.rs +++ b/bin/node/executor/tests/fees.rs @@ -19,16 +19,15 @@ use codec::{Encode, Joiner}; use frame_support::{ StorageValue, StorageMap, traits::Currency, - weights::{GetDispatchInfo, constants::ExtrinsicBaseWeight}, + weights::{GetDispatchInfo, constants::ExtrinsicBaseWeight, IdentityFee, WeightToFeePolynomial}, }; use sp_core::NeverNativeValue; -use sp_runtime::{Fixed128, Perbill, traits::Convert}; +use sp_runtime::{Perbill, FixedPointNumber}; use node_runtime::{ - CheckedExtrinsic, Call, Runtime, Balances, TransactionPayment, - TransactionByteFee, WeightFeeCoefficient, + CheckedExtrinsic, Call, Runtime, Balances, TransactionPayment, Multiplier, + TransactionByteFee, constants::currency::*, }; -use node_runtime::impls::LinearWeightToFee; use node_primitives::Balance; use node_testing::keyring::*; @@ -39,8 +38,8 @@ use self::common::{*, sign}; fn fee_multiplier_increases_and_decreases_on_big_weight() { let mut t = new_test_ext(COMPACT_CODE, false); - // initial fee multiplier must be zero - let mut prev_multiplier = Fixed128::from_parts(0); + // initial fee multiplier must be one. + let mut prev_multiplier = Multiplier::one(); t.execute_with(|| { assert_eq!(TransactionPayment::next_fee_multiplier(), prev_multiplier); @@ -60,7 +59,7 @@ fn fee_multiplier_increases_and_decreases_on_big_weight() { }, CheckedExtrinsic { signed: Some((charlie(), signed_extra(0, 0))), - function: Call::System(frame_system::Call::fill_block(Perbill::from_percent(90))), + function: Call::System(frame_system::Call::fill_block(Perbill::from_percent(60))), } ] ); @@ -123,7 +122,7 @@ fn fee_multiplier_increases_and_decreases_on_big_weight() { } #[test] -fn transaction_fee_is_correct_ultimate() { +fn transaction_fee_is_correct() { // This uses the exact values of substrate-node. // // weight of transfer call as of now: 1_000_000 @@ -181,13 +180,13 @@ fn transaction_fee_is_correct_ultimate() { let mut balance_alice = (100 - 69) * DOLLARS; let base_weight = ExtrinsicBaseWeight::get(); - let base_fee = LinearWeightToFee::::convert(base_weight); + let base_fee = IdentityFee::::calc(&base_weight); let length_fee = TransactionByteFee::get() * (xt.clone().encode().len() as Balance); balance_alice -= length_fee; let weight = default_transfer_call().get_dispatch_info().weight; - let weight_fee = LinearWeightToFee::::convert(weight); + let weight_fee = IdentityFee::::calc(&weight); // we know that weight to fee multiplier is effect-less in block 1. // current weight of transfer = 200_000_000 diff --git a/bin/node/inspect/Cargo.toml b/bin/node/inspect/Cargo.toml index c16bb0493ae05f29eb533b8478470e99ce1735b0..e76f215a998142516e52d2996300a073a02cdf59 100644 --- a/bin/node/inspect/Cargo.toml +++ b/bin/node/inspect/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-inspect" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -11,13 +11,13 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0" } +codec = { package = "parity-scale-codec", version = "1.3.1" } derive_more = "0.99" log = "0.4.8" -sc-cli = { version = "0.8.0-alpha.8", path = "../../../client/cli" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../../../client/api" } -sc-service = { version = "0.8.0-alpha.8", default-features = false, path = "../../../client/service" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } +sc-cli = { version = "0.8.0-rc4", path = "../../../client/cli" } +sc-client-api = { version = "2.0.0-rc4", path = "../../../client/api" } +sc-service = { version = "0.8.0-rc4", default-features = false, path = "../../../client/service" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } structopt = "0.3.8" diff --git a/bin/node/inspect/src/cli.rs b/bin/node/inspect/src/cli.rs index 3a71407ab03334e34c1297b0f18d86c73a488afd..d66644bab52fa78788098e3713ceae1576f25b6c 100644 --- a/bin/node/inspect/src/cli.rs +++ b/bin/node/inspect/src/cli.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -23,7 +23,7 @@ use sc_cli::{ImportParams, SharedParams}; use structopt::StructOpt; /// The `inspect` command used to print decoded chain data. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct InspectCmd { #[allow(missing_docs)] #[structopt(flatten)] @@ -39,7 +39,7 @@ pub struct InspectCmd { } /// A possible inspect sub-commands. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub enum InspectSubCmd { /// Decode block with native version of runtime and print out the details. Block { diff --git a/bin/node/inspect/src/command.rs b/bin/node/inspect/src/command.rs index c409aee86ea45040e05b974bd00837136a595536..fae6c10c7fe7846dc63e990dcf03254655467705 100644 --- a/bin/node/inspect/src/command.rs +++ b/bin/node/inspect/src/command.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/node/inspect/src/lib.rs b/bin/node/inspect/src/lib.rs index 5123fc595987251a9c4061e7b0b6f9b746a04ff8..02f5614b81a78044d1418d9d23f87322da2c298c 100644 --- a/bin/node/inspect/src/lib.rs +++ b/bin/node/inspect/src/lib.rs @@ -1,18 +1,18 @@ // This file is part of Substrate. -// +// // Copyright (C) 2020 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 -// +// // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. -// +// // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. -// +// // You should have received a copy of the GNU General Public License // along with this program. If not, see . diff --git a/bin/node/primitives/Cargo.toml b/bin/node/primitives/Cargo.toml index 96f8428a1c4fa6a50434c9549dfa91a155f2d87b..0a66336046caafc81ef7dd9a4d9da1e122617407 100644 --- a/bin/node/primitives/Cargo.toml +++ b/bin/node/primitives/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-primitives" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -11,14 +11,14 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/system" } -sp-application-crypto = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/application-crypto" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/system" } +sp-application-crypto = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/application-crypto" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/runtime" } [dev-dependencies] -sp-serializer = { version = "2.0.0-alpha.8", path = "../../../primitives/serializer" } +sp-serializer = { version = "2.0.0-rc4", path = "../../../primitives/serializer" } pretty_assertions = "0.6.1" [features] diff --git a/bin/node/rpc-client/Cargo.toml b/bin/node/rpc-client/Cargo.toml index 80ca0063b2cd94a50af2b4e88d141756121d6dba..2d21746f2aeb275b7db28037e59e9aa02a3c9c14 100644 --- a/bin/node/rpc-client/Cargo.toml +++ b/bin/node/rpc-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-rpc-client" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -14,7 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"] env_logger = "0.7.0" futures = "0.1.29" hyper = "0.12.35" -jsonrpc-core-client = { version = "14.0.5", default-features = false, features = ["http"] } +jsonrpc-core-client = { version = "14.2.0", default-features = false, features = ["http"] } log = "0.4.8" -node-primitives = { version = "2.0.0-alpha.8", path = "../primitives" } -sc-rpc = { version = "2.0.0-alpha.8", path = "../../../client/rpc" } +node-primitives = { version = "2.0.0-rc4", path = "../primitives" } +sc-rpc = { version = "2.0.0-rc4", path = "../../../client/rpc" } diff --git a/bin/node/rpc/Cargo.toml b/bin/node/rpc/Cargo.toml index bd8a8bd3127287828ddf4a173bc0f1508db5aac2..95d55fab6402c9a0dff73bbbb8c19bd6fd8823ef 100644 --- a/bin/node/rpc/Cargo.toml +++ b/bin/node/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-rpc" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -11,22 +11,24 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sc-client-api = { version = "2.0.0-alpha.8", path = "../../../client/api" } -jsonrpc-core = "14.0.3" -node-primitives = { version = "2.0.0-alpha.8", path = "../primitives" } -node-runtime = { version = "2.0.0-alpha.8", path = "../runtime" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.8", path = "../../../primitives/api" } -pallet-contracts-rpc = { version = "0.8.0-alpha.8", path = "../../../frame/contracts/rpc/" } -pallet-transaction-payment-rpc = { version = "2.0.0-alpha.8", path = "../../../frame/transaction-payment/rpc/" } -substrate-frame-rpc-system = { version = "2.0.0-alpha.8", path = "../../../utils/frame/rpc/system" } -sp-transaction-pool = { version = "2.0.0-alpha.8", path = "../../../primitives/transaction-pool" } -sc-consensus-babe = { version = "0.8.0-alpha.8", path = "../../../client/consensus/babe" } -sc-consensus-babe-rpc = { version = "0.8.0-alpha.8", path = "../../../client/consensus/babe/rpc" } -sp-consensus-babe = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/babe" } -sc-keystore = { version = "2.0.0-alpha.8", path = "../../../client/keystore" } -sc-consensus-epochs = { version = "0.8.0-alpha.8", path = "../../../client/consensus/epochs" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/common" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } -sc-finality-grandpa = { version = "0.8.0-alpha.8", path = "../../../client/finality-grandpa" } -sc-finality-grandpa-rpc = { version = "0.8.0-alpha.8", path = "../../../client/finality-grandpa/rpc" } +sc-client-api = { version = "2.0.0-rc4", path = "../../../client/api" } +jsonrpc-core = "14.2.0" +node-primitives = { version = "2.0.0-rc4", path = "../primitives" } +node-runtime = { version = "2.0.0-rc4", path = "../runtime" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-rc4", path = "../../../primitives/api" } +pallet-contracts-rpc = { version = "0.8.0-rc4", path = "../../../frame/contracts/rpc/" } +pallet-transaction-payment-rpc = { version = "2.0.0-rc4", path = "../../../frame/transaction-payment/rpc/" } +substrate-frame-rpc-system = { version = "2.0.0-rc4", path = "../../../utils/frame/rpc/system" } +sp-transaction-pool = { version = "2.0.0-rc4", path = "../../../primitives/transaction-pool" } +sc-consensus-babe = { version = "0.8.0-rc4", path = "../../../client/consensus/babe" } +sc-consensus-babe-rpc = { version = "0.8.0-rc4", path = "../../../client/consensus/babe/rpc" } +sp-consensus-babe = { version = "0.8.0-rc4", path = "../../../primitives/consensus/babe" } +sc-keystore = { version = "2.0.0-rc4", path = "../../../client/keystore" } +sc-consensus-epochs = { version = "0.8.0-rc4", path = "../../../client/consensus/epochs" } +sp-consensus = { version = "0.8.0-rc4", path = "../../../primitives/consensus/common" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } +sc-finality-grandpa = { version = "0.8.0-rc4", path = "../../../client/finality-grandpa" } +sc-finality-grandpa-rpc = { version = "0.8.0-rc4", path = "../../../client/finality-grandpa/rpc" } +sc-rpc-api = { version = "0.8.0-rc4", path = "../../../client/rpc-api" } +sp-block-builder = { version = "2.0.0-rc4", path = "../../../primitives/block-builder" } diff --git a/bin/node/rpc/src/lib.rs b/bin/node/rpc/src/lib.rs index 02cb44d4020f761fd8d3f2923b14ef5311c5c4c4..9b6b5991748f96e60ff0ea20d126faaa315637fb 100644 --- a/bin/node/rpc/src/lib.rs +++ b/bin/node/rpc/src/lib.rs @@ -30,7 +30,7 @@ #![warn(missing_docs)] -use std::{sync::Arc, fmt}; +use std::sync::Arc; use node_primitives::{Block, BlockNumber, AccountId, Index, Balance, Hash}; use node_runtime::UncheckedExtrinsic; @@ -42,9 +42,11 @@ use sc_keystore::KeyStorePtr; use sp_consensus_babe::BabeApi; use sc_consensus_epochs::SharedEpochChanges; use sc_consensus_babe::{Config, Epoch}; -use sc_consensus_babe_rpc::BabeRPCHandler; +use sc_consensus_babe_rpc::BabeRpcHandler; use sc_finality_grandpa::{SharedVoterState, SharedAuthoritySet}; use sc_finality_grandpa_rpc::GrandpaRpcHandler; +use sc_rpc_api::DenyUnsafe; +use sp_block_builder::BlockBuilder; /// Light client extra dependencies. pub struct LightDeps { @@ -84,6 +86,8 @@ pub struct FullDeps { pub pool: Arc

, /// The SelectChain Strategy pub select_chain: SC, + /// Whether to deny unsafe calls + pub deny_unsafe: DenyUnsafe, /// BABE specific dependencies. pub babe: BabeDeps, /// GRANDPA specific dependencies. @@ -101,7 +105,7 @@ pub fn create_full( C::Api: pallet_contracts_rpc::ContractsRuntimeApi, C::Api: pallet_transaction_payment_rpc::TransactionPaymentRuntimeApi, C::Api: BabeApi, - ::Error: fmt::Debug, + C::Api: BlockBuilder, P: TransactionPool + 'static, M: jsonrpc_core::Metadata + Default, SC: SelectChain +'static, @@ -115,6 +119,7 @@ pub fn create_full( client, pool, select_chain, + deny_unsafe, babe, grandpa, } = deps; @@ -129,7 +134,7 @@ pub fn create_full( } = grandpa; io.extend_with( - SystemApi::to_delegate(FullSystem::new(client.clone(), pool)) + SystemApi::to_delegate(FullSystem::new(client.clone(), pool, deny_unsafe)) ); // Making synchronous calls in light client freezes the browser currently, // more context: https://github.com/paritytech/substrate/pull/3480 @@ -142,7 +147,14 @@ pub fn create_full( ); io.extend_with( sc_consensus_babe_rpc::BabeApi::to_delegate( - BabeRPCHandler::new(client, shared_epoch_changes, keystore, babe_config, select_chain) + BabeRpcHandler::new( + client, + shared_epoch_changes, + keystore, + babe_config, + select_chain, + deny_unsafe, + ), ) ); io.extend_with( @@ -174,7 +186,7 @@ pub fn create_light( } = deps; let mut io = jsonrpc_core::IoHandler::default(); io.extend_with( - SystemApi::::to_delegate(LightSystem::new(client, remote_blockchain, fetcher, pool)) + SystemApi::::to_delegate(LightSystem::new(client, remote_blockchain, fetcher, pool)) ); io diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index afcc63d5d71a58b18e84f1327659b9cdb1f83548..568b1afb5eb3d1990ef88947e335e3e942a1098d 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-runtime" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" @@ -14,74 +14,77 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] # third-party dependencies -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } integer-sqrt = { version = "0.1.2" } serde = { version = "1.0.102", optional = true } static_assertions = "1.1.0" +hex-literal = { version = "0.2.1", optional = true } # primitives -sp-authority-discovery = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/authority-discovery" } -sp-consensus-babe = { version = "0.8.0-alpha.8", default-features = false, path = "../../../primitives/consensus/babe" } -sp-block-builder = { path = "../../../primitives/block-builder", default-features = false, version = "2.0.0-alpha.8"} -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/inherents" } -node-primitives = { version = "2.0.0-alpha.8", default-features = false, path = "../primitives" } -sp-offchain = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/offchain" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/core" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/std" } -sp-api = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/api" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/runtime" } -sp-staking = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/staking" } -sp-keyring = { version = "2.0.0-alpha.8", optional = true, path = "../../../primitives/keyring" } -sp-session = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/session" } -sp-transaction-pool = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/transaction-pool" } -sp-version = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/version" } +sp-authority-discovery = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/authority-discovery" } +sp-consensus-babe = { version = "0.8.0-rc4", default-features = false, path = "../../../primitives/consensus/babe" } +sp-block-builder = { path = "../../../primitives/block-builder", default-features = false, version = "2.0.0-rc4"} +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/inherents" } +node-primitives = { version = "2.0.0-rc4", default-features = false, path = "../primitives" } +sp-offchain = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/offchain" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/core" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/std" } +sp-api = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/api" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/runtime" } +sp-staking = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/staking" } +sp-keyring = { version = "2.0.0-rc4", optional = true, path = "../../../primitives/keyring" } +sp-session = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/session" } +sp-transaction-pool = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/transaction-pool" } +sp-version = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/version" } # frame dependencies -frame-executive = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/executive" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/benchmarking", optional = true } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/system" } -frame-system-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/system/benchmarking", optional = true } -frame-system-rpc-runtime-api = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } -pallet-authority-discovery = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/authority-discovery" } -pallet-authorship = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/authorship" } -pallet-babe = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/babe" } -pallet-balances = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/balances" } -pallet-collective = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/collective" } -pallet-contracts = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/contracts" } -pallet-contracts-primitives = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/contracts/common/" } -pallet-contracts-rpc-runtime-api = { version = "0.8.0-alpha.8", default-features = false, path = "../../../frame/contracts/rpc/runtime-api/" } -pallet-democracy = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/democracy" } -pallet-elections-phragmen = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/elections-phragmen" } -pallet-finality-tracker = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/finality-tracker" } -pallet-grandpa = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/grandpa" } -pallet-im-online = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/im-online" } -pallet-indices = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/indices" } -pallet-identity = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/identity" } -pallet-membership = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/membership" } -pallet-offences = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/offences" } -pallet-offences-benchmarking = { version = "2.0.0-alpha.8", path = "../../../frame/offences/benchmarking", default-features = false, optional = true } -pallet-randomness-collective-flip = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/randomness-collective-flip" } -pallet-recovery = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/recovery" } -pallet-session = { version = "2.0.0-alpha.8", features = ["historical"], path = "../../../frame/session", default-features = false } -pallet-session-benchmarking = { version = "2.0.0-alpha.8", path = "../../../frame/session/benchmarking", default-features = false, optional = true } -pallet-staking = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/staking" } -pallet-staking-reward-curve = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/staking/reward-curve" } -pallet-scheduler = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/scheduler" } -pallet-society = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/society" } -pallet-sudo = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/sudo" } -pallet-timestamp = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/timestamp" } -pallet-treasury = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/treasury" } -pallet-utility = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/utility" } -pallet-transaction-payment = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/transaction-payment" } -pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" } -pallet-vesting = { version = "2.0.0-alpha.8", default-features = false, path = "../../../frame/vesting" } +frame-executive = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/executive" } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/benchmarking", optional = true } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/system" } +frame-system-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/system/benchmarking", optional = true } +frame-system-rpc-runtime-api = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } +pallet-authority-discovery = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/authority-discovery" } +pallet-authorship = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/authorship" } +pallet-babe = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/babe" } +pallet-balances = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/balances" } +pallet-collective = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/collective" } +pallet-contracts = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/contracts" } +pallet-contracts-primitives = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/contracts/common/" } +pallet-contracts-rpc-runtime-api = { version = "0.8.0-rc4", default-features = false, path = "../../../frame/contracts/rpc/runtime-api/" } +pallet-democracy = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/democracy" } +pallet-elections-phragmen = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/elections-phragmen" } +pallet-finality-tracker = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/finality-tracker" } +pallet-grandpa = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/grandpa" } +pallet-im-online = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/im-online" } +pallet-indices = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/indices" } +pallet-identity = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/identity" } +pallet-membership = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/membership" } +pallet-multisig = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/multisig" } +pallet-offences = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/offences" } +pallet-offences-benchmarking = { version = "2.0.0-rc4", path = "../../../frame/offences/benchmarking", default-features = false, optional = true } +pallet-proxy = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/proxy" } +pallet-randomness-collective-flip = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/randomness-collective-flip" } +pallet-recovery = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/recovery" } +pallet-session = { version = "2.0.0-rc4", features = ["historical"], path = "../../../frame/session", default-features = false } +pallet-session-benchmarking = { version = "2.0.0-rc4", path = "../../../frame/session/benchmarking", default-features = false, optional = true } +pallet-staking = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/staking" } +pallet-staking-reward-curve = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/staking/reward-curve" } +pallet-scheduler = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/scheduler" } +pallet-society = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/society" } +pallet-sudo = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/sudo" } +pallet-timestamp = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/timestamp" } +pallet-treasury = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/treasury" } +pallet-utility = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/utility" } +pallet-transaction-payment = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/transaction-payment" } +pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" } +pallet-vesting = { version = "2.0.0-rc4", default-features = false, path = "../../../frame/vesting" } [build-dependencies] wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } [dev-dependencies] -sp-io = { version = "2.0.0-alpha.8", path = "../../../primitives/io" } +sp-io = { version = "2.0.0-rc4", path = "../../../primitives/io" } [features] default = ["std"] @@ -107,10 +110,13 @@ std = [ "pallet-indices/std", "sp-inherents/std", "pallet-membership/std", + "pallet-multisig/std", "pallet-identity/std", + "pallet-scheduler/std", "node-primitives/std", "sp-offchain/std", "pallet-offences/std", + "pallet-proxy/std", "sp-core/std", "pallet-randomness-collective-flip/std", "sp-std/std", @@ -149,6 +155,9 @@ runtime-benchmarks = [ "pallet-elections-phragmen/runtime-benchmarks", "pallet-identity/runtime-benchmarks", "pallet-im-online/runtime-benchmarks", + "pallet-indices/runtime-benchmarks", + "pallet-multisig/runtime-benchmarks", + "pallet-proxy/runtime-benchmarks", "pallet-scheduler/runtime-benchmarks", "pallet-society/runtime-benchmarks", "pallet-staking/runtime-benchmarks", @@ -159,4 +168,5 @@ runtime-benchmarks = [ "pallet-offences-benchmarking", "pallet-session-benchmarking", "frame-system-benchmarking", + "hex-literal", ] diff --git a/bin/node/runtime/build.rs b/bin/node/runtime/build.rs index cd5db582f385c8e0310a9dcbad07857df566539d..a4f323566006cf7c003cec2e413e1d23ecffead1 100644 --- a/bin/node/runtime/build.rs +++ b/bin/node/runtime/build.rs @@ -20,7 +20,7 @@ use wasm_builder_runner::WasmBuilder; fn main() { WasmBuilder::new() .with_current_project() - .with_wasm_builder_from_crates_or_path("1.0.9", "../../../utils/wasm-builder") + .with_wasm_builder_from_crates_or_path("1.0.11", "../../../utils/wasm-builder") .export_heap_base() .import_memory() .build() diff --git a/bin/node/runtime/src/constants.rs b/bin/node/runtime/src/constants.rs index 45f1ab19a450ecc69cbe54de6ba09b32428dafab..8e87d61c1e6b597ccb0eb114351befe969264ddf 100644 --- a/bin/node/runtime/src/constants.rs +++ b/bin/node/runtime/src/constants.rs @@ -24,6 +24,10 @@ pub mod currency { pub const MILLICENTS: Balance = 1_000_000_000; pub const CENTS: Balance = 1_000 * MILLICENTS; // assume this is worth about a cent. pub const DOLLARS: Balance = 100 * CENTS; + + pub const fn deposit(items: u32, bytes: u32) -> Balance { + items as Balance * 15 * CENTS + (bytes as Balance) * 6 * CENTS + } } /// Time. diff --git a/bin/node/runtime/src/impls.rs b/bin/node/runtime/src/impls.rs index ef994392b52c0a882096d79325414964f76c189a..039093ddee69700594b70200595a39b07f7e7fa4 100644 --- a/bin/node/runtime/src/impls.rs +++ b/bin/node/runtime/src/impls.rs @@ -17,12 +17,10 @@ //! Some configurable implementations as associated type for the substrate runtime. -use core::num::NonZeroI128; use node_primitives::Balance; -use sp_runtime::traits::{Convert, Saturating}; -use sp_runtime::{Fixed128, Perquintill}; -use frame_support::{traits::{OnUnbalanced, Currency, Get}, weights::Weight}; -use crate::{Balances, System, Authorship, MaximumBlockWeight, NegativeImbalance}; +use sp_runtime::traits::Convert; +use frame_support::traits::{OnUnbalanced, Currency}; +use crate::{Balances, Authorship, NegativeImbalance}; pub struct Author; impl OnUnbalanced for Author { @@ -47,106 +45,63 @@ impl Convert for CurrencyToVoteHandler { fn convert(x: u128) -> Balance { x * Self::factor() } } -/// Convert from weight to balance via a simple coefficient multiplication -/// The associated type C encapsulates a constant in units of balance per weight -pub struct LinearWeightToFee(sp_std::marker::PhantomData); - -impl> Convert for LinearWeightToFee { - fn convert(w: Weight) -> Balance { - // setting this to zero will disable the weight fee. - let coefficient = C::get(); - Balance::from(w).saturating_mul(coefficient) - } -} - -/// Update the given multiplier based on the following formula -/// -/// diff = (previous_block_weight - target_weight)/max_weight -/// v = 0.00004 -/// next_weight = weight * (1 + (v * diff) + (v * diff)^2 / 2) -/// -/// Where `target_weight` must be given as the `Get` implementation of the `T` generic type. -/// https://research.web3.foundation/en/latest/polkadot/Token%20Economics/#relay-chain-transaction-fees -pub struct TargetedFeeAdjustment(sp_std::marker::PhantomData); - -impl> Convert for TargetedFeeAdjustment { - fn convert(multiplier: Fixed128) -> Fixed128 { - let max_weight = MaximumBlockWeight::get(); - let block_weight = System::all_extrinsics_weight().total().min(max_weight); - let target_weight = (T::get() * max_weight) as u128; - let block_weight = block_weight as u128; - - // determines if the first_term is positive - let positive = block_weight >= target_weight; - let diff_abs = block_weight.max(target_weight) - block_weight.min(target_weight); - // safe, diff_abs cannot exceed u64 and it can always be computed safely even with the lossy - // `Fixed128::from_rational`. - let diff = Fixed128::from_rational( - diff_abs as i128, - NonZeroI128::new(max_weight.max(1) as i128).unwrap(), - ); - let diff_squared = diff.saturating_mul(diff); - - // 0.00004 = 4/100_000 = 40_000/10^9 - let v = Fixed128::from_rational(4, NonZeroI128::new(100_000).unwrap()); - // 0.00004^2 = 16/10^10 Taking the future /2 into account... 8/10^10 - let v_squared_2 = Fixed128::from_rational(8, NonZeroI128::new(10_000_000_000).unwrap()); - - let first_term = v.saturating_mul(diff); - let second_term = v_squared_2.saturating_mul(diff_squared); - - if positive { - // Note: this is merely bounded by how big the multiplier and the inner value can go, - // not by any economical reasoning. - let excess = first_term.saturating_add(second_term); - multiplier.saturating_add(excess) - } else { - // Defensive-only: first_term > second_term. Safe subtraction. - let negative = first_term.saturating_sub(second_term); - multiplier.saturating_sub(negative) - // despite the fact that apply_to saturates weight (final fee cannot go below 0) - // it is crucially important to stop here and don't further reduce the weight fee - // multiplier. While at -1, it means that the network is so un-congested that all - // transactions have no weight fee. We stop here and only increase if the network - // became more busy. - .max(Fixed128::from_natural(-1)) - } - } -} - #[cfg(test)] -mod tests { +mod multiplier_tests { use super::*; - use sp_runtime::assert_eq_error_rate; - use crate::{MaximumBlockWeight, AvailableBlockRatio, Runtime}; - use crate::{constants::currency::*, TransactionPayment, TargetBlockFullness}; - use frame_support::weights::Weight; - use core::num::NonZeroI128; + use sp_runtime::{assert_eq_error_rate, FixedPointNumber}; + use pallet_transaction_payment::{Multiplier, TargetedFeeAdjustment}; + + use crate::{ + constants::{currency::*, time::*}, + TransactionPayment, MaximumBlockWeight, AvailableBlockRatio, Runtime, TargetBlockFullness, + AdjustmentVariable, System, MinimumMultiplier, + }; + use frame_support::weights::{Weight, WeightToFeePolynomial}; fn max() -> Weight { - MaximumBlockWeight::get() + AvailableBlockRatio::get() * MaximumBlockWeight::get() + } + + fn min_multiplier() -> Multiplier { + MinimumMultiplier::get() } fn target() -> Weight { TargetBlockFullness::get() * max() } - // poc reference implementation. - fn fee_multiplier_update(block_weight: Weight, previous: Fixed128) -> Fixed128 { + // update based on runtime impl. + fn runtime_multiplier_update(fm: Multiplier) -> Multiplier { + TargetedFeeAdjustment::< + Runtime, + TargetBlockFullness, + AdjustmentVariable, + MinimumMultiplier, + >::convert(fm) + } + + // update based on reference impl. + fn truth_value_update(block_weight: Weight, previous: Multiplier) -> Multiplier { + let accuracy = Multiplier::accuracy() as f64; + let previous_float = previous.into_inner() as f64 / accuracy; + // bump if it is zero. + let previous_float = previous_float.max(min_multiplier().into_inner() as f64 / accuracy); + // maximum tx weight let m = max() as f64; // block weight always truncated to max weight let block_weight = (block_weight as f64).min(m); - let v: f64 = 0.00004; + let v: f64 = AdjustmentVariable::get().to_fraction(); // Ideal saturation in terms of weight let ss = target() as f64; // Current saturation in terms of weight let s = block_weight; - let fm = v * (s/m - ss/m) + v.powi(2) * (s/m - ss/m).powi(2) / 2.0; - let addition_fm = Fixed128::from_parts((fm * Fixed128::accuracy() as f64).round() as i128); - previous.saturating_add(addition_fm) + let t1 = v * (s/m - ss/m); + let t2 = v.powi(2) * (s/m - ss/m).powi(2) / 2.0; + let next_float = previous_float * (1.0 + t1 + t2); + Multiplier::from_fraction(next_float) } fn run_with_system_weight(w: Weight, assertions: F) where F: Fn() -> () { @@ -159,11 +114,12 @@ mod tests { } #[test] - fn fee_multiplier_update_poc_works() { - let fm = Fixed128::from_rational(0, NonZeroI128::new(1).unwrap()); + fn truth_value_update_poc_works() { + let fm = Multiplier::saturating_from_rational(1, 2); let test_set = vec![ (0, fm.clone()), (100, fm.clone()), + (1000, fm.clone()), (target(), fm.clone()), (max() / 2, fm.clone()), (max(), fm.clone()), @@ -171,37 +127,71 @@ mod tests { test_set.into_iter().for_each(|(w, fm)| { run_with_system_weight(w, || { assert_eq_error_rate!( - fee_multiplier_update(w, fm), - TargetedFeeAdjustment::::convert(fm), - // Error is only 1 in 10^18 - Fixed128::from_parts(1), + truth_value_update(w, fm), + runtime_multiplier_update(fm), + // Error is only 1 in 100^18 + Multiplier::from_inner(100), ); }) }) } #[test] - fn empty_chain_simulation() { - // just a few txs per_block. - let block_weight = 0; - run_with_system_weight(block_weight, || { - let mut fm = Fixed128::default(); + fn multiplier_can_grow_from_zero() { + // if the min is too small, then this will not change, and we are doomed forever. + // the weight is 1/100th bigger than target. + run_with_system_weight(target() * 101 / 100, || { + let next = runtime_multiplier_update(min_multiplier()); + assert!(next > min_multiplier(), "{:?} !>= {:?}", next, min_multiplier()); + }) + } + + #[test] + fn multiplier_cannot_go_below_limit() { + // will not go any further below even if block is empty. + run_with_system_weight(0, || { + let next = runtime_multiplier_update(min_multiplier()); + assert_eq!(next, min_multiplier()); + }) + } + + #[test] + fn time_to_reach_zero() { + // blocks per 24h in substrate-node: 28,800 (k) + // s* = 0.1875 + // The bound from the research in an empty chain is: + // v <~ (p / k(0 - s*)) + // p > v * k * -0.1875 + // to get p == -1 we'd need + // -1 > 0.00001 * k * -0.1875 + // 1 < 0.00001 * k * 0.1875 + // 10^9 / 1875 < k + // k > 533_333 ~ 18,5 days. + run_with_system_weight(0, || { + // start from 1, the default. + let mut fm = Multiplier::one(); let mut iterations: u64 = 0; loop { - let next = TargetedFeeAdjustment::::convert(fm); + let next = runtime_multiplier_update(fm); fm = next; - if fm == Fixed128::from_natural(-1) { break; } + if fm == min_multiplier() { break; } iterations += 1; } - println!("iteration {}, new fm = {:?}. Weight fee is now zero", iterations, fm); - assert!(iterations > 50_000, "This assertion is just a warning; Don't panic. \ - Current substrate/polkadot node are configured with a _slow adjusting fee_ \ - mechanism. Hence, it is really unlikely that fees collapse to zero even on an \ - empty chain in less than at least of couple of thousands of empty blocks. But this \ - simulation indicates that fees collapsed to zero after {} almost-empty blocks. \ - Check it", - iterations, - ); + assert!(iterations > 533_333); + }) + } + + #[test] + fn min_change_per_day() { + run_with_system_weight(max(), || { + let mut fm = Multiplier::one(); + // See the example in the doc of `TargetedFeeAdjustment`. are at least 0.234, hence + // `fm > 1.234`. + for _ in 0..DAYS { + let next = runtime_multiplier_update(fm); + fm = next; + } + assert!(fm > Multiplier::saturating_from_rational(1234, 1000)); }) } @@ -213,23 +203,24 @@ mod tests { // almost full. The entire quota of normal transactions is taken. let block_weight = AvailableBlockRatio::get() * max() - 100; - // Default substrate minimum. - let tx_weight = 10_000; + // Default substrate weight. + let tx_weight = frame_support::weights::constants::ExtrinsicBaseWeight::get(); run_with_system_weight(block_weight, || { // initial value configured on module - let mut fm = Fixed128::default(); + let mut fm = Multiplier::one(); assert_eq!(fm, TransactionPayment::next_fee_multiplier()); let mut iterations: u64 = 0; loop { - let next = TargetedFeeAdjustment::::convert(fm); + let next = runtime_multiplier_update(fm); // if no change, panic. This should never happen in this case. if fm == next { panic!("The fee should ever increase"); } fm = next; iterations += 1; - let fee = ::WeightToFee::convert(tx_weight); - let adjusted_fee = fm.saturated_multiply_accumulate(fee); + let fee = + ::WeightToFee::calc(&tx_weight); + let adjusted_fee = fm.saturating_mul_acc_int(fee); println!( "iteration {}, new fm = {:?}. Fee at this point is: {} units / {} millicents, \ {} cents, {} dollars", @@ -246,95 +237,86 @@ mod tests { #[test] fn stateless_weight_mul() { - // This test will show that heavy blocks have a weight multiplier greater than 0 - // and light blocks will have a weight multiplier less than 0. + let fm = Multiplier::saturating_from_rational(1, 2); run_with_system_weight(target() / 4, || { - // `fee_multiplier_update` is enough as it is the absolute truth value. - let next = TargetedFeeAdjustment::::convert(Fixed128::default()); - assert_eq!( + let next = runtime_multiplier_update(fm); + assert_eq_error_rate!( next, - fee_multiplier_update(target() / 4 ,Fixed128::default()) + truth_value_update(target() / 4 , fm), + Multiplier::from_inner(100), ); - // Light block. Fee is reduced a little. - assert!(next < Fixed128::zero()) + // Light block. Multiplier is reduced a little. + assert!(next < fm); }); + run_with_system_weight(target() / 2, || { - let next = TargetedFeeAdjustment::::convert(Fixed128::default()); - assert_eq!( + let next = runtime_multiplier_update(fm); + assert_eq_error_rate!( next, - fee_multiplier_update(target() / 2 ,Fixed128::default()) + truth_value_update(target() / 2 , fm), + Multiplier::from_inner(100), ); - - // Light block. Fee is reduced a little. - assert!(next < Fixed128::zero()) + // Light block. Multiplier is reduced a little. + assert!(next < fm); }); run_with_system_weight(target(), || { - // ideal. Original fee. No changes. - let next = TargetedFeeAdjustment::::convert(Fixed128::default()); - assert_eq!(next, Fixed128::zero()) + let next = runtime_multiplier_update(fm); + assert_eq_error_rate!( + next, + truth_value_update(target(), fm), + Multiplier::from_inner(100), + ); + // ideal. No changes. + assert_eq!(next, fm) }); run_with_system_weight(target() * 2, || { // More than ideal. Fee is increased. - let next = TargetedFeeAdjustment::::convert(Fixed128::default()); - assert_eq!( + let next = runtime_multiplier_update(fm); + assert_eq_error_rate!( next, - fee_multiplier_update(target() * 2 ,Fixed128::default()) + truth_value_update(target() * 2 , fm), + Multiplier::from_inner(100), ); // Heavy block. Fee is increased a little. - assert!(next > Fixed128::zero()) + assert!(next > fm); }); } #[test] - fn stateful_weight_mul_grow_to_infinity() { + fn weight_mul_grow_on_big_block() { run_with_system_weight(target() * 2, || { - let mut original = Fixed128::default(); - let mut next = Fixed128::default(); + let mut original = Multiplier::zero(); + let mut next = Multiplier::default(); (0..1_000).for_each(|_| { - next = TargetedFeeAdjustment::::convert(original); - assert_eq!( + next = runtime_multiplier_update(original); + assert_eq_error_rate!( next, - fee_multiplier_update(target() * 2, original), + truth_value_update(target() * 2, original), + Multiplier::from_inner(100), ); // must always increase - assert!(next > original); + assert!(next > original, "{:?} !>= {:?}", next, original); original = next; }); }); } #[test] - fn stateful_weight_mil_collapse_to_minus_one() { - run_with_system_weight(0, || { - let mut original = Fixed128::default(); // 0 + fn weight_mul_decrease_on_small_block() { + run_with_system_weight(target() / 2, || { + let mut original = Multiplier::saturating_from_rational(1, 2); let mut next; - // decreases - next = TargetedFeeAdjustment::::convert(original); - assert_eq!( - next, - fee_multiplier_update(0, original), - ); - assert!(next < original); - original = next; - - // keeps decreasing - next = TargetedFeeAdjustment::::convert(original); - assert_eq!( - next, - fee_multiplier_update(0, original), - ); - assert!(next < original); - - // ... stops going down at -1 - assert_eq!( - TargetedFeeAdjustment::::convert(Fixed128::from_natural(-1)), - Fixed128::from_natural(-1) - ); + for _ in 0..100 { + // decreases + next = runtime_multiplier_update(original); + assert!(next < original, "{:?} !<= {:?}", next, original); + original = next; + } }) } @@ -342,7 +324,7 @@ mod tests { fn weight_to_fee_should_not_overflow_on_large_weights() { let kb = 1024 as Weight; let mb = kb * kb; - let max_fm = Fixed128::from_natural(i128::max_value()); + let max_fm = Multiplier::saturating_from_integer(i128::max_value()); // check that for all values it can compute, correctly. vec![ @@ -363,9 +345,9 @@ mod tests { Weight::max_value(), ].into_iter().for_each(|i| { run_with_system_weight(i, || { - let next = TargetedFeeAdjustment::::convert(Fixed128::default()); - let truth = fee_multiplier_update(i, Fixed128::default()); - assert_eq_error_rate!(truth, next, Fixed128::from_parts(50_000_000)); + let next = runtime_multiplier_update(Multiplier::one()); + let truth = truth_value_update(i, Multiplier::one()); + assert_eq_error_rate!(truth, next, Multiplier::from_inner(50_000_000)); }); }); @@ -375,7 +357,7 @@ mod tests { .into_iter() .for_each(|i| { run_with_system_weight(i, || { - let fm = TargetedFeeAdjustment::::convert(max_fm); + let fm = runtime_multiplier_update(max_fm); // won't grow. The convert saturates everything. assert_eq!(fm, max_fm); }) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index bae2e587542dd0c9e35308aaa000b38acadb3157..39b63f2ed2568d147c0c5fc4f91b6f1c0ba10401 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -23,14 +23,18 @@ #![recursion_limit="256"] use sp_std::prelude::*; + use frame_support::{ - construct_runtime, parameter_types, debug, + construct_runtime, parameter_types, debug, RuntimeDebug, weights::{ - Weight, + Weight, IdentityFee, constants::{BlockExecutionWeight, ExtrinsicBaseWeight, RocksDbWeight, WEIGHT_PER_SECOND}, }, traits::{Currency, Imbalance, KeyOwnerProofSystem, OnUnbalanced, Randomness, LockIdentifier}, }; +use frame_system::{EnsureRoot, EnsureOneOf}; +use frame_support::traits::InstanceFilter; +use codec::{Encode, Decode}; use sp_core::{ crypto::KeyTypeId, u32_trait::{_1, _2, _3, _4}, @@ -41,13 +45,13 @@ use node_primitives::{AccountIndex, Balance, BlockNumber, Hash, Index, Moment}; use sp_api::impl_runtime_apis; use sp_runtime::{ Permill, Perbill, Perquintill, Percent, ApplyExtrinsicResult, - impl_opaque_keys, generic, create_runtime_str, ModuleId, + impl_opaque_keys, generic, create_runtime_str, ModuleId, FixedPointNumber, }; use sp_runtime::curve::PiecewiseLinear; use sp_runtime::transaction_validity::{TransactionValidity, TransactionSource, TransactionPriority}; use sp_runtime::traits::{ self, BlakeTwo256, Block as BlockT, StaticLookup, SaturatedConversion, - ConvertInto, OpaqueKeys, NumberFor, + ConvertInto, OpaqueKeys, NumberFor, Saturating, }; use sp_version::RuntimeVersion; #[cfg(any(feature = "std", test))] @@ -57,24 +61,24 @@ use pallet_grandpa::fg_primitives; use pallet_im_online::sr25519::AuthorityId as ImOnlineId; use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId; use pallet_transaction_payment_rpc_runtime_api::RuntimeDispatchInfo; +pub use pallet_transaction_payment::{Multiplier, TargetedFeeAdjustment}; use pallet_contracts_rpc_runtime_api::ContractExecResult; use pallet_session::{historical as pallet_session_historical}; use sp_inherents::{InherentData, CheckInherentsResult}; -use codec::Encode; use static_assertions::const_assert; #[cfg(any(feature = "std", test))] pub use sp_runtime::BuildStorage; -pub use pallet_timestamp::Call as TimestampCall; +#[cfg(any(feature = "std", test))] pub use pallet_balances::Call as BalancesCall; +#[cfg(any(feature = "std", test))] pub use frame_system::Call as SystemCall; -pub use pallet_contracts::Gas; -pub use frame_support::StorageValue; +#[cfg(any(feature = "std", test))] pub use pallet_staking::StakerStatus; /// Implementations of some helper traits passed into runtime modules as associated types. pub mod impls; -use impls::{CurrencyToVoteHandler, Author, LinearWeightToFee, TargetedFeeAdjustment}; +use impls::{CurrencyToVoteHandler, Author}; /// Constant values used within the runtime. pub mod constants; @@ -93,8 +97,8 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // and set impl_version to 0. If only runtime // implementation changes and behavior does not, then leave spec_version as // is and increment impl_version. - spec_version: 248, - impl_version: 1, + spec_version: 255, + impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 1, }; @@ -126,16 +130,24 @@ impl OnUnbalanced for DealWithFees { } } +const AVERAGE_ON_INITIALIZE_WEIGHT: Perbill = Perbill::from_percent(10); parameter_types! { pub const BlockHashCount: BlockNumber = 2400; /// We allow for 2 seconds of compute with a 6 second average block time. pub const MaximumBlockWeight: Weight = 2 * WEIGHT_PER_SECOND; + pub const AvailableBlockRatio: Perbill = Perbill::from_percent(75); + /// Assume 10% of weight for average on_initialize calls. + pub MaximumExtrinsicWeight: Weight = + AvailableBlockRatio::get().saturating_sub(AVERAGE_ON_INITIALIZE_WEIGHT) + * MaximumBlockWeight::get(); pub const MaximumBlockLength: u32 = 5 * 1024 * 1024; pub const Version: RuntimeVersion = VERSION; - pub const AvailableBlockRatio: Perbill = Perbill::from_percent(75); } +const_assert!(AvailableBlockRatio::get().deconstruct() >= AVERAGE_ON_INITIALIZE_WEIGHT.deconstruct()); + impl frame_system::Trait for Runtime { + type BaseCallFilter = (); type Origin = Origin; type Call = Call; type Index = Index; @@ -151,6 +163,7 @@ impl frame_system::Trait for Runtime { type DbWeight = RocksDbWeight; type BlockExecutionWeight = BlockExecutionWeight; type ExtrinsicBaseWeight = ExtrinsicBaseWeight; + type MaximumExtrinsicWeight = MaximumExtrinsicWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = Version; @@ -160,25 +173,83 @@ impl frame_system::Trait for Runtime { type OnKilledAccount = (); } +impl pallet_utility::Trait for Runtime { + type Event = Event; + type Call = Call; +} + parameter_types! { - // One storage item; value is size 4+4+16+32 bytes = 56 bytes. - pub const MultisigDepositBase: Balance = 30 * CENTS; + // One storage item; key size is 32; value is size 4+4+16+32 bytes = 56 bytes. + pub const DepositBase: Balance = deposit(1, 88); // Additional storage item size of 32 bytes. - pub const MultisigDepositFactor: Balance = 5 * CENTS; + pub const DepositFactor: Balance = deposit(0, 32); pub const MaxSignatories: u16 = 100; } -impl pallet_utility::Trait for Runtime { +impl pallet_multisig::Trait for Runtime { type Event = Event; type Call = Call; type Currency = Balances; - type MultisigDepositBase = MultisigDepositBase; - type MultisigDepositFactor = MultisigDepositFactor; + type DepositBase = DepositBase; + type DepositFactor = DepositFactor; type MaxSignatories = MaxSignatories; } parameter_types! { - pub const MaximumSchedulerWeight: Weight = Perbill::from_percent(80) * MaximumBlockWeight::get(); + // One storage item; key size 32, value size 8; . + pub const ProxyDepositBase: Balance = deposit(1, 8); + // Additional storage item size of 33 bytes. + pub const ProxyDepositFactor: Balance = deposit(0, 33); + pub const MaxProxies: u16 = 32; +} + +/// The type used to represent the kinds of proxying allowed. +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Encode, Decode, RuntimeDebug)] +pub enum ProxyType { + Any, + NonTransfer, + Governance, + Staking, +} +impl Default for ProxyType { fn default() -> Self { Self::Any } } +impl InstanceFilter for ProxyType { + fn filter(&self, c: &Call) -> bool { + match self { + ProxyType::Any => true, + ProxyType::NonTransfer => !matches!(c, + Call::Balances(..) | Call::Vesting(pallet_vesting::Call::vested_transfer(..)) + | Call::Indices(pallet_indices::Call::transfer(..)) + ), + ProxyType::Governance => matches!(c, + Call::Democracy(..) | Call::Council(..) | Call::Society(..) + | Call::TechnicalCommittee(..) | Call::Elections(..) | Call::Treasury(..) + ), + ProxyType::Staking => matches!(c, Call::Staking(..)), + } + } + fn is_superset(&self, o: &Self) -> bool { + match (self, o) { + (x, y) if x == y => true, + (ProxyType::Any, _) => true, + (_, ProxyType::Any) => false, + (ProxyType::NonTransfer, _) => true, + _ => false, + } + } +} + +impl pallet_proxy::Trait for Runtime { + type Event = Event; + type Call = Call; + type Currency = Balances; + type ProxyType = ProxyType; + type ProxyDepositBase = ProxyDepositBase; + type ProxyDepositFactor = ProxyDepositFactor; + type MaxProxies = MaxProxies; +} + +parameter_types! { + pub MaximumSchedulerWeight: Weight = Perbill::from_percent(80) * MaximumBlockWeight::get(); } impl pallet_scheduler::Trait for Runtime { @@ -205,9 +276,9 @@ parameter_types! { impl pallet_indices::Trait for Runtime { type AccountIndex = AccountIndex; - type Event = Event; type Currency = Balances; type Deposit = IndexDeposit; + type Event = Event; } parameter_types! { @@ -224,19 +295,18 @@ impl pallet_balances::Trait for Runtime { parameter_types! { pub const TransactionByteFee: Balance = 10 * MILLICENTS; - // In the Substrate node, a weight of 10_000_000 (smallest non-zero weight) - // is mapped to 10_000_000 units of fees, hence: - pub const WeightFeeCoefficient: Balance = 1; - // for a sane configuration, this should always be less than `AvailableBlockRatio`. pub const TargetBlockFullness: Perquintill = Perquintill::from_percent(25); + pub AdjustmentVariable: Multiplier = Multiplier::saturating_from_rational(1, 100_000); + pub MinimumMultiplier: Multiplier = Multiplier::saturating_from_rational(1, 1_000_000_000u128); } impl pallet_transaction_payment::Trait for Runtime { type Currency = Balances; type OnTransactionPayment = DealWithFees; type TransactionByteFee = TransactionByteFee; - type WeightToFee = LinearWeightToFee; - type FeeMultiplierUpdate = TargetedFeeAdjustment; + type WeightToFee = IdentityFee; + type FeeMultiplierUpdate = + TargetedFeeAdjustment; } parameter_types! { @@ -278,11 +348,11 @@ impl pallet_session::Trait for Runtime { type ValidatorId = ::AccountId; type ValidatorIdOf = pallet_staking::StashOf; type ShouldEndSession = Babe; + type NextSessionRotation = Babe; type SessionManager = pallet_session::historical::NoteHistoricalRoot; type SessionHandler = ::KeyTypeIdProviders; type Keys = SessionKeys; type DisabledValidatorsThreshold = DisabledValidatorsThreshold; - type NextSessionRotation = Babe; } impl pallet_session::historical::Trait for Runtime { @@ -306,9 +376,11 @@ parameter_types! { pub const BondingDuration: pallet_staking::EraIndex = 24 * 28; pub const SlashDeferDuration: pallet_staking::EraIndex = 24 * 7; // 1/4 the bonding duration. pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; - pub const ElectionLookahead: BlockNumber = EPOCH_DURATION_IN_BLOCKS / 4; pub const MaxNominatorRewardedPerValidator: u32 = 64; - pub const MaxIterations: u32 = 5; + pub const ElectionLookahead: BlockNumber = EPOCH_DURATION_IN_BLOCKS / 4; + pub const MaxIterations: u32 = 10; + // 0.05%. The higher the value, the more strict solution acceptance becomes. + pub MinSolutionScoreBump: Perbill = Perbill::from_rational_approximation(5u32, 10_000); } impl pallet_staking::Trait for Runtime { @@ -323,13 +395,18 @@ impl pallet_staking::Trait for Runtime { type BondingDuration = BondingDuration; type SlashDeferDuration = SlashDeferDuration; /// A super-majority of the council can cancel the slash. - type SlashCancelOrigin = pallet_collective::EnsureProportionAtLeast<_3, _4, AccountId, CouncilCollective>; + type SlashCancelOrigin = EnsureOneOf< + AccountId, + EnsureRoot, + pallet_collective::EnsureProportionAtLeast<_3, _4, AccountId, CouncilCollective> + >; type SessionInterface = Self; type RewardCurve = RewardCurve; type NextNewSession = Session; type ElectionLookahead = ElectionLookahead; type Call = Call; type MaxIterations = MaxIterations; + type MinSolutionScoreBump = MinSolutionScoreBump; type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; type UnsignedPriority = StakingUnsignedPriority; } @@ -375,6 +452,7 @@ impl pallet_democracy::Trait for Runtime { type VetoOrigin = pallet_collective::EnsureMember; type CooloffPeriod = CooloffPeriod; type PreimageByteDeposit = PreimageByteDeposit; + type OperationalPreimageOrigin = pallet_collective::EnsureMember; type Slash = Treasury; type Scheduler = Scheduler; type MaxVotes = MaxVotes; @@ -394,21 +472,21 @@ impl pallet_collective::Trait for Runtime { type MaxProposals = CouncilMaxProposals; } -const DESIRED_MEMBERS: u32 = 13; parameter_types! { pub const CandidacyBond: Balance = 10 * DOLLARS; pub const VotingBond: Balance = 1 * DOLLARS; pub const TermDuration: BlockNumber = 7 * DAYS; - pub const DesiredMembers: u32 = DESIRED_MEMBERS; + pub const DesiredMembers: u32 = 13; pub const DesiredRunnersUp: u32 = 7; pub const ElectionsPhragmenModuleId: LockIdentifier = *b"phrelect"; } -// Make sure that there are no more than `MAX_MEMBERS` members elected via phragmen. -const_assert!(DESIRED_MEMBERS <= pallet_collective::MAX_MEMBERS); + +// Make sure that there are no more than `MAX_MEMBERS` members elected via elections-phragmen. +const_assert!(DesiredMembers::get() <= pallet_collective::MAX_MEMBERS); impl pallet_elections_phragmen::Trait for Runtime { - type ModuleId = ElectionsPhragmenModuleId; type Event = Event; + type ModuleId = ElectionsPhragmenModuleId; type Currency = Balances; type ChangeMembers = Council; // NOTE: this implies that council's genesis members cannot be set directly and must come from @@ -439,13 +517,18 @@ impl pallet_collective::Trait for Runtime { type MaxProposals = TechnicalMaxProposals; } +type EnsureRootOrHalfCouncil = EnsureOneOf< + AccountId, + EnsureRoot, + pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective> +>; impl pallet_membership::Trait for Runtime { type Event = Event; - type AddOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; - type RemoveOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; - type SwapOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; - type ResetOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; - type PrimeOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; + type AddOrigin = EnsureRootOrHalfCouncil; + type RemoveOrigin = EnsureRootOrHalfCouncil; + type SwapOrigin = EnsureRootOrHalfCouncil; + type ResetOrigin = EnsureRootOrHalfCouncil; + type PrimeOrigin = EnsureRootOrHalfCouncil; type MembershipInitialized = TechnicalCommittee; type MembershipChanged = TechnicalCommittee; } @@ -463,9 +546,18 @@ parameter_types! { } impl pallet_treasury::Trait for Runtime { + type ModuleId = TreasuryModuleId; type Currency = Balances; - type ApproveOrigin = pallet_collective::EnsureMembers<_4, AccountId, CouncilCollective>; - type RejectOrigin = pallet_collective::EnsureMembers<_2, AccountId, CouncilCollective>; + type ApproveOrigin = EnsureOneOf< + AccountId, + EnsureRoot, + pallet_collective::EnsureMembers<_4, AccountId, CouncilCollective> + >; + type RejectOrigin = EnsureOneOf< + AccountId, + EnsureRoot, + pallet_collective::EnsureMembers<_2, AccountId, CouncilCollective> + >; type Tippers = Elections; type TipCountdown = TipCountdown; type TipFindersFee = TipFindersFee; @@ -477,20 +569,19 @@ impl pallet_treasury::Trait for Runtime { type ProposalBondMinimum = ProposalBondMinimum; type SpendPeriod = SpendPeriod; type Burn = Burn; - type ModuleId = TreasuryModuleId; } parameter_types! { - pub const TombstoneDeposit: Balance = 1 * DOLLARS; - pub const RentByteFee: Balance = 1 * DOLLARS; - pub const RentDepositOffset: Balance = 1000 * DOLLARS; - pub const SurchargeReward: Balance = 150 * DOLLARS; + pub const TombstoneDeposit: Balance = 16 * MILLICENTS; + pub const RentByteFee: Balance = 4 * MILLICENTS; + pub const RentDepositOffset: Balance = 1000 * MILLICENTS; + pub const SurchargeReward: Balance = 150 * MILLICENTS; } impl pallet_contracts::Trait for Runtime { type Time = Timestamp; type Randomness = RandomnessCollectiveFlip; - type Call = Call; + type Currency = Balances; type Event = Event; type DetermineContractAddress = pallet_contracts::SimpleAddressDeterminer; type TrieIdGenerator = pallet_contracts::TrieIdFromParentCounter; @@ -503,6 +594,7 @@ impl pallet_contracts::Trait for Runtime { type SurchargeReward = SurchargeReward; type MaxDepth = pallet_contracts::DefaultMaxDepth; type MaxValueSize = pallet_contracts::DefaultMaxValueSize; + type WeightPrice = pallet_transaction_payment::Module; } impl pallet_sudo::Trait for Runtime { @@ -513,7 +605,7 @@ impl pallet_sudo::Trait for Runtime { parameter_types! { pub const SessionDuration: BlockNumber = EPOCH_DURATION_IN_SLOTS as _; pub const ImOnlineUnsignedPriority: TransactionPriority = TransactionPriority::max_value(); - /// We prioritize im-online heartbeats over phragmen solution submission. + /// We prioritize im-online heartbeats over election solution submission. pub const StakingUnsignedPriority: TransactionPriority = TransactionPriority::max_value() / 2; } @@ -568,8 +660,8 @@ impl frame_system::offchain::SigningTypes for Runtime { impl frame_system::offchain::SendTransactionTypes for Runtime where Call: From, { - type OverarchingCall = Call; type Extrinsic = UncheckedExtrinsic; + type OverarchingCall = Call; } impl pallet_im_online::Trait for Runtime { @@ -580,10 +672,15 @@ impl pallet_im_online::Trait for Runtime { type UnsignedPriority = ImOnlineUnsignedPriority; } +parameter_types! { + pub OffencesWeightSoftLimit: Weight = Perbill::from_percent(60) * MaximumBlockWeight::get(); +} + impl pallet_offences::Trait for Runtime { type Event = Event; type IdentificationTuple = pallet_session::historical::IdentificationTuple; type OnOffenceHandler = Staking; + type WeightSoftLimit = OffencesWeightSoftLimit; } impl pallet_authority_discovery::Trait for Runtime {} @@ -640,8 +737,8 @@ impl pallet_identity::Trait for Runtime { type MaxAdditionalFields = MaxAdditionalFields; type MaxRegistrars = MaxRegistrars; type Slashed = Treasury; - type ForceOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; - type RegistrarOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; + type ForceOrigin = EnsureRootOrHalfCouncil; + type RegistrarOrigin = EnsureRootOrHalfCouncil; } parameter_types! { @@ -674,6 +771,7 @@ parameter_types! { impl pallet_society::Trait for Runtime { type Event = Event; + type ModuleId = SocietyModuleId; type Currency = Balances; type Randomness = RandomnessCollectiveFlip; type CandidateDeposit = CandidateDeposit; @@ -686,7 +784,6 @@ impl pallet_society::Trait for Runtime { type FounderSetOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; type SuspensionJudgementOrigin = pallet_society::EnsureFounder; type ChallengePeriod = ChallengePeriod; - type ModuleId = SocietyModuleId; } parameter_types! { @@ -707,7 +804,7 @@ construct_runtime!( UncheckedExtrinsic = UncheckedExtrinsic { System: frame_system::{Module, Call, Config, Storage, Event}, - Utility: pallet_utility::{Module, Call, Storage, Event}, + Utility: pallet_utility::{Module, Call, Event}, Babe: pallet_babe::{Module, Call, Storage, Config, Inherent}, Timestamp: pallet_timestamp::{Module, Call, Storage, Inherent}, Authorship: pallet_authorship::{Module, Call, Storage, Inherent}, @@ -736,6 +833,8 @@ construct_runtime!( Recovery: pallet_recovery::{Module, Call, Storage, Event}, Vesting: pallet_vesting::{Module, Call, Storage, Event, Config}, Scheduler: pallet_scheduler::{Module, Call, Storage, Event}, + Proxy: pallet_proxy::{Module, Call, Storage, Event}, + Multisig: pallet_multisig::{Module, Call, Storage, Event}, } ); @@ -976,8 +1075,26 @@ impl_runtime_apis! { impl pallet_offences_benchmarking::Trait for Runtime {} impl frame_system_benchmarking::Trait for Runtime {} + let whitelist: Vec> = vec![ + // Block Number + // frame_system::Number::::hashed_key().to_vec(), + hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef702a5c1b19ab7a04f536c519aca4983ac").to_vec(), + // Total Issuance + hex_literal::hex!("c2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80").to_vec(), + // Execution Phase + hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef7ff553b5a9862a516939d82b3d3d8661a").to_vec(), + // Event Count + hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef70a98fdbe9ce6c55837576c60c7af3850").to_vec(), + // System Events + hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef780d41e5e16056765bc8461851072c9d7").to_vec(), + // Caller 0 Account + hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da946c154ffd9992e395af90b5b13cc6f295c77033fce8a9045824a6690bbf99c6db269502f0a8d1d2a008542d5690a0749").to_vec(), + // Treasury Account + hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da95ecffd7b6c0f78751baa9d281e0bfa3a6d6f646c70792f74727372790000000000000000000000000000000000000000").to_vec(), + ]; + let mut batches = Vec::::new(); - let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat); + let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat, &whitelist); add_benchmark!(params, batches, b"balances", Balances); add_benchmark!(params, batches, b"collective", Council); @@ -985,7 +1102,10 @@ impl_runtime_apis! { add_benchmark!(params, batches, b"elections", Elections); add_benchmark!(params, batches, b"identity", Identity); add_benchmark!(params, batches, b"im-online", ImOnline); + add_benchmark!(params, batches, b"indices", Indices); + add_benchmark!(params, batches, b"multisig", Multisig); add_benchmark!(params, batches, b"offences", OffencesBench::); + add_benchmark!(params, batches, b"proxy", Proxy); add_benchmark!(params, batches, b"scheduler", Scheduler); add_benchmark!(params, batches, b"session", SessionBench::); add_benchmark!(params, batches, b"staking", Staking); diff --git a/bin/node/testing/Cargo.toml b/bin/node/testing/Cargo.toml index 37518ebe804431a6994c64cf86c056d03e9a55b6..fbf369cc3b4e4cfd60303afea26a8380a25ece18 100644 --- a/bin/node/testing/Cargo.toml +++ b/bin/node/testing/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-testing" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] description = "Test utilities for Substrate node." edition = "2018" @@ -13,40 +13,40 @@ publish = true targets = ["x86_64-unknown-linux-gnu"] [dependencies] -pallet-balances = { version = "2.0.0-alpha.8", path = "../../../frame/balances" } -sc-service = { version = "0.8.0-alpha.8", features = ["test-helpers", "db"], path = "../../../client/service" } -sc-client-db = { version = "0.8.0-alpha.8", path = "../../../client/db/", features = ["kvdb-rocksdb", "parity-db"] } -sc-client-api = { version = "2.0.0-alpha.8", path = "../../../client/api/" } -codec = { package = "parity-scale-codec", version = "1.3.0" } -pallet-contracts = { version = "2.0.0-alpha.8", path = "../../../frame/contracts" } -pallet-grandpa = { version = "2.0.0-alpha.8", path = "../../../frame/grandpa" } -pallet-indices = { version = "2.0.0-alpha.8", path = "../../../frame/indices" } -sp-keyring = { version = "2.0.0-alpha.8", path = "../../../primitives/keyring" } -node-executor = { version = "2.0.0-alpha.8", path = "../executor" } -node-primitives = { version = "2.0.0-alpha.8", path = "../primitives" } -node-runtime = { version = "2.0.0-alpha.8", path = "../runtime" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-io = { version = "2.0.0-alpha.8", path = "../../../primitives/io" } -frame-support = { version = "2.0.0-alpha.8", path = "../../../frame/support" } -pallet-session = { version = "2.0.0-alpha.8", path = "../../../frame/session" } -pallet-society = { version = "2.0.0-alpha.8", path = "../../../frame/society" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -pallet-staking = { version = "2.0.0-alpha.8", path = "../../../frame/staking" } -sc-executor = { version = "0.8.0-alpha.8", path = "../../../client/executor", features = ["wasmtime"] } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/common" } -frame-system = { version = "2.0.0-alpha.8", path = "../../../frame/system" } -substrate-test-client = { version = "2.0.0-alpha.8", path = "../../../test-utils/client" } -pallet-timestamp = { version = "2.0.0-alpha.8", path = "../../../frame/timestamp" } -pallet-transaction-payment = { version = "2.0.0-alpha.8", path = "../../../frame/transaction-payment" } -pallet-treasury = { version = "2.0.0-alpha.8", path = "../../../frame/treasury" } +pallet-balances = { version = "2.0.0-rc4", path = "../../../frame/balances" } +sc-service = { version = "0.8.0-rc4", features = ["test-helpers", "db"], path = "../../../client/service" } +sc-client-db = { version = "0.8.0-rc4", path = "../../../client/db/", features = ["kvdb-rocksdb", "parity-db"] } +sc-client-api = { version = "2.0.0-rc4", path = "../../../client/api/" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +pallet-contracts = { version = "2.0.0-rc4", path = "../../../frame/contracts" } +pallet-grandpa = { version = "2.0.0-rc4", path = "../../../frame/grandpa" } +pallet-indices = { version = "2.0.0-rc4", path = "../../../frame/indices" } +sp-keyring = { version = "2.0.0-rc4", path = "../../../primitives/keyring" } +node-executor = { version = "2.0.0-rc4", path = "../executor" } +node-primitives = { version = "2.0.0-rc4", path = "../primitives" } +node-runtime = { version = "2.0.0-rc4", path = "../runtime" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-io = { version = "2.0.0-rc4", path = "../../../primitives/io" } +frame-support = { version = "2.0.0-rc4", path = "../../../frame/support" } +pallet-session = { version = "2.0.0-rc4", path = "../../../frame/session" } +pallet-society = { version = "2.0.0-rc4", path = "../../../frame/society" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +pallet-staking = { version = "2.0.0-rc4", path = "../../../frame/staking" } +sc-executor = { version = "0.8.0-rc4", path = "../../../client/executor", features = ["wasmtime"] } +sp-consensus = { version = "0.8.0-rc4", path = "../../../primitives/consensus/common" } +frame-system = { version = "2.0.0-rc4", path = "../../../frame/system" } +substrate-test-client = { version = "2.0.0-rc4", path = "../../../test-utils/client" } +pallet-timestamp = { version = "2.0.0-rc4", path = "../../../frame/timestamp" } +pallet-transaction-payment = { version = "2.0.0-rc4", path = "../../../frame/transaction-payment" } +pallet-treasury = { version = "2.0.0-rc4", path = "../../../frame/treasury" } wabt = "0.9.2" -sp-api = { version = "2.0.0-alpha.8", path = "../../../primitives/api" } -sp-finality-tracker = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/finality-tracker" } -sp-timestamp = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/timestamp" } -sp-block-builder = { version = "2.0.0-alpha.8", path = "../../../primitives/block-builder" } -sc-block-builder = { version = "0.8.0-alpha.8", path = "../../../client/block-builder" } -sp-inherents = { version = "2.0.0-alpha.8", path = "../../../primitives/inherents" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } +sp-api = { version = "2.0.0-rc4", path = "../../../primitives/api" } +sp-finality-tracker = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/finality-tracker" } +sp-timestamp = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/timestamp" } +sp-block-builder = { version = "2.0.0-rc4", path = "../../../primitives/block-builder" } +sc-block-builder = { version = "0.8.0-rc4", path = "../../../client/block-builder" } +sp-inherents = { version = "2.0.0-rc4", path = "../../../primitives/inherents" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } log = "0.4.8" tempfile = "3.1.0" fs_extra = "1" @@ -54,4 +54,4 @@ futures = "0.3.1" [dev-dependencies] criterion = "0.3.0" -sc-cli = { version = "0.8.0-alpha.8", path = "../../../client/cli" } +sc-cli = { version = "0.8.0-rc4", path = "../../../client/cli" } diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index 888ec71cdabe178e398b0e30d7cccee5c9ca397d..507d3420d835470e41fd1f1bb71fef9eaee9ad60 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -55,7 +55,7 @@ use sp_api::ProvideRuntimeApi; use sp_block_builder::BlockBuilder; use sp_inherents::InherentData; use sc_client_api::{ - ExecutionStrategy, + ExecutionStrategy, BlockBackend, execution_extensions::{ExecutionExtensions, ExecutionStrategies}, }; use sp_core::{Pair, Public, sr25519, ed25519}; @@ -152,20 +152,12 @@ impl BlockType { } /// Content of the generated block. +#[derive(Clone, Debug)] pub struct BlockContent { block_type: BlockType, size: Option, } -impl BlockContent { - fn iter_while(&self, mut f: impl FnMut(usize) -> bool) { - match self.size { - Some(v) => { for i in 0..v { if !f(i) { break; }}} - None => { for i in 0.. { if !f(i) { break; }}} - } - } -} - /// Type of backend database. #[derive(Debug, PartialEq, Clone, Copy)] pub enum DatabaseType { @@ -219,10 +211,97 @@ impl CloneableSpawn for TaskExecutor { } } +/// Iterator for block content. +pub struct BlockContentIterator<'a> { + iteration: usize, + content: BlockContent, + runtime_version: sc_executor::RuntimeVersion, + genesis_hash: node_primitives::Hash, + keyring: &'a BenchKeyring, +} + +impl<'a> BlockContentIterator<'a> { + fn new(content: BlockContent, keyring: &'a BenchKeyring, client: &Client) -> Self { + let runtime_version = client.runtime_version_at(&BlockId::number(0)) + .expect("There should be runtime version at 0"); + + let genesis_hash = client.block_hash(Zero::zero()) + .expect("Database error?") + .expect("Genesis block always exists; qed") + .into(); + + BlockContentIterator { + iteration: 0, + content, + keyring, + runtime_version, + genesis_hash, + } + } +} + +impl<'a> Iterator for BlockContentIterator<'a> { + type Item = OpaqueExtrinsic; + + fn next(&mut self) -> Option { + if self.content.size.map(|size| size <= self.iteration).unwrap_or(false) { + return None; + } + + let sender = self.keyring.at(self.iteration); + let receiver = get_account_id_from_seed::( + &format!("random-user//{}", self.iteration) + ); + + let signed = self.keyring.sign( + CheckedExtrinsic { + signed: Some((sender, signed_extra(0, node_runtime::ExistentialDeposit::get() + 1))), + function: match self.content.block_type { + BlockType::RandomTransfersKeepAlive => { + Call::Balances( + BalancesCall::transfer_keep_alive( + pallet_indices::address::Address::Id(receiver), + node_runtime::ExistentialDeposit::get() + 1, + ) + ) + }, + BlockType::RandomTransfersReaping => { + Call::Balances( + BalancesCall::transfer( + pallet_indices::address::Address::Id(receiver), + // Transfer so that ending balance would be 1 less than existential deposit + // so that we kill the sender account. + 100*DOLLARS - (node_runtime::ExistentialDeposit::get() - 1), + ) + ) + }, + BlockType::Noop => { + Call::System( + SystemCall::remark(Vec::new()) + ) + }, + }, + }, + self.runtime_version.spec_version, + self.runtime_version.transaction_version, + self.genesis_hash.into(), + ); + + let encoded = Encode::encode(&signed); + + let opaque = OpaqueExtrinsic::decode(&mut &encoded[..]) + .expect("Failed to decode opaque"); + + self.iteration += 1; + + Some(opaque) + } +} + impl BenchDb { /// New immutable benchmarking database. /// - /// See [`new`] method documentation for more information about the purpose + /// See [`BenchDb::new`] method documentation for more information about the purpose /// of this structure. pub fn with_key_types( database_type: DatabaseType, @@ -288,8 +367,33 @@ impl BenchDb { (client, backend) } - /// Generate new block using this database. - pub fn generate_block(&mut self, content: BlockContent) -> Block { + /// Generate list of required inherents. + /// + /// Uses already instantiated Client. + pub fn generate_inherents(&mut self, client: &Client) -> Vec { + let mut inherent_data = InherentData::new(); + let timestamp = 1 * MinimumPeriod::get(); + + inherent_data.put_data(sp_timestamp::INHERENT_IDENTIFIER, ×tamp) + .expect("Put timestamp failed"); + inherent_data.put_data(sp_finality_tracker::INHERENT_IDENTIFIER, &0) + .expect("Put finality tracker failed"); + + client.runtime_api() + .inherent_extrinsics_with_context( + &BlockId::number(0), + ExecutionContext::BlockConstruction, + inherent_data, + ).expect("Get inherents failed") + } + + /// Iterate over some block content with transaction signed using this database keyring. + pub fn block_content(&self, content: BlockContent, client: &Client) -> BlockContentIterator { + BlockContentIterator::new(content, &self.keyring, client) + } + + /// Get cliet for this database operations. + pub fn client(&mut self) -> Client { let (client, _backend) = Self::bench_client( self.database_type, self.directory_guard.path(), @@ -297,92 +401,33 @@ impl BenchDb { &self.keyring, ); - let version = client.runtime_version_at(&BlockId::number(0)) - .expect("There should be runtime version at 0") - .spec_version; + client + } - let genesis_hash = client.block_hash(Zero::zero()) - .expect("Database error?") - .expect("Genesis block always exists; qed") - .into(); + /// Generate new block using this database. + pub fn generate_block(&mut self, content: BlockContent) -> Block { + let client = self.client(); let mut block = client .new_block(Default::default()) .expect("Block creation failed"); - let timestamp = 1 * MinimumPeriod::get(); - - let mut inherent_data = InherentData::new(); - inherent_data.put_data(sp_timestamp::INHERENT_IDENTIFIER, ×tamp) - .expect("Put timestamp failed"); - inherent_data.put_data(sp_finality_tracker::INHERENT_IDENTIFIER, &0) - .expect("Put finality tracker failed"); - - for extrinsic in client.runtime_api() - .inherent_extrinsics_with_context( - &BlockId::number(0), - ExecutionContext::BlockConstruction, - inherent_data, - ).expect("Get inherents failed") - { + for extrinsic in self.generate_inherents(&client) { block.push(extrinsic).expect("Push inherent failed"); } let start = std::time::Instant::now(); - content.iter_while(|iteration| { - let sender = self.keyring.at(iteration); - let receiver = get_account_id_from_seed::( - &format!("random-user//{}", iteration) - ); - - let signed = self.keyring.sign( - CheckedExtrinsic { - signed: Some((sender, signed_extra(0, node_runtime::ExistentialDeposit::get() + 1))), - function: match content.block_type { - BlockType::RandomTransfersKeepAlive => { - Call::Balances( - BalancesCall::transfer_keep_alive( - pallet_indices::address::Address::Id(receiver), - node_runtime::ExistentialDeposit::get() + 1, - ) - ) - }, - BlockType::RandomTransfersReaping => { - Call::Balances( - BalancesCall::transfer( - pallet_indices::address::Address::Id(receiver), - // Transfer so that ending balance would be 1 less than existential deposit - // so that we kill the sender account. - 100*DOLLARS - (node_runtime::ExistentialDeposit::get() - 1), - ) - ) - }, - BlockType::Noop => { - Call::System( - SystemCall::remark(Vec::new()) - ) - }, - }, - }, - version, - genesis_hash, - ); - - let encoded = Encode::encode(&signed); - - let opaque = OpaqueExtrinsic::decode(&mut &encoded[..]) - .expect("Failed to decode opaque"); - + for opaque in self.block_content(content, &client) { match block.push(opaque) { Err(sp_blockchain::Error::ApplyExtrinsicFailed( sp_blockchain::ApplyExtrinsicFailed::Validity(e) )) if e.exhausted_resources() => { - return false; + break; }, Err(err) => panic!("Error pushing transaction: {:?}", err), - Ok(_) => true, + Ok(_) => {}, } - }); + }; let block = block.build().expect("Block build failed").block; @@ -411,7 +456,7 @@ impl BenchDb { ); BenchContext { - client, backend, db_guard: directory_guard, + client: Arc::new(client), backend, db_guard: directory_guard, } } } @@ -462,10 +507,16 @@ impl BenchKeyring { } /// Sign transaction with keypair from this keyring. - pub fn sign(&self, xt: CheckedExtrinsic, version: u32, genesis_hash: [u8; 32]) -> UncheckedExtrinsic { + pub fn sign( + &self, + xt: CheckedExtrinsic, + spec_version: u32, + tx_version: u32, + genesis_hash: [u8; 32] + ) -> UncheckedExtrinsic { match xt.signed { Some((signed, extra)) => { - let payload = (xt.function, extra.clone(), version, genesis_hash, genesis_hash); + let payload = (xt.function, extra.clone(), spec_version, tx_version, genesis_hash, genesis_hash); let key = self.accounts.get(&signed).expect("Account id not found in keyring"); let signature = payload.using_encoded(|b| { if b.len() > 256 { @@ -537,7 +588,7 @@ impl Guard { /// Benchmarking/test context holding instantiated client and backend references. pub struct BenchContext { /// Node client. - pub client: Client, + pub client: Arc, /// Node backend. pub backend: Arc, diff --git a/bin/node/testing/src/client.rs b/bin/node/testing/src/client.rs index 5d2795a6f6c818b3ca2abe4d5ac1caa79f2b68a5..f44747b26b7a6eb6409e76063b23914f3fb43f66 100644 --- a/bin/node/testing/src/client.rs +++ b/bin/node/testing/src/client.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/node/testing/src/genesis.rs b/bin/node/testing/src/genesis.rs index f99b559a254d8b67cdbef8a9723f2020ee3101c7..2bbae96cf4332f08c24469857388d72877877653 100644 --- a/bin/node/testing/src/genesis.rs +++ b/bin/node/testing/src/genesis.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -23,7 +23,7 @@ use sp_keyring::{Ed25519Keyring, Sr25519Keyring}; use node_runtime::{ GenesisConfig, BalancesConfig, SessionConfig, StakingConfig, SystemConfig, GrandpaConfig, IndicesConfig, ContractsConfig, SocietyConfig, WASM_BINARY, - AccountId, + AccountId, StakerStatus, }; use node_runtime::constants::currency::*; use sp_core::ChangesTrieConfiguration; @@ -87,9 +87,9 @@ pub fn config_endowed( }), pallet_staking: Some(StakingConfig { stakers: vec![ - (dave(), alice(), 111 * DOLLARS, pallet_staking::StakerStatus::Validator), - (eve(), bob(), 100 * DOLLARS, pallet_staking::StakerStatus::Validator), - (ferdie(), charlie(), 100 * DOLLARS, pallet_staking::StakerStatus::Validator) + (dave(), alice(), 111 * DOLLARS, StakerStatus::Validator), + (eve(), bob(), 100 * DOLLARS, StakerStatus::Validator), + (ferdie(), charlie(), 100 * DOLLARS, StakerStatus::Validator) ], validator_count: 3, minimum_validator_count: 0, diff --git a/bin/node/testing/src/keyring.rs b/bin/node/testing/src/keyring.rs index 99a44e065d2b2f69613d4c07a7398c80bb311fc5..efa47a59821945caa16a271b81bc7dcd344b5292 100644 --- a/bin/node/testing/src/keyring.rs +++ b/bin/node/testing/src/keyring.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/node/testing/src/lib.rs b/bin/node/testing/src/lib.rs index 07859f5fed6e85e390ccadbb4162b513a9e0f442..d682347e40019dd4a7ff74a8716929130623f31e 100644 --- a/bin/node/testing/src/lib.rs +++ b/bin/node/testing/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/utils/chain-spec-builder/Cargo.toml b/bin/utils/chain-spec-builder/Cargo.toml index d4faa4f1e3b54fc69a38bbe18e0cebc9d8195d54..b633ffa96634e19aadbe082d3bafc971cf812654 100644 --- a/bin/utils/chain-spec-builder/Cargo.toml +++ b/bin/utils/chain-spec-builder/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chain-spec-builder" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" @@ -13,9 +13,9 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] ansi_term = "0.12.1" -sc-keystore = { version = "2.0.0-alpha.8", path = "../../../client/keystore" } -sc-chain-spec = { version = "2.0.0-alpha.8", path = "../../../client/chain-spec" } -node-cli = { version = "2.0.0-alpha.8", path = "../../node/cli" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } +sc-keystore = { version = "2.0.0-rc4", path = "../../../client/keystore" } +sc-chain-spec = { version = "2.0.0-rc4", path = "../../../client/chain-spec" } +node-cli = { version = "2.0.0-rc4", path = "../../node/cli" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } rand = "0.7.2" structopt = "0.3.8" diff --git a/bin/utils/chain-spec-builder/build.rs b/bin/utils/chain-spec-builder/build.rs index cf8afddb54d4ecf718e9fb25f71994baf0d99ab9..8d5aac1a08742486a9b0e5d55aaf7959941abd77 100644 --- a/bin/utils/chain-spec-builder/build.rs +++ b/bin/utils/chain-spec-builder/build.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/utils/chain-spec-builder/src/main.rs b/bin/utils/chain-spec-builder/src/main.rs index 1561b3a6b0624eab3a93df8ccef287c629fc7d18..4fbcc1e850723982de125067c931f27110f82ec9 100644 --- a/bin/utils/chain-spec-builder/src/main.rs +++ b/bin/utils/chain-spec-builder/src/main.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/utils/subkey/Cargo.toml b/bin/utils/subkey/Cargo.toml index 91643b455198b91c053bcef07ef0d14852a07298..5ade94275eb4bb5da0d75a8e3c7e96bd77eae1de 100644 --- a/bin/utils/subkey/Cargo.toml +++ b/bin/utils/subkey/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subkey" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -12,28 +12,28 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] futures = "0.1.29" -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -node-runtime = { version = "2.0.0-alpha.8", path = "../../node/runtime" } -node-primitives = { version = "2.0.0-alpha.8", path = "../../node/primitives" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +node-runtime = { version = "2.0.0-rc4", path = "../../node/runtime" } +node-primitives = { version = "2.0.0-rc4", path = "../../node/primitives" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } rand = "0.7.2" clap = "2.33.0" tiny-bip39 = "0.7" substrate-bip39 = "0.4.1" hex = "0.4.0" hex-literal = "0.2.1" -codec = { package = "parity-scale-codec", version = "1.3.0" } -frame-system = { version = "2.0.0-alpha.8", path = "../../../frame/system" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../../../frame/balances" } -pallet-transaction-payment = { version = "2.0.0-alpha.8", path = "../../../frame/transaction-payment" } -pallet-grandpa = { version = "2.0.0-alpha.8", path = "../../../frame/grandpa" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +frame-system = { version = "2.0.0-rc4", path = "../../../frame/system" } +pallet-balances = { version = "2.0.0-rc4", path = "../../../frame/balances" } +pallet-transaction-payment = { version = "2.0.0-rc4", path = "../../../frame/transaction-payment" } +pallet-grandpa = { version = "2.0.0-rc4", path = "../../../frame/grandpa" } rpassword = "4.0.1" itertools = "0.8.2" derive_more = { version = "0.99.2" } -sc-rpc = { version = "2.0.0-alpha.8", path = "../../../client/rpc" } -jsonrpc-core-client = { version = "14.0.3", features = ["http"] } +sc-rpc = { version = "2.0.0-rc4", path = "../../../client/rpc" } +jsonrpc-core-client = { version = "14.2.0", features = ["http"] } hyper = "0.12.35" -libp2p = "0.18.1" +libp2p = { version = "0.20.1", default-features = false } serde_json = "1.0" [features] diff --git a/bin/utils/subkey/src/main.rs b/bin/utils/subkey/src/main.rs index da4f59430f9fc3fca0b22058677a4580d1ca70da..4153e769c97f57ce65e3f6eb2c36c1991c586224 100644 --- a/bin/utils/subkey/src/main.rs +++ b/bin/utils/subkey/src/main.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -267,6 +267,9 @@ fn get_app<'a, 'b>(usage: &'a str) -> App<'a, 'b> { If the value is a file, the file content is used as URI. \ If not given, you will be prompted for the URI.' "), + SubCommand::with_name("inspect-node-key") + .about("Print the peer ID corresponding to the node key in the given file") + .args_from_usage("[file] 'Name of file to read the secret key from'"), SubCommand::with_name("sign") .about("Sign a message, provided on STDIN, with a given (secret) key") .args_from_usage(" @@ -439,6 +442,17 @@ where ("inspect", Some(matches)) => { C::print_from_uri(&get_uri("uri", &matches)?, password, maybe_network, output); } + ("inspect-node-key", Some(matches)) => { + let file = matches.value_of("file").ok_or(Error::Static("Input file name is required"))?; + + let mut file_content = fs::read(file)?; + let secret = libp2p_ed25519::SecretKey::from_bytes(&mut file_content) + .map_err(|_| Error::Static("Bad node key file"))?; + let keypair = libp2p_ed25519::Keypair::from(secret); + let peer_id = PublicKey::Ed25519(keypair.public()).into_peer_id(); + + println!("{}", peer_id); + } ("sign", Some(matches)) => { let suri = get_uri("suri", &matches)?; let should_decode = matches.is_present("hex"); @@ -525,7 +539,7 @@ where let account_id: AccountId = ModuleId(id_fixed_array).into_account(); let v = maybe_network.unwrap_or(Ss58AddressFormat::SubstrateAccount); - + C::print_from_uri(&account_id.to_ss58check_with_version(v), password, maybe_network, output); } _ => print_usage(&matches), diff --git a/bin/utils/subkey/src/rpc.rs b/bin/utils/subkey/src/rpc.rs index c6c63be4d1aa37015866ccba38247901a9263caf..e24cf50dc45026615891782cc2440a3eda5749a0 100644 --- a/bin/utils/subkey/src/rpc.rs +++ b/bin/utils/subkey/src/rpc.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/bin/utils/subkey/src/vanity.rs b/bin/utils/subkey/src/vanity.rs index 83a71659d857eb95796b2c103e3224b934c9025a..d09aeeef25a47a79ff434035a4d43a20d88c53a6 100644 --- a/bin/utils/subkey/src/vanity.rs +++ b/bin/utils/subkey/src/vanity.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/client/api/Cargo.toml b/client/api/Cargo.toml index 9d79e2ca5ab8443864d8a86ffe664a7e16cba97a..a32623ffdbd51236ea15b100d0fbe19c1f8a52f1 100644 --- a/client/api/Cargo.toml +++ b/client/api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-client-api" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -13,37 +13,37 @@ documentation = "https://docs.rs/sc-client-api" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../primitives/consensus/common" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-consensus = { version = "0.8.0-rc4", path = "../../primitives/consensus/common" } derive_more = { version = "0.99.2" } -sc-executor = { version = "0.8.0-alpha.8", path = "../executor" } -sp-externalities = { version = "0.8.0-alpha.8", path = "../../primitives/externalities" } +sc-executor = { version = "0.8.0-rc4", path = "../executor" } +sp-externalities = { version = "0.8.0-rc4", path = "../../primitives/externalities" } fnv = { version = "1.0.6" } futures = { version = "0.3.1" } hash-db = { version = "0.15.2", default-features = false } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../primitives/blockchain" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../primitives/blockchain" } hex-literal = { version = "0.2.1" } -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/inherents" } -sp-keyring = { version = "2.0.0-alpha.8", path = "../../primitives/keyring" } +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/inherents" } +sp-keyring = { version = "2.0.0-rc4", path = "../../primitives/keyring" } kvdb = "0.6.0" log = { version = "0.4.8" } parking_lot = "0.10.0" lazy_static = "1.4.0" -sp-database = { version = "2.0.0-alpha.8", path = "../../primitives/database" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-version = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/version" } -sp-api = { version = "2.0.0-alpha.8", path = "../../primitives/api" } -sp-utils = { version = "2.0.0-alpha.8", path = "../../primitives/utils" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../primitives/state-machine" } -sc-telemetry = { version = "2.0.0-alpha.8", path = "../telemetry" } -sp-trie = { version = "2.0.0-alpha.8", path = "../../primitives/trie" } -sp-storage = { version = "2.0.0-alpha.8", path = "../../primitives/storage" } -sp-transaction-pool = { version = "2.0.0-alpha.8", path = "../../primitives/transaction-pool" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.8.0-alpha.8", path = "../../utils/prometheus" } +sp-database = { version = "2.0.0-rc4", path = "../../primitives/database" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-version = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/version" } +sp-api = { version = "2.0.0-rc4", path = "../../primitives/api" } +sp-utils = { version = "2.0.0-rc4", path = "../../primitives/utils" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../primitives/state-machine" } +sc-telemetry = { version = "2.0.0-rc4", path = "../telemetry" } +sp-trie = { version = "2.0.0-rc4", path = "../../primitives/trie" } +sp-storage = { version = "2.0.0-rc4", path = "../../primitives/storage" } +sp-transaction-pool = { version = "2.0.0-rc4", path = "../../primitives/transaction-pool" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.8.0-rc4", path = "../../utils/prometheus" } [dev-dependencies] kvdb-memorydb = "0.6.0" -sp-test-primitives = { version = "2.0.0-alpha.8", path = "../../primitives/test-primitives" } -substrate-test-runtime = { version = "2.0.0-alpha.8", path = "../../test-utils/runtime" } +sp-test-primitives = { version = "2.0.0-rc4", path = "../../primitives/test-primitives" } +substrate-test-runtime = { version = "2.0.0-rc4", path = "../../test-utils/runtime" } diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index fd9577695ae8704dec1ae0a2f257dadb40d14489..9482a6118d71a21415c74f7410d27ead150e06c3 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -67,8 +67,10 @@ pub struct ImportSummary { pub is_new_best: bool, /// Optional storage changes. pub storage_changes: Option<(StorageCollection, ChildStorageCollection)>, - /// Blocks that got retracted because of this one got imported. - pub retracted: Vec, + /// Tree route from old best to new best. + /// + /// If `None`, there was no re-org while importing. + pub tree_route: Option>, } /// Import operation wrapper diff --git a/client/api/src/call_executor.rs b/client/api/src/call_executor.rs index 8c69eb8caa85aa20033f8eab3b696b51e6831704..d9d43900dfc94f2732db7be0051401a832bf8c5b 100644 --- a/client/api/src/call_executor.rs +++ b/client/api/src/call_executor.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! A method call executor interface. use std::{panic::UnwindSafe, result, cell::RefCell}; diff --git a/client/api/src/cht.rs b/client/api/src/cht.rs index 55a38a514990b9432b9a8a5bfe5b7659aa485b00..30cfd3a1b671b8f159d40bdcb499137de8fe5a69 100644 --- a/client/api/src/cht.rs +++ b/client/api/src/cht.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Canonical hash trie definitions and helper functions. //! //! Each CHT is a trie mapping block numbers to canonical hash. diff --git a/client/api/src/client.rs b/client/api/src/client.rs index c855cd3a0832883c3a66b5c986871909aece048a..35d40965e6425096790e6e88c37045724b2b5578 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -16,7 +16,7 @@ //! A set of APIs supported by the client along with their primitives. -use std::{fmt, collections::HashSet}; +use std::{fmt, collections::HashSet, sync::Arc}; use sp_core::storage::StorageKey; use sp_runtime::{ traits::{Block as BlockT, NumberFor}, @@ -90,6 +90,9 @@ pub trait BlockBackend { /// Get block justification set by id. fn justification(&self, id: &BlockId) -> sp_blockchain::Result>; + + /// Get block hash by number. + fn block_hash(&self, number: NumberFor) -> sp_blockchain::Result>; } /// Provide a list of potential uncle headers for a given block. @@ -234,8 +237,10 @@ pub struct BlockImportNotification { pub header: Block::Header, /// Is this the new best block. pub is_new_best: bool, - /// List of retracted blocks ordered by block number. - pub retracted: Vec, + /// Tree route from old best to new best parent. + /// + /// If `None`, there was no re-org while importing. + pub tree_route: Option>>, } /// Summary of a finalized block. @@ -246,3 +251,22 @@ pub struct FinalityNotification { /// Imported block header. pub header: Block::Header, } + +impl From> for sp_transaction_pool::ChainEvent { + fn from(n: BlockImportNotification) -> Self { + Self::NewBlock { + is_new_best: n.is_new_best, + hash: n.hash, + header: n.header, + tree_route: n.tree_route, + } + } +} + +impl From> for sp_transaction_pool::ChainEvent { + fn from(n: FinalityNotification) -> Self { + Self::Finalized { + hash: n.hash, + } + } +} diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 07df822488cf87c126411814be0af185c1a250b8..1de2747eb4c76dd42ec6ebe28fbd90183cd61615 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,14 +15,15 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! In memory client backend use std::collections::HashMap; +use std::ptr; use std::sync::Arc; use parking_lot::RwLock; -use sp_core::storage::well_known_keys; -use sp_core::offchain::storage::{ - InMemOffchainStorage as OffchainStorage +use sp_core::{ + storage::well_known_keys, offchain::storage::InMemOffchainStorage as OffchainStorage, }; use sp_runtime::generic::BlockId; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor, HashFor}; @@ -191,11 +192,19 @@ impl Blockchain { /// Compare this blockchain with another in-mem blockchain pub fn equals_to(&self, other: &Self) -> bool { + // Check ptr equality first to avoid double read locks. + if ptr::eq(self, other) { + return true; + } self.canon_equals_to(other) && self.storage.read().blocks == other.storage.read().blocks } /// Compare canonical chain to other canonical chain. pub fn canon_equals_to(&self, other: &Self) -> bool { + // Check ptr equality first to avoid double read locks. + if ptr::eq(self, other) { + return true; + } let this = self.storage.read(); let other = other.storage.read(); this.hashes == other.hashes @@ -518,13 +527,17 @@ impl backend::BlockImportOperation for BlockImportOperatio fn reset_storage(&mut self, storage: Storage) -> sp_blockchain::Result { check_genesis_storage(&storage)?; - let child_delta = storage.children_default.into_iter() + let child_delta = storage.children_default.iter() .map(|(_storage_key, child_content)| - (child_content.child_info, child_content.data.into_iter().map(|(k, v)| (k, Some(v))))); + ( + &child_content.child_info, + child_content.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))) + ) + ); let (root, transaction) = self.old_state.full_storage_root( - storage.top.into_iter().map(|(k, v)| (k, Some(v))), - child_delta + storage.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), + child_delta, ); self.new_state = Some(transaction); diff --git a/client/api/src/leaves.rs b/client/api/src/leaves.rs index e89aeed8d135abe311d19d5df2659042a05fdd63..25f9f3d29b02259d82ce8f7e9b0a8b8bd38c1a12 100644 --- a/client/api/src/leaves.rs +++ b/client/api/src/leaves.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Helper for managing the set of available leaves in the chain for DB implementations. use std::collections::BTreeMap; diff --git a/client/api/src/notifications.rs b/client/api/src/notifications.rs index 72071eb20c48e151c8ded711e5377b188895afc8..ec63c372c7e5969b28e5dfdbd54afae0b757b1b6 100644 --- a/client/api/src/notifications.rs +++ b/client/api/src/notifications.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Storage notifications use std::{ diff --git a/client/api/src/proof_provider.rs b/client/api/src/proof_provider.rs index 66039b960155f3865666e6281ef8c59b79fee4ee..5749ae0576fc38799d8c2eeb9b2aeaccd49b2bca 100644 --- a/client/api/src/proof_provider.rs +++ b/client/api/src/proof_provider.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/client/authority-discovery/Cargo.toml b/client/authority-discovery/Cargo.toml index ad567bf0e67c2d7936fe2dcc5f97a440349e1e78..a3ff17d9e0a4d08795d903619a282c676c4d7c7d 100644 --- a/client/authority-discovery/Cargo.toml +++ b/client/authority-discovery/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-authority-discovery" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" @@ -17,27 +17,27 @@ prost-build = "0.6.1" [dependencies] bytes = "0.5.0" -codec = { package = "parity-scale-codec", default-features = false, version = "1.3.0" } +codec = { package = "parity-scale-codec", default-features = false, version = "1.3.1" } derive_more = "0.99.2" futures = "0.3.4" futures-timer = "3.0.1" -libp2p = { version = "0.18.1", default-features = false, features = ["secp256k1", "libp2p-websocket"] } +libp2p = { version = "0.20.1", default-features = false, features = ["kad"] } log = "0.4.8" -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-alpha.8"} +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-rc4"} prost = "0.6.1" rand = "0.7.2" -sc-client-api = { version = "2.0.0-alpha.8", path = "../api" } -sc-keystore = { version = "2.0.0-alpha.8", path = "../keystore" } -sc-network = { version = "0.8.0-alpha.8", path = "../network" } +sc-client-api = { version = "2.0.0-rc4", path = "../api" } +sc-keystore = { version = "2.0.0-rc4", path = "../keystore" } +sc-network = { version = "0.8.0-rc4", path = "../network" } serde_json = "1.0.41" -sp-authority-discovery = { version = "2.0.0-alpha.8", path = "../../primitives/authority-discovery" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../primitives/blockchain" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.8", path = "../../primitives/api" } +sp-authority-discovery = { version = "2.0.0-rc4", path = "../../primitives/authority-discovery" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../primitives/blockchain" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } +sp-api = { version = "2.0.0-rc4", path = "../../primitives/api" } [dev-dependencies] env_logger = "0.7.0" quickcheck = "0.9.0" -sc-peerset = { version = "2.0.0-alpha.8", path = "../peerset" } -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../test-utils/runtime/client"} +sc-peerset = { version = "2.0.0-rc4", path = "../peerset" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../test-utils/runtime/client"} diff --git a/client/authority-discovery/src/lib.rs b/client/authority-discovery/src/lib.rs index bc76c1433140331b07c276fe132454b476b076d5..ba1c9f0fa8dce636b6dc7c6d5b308cbe84935469 100644 --- a/client/authority-discovery/src/lib.rs +++ b/client/authority-discovery/src/lib.rs @@ -58,19 +58,27 @@ use futures::task::{Context, Poll}; use futures::{Future, FutureExt, ready, Stream, StreamExt}; use futures_timer::Delay; +use addr_cache::AddrCache; use codec::Decode; use error::{Error, Result}; +use libp2p::core::multiaddr; use log::{debug, error, log_enabled}; use prometheus_endpoint::{Counter, CounterVec, Gauge, Opts, U64, register}; use prost::Message; use sc_client_api::blockchain::HeaderBackend; -use sc_network::{Multiaddr, config::MultiaddrWithPeerId, DhtEvent, ExHashT, NetworkStateInfo}; +use sc_network::{ + config::MultiaddrWithPeerId, + DhtEvent, + ExHashT, + Multiaddr, + NetworkStateInfo, + PeerId, +}; use sp_authority_discovery::{AuthorityDiscoveryApi, AuthorityId, AuthoritySignature, AuthorityPair}; use sp_core::crypto::{key_types, Pair}; use sp_core::traits::BareCryptoStorePtr; use sp_runtime::{traits::Block as BlockT, generic::BlockId}; use sp_api::ProvideRuntimeApi; -use addr_cache::AddrCache; #[cfg(test)] mod tests; @@ -233,7 +241,7 @@ where .collect(), None => self.network.external_addresses() .into_iter() - .map(|a| a.with(libp2p::core::multiaddr::Protocol::P2p( + .map(|a| a.with(multiaddr::Protocol::P2p( self.network.local_peer_id().into(), ))) .map(|a| a.to_vec()) @@ -294,13 +302,26 @@ where .authorities(&id) .map_err(Error::CallingRuntime)?; + let local_keys = match &self.role { + Role::Authority(key_store) => { + key_store.read() + .sr25519_public_keys(key_types::AUTHORITY_DISCOVERY) + .into_iter() + .collect::>() + }, + Role::Sentry => HashSet::new(), + }; + for authority_id in authorities.iter() { - if let Some(metrics) = &self.metrics { - metrics.request.inc(); - } + // Make sure we don't look up our own keys. + if !local_keys.contains(authority_id.as_ref()) { + if let Some(metrics) = &self.metrics { + metrics.request.inc(); + } - self.network - .get_value(&hash_authority_id(authority_id.as_ref())); + self.network + .get_value(&hash_authority_id(authority_id.as_ref())); + } } Ok(()) @@ -410,6 +431,8 @@ where .get(&remote_key) .ok_or(Error::MatchingHashedAuthorityIdWithAuthorityId)?; + let local_peer_id = self.network.local_peer_id(); + let remote_addresses: Vec = values.into_iter() .map(|(_k, v)| { let schema::SignedAuthorityAddresses { signature, addresses } = @@ -434,7 +457,27 @@ where Ok(addresses) }) .collect::>>>()? - .into_iter().flatten().collect(); + .into_iter() + .flatten() + // Ignore own addresses. + .filter(|addr| !addr.iter().any(|protocol| { + // Parse to PeerId first as Multihashes of old and new PeerId + // representation don't equal. + // + // See https://github.com/libp2p/rust-libp2p/issues/555 for + // details. + if let multiaddr::Protocol::P2p(hash) = protocol { + let peer_id = match PeerId::from_multihash(hash) { + Ok(peer_id) => peer_id, + Err(_) => return true, // Discard address. + }; + + return peer_id == local_peer_id; + } + + false // Multiaddr does not contain a PeerId. + })) + .collect(); if !remote_addresses.is_empty() { self.addr_cache.insert(authority_id.clone(), remote_addresses); diff --git a/client/authority-discovery/src/tests.rs b/client/authority-discovery/src/tests.rs index 09c0319d299a9e6ed4fe3016c4f16ac7996a6c11..09a65fd138c11ad7178fe91237a644e1d6e71a1b 100644 --- a/client/authority-discovery/src/tests.rs +++ b/client/authority-discovery/src/tests.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use std::{iter::FromIterator, sync::{Arc, Mutex}}; use futures::channel::mpsc::channel; @@ -23,10 +24,10 @@ use futures::future::{poll_fn, FutureExt}; use futures::sink::SinkExt; use futures::task::LocalSpawn; use futures::poll; -use libp2p::{kad, PeerId}; +use libp2p::{kad, core::multiaddr, PeerId}; use sp_api::{ProvideRuntimeApi, ApiRef}; -use sp_core::testing::KeyStore; +use sp_core::{crypto::Public, testing::KeyStore}; use sp_runtime::traits::{Zero, Block as BlockT, NumberFor}; use substrate_test_runtime_client::runtime::Block; @@ -209,7 +210,7 @@ impl NetworkStateInfo for TestNetwork { } fn external_addresses(&self) -> Vec { - vec!["/ip6/2001:db8::".parse().unwrap()] + vec!["/ip6/2001:db8::/tcp/30333".parse().unwrap()] } } @@ -280,7 +281,7 @@ fn publish_discover_cycle() { let peer_id = network.local_peer_id(); let address = network.external_addresses().pop().unwrap(); - address.with(libp2p::core::multiaddr::Protocol::P2p( + address.with(multiaddr::Protocol::P2p( peer_id.into(), )) }; @@ -460,3 +461,102 @@ fn dont_stop_polling_when_error_is_returned() { } ); } + +/// In the scenario of a validator publishing the address of its sentry node to +/// the DHT, said sentry node should not add its own Multiaddr to the +/// peerset "authority" priority group. +#[test] +fn never_add_own_address_to_priority_group() { + let validator_key_store = KeyStore::new(); + let validator_public = validator_key_store + .write() + .sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None) + .unwrap(); + + let sentry_network: Arc = Arc::new(Default::default()); + + let sentry_multiaddr = { + let peer_id = sentry_network.local_peer_id(); + let address: Multiaddr = "/ip6/2001:db8:0:0:0:0:0:2/tcp/30333".parse().unwrap(); + + address.with(multiaddr::Protocol::P2p( + peer_id.into(), + )) + }; + + // Address of some other sentry node of `validator`. + let random_multiaddr = { + let peer_id = PeerId::random(); + let address: Multiaddr = "/ip6/2001:db8:0:0:0:0:0:1/tcp/30333".parse().unwrap(); + + address.with(multiaddr::Protocol::P2p( + peer_id.into(), + )) + }; + + let dht_event = { + let addresses = vec![ + sentry_multiaddr.to_vec(), + random_multiaddr.to_vec(), + ]; + + let mut serialized_addresses = vec![]; + schema::AuthorityAddresses { addresses } + .encode(&mut serialized_addresses) + .map_err(Error::EncodingProto) + .unwrap(); + + let signature = validator_key_store.read() + .sign_with( + key_types::AUTHORITY_DISCOVERY, + &validator_public.clone().into(), + serialized_addresses.as_slice(), + ) + .map_err(|_| Error::Signing) + .unwrap(); + + let mut signed_addresses = vec![]; + schema::SignedAuthorityAddresses { + addresses: serialized_addresses.clone(), + signature, + } + .encode(&mut signed_addresses) + .map_err(Error::EncodingProto) + .unwrap(); + + let key = hash_authority_id(&validator_public.to_raw_vec()); + let value = signed_addresses; + (key, value) + }; + + let (_dht_event_tx, dht_event_rx) = channel(1); + let sentry_test_api = Arc::new(TestApi { + // Make sure the sentry node identifies its validator as an authority. + authorities: vec![validator_public.into()], + }); + + let mut sentry_authority_discovery = AuthorityDiscovery::new( + sentry_test_api, + sentry_network.clone(), + vec![], + dht_event_rx.boxed(), + Role::Sentry, + None, + ); + + sentry_authority_discovery.handle_dht_value_found_event(vec![dht_event]).unwrap(); + + assert_eq!( + sentry_network.set_priority_group_call.lock().unwrap().len(), 1, + "Expect authority discovery to set the priority set.", + ); + + assert_eq!( + sentry_network.set_priority_group_call.lock().unwrap()[0], + ( + "authorities".to_string(), + HashSet::from_iter(vec![random_multiaddr.clone()].into_iter(),) + ), + "Expect authority discovery to only add `random_multiaddr`." + ); +} diff --git a/client/basic-authorship/Cargo.toml b/client/basic-authorship/Cargo.toml index 5a5aaacd5f0d432c72a7b211d37b7cd1296bf1da..b6a853a1a14ef05dfe176e4da8cd48ed78596f82 100644 --- a/client/basic-authorship/Cargo.toml +++ b/client/basic-authorship/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-basic-authorship" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -12,23 +12,25 @@ description = "Basic implementation of block-authoring logic." targets = ["x86_64-unknown-linux-gnu"] [dependencies] -log = "0.4.8" +codec = { package = "parity-scale-codec", version = "1.3.1" } futures = "0.3.4" -codec = { package = "parity-scale-codec", version = "1.3.0" } -sp-api = { version = "2.0.0-alpha.8", path = "../../primitives/api" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../primitives/blockchain" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../api" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../primitives/consensus/common" } -sp-inherents = { version = "2.0.0-alpha.8", path = "../../primitives/inherents" } -sc-telemetry = { version = "2.0.0-alpha.8", path = "../telemetry" } -sp-transaction-pool = { version = "2.0.0-alpha.8", path = "../../primitives/transaction-pool" } -sc-block-builder = { version = "0.8.0-alpha.8", path = "../block-builder" } -tokio-executor = { version = "0.2.0-alpha.6", features = ["blocking"] } futures-timer = "3.0.1" +log = "0.4.8" +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-rc4"} +sp-api = { version = "2.0.0-rc4", path = "../../primitives/api" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../primitives/blockchain" } +sc-client-api = { version = "2.0.0-rc4", path = "../api" } +sp-consensus = { version = "0.8.0-rc4", path = "../../primitives/consensus/common" } +sp-inherents = { version = "2.0.0-rc4", path = "../../primitives/inherents" } +sc-telemetry = { version = "2.0.0-rc4", path = "../telemetry" } +sp-transaction-pool = { version = "2.0.0-rc4", path = "../../primitives/transaction-pool" } +sc-block-builder = { version = "0.8.0-rc4", path = "../block-builder" } +sc-proposer-metrics = { version = "0.8.0-rc4", path = "../proposer-metrics" } +tokio-executor = { version = "0.2.0-alpha.6", features = ["blocking"] } [dev-dependencies] -sc-transaction-pool = { version = "2.0.0-alpha.8", path = "../../client/transaction-pool" } -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../test-utils/runtime/client" } +sc-transaction-pool = { version = "2.0.0-rc4", path = "../../client/transaction-pool" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../test-utils/runtime/client" } parking_lot = "0.10.0" diff --git a/client/basic-authorship/src/basic_authorship.rs b/client/basic-authorship/src/basic_authorship.rs index 2deecc72b9876d3b5e17f14605169687c6409abd..383d0ea6fcad4ce01e6672368668aab6164048e3 100644 --- a/client/basic-authorship/src/basic_authorship.rs +++ b/client/basic-authorship/src/basic_authorship.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! A consensus proposer for "basic" chains which use the primitive inherent-data. // FIXME #1021 move this into sp-consensus @@ -38,21 +39,31 @@ use futures::{executor, future, future::Either}; use sp_blockchain::{HeaderBackend, ApplyExtrinsicFailed::Validity, Error::ApplyExtrinsicFailed}; use std::marker::PhantomData; +use prometheus_endpoint::Registry as PrometheusRegistry; +use sc_proposer_metrics::MetricsLink as PrometheusMetrics; + /// Proposer factory. pub struct ProposerFactory { /// The client instance. client: Arc, /// The transaction pool. transaction_pool: Arc, + /// Prometheus Link, + metrics: PrometheusMetrics, /// phantom member to pin the `Backend` type. _phantom: PhantomData, } impl ProposerFactory { - pub fn new(client: Arc, transaction_pool: Arc) -> Self { + pub fn new( + client: Arc, + transaction_pool: Arc, + prometheus: Option<&PrometheusRegistry>, + ) -> Self { ProposerFactory { client, transaction_pool, + metrics: PrometheusMetrics::new(prometheus), _phantom: PhantomData, } } @@ -80,15 +91,14 @@ impl ProposerFactory info!("🙌 Starting consensus session on top of parent {:?}", parent_hash); let proposer = Proposer { - inner: Arc::new(ProposerInner { - client: self.client.clone(), - parent_hash, - parent_id: id, - parent_number: *parent_header.number(), - transaction_pool: self.transaction_pool.clone(), - now, - _phantom: PhantomData, - }), + client: self.client.clone(), + parent_hash, + parent_id: id, + parent_number: *parent_header.number(), + transaction_pool: self.transaction_pool.clone(), + now, + metrics: self.metrics.clone(), + _phantom: PhantomData, }; proposer @@ -120,17 +130,13 @@ impl sp_consensus::Environment for /// The proposer logic. pub struct Proposer { - inner: Arc>, -} - -/// Proposer inner, to wrap parameters under Arc. -struct ProposerInner { client: Arc, parent_hash: ::Hash, parent_id: BlockId, parent_number: <::Header as HeaderT>::Number, transaction_pool: Arc, now: Box time::Instant + Send + Sync>, + metrics: PrometheusMetrics, _phantom: PhantomData, } @@ -152,22 +158,21 @@ impl sp_consensus::Proposer for type Error = sp_blockchain::Error; fn propose( - &mut self, + self, inherent_data: InherentData, inherent_digests: DigestFor, max_duration: time::Duration, record_proof: RecordProof, ) -> Self::Proposal { - let inner = self.inner.clone(); tokio_executor::blocking::run(move || { // leave some time for evaluation and block finalization (33%) - let deadline = (inner.now)() + max_duration - max_duration / 3; - inner.propose_with(inherent_data, inherent_digests, deadline, record_proof) + let deadline = (self.now)() + max_duration - max_duration / 3; + self.propose_with(inherent_data, inherent_digests, deadline, record_proof) }) } } -impl ProposerInner +impl Proposer where A: TransactionPool, B: backend::Backend + Send + Sync + 'static, @@ -178,7 +183,7 @@ impl ProposerInner + BlockBuilderApi, { fn propose_with( - &self, + self, inherent_data: InherentData, inherent_digests: DigestFor, deadline: time::Instant, @@ -219,7 +224,7 @@ impl ProposerInner } // proceed with transactions - let mut is_first = true; + let block_timer = time::Instant::now(); let mut skipped = 0; let mut unqueue_invalid = Vec::new(); let pending_iterator = match executor::block_on(future::select( @@ -229,7 +234,8 @@ impl ProposerInner Either::Left((iterator, _)) => iterator, Either::Right(_) => { log::warn!( - "Timeout fired waiting for transaction pool to be ready. Proceeding to block production anyway.", + "Timeout fired waiting for transaction pool at block #{}. Proceeding with production.", + self.parent_number, ); self.transaction_pool.ready() } @@ -255,10 +261,7 @@ impl ProposerInner } Err(ApplyExtrinsicFailed(Validity(e))) if e.exhausted_resources() => { - if is_first { - debug!("[{:?}] Invalid transaction: FullBlock on empty block", pending_tx_hash); - unqueue_invalid.push(pending_tx_hash); - } else if skipped < MAX_SKIPPED_TRANSACTIONS { + if skipped < MAX_SKIPPED_TRANSACTIONS { skipped += 1; debug!( "Block seems full, but will try {} more transactions before quitting.", @@ -281,14 +284,19 @@ impl ProposerInner unqueue_invalid.push(pending_tx_hash); } } - - is_first = false; } self.transaction_pool.remove_invalid(&unqueue_invalid); let (block, storage_changes, proof) = block_builder.build()?.into_inner(); + self.metrics.report( + |metrics| { + metrics.number_of_transactions.set(block.extrinsics().len() as u64); + metrics.block_constructed.observe(block_timer.elapsed().as_secs_f64()); + } + ); + info!("🎁 Prepared block for proposing at {} [hash: {:?}; parent_hash: {}; extrinsics ({}): [{}]]", block.header().number(), ::Hash::from(block.header().hash()), @@ -324,15 +332,14 @@ mod tests { use parking_lot::Mutex; use sp_consensus::{BlockOrigin, Proposer}; use substrate_test_runtime_client::{ - prelude::*, - runtime::{Extrinsic, Transfer}, + prelude::*, TestClientBuilder, runtime::{Extrinsic, Transfer}, TestClientBuilderExt, }; use sp_transaction_pool::{ChainEvent, MaintainedTransactionPool, TransactionSource}; use sc_transaction_pool::{BasicPool, FullChainApi}; use sp_api::Core; - use backend::Backend; use sp_blockchain::HeaderBackend; use sp_runtime::traits::NumberFor; + use sc_client_api::Backend; const SOURCE: TransactionSource = TransactionSource::External; @@ -345,12 +352,12 @@ mod tests { }.into_signed_tx() } - fn chain_event(block_number: u64, header: B::Header) -> ChainEvent + fn chain_event(header: B::Header) -> ChainEvent where NumberFor: From { ChainEvent::NewBlock { - id: BlockId::Number(block_number.into()), - retracted: vec![], + hash: header.hash(), + tree_route: None, is_new_best: true, header, } @@ -374,15 +381,16 @@ mod tests { futures::executor::block_on( txpool.maintain(chain_event( - 0, - client.header(&BlockId::Number(0u64)).expect("header get error").expect("there should be header") + client.header(&BlockId::Number(0u64)) + .expect("header get error") + .expect("there should be header") )) ); - let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone()); + let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone(), None); let cell = Mutex::new((false, time::Instant::now())); - let mut proposer = proposer_factory.init_with_now( + let proposer = proposer_factory.init_with_now( &client.header(&BlockId::number(0)).unwrap().unwrap(), Box::new(move || { let mut value = cell.lock(); @@ -420,10 +428,10 @@ mod tests { ).0 ); - let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone()); + let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone(), None); let cell = Mutex::new((false, time::Instant::now())); - let mut proposer = proposer_factory.init_with_now( + let proposer = proposer_factory.init_with_now( &client.header(&BlockId::number(0)).unwrap().unwrap(), Box::new(move || { let mut value = cell.lock(); @@ -445,8 +453,7 @@ mod tests { #[test] fn proposed_storage_changes_should_match_execute_block_storage_changes() { - let (client, backend) = substrate_test_runtime_client::TestClientBuilder::new() - .build_with_backend(); + let (client, backend) = TestClientBuilder::new().build_with_backend(); let client = Arc::new(client); let txpool = Arc::new( BasicPool::new( @@ -465,14 +472,15 @@ mod tests { futures::executor::block_on( txpool.maintain(chain_event( - 0, - client.header(&BlockId::Number(0u64)).expect("header get error").expect("there should be header") + client.header(&BlockId::Number(0u64)) + .expect("header get error") + .expect("there should be header"), )) ); - let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone()); + let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone(), None); - let mut proposer = proposer_factory.init_with_now( + let proposer = proposer_factory.init_with_now( &client.header(&block_id).unwrap().unwrap(), Box::new(move || time::Instant::now()), ); @@ -493,8 +501,11 @@ mod tests { backend.changes_trie_storage(), ).unwrap(); - let storage_changes = api.into_storage_changes(&state, changes_trie_state.as_ref(), genesis_hash) - .unwrap(); + let storage_changes = api.into_storage_changes( + &state, + changes_trie_state.as_ref(), + genesis_hash, + ).unwrap(); assert_eq!( proposal.storage_changes.transaction_storage_root, @@ -536,14 +547,14 @@ mod tests { ]) ).unwrap(); - let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone()); + let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone(), None); let mut propose_block = | client: &TestClient, number, expected_block_extrinsics, expected_pool_transactions, | { - let mut proposer = proposer_factory.init_with_now( + let proposer = proposer_factory.init_with_now( &client.header(&BlockId::number(number)).unwrap().unwrap(), Box::new(move || time::Instant::now()), ); @@ -564,8 +575,9 @@ mod tests { futures::executor::block_on( txpool.maintain(chain_event( - 0, - client.header(&BlockId::Number(0u64)).expect("header get error").expect("there should be header") + client.header(&BlockId::Number(0u64)) + .expect("header get error") + .expect("there should be header") )) ); @@ -575,8 +587,9 @@ mod tests { futures::executor::block_on( txpool.maintain(chain_event( - 1, - client.header(&BlockId::Number(1)).expect("header get error").expect("there should be header") + client.header(&BlockId::Number(1)) + .expect("header get error") + .expect("there should be header") )) ); diff --git a/client/basic-authorship/src/lib.rs b/client/basic-authorship/src/lib.rs index bebce17059fc2faff758af247f226c8348b4d7c7..4f53c87de3979d7230629455ff761ca39d398657 100644 --- a/client/basic-authorship/src/lib.rs +++ b/client/basic-authorship/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Basic implementation of block-authoring logic. //! //! # Example @@ -24,12 +25,15 @@ //! # use sp_consensus::{Environment, Proposer, RecordProof}; //! # use sp_runtime::generic::BlockId; //! # use std::{sync::Arc, time::Duration}; -//! # use substrate_test_runtime_client::{self, runtime::{Extrinsic, Transfer}, AccountKeyring}; +//! # use substrate_test_runtime_client::{ +//! # runtime::{Extrinsic, Transfer}, AccountKeyring, +//! # DefaultTestClientBuilderExt, TestClientBuilderExt, +//! # }; //! # use sc_transaction_pool::{BasicPool, FullChainApi}; //! # let client = Arc::new(substrate_test_runtime_client::new()); //! # let txpool = Arc::new(BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone())), None).0); //! // The first step is to create a `ProposerFactory`. -//! let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone()); +//! let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone(), None); //! //! // From this factory, we create a `Proposer`. //! let proposer = proposer_factory.init( @@ -37,7 +41,7 @@ //! ); //! //! // The proposer is created asynchronously. -//! let mut proposer = futures::executor::block_on(proposer).unwrap(); +//! let proposer = futures::executor::block_on(proposer).unwrap(); //! //! // This `Proposer` allows us to create a block proposition. //! // The proposer will grab transactions from the transaction pool, and put them into the block. diff --git a/client/block-builder/Cargo.toml b/client/block-builder/Cargo.toml index 07eb6fc82af289b0c2ef18ab3077cf8c1b25dff5..1e733355f752689fd5a2edc400f79d0656de325a 100644 --- a/client/block-builder/Cargo.toml +++ b/client/block-builder/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-block-builder" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -13,16 +13,16 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../primitives/state-machine" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.8", path = "../../primitives/api" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../primitives/consensus/common" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../primitives/blockchain" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-block-builder = { version = "2.0.0-alpha.8", path = "../../primitives/block-builder" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../api" } -codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } +sp-state-machine = { version = "0.8.0-rc4", path = "../../primitives/state-machine" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } +sp-api = { version = "2.0.0-rc4", path = "../../primitives/api" } +sp-consensus = { version = "0.8.0-rc4", path = "../../primitives/consensus/common" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../primitives/blockchain" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-block-builder = { version = "2.0.0-rc4", path = "../../primitives/block-builder" } +sc-client-api = { version = "2.0.0-rc4", path = "../api" } +codec = { package = "parity-scale-codec", version = "1.3.1", features = ["derive"] } [dev-dependencies] substrate-test-runtime-client = { path = "../../test-utils/runtime/client" } -sp-trie = { version = "2.0.0-alpha.8", path = "../../primitives/trie" } +sp-trie = { version = "2.0.0-rc4", path = "../../primitives/trie" } diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index 730fe0258ee4f2e40cf4036dfc5111625658fa2a..af40b336623d85ac441c8e76a7f9619688726746 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate block builder //! //! This crate provides the [`BlockBuilder`] utility and the corresponding runtime api diff --git a/client/chain-spec/Cargo.toml b/client/chain-spec/Cargo.toml index 305de23b1d13993cd720dfb42d6e6afdc61e564e..a3176deee5ed7666f650df8a2bd6024d4704580b 100644 --- a/client/chain-spec/Cargo.toml +++ b/client/chain-spec/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-chain-spec" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -12,12 +12,12 @@ description = "Substrate chain configurations." targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sc-chain-spec-derive = { version = "2.0.0-alpha.8", path = "./derive" } +sc-chain-spec-derive = { version = "2.0.0-rc4", path = "./derive" } impl-trait-for-tuples = "0.1.3" -sc-network = { version = "0.8.0-alpha.8", path = "../network" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } +sc-network = { version = "0.8.0-rc4", path = "../network" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } -sp-chain-spec = { version = "2.0.0-alpha.8", path = "../../primitives/chain-spec" } -sc-telemetry = { version = "2.0.0-alpha.8", path = "../telemetry" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } +sp-chain-spec = { version = "2.0.0-rc4", path = "../../primitives/chain-spec" } +sc-telemetry = { version = "2.0.0-rc4", path = "../telemetry" } diff --git a/client/chain-spec/derive/Cargo.toml b/client/chain-spec/derive/Cargo.toml index beacd7ffda73ca945e153f499f7ee3e4bd9343eb..75a290dc98332178df5290616b9fd1bb81755bee 100644 --- a/client/chain-spec/derive/Cargo.toml +++ b/client/chain-spec/derive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-chain-spec-derive" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 333aa6fb5cb35994e5e21882e9cc316d7f567d7d..52414f8687c96ed97f56e84e1345d3c0f8423ecc 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate chain configurations. use std::{borrow::Cow, fs::File, path::PathBuf, sync::Arc, collections::HashMap}; diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index 6fb26942612d3637202a0645c2dbbba32801ff7b..66bce2b1363c2e701dec8c5e3be1379bacb51cc0 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -158,3 +158,9 @@ pub trait ChainSpec: BuildStorage + Send { /// This will be used as storage at genesis. fn set_storage(&mut self, storage: Storage); } + +impl std::fmt::Debug for dyn ChainSpec { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "ChainSpec(name = {:?}, id = {:?})", self.name(), self.id()) + } +} diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml index 8be0381ee88b7744315833e9a4ea52d4b4d4ff22..616b4f3481324f6c4a3d157df663f3f99972750e 100644 --- a/client/cli/Cargo.toml +++ b/client/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-cli" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "Substrate CLI interface." edition = "2018" @@ -12,7 +12,6 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -clap = "2.33.0" derive_more = "0.99.2" env_logger = "0.7.0" log = "0.4.8" @@ -21,28 +20,27 @@ regex = "1.3.1" time = "0.1.42" ansi_term = "0.12.1" lazy_static = "1.4.0" -app_dirs = "1.2.1" tokio = { version = "0.2.9", features = [ "signal", "rt-core", "rt-threaded" ] } futures = "0.3.4" fdlimit = "0.1.4" serde_json = "1.0.41" -sc-informant = { version = "0.8.0-alpha.8", path = "../informant" } -sp-panic-handler = { version = "2.0.0-alpha.8", path = "../../primitives/panic-handler" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../api" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../primitives/blockchain" } -sc-network = { version = "0.8.0-alpha.8", path = "../network" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } -sp-utils = { version = "2.0.0-alpha.8", path = "../../primitives/utils" } -sp-version = { version = "2.0.0-alpha.8", path = "../../primitives/version" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sc-service = { version = "0.8.0-alpha.8", default-features = false, path = "../service" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../primitives/state-machine" } -sc-telemetry = { version = "2.0.0-alpha.8", path = "../telemetry" } -substrate-prometheus-endpoint = { path = "../../utils/prometheus" , version = "0.8.0-alpha.8"} -sp-keyring = { version = "2.0.0-alpha.8", path = "../../primitives/keyring" } +sc-informant = { version = "0.8.0-rc4", path = "../informant" } +sp-panic-handler = { version = "2.0.0-rc4", path = "../../primitives/panic-handler" } +sc-client-api = { version = "2.0.0-rc4", path = "../api" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../primitives/blockchain" } +sc-network = { version = "0.8.0-rc4", path = "../network" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-rc4", path = "../../primitives/utils" } +sp-version = { version = "2.0.0-rc4", path = "../../primitives/version" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sc-service = { version = "0.8.0-rc4", default-features = false, path = "../service" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../primitives/state-machine" } +sc-telemetry = { version = "2.0.0-rc4", path = "../telemetry" } +substrate-prometheus-endpoint = { path = "../../utils/prometheus" , version = "0.8.0-rc4"} +sp-keyring = { version = "2.0.0-rc4", path = "../../primitives/keyring" } names = "0.11.0" structopt = "0.3.8" -sc-tracing = { version = "2.0.0-alpha.8", path = "../tracing" } +sc-tracing = { version = "2.0.0-rc4", path = "../tracing" } chrono = "0.4.10" parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } diff --git a/client/cli/src/arg_enums.rs b/client/cli/src/arg_enums.rs index 3269a702fdbf035a423b5529bd9925d8b719c629..db13fff76148e6acbfc74212e0c4a4cacc53e525 100644 --- a/client/cli/src/arg_enums.rs +++ b/client/cli/src/arg_enums.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -178,6 +178,8 @@ arg_enum! { pub const DEFAULT_EXECUTION_SYNCING: ExecutionStrategy = ExecutionStrategy::NativeElseWasm; /// Default value for the `--execution-import-block` parameter. pub const DEFAULT_EXECUTION_IMPORT_BLOCK: ExecutionStrategy = ExecutionStrategy::NativeElseWasm; +/// Default value for the `--execution-import-block` parameter when the node is a validator. +pub const DEFAULT_EXECUTION_IMPORT_BLOCK_VALIDATOR: ExecutionStrategy = ExecutionStrategy::Wasm; /// Default value for the `--execution-block-construction` parameter. pub const DEFAULT_EXECUTION_BLOCK_CONSTRUCTION: ExecutionStrategy = ExecutionStrategy::Wasm; /// Default value for the `--execution-offchain-worker` parameter. diff --git a/client/cli/src/commands/build_spec_cmd.rs b/client/cli/src/commands/build_spec_cmd.rs index ca927adacd6fb0711afb8cb990930adebb7f2359..23626359ff131cb4c131f0cc51633ca99dc855bc 100644 --- a/client/cli/src/commands/build_spec_cmd.rs +++ b/client/cli/src/commands/build_spec_cmd.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::error; use crate::params::NodeKeyParams; use crate::params::SharedParams; @@ -23,9 +24,10 @@ use log::info; use sc_network::config::build_multiaddr; use sc_service::{config::MultiaddrWithPeerId, Configuration}; use structopt::StructOpt; +use std::io::Write; /// The `build-spec` command used to build a specification. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct BuildSpecCmd { /// Force raw genesis storage output. #[structopt(long = "raw")] @@ -65,9 +67,9 @@ impl BuildSpecCmd { } let json = sc_service::chain_ops::build_spec(&*spec, raw_output)?; - - print!("{}", json); - + if std::io::stdout().write_all(json.as_bytes()).is_err() { + let _ = std::io::stderr().write_all(b"Error writing to stdout\n"); + } Ok(()) } } diff --git a/client/cli/src/commands/check_block_cmd.rs b/client/cli/src/commands/check_block_cmd.rs index 7d5110a29c149477d2be29fe31512a823fc9d93c..c000ea7fb11eec63ccc9e4d1c494ed620de196cb 100644 --- a/client/cli/src/commands/check_block_cmd.rs +++ b/client/cli/src/commands/check_block_cmd.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::{ CliConfiguration, error, params::{ImportParams, SharedParams, BlockNumberOrHash}, }; @@ -24,7 +25,7 @@ use std::{fmt::Debug, str::FromStr}; use structopt::StructOpt; /// The `check-block` command used to validate blocks. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct CheckBlockCmd { /// Block hash or number #[structopt(value_name = "HASH or NUMBER")] diff --git a/client/cli/src/commands/export_blocks_cmd.rs b/client/cli/src/commands/export_blocks_cmd.rs index b23f16d88461974e7817f812c7175252fbea987b..7c523c0555d55516aca6daad20a7c4c7fd3747a1 100644 --- a/client/cli/src/commands/export_blocks_cmd.rs +++ b/client/cli/src/commands/export_blocks_cmd.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::error; use crate::params::{BlockNumber, DatabaseParams, PruningParams, SharedParams}; use crate::CliConfiguration; @@ -30,7 +31,7 @@ use std::path::PathBuf; use structopt::StructOpt; /// The `export-blocks` command used to export blocks. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct ExportBlocksCmd { /// Output file name or stdout if unspecified. #[structopt(parse(from_os_str))] diff --git a/client/cli/src/commands/export_state_cmd.rs b/client/cli/src/commands/export_state_cmd.rs index 45b943ba3ea38a4837ea8568ca03ce25f8691ed7..23a43a178abe5c532d8e8462748ba5939c7e14a5 100644 --- a/client/cli/src/commands/export_state_cmd.rs +++ b/client/cli/src/commands/export_state_cmd.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,18 +15,19 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::{ CliConfiguration, error, params::{PruningParams, SharedParams, BlockNumberOrHash}, }; use log::info; use sc_service::{Configuration, ServiceBuilderCommand}; use sp_runtime::traits::{Block as BlockT, NumberFor}; -use std::{fmt::Debug, str::FromStr}; +use std::{fmt::Debug, str::FromStr, io::Write}; use structopt::StructOpt; /// The `export-state` command used to export the state of a given block into /// a chain spec. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct ExportStateCmd { /// Block hash or number. #[structopt(value_name = "HASH or NUMBER")] @@ -58,15 +59,15 @@ impl ExportStateCmd { { info!("Exporting raw state..."); let mut input_spec = config.chain_spec.cloned_box(); - let block_id = self.input.clone().map(|b| b.parse()).transpose()?; + let block_id = self.input.as_ref().map(|b| b.parse()).transpose()?; let raw_state = builder(config)?.export_raw_state(block_id)?; input_spec.set_storage(raw_state); info!("Generating new chain spec..."); let json = sc_service::chain_ops::build_spec(&*input_spec, true)?; - - print!("{}", json); - + if std::io::stdout().write_all(json.as_bytes()).is_err() { + let _ = std::io::stderr().write_all(b"Error writing to stdout\n"); + } Ok(()) } } diff --git a/client/cli/src/commands/import_blocks_cmd.rs b/client/cli/src/commands/import_blocks_cmd.rs index 4690eb23f0f99d32f92ed016d1531fffa0d134c0..8e178c4b97964040d2ca26ba7ee1d6d4105d9008 100644 --- a/client/cli/src/commands/import_blocks_cmd.rs +++ b/client/cli/src/commands/import_blocks_cmd.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::error; use crate::params::ImportParams; use crate::params::SharedParams; @@ -28,7 +29,7 @@ use std::path::PathBuf; use structopt::StructOpt; /// The `import-blocks` command used to import blocks. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct ImportBlocksCmd { /// Input file or stdin if unspecified. #[structopt(parse(from_os_str))] @@ -40,6 +41,10 @@ pub struct ImportBlocksCmd { #[structopt(long = "default-heap-pages", value_name = "COUNT")] pub default_heap_pages: Option, + /// Try importing blocks from binary format rather than JSON. + #[structopt(long)] + pub binary: bool, + #[allow(missing_docs)] #[structopt(flatten)] pub shared_params: SharedParams, @@ -78,7 +83,7 @@ impl ImportBlocksCmd { }; builder(config)? - .import_blocks(file, false) + .import_blocks(file, false, self.binary) .await .map_err(Into::into) } diff --git a/client/cli/src/commands/mod.rs b/client/cli/src/commands/mod.rs index c1fd9e03283ed2f4cca30208b39cd51471b0c362..04cce66bef80d231a2b9d63509d1133914c7e5a1 100644 --- a/client/cli/src/commands/mod.rs +++ b/client/cli/src/commands/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -27,11 +27,11 @@ mod run_cmd; pub use self::build_spec_cmd::BuildSpecCmd; pub use self::check_block_cmd::CheckBlockCmd; pub use self::export_blocks_cmd::ExportBlocksCmd; +pub use self::export_state_cmd::ExportStateCmd; pub use self::import_blocks_cmd::ImportBlocksCmd; pub use self::purge_chain_cmd::PurgeChainCmd; pub use self::revert_cmd::RevertCmd; pub use self::run_cmd::RunCmd; -pub use self::export_state_cmd::ExportStateCmd; use std::fmt::Debug; use structopt::StructOpt; @@ -40,7 +40,7 @@ use structopt::StructOpt; /// The core commands are split into multiple subcommands and `Run` is the default subcommand. From /// the CLI user perspective, it is not visible that `Run` is a subcommand. So, all parameters of /// `Run` are exported as main executable parameters. -#[derive(Debug, Clone, StructOpt)] +#[derive(Debug, StructOpt)] pub enum Subcommand { /// Build a spec.json file, outputs to stdout. BuildSpec(BuildSpecCmd), @@ -162,7 +162,7 @@ macro_rules! substrate_cli_subcommands { } } - fn base_path(&self) -> $crate::Result<::std::option::Option<::std::path::PathBuf>> { + fn base_path(&self) -> $crate::Result<::std::option::Option> { match self { $($enum::$variant(cmd) => cmd.base_path()),* } @@ -278,10 +278,16 @@ macro_rules! substrate_cli_subcommands { } } - fn execution_strategies(&self, is_dev: bool) + fn execution_strategies(&self, is_dev: bool, is_validator: bool) -> $crate::Result<::sc_client_api::execution_extensions::ExecutionStrategies> { match self { - $($enum::$variant(cmd) => cmd.execution_strategies(is_dev)),* + $($enum::$variant(cmd) => cmd.execution_strategies(is_dev, is_validator)),* + } + } + + fn rpc_ipc(&self) -> $crate::Result<::std::option::Option<::std::string::String>> { + match self { + $($enum::$variant(cmd) => cmd.rpc_ipc()),* } } @@ -409,4 +415,3 @@ macro_rules! substrate_cli_subcommands { substrate_cli_subcommands!( Subcommand => BuildSpec, ExportBlocks, ImportBlocks, CheckBlock, Revert, PurgeChain, ExportState ); - diff --git a/client/cli/src/commands/purge_chain_cmd.rs b/client/cli/src/commands/purge_chain_cmd.rs index 9b77906e38449cde2b5a3fbb227d9c58058f5ac4..053f427309828c12df5b07693847feea3ec28b5a 100644 --- a/client/cli/src/commands/purge_chain_cmd.rs +++ b/client/cli/src/commands/purge_chain_cmd.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::error; use crate::params::{DatabaseParams, SharedParams}; use crate::CliConfiguration; @@ -25,7 +26,7 @@ use std::io::{self, Write}; use structopt::StructOpt; /// The `purge-chain` command used to remove the whole chain. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct PurgeChainCmd { /// Skip interactive prompt by answering yes automatically. #[structopt(short = "y")] diff --git a/client/cli/src/commands/revert_cmd.rs b/client/cli/src/commands/revert_cmd.rs index 03b16578b33a9e995f69f3627fc6023b86adf351..1b5489df708a74fa59ba6188e8c1707a7d98b1c1 100644 --- a/client/cli/src/commands/revert_cmd.rs +++ b/client/cli/src/commands/revert_cmd.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::error; use crate::params::{BlockNumber, PruningParams, SharedParams}; use crate::CliConfiguration; @@ -24,7 +25,7 @@ use std::fmt::Debug; use structopt::StructOpt; /// The `revert` command used revert the chain to a previous state. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct RevertCmd { /// Number of blocks to revert. #[structopt(default_value = "256")] diff --git a/client/cli/src/commands/run_cmd.rs b/client/cli/src/commands/run_cmd.rs index ea8bd640a548bfa7909d6fae9cfc2da37b7c7197..de5589196f27fa0ecb6d2fa9b749f7e367f0cf66 100644 --- a/client/cli/src/commands/run_cmd.rs +++ b/client/cli/src/commands/run_cmd.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,18 +15,19 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::arg_enums::RpcMethods; use crate::error::{Error, Result}; use crate::params::ImportParams; use crate::params::KeystoreParams; use crate::params::NetworkParams; +use crate::params::OffchainWorkerParams; use crate::params::SharedParams; use crate::params::TransactionPoolParams; -use crate::params::OffchainWorkerParams; use crate::CliConfiguration; use regex::Regex; use sc_service::{ - config::{MultiaddrWithPeerId, PrometheusConfig, TransactionPoolOptions}, + config::{BasePath, MultiaddrWithPeerId, PrometheusConfig, TransactionPoolOptions}, ChainSpec, Role, }; use sc_telemetry::TelemetryEndpoints; @@ -34,7 +35,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use structopt::StructOpt; /// The `run` command used to run a node. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct RunCmd { /// Enable validator mode. /// @@ -64,7 +65,7 @@ pub struct RunCmd { pub sentry: Vec, /// Disable GRANDPA voter when running in validator mode, otherwise disable the GRANDPA observer. - #[structopt(long = "no-grandpa")] + #[structopt(long)] pub no_grandpa: bool, /// Experimental: Run in light client mode. @@ -121,6 +122,10 @@ pub struct RunCmd { #[structopt(long = "prometheus-external")] pub prometheus_external: bool, + /// Specify IPC RPC server path + #[structopt(long = "ipc-path", value_name = "PATH")] + pub ipc_path: Option, + /// Specify HTTP RPC server TCP port. #[structopt(long = "rpc-port", value_name = "PORT")] pub rpc_port: Option, @@ -249,6 +254,16 @@ pub struct RunCmd { conflicts_with_all = &[ "sentry", "public-addr" ] )] pub sentry_nodes: Vec, + + /// Run a temporary node. + /// + /// A temporary directory will be created to store the configuration and will be deleted + /// at the end of the process. + /// + /// Note: the directory is random per process execution. This directory is used as base path + /// which includes: database, node key and keystore. + #[structopt(long, conflicts_with = "base-path")] + pub tmp: bool, } impl RunCmd { @@ -310,7 +325,7 @@ impl CliConfiguration for RunCmd { Error::Input(format!( "Invalid node name '{}'. Reason: {}. If unsure, use none.", name, msg - )); + )) })?; Ok(name) @@ -423,6 +438,10 @@ impl CliConfiguration for RunCmd { Ok(Some(SocketAddr::new(interface, self.rpc_port.unwrap_or(9933)))) } + fn rpc_ipc(&self) -> Result> { + Ok(self.ipc_path.clone()) + } + fn rpc_ws(&self) -> Result> { let interface = rpc_interface( self.ws_external, @@ -445,6 +464,14 @@ impl CliConfiguration for RunCmd { fn max_runtime_instances(&self) -> Result> { Ok(self.max_runtime_instances.map(|x| x.min(256))) } + + fn base_path(&self) -> Result> { + Ok(if self.tmp { + Some(BasePath::new_temp_dir()?) + } else { + self.shared_params().base_path() + }) + } } /// Check whether a node name is considered as valid. @@ -583,7 +610,9 @@ mod tests { #[test] fn tests_node_name_bad() { - assert!(is_node_name_valid("long names are not very cool for the ui").is_err()); + assert!(is_node_name_valid( + "very very long names are really not very cool for the ui at all, really they're not" + ).is_err()); assert!(is_node_name_valid("Dots.not.Ok").is_err()); assert!(is_node_name_valid("http://visit.me").is_err()); assert!(is_node_name_valid("https://visit.me").is_err()); diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs index d309c1e34aa062a48626613af8850185429002c5..5563f46115bfd2b1ed556fe441ae811df389373d 100644 --- a/client/cli/src/config.rs +++ b/client/cli/src/config.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Configuration trait for a CLI based on substrate use crate::arg_enums::Database; @@ -23,23 +24,19 @@ use crate::{ init_logger, DatabaseParams, ImportParams, KeystoreParams, NetworkParams, NodeKeyParams, OffchainWorkerParams, PruningParams, SharedParams, SubstrateCli, }; -use app_dirs::{AppDataType, AppInfo}; use names::{Generator, Name}; use sc_client_api::execution_extensions::ExecutionStrategies; use sc_service::config::{ - Configuration, DatabaseConfig, ExtTransport, KeystoreConfig, NetworkConfiguration, + BasePath, Configuration, DatabaseConfig, ExtTransport, KeystoreConfig, NetworkConfiguration, NodeKeyConfig, OffchainWorkerConfig, PrometheusConfig, PruningMode, Role, RpcMethods, - TaskType, TelemetryEndpoints, TransactionPoolOptions, WasmExecutionMethod, + TaskExecutor, TelemetryEndpoints, TransactionPoolOptions, WasmExecutionMethod, }; use sc_service::{ChainSpec, TracingReceiver}; -use std::future::Future; use std::net::SocketAddr; use std::path::PathBuf; -use std::pin::Pin; -use std::sync::Arc; /// The maximum number of characters for a node name. -pub(crate) const NODE_NAME_MAX_LENGTH: usize = 32; +pub(crate) const NODE_NAME_MAX_LENGTH: usize = 64; /// default sub directory to store network config pub(crate) const DEFAULT_NETWORK_CONFIG_PATH: &'static str = "network"; @@ -87,7 +84,7 @@ pub trait CliConfiguration: Sized { /// Get the base path of the configuration (if any) /// /// By default this is retrieved from `SharedParams`. - fn base_path(&self) -> Result> { + fn base_path(&self) -> Result> { Ok(self.shared_params().base_path()) } @@ -246,9 +243,14 @@ pub trait CliConfiguration: Sized { /// /// By default this is retrieved from `ImportParams` if it is available. Otherwise its /// `ExecutionStrategies::default()`. - fn execution_strategies(&self, is_dev: bool) -> Result { - Ok(self.import_params() - .map(|x| x.execution_strategies(is_dev)) + fn execution_strategies( + &self, + is_dev: bool, + is_validator: bool, + ) -> Result { + Ok(self + .import_params() + .map(|x| x.execution_strategies(is_dev, is_validator)) .unwrap_or(Default::default())) } @@ -259,6 +261,13 @@ pub trait CliConfiguration: Sized { Ok(Default::default()) } + /// Get the RPC IPC path (`None` if disabled). + /// + /// By default this is `None`. + fn rpc_ipc(&self) -> Result> { + Ok(Default::default()) + } + /// Get the RPC websocket address (`None` if disabled). /// /// By default this is `None`. @@ -397,23 +406,17 @@ pub trait CliConfiguration: Sized { fn create_configuration( &self, cli: &C, - task_executor: Arc + Send>>, TaskType) + Send + Sync>, + task_executor: TaskExecutor, ) -> Result { let is_dev = self.is_dev()?; let chain_id = self.chain_id(is_dev)?; let chain_spec = cli.load_spec(chain_id.as_str())?; - let config_dir = self + let base_path = self .base_path()? - .unwrap_or_else(|| { - app_dirs::get_app_root( - AppDataType::UserData, - &AppInfo { - name: C::executable_name(), - author: C::author(), - }, - ) - .expect("app directories exist on all supported platforms; qed") - }) + .unwrap_or_else(|| BasePath::from_project("", "", C::executable_name())); + let config_dir = base_path + .path() + .to_path_buf() .join("chains") .join(chain_spec.id()); let net_config_dir = config_dir.join(DEFAULT_NETWORK_CONFIG_PATH); @@ -423,6 +426,7 @@ pub trait CliConfiguration: Sized { let node_key = self.node_key(&net_config_dir)?; let role = self.role(is_dev)?; let max_runtime_instances = self.max_runtime_instances()?.unwrap_or(8); + let is_validator = role.is_network_authority(); let unsafe_pruning = self .import_params() @@ -448,9 +452,10 @@ pub trait CliConfiguration: Sized { state_cache_child_ratio: self.state_cache_child_ratio()?, pruning: self.pruning(unsafe_pruning, &role)?, wasm_method: self.wasm_method()?, - execution_strategies: self.execution_strategies(is_dev)?, + execution_strategies: self.execution_strategies(is_dev, is_validator)?, rpc_http: self.rpc_http()?, rpc_ws: self.rpc_ws()?, + rpc_ipc: self.rpc_ipc()?, rpc_methods: self.rpc_methods()?, rpc_ws_max_connections: self.rpc_ws_max_connections()?, rpc_cors: self.rpc_cors(is_dev)?, @@ -468,6 +473,8 @@ pub trait CliConfiguration: Sized { max_runtime_instances, announce_block: self.announce_block()?, role, + base_path: Some(base_path), + informant_output_format: Default::default(), }) } @@ -511,5 +518,5 @@ pub fn generate_node_name() -> String { if count < NODE_NAME_MAX_LENGTH { return node_name; } - }; + } } diff --git a/client/cli/src/error.rs b/client/cli/src/error.rs index 8839ff79c462b5597357078829db9c6271611053..f091354be154bbd62856143ed9c23e8c2444c82c 100644 --- a/client/cli/src/error.rs +++ b/client/cli/src/error.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Initialization errors. /// Result type alias for the CLI. @@ -26,7 +27,7 @@ pub enum Error { /// Io error Io(std::io::Error), /// Cli error - Cli(clap::Error), + Cli(structopt::clap::Error), /// Service error Service(sc_service::Error), /// Client error @@ -36,6 +37,7 @@ pub enum Error { Input(String), /// Invalid listen multiaddress #[display(fmt="Invalid listen multiaddress")] + #[from(ignore)] InvalidListenMultiaddress, /// Other uncategorized error. #[from(ignore)] diff --git a/client/cli/src/lib.rs b/client/cli/src/lib.rs index a9195ab383e8c454a353333cee9d37c0ef481c55..a702edba65784c717e8209715edffbb3934e85b6 100644 --- a/client/cli/src/lib.rs +++ b/client/cli/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate CLI library. #![warn(missing_docs)] @@ -36,11 +37,10 @@ use log::info; pub use params::*; use regex::Regex; pub use runner::*; -use sc_service::{ChainSpec, Configuration, TaskType}; -use std::future::Future; +use sc_service::{Configuration, TaskExecutor}; +pub use sc_service::{ChainSpec, Role}; +pub use sp_version::RuntimeVersion; use std::io::Write; -use std::pin::Pin; -use std::sync::Arc; pub use structopt; use structopt::{ clap::{self, AppSettings}, @@ -164,9 +164,9 @@ pub trait SubstrateCli: Sized { /// Print the error message and quit the program in case of failure. /// /// **NOTE:** This method WILL NOT exit when `--help` or `--version` (or short versions) are - /// used. It will return a [`clap::Error`], where the [`kind`] is a - /// [`ErrorKind::HelpDisplayed`] or [`ErrorKind::VersionDisplayed`] respectively. You must call - /// [`Error::exit`] or perform a [`std::process::exit`]. + /// used. It will return a [`clap::Error`], where the [`clap::Error::kind`] is a + /// [`clap::ErrorKind::HelpDisplayed`] or [`clap::ErrorKind::VersionDisplayed`] respectively. + /// You must call [`clap::Error::exit`] or perform a [`std::process::exit`]. fn try_from_iter(iter: I) -> clap::Result where Self: StructOpt + Sized, @@ -198,7 +198,7 @@ pub trait SubstrateCli: Sized { fn create_configuration( &self, command: &T, - task_executor: Arc + Send>>, TaskType) + Send + Sync>, + task_executor: TaskExecutor, ) -> error::Result { command.create_configuration(self, task_executor) } @@ -209,6 +209,9 @@ pub trait SubstrateCli: Sized { command.init::()?; Runner::new(self, command) } + + /// Native runtime version. + fn native_runtime_version(chain_spec: &Box) -> &'static RuntimeVersion; } /// Initialize the logger @@ -252,7 +255,7 @@ pub fn init_logger(pattern: &str) { format!("{}", Colour::Blue.bold().paint(x)) }); let millis = (now.tm_nsec as f32 / 1000000.0).floor() as usize; - let timestamp = format!("{}.{}", timestamp, millis); + let timestamp = format!("{}.{:03}", timestamp, millis); format!( "{} {} {} {} {}", Colour::Black.bold().paint(timestamp), diff --git a/client/cli/src/params/database_params.rs b/client/cli/src/params/database_params.rs index 76e38e424d07c0dab9acaa1434e3083d7822f5e4..24b23f6076a02302071c76a6da614c8e5c11e6a0 100644 --- a/client/cli/src/params/database_params.rs +++ b/client/cli/src/params/database_params.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,11 +15,12 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::arg_enums::Database; use structopt::StructOpt; /// Parameters for block import. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct DatabaseParams { /// Select database backend to use. #[structopt( diff --git a/client/cli/src/params/import_params.rs b/client/cli/src/params/import_params.rs index 5602f098d8902c0798c27e1431c5b23262dd69fd..c2fb34f90e6fc84c247d535d9323679e5608dc1c 100644 --- a/client/cli/src/params/import_params.rs +++ b/client/cli/src/params/import_params.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,9 +15,10 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::arg_enums::{ - ExecutionStrategy, TracingReceiver, WasmExecutionMethod, - DEFAULT_EXECUTION_BLOCK_CONSTRUCTION, DEFAULT_EXECUTION_IMPORT_BLOCK, + ExecutionStrategy, TracingReceiver, WasmExecutionMethod, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION, + DEFAULT_EXECUTION_IMPORT_BLOCK, DEFAULT_EXECUTION_IMPORT_BLOCK_VALIDATOR, DEFAULT_EXECUTION_OFFCHAIN_WORKER, DEFAULT_EXECUTION_OTHER, DEFAULT_EXECUTION_SYNCING, }; use crate::params::DatabaseParams; @@ -26,7 +27,7 @@ use sc_client_api::execution_extensions::ExecutionStrategies; use structopt::StructOpt; /// Parameters for block import. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct ImportParams { #[allow(missing_docs)] #[structopt(flatten)] @@ -103,22 +104,27 @@ impl ImportParams { } /// Get execution strategies for the parameters - pub fn execution_strategies( - &self, - is_dev: bool, - ) -> ExecutionStrategies { + pub fn execution_strategies(&self, is_dev: bool, is_validator: bool) -> ExecutionStrategies { let exec = &self.execution_strategies; - let exec_all_or = |strat: ExecutionStrategy, default: ExecutionStrategy| { - exec.execution.unwrap_or(if strat == default && is_dev { + let exec_all_or = |strat: Option, default: ExecutionStrategy| { + let default = if is_dev { ExecutionStrategy::Native } else { - strat - }).into() + default + }; + + exec.execution.unwrap_or(strat.unwrap_or(default)).into() + }; + + let default_execution_import_block = if is_validator { + DEFAULT_EXECUTION_IMPORT_BLOCK_VALIDATOR + } else { + DEFAULT_EXECUTION_IMPORT_BLOCK }; ExecutionStrategies { syncing: exec_all_or(exec.execution_syncing, DEFAULT_EXECUTION_SYNCING), - importing: exec_all_or(exec.execution_import_block, DEFAULT_EXECUTION_IMPORT_BLOCK), + importing: exec_all_or(exec.execution_import_block, default_execution_import_block), block_construction: exec_all_or(exec.execution_block_construction, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION), offchain_worker: @@ -129,27 +135,27 @@ impl ImportParams { } /// Execution strategies parameters. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct ExecutionStrategiesParams { - /// The means of execution used when calling into the runtime while syncing blocks. + /// The means of execution used when calling into the runtime for importing blocks as + /// part of an initial sync. #[structopt( long = "execution-syncing", value_name = "STRATEGY", possible_values = &ExecutionStrategy::variants(), case_insensitive = true, - default_value = DEFAULT_EXECUTION_SYNCING.as_str(), )] - pub execution_syncing: ExecutionStrategy, + pub execution_syncing: Option, - /// The means of execution used when calling into the runtime while importing blocks. + /// The means of execution used when calling into the runtime for general block import + /// (including locally authored blocks). #[structopt( long = "execution-import-block", value_name = "STRATEGY", possible_values = &ExecutionStrategy::variants(), case_insensitive = true, - default_value = DEFAULT_EXECUTION_IMPORT_BLOCK.as_str(), )] - pub execution_import_block: ExecutionStrategy, + pub execution_import_block: Option, /// The means of execution used when calling into the runtime while constructing blocks. #[structopt( @@ -157,9 +163,8 @@ pub struct ExecutionStrategiesParams { value_name = "STRATEGY", possible_values = &ExecutionStrategy::variants(), case_insensitive = true, - default_value = DEFAULT_EXECUTION_BLOCK_CONSTRUCTION.as_str(), )] - pub execution_block_construction: ExecutionStrategy, + pub execution_block_construction: Option, /// The means of execution used when calling into the runtime while using an off-chain worker. #[structopt( @@ -167,9 +172,8 @@ pub struct ExecutionStrategiesParams { value_name = "STRATEGY", possible_values = &ExecutionStrategy::variants(), case_insensitive = true, - default_value = DEFAULT_EXECUTION_OFFCHAIN_WORKER.as_str(), )] - pub execution_offchain_worker: ExecutionStrategy, + pub execution_offchain_worker: Option, /// The means of execution used when calling into the runtime while not syncing, importing or constructing blocks. #[structopt( @@ -177,9 +181,8 @@ pub struct ExecutionStrategiesParams { value_name = "STRATEGY", possible_values = &ExecutionStrategy::variants(), case_insensitive = true, - default_value = DEFAULT_EXECUTION_OTHER.as_str(), )] - pub execution_other: ExecutionStrategy, + pub execution_other: Option, /// The execution strategy that should be used by all execution contexts. #[structopt( diff --git a/client/cli/src/params/keystore_params.rs b/client/cli/src/params/keystore_params.rs index aa9ddeef499d9021acde0ca0518448637ea847df..8b20dd247aec6d6e3bb4b8c2b49026ae114e2fe5 100644 --- a/client/cli/src/params/keystore_params.rs +++ b/client/cli/src/params/keystore_params.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,17 +15,19 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::error::Result; use sc_service::config::KeystoreConfig; use std::fs; use std::path::PathBuf; use structopt::StructOpt; +use sp_core::crypto::SecretString; /// default sub directory for the key store const DEFAULT_KEYSTORE_CONFIG_PATH: &'static str = "keystore"; /// Parameters of the keystore -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct KeystoreParams { /// Specify custom keystore path. #[structopt(long = "keystore-path", value_name = "PATH", parse(from_os_str))] @@ -41,9 +43,10 @@ pub struct KeystoreParams { /// Password used by the keystore. #[structopt( long = "password", + parse(try_from_str = secret_string_from_str), conflicts_with_all = &[ "password-interactive", "password-filename" ] )] - pub password: Option, + pub password: Option, /// File that contains the password used by the keystore. #[structopt( @@ -55,26 +58,37 @@ pub struct KeystoreParams { pub password_filename: Option, } +/// Parse a sercret string, returning a displayable error. +pub fn secret_string_from_str(s: &str) -> std::result::Result { + Ok(std::str::FromStr::from_str(s) + .map_err(|_e| "Could not get SecretString".to_string())?) +} + impl KeystoreParams { /// Get the keystore configuration for the parameters pub fn keystore_config(&self, base_path: &PathBuf) -> Result { let password = if self.password_interactive { #[cfg(not(target_os = "unknown"))] { - Some(input_keystore_password()?.into()) + let mut password = input_keystore_password()?; + let secret = std::str::FromStr::from_str(password.as_str()) + .map_err(|()| "Error reading password")?; + use sp_core::crypto::Zeroize; + password.zeroize(); + Some(secret) } #[cfg(target_os = "unknown")] None } else if let Some(ref file) = self.password_filename { - Some( - fs::read_to_string(file) - .map_err(|e| format!("{}", e))? - .into(), - ) - } else if let Some(ref password) = self.password { - Some(password.clone().into()) + let mut password = fs::read_to_string(file) + .map_err(|e| format!("{}", e))?; + let secret = std::str::FromStr::from_str(password.as_str()) + .map_err(|()| "Error reading password")?; + use sp_core::crypto::Zeroize; + password.zeroize(); + Some(secret) } else { - None + self.password.clone() }; let path = self diff --git a/client/cli/src/params/mod.rs b/client/cli/src/params/mod.rs index 152fe4c93c1fe3629b35f03826b23aa6f94e3d49..f648337ed0af6846193d1afaf8518d223d7868c9 100644 --- a/client/cli/src/params/mod.rs +++ b/client/cli/src/params/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -39,7 +39,7 @@ pub use crate::params::shared_params::*; pub use crate::params::transaction_pool_params::*; /// Wrapper type of `String` that holds an unsigned integer of arbitrary size, formatted as a decimal. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct BlockNumber(String); impl FromStr for BlockNumber { @@ -72,7 +72,7 @@ impl BlockNumber { } /// Wrapper type that is either a `Hash` or the number of a `Block`. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct BlockNumberOrHash(String); impl FromStr for BlockNumberOrHash { diff --git a/client/cli/src/params/network_params.rs b/client/cli/src/params/network_params.rs index 6f5aea15d39e34167283c4f0fd8a8bdd5316a3c2..253585544d2600dd00bf9ba7fea3a6bca60b9a7d 100644 --- a/client/cli/src/params/network_params.rs +++ b/client/cli/src/params/network_params.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::params::node_key_params::NodeKeyParams; use sc_network::{ config::{NetworkConfiguration, NodeKeyConfig, NonReservedPeerMode, TransportConfig}, @@ -25,7 +26,7 @@ use std::path::PathBuf; use structopt::StructOpt; /// Parameters used to create the network configuration. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct NetworkParams { /// Specify a list of bootnodes. #[structopt(long = "bootnodes", value_name = "ADDR")] @@ -101,11 +102,6 @@ pub struct NetworkParams { /// By default this option is true for `--dev` and false otherwise. #[structopt(long)] pub discover_local: bool, - - /// Use the legacy "pre-mainnet-launch" networking protocol. Enable if things seem broken. - /// This option will be removed in the future. - #[structopt(long)] - pub legacy_network_protocol: bool, } impl NetworkParams { @@ -123,6 +119,9 @@ impl NetworkParams { let listen_addresses = if self.listen_addr.is_empty() { vec![ + Multiaddr::empty() + .with(Protocol::Ip6([0, 0, 0, 0, 0, 0, 0, 0].into())) + .with(Protocol::Tcp(port)), Multiaddr::empty() .with(Protocol::Ip4([0, 0, 0, 0].into())) .with(Protocol::Tcp(port)), @@ -161,7 +160,6 @@ impl NetworkParams { }, max_parallel_downloads: self.max_parallel_downloads, allow_non_globals_in_dht: self.discover_local || is_dev, - use_new_block_requests_protocol: !self.legacy_network_protocol, } } } diff --git a/client/cli/src/params/node_key_params.rs b/client/cli/src/params/node_key_params.rs index 34d5d618fbfc2a284a537a5bd1bfba295ffb55ba..689cc6c681c8339e79937e3dbb8ac9e3cfe04c6f 100644 --- a/client/cli/src/params/node_key_params.rs +++ b/client/cli/src/params/node_key_params.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use sc_network::config::NodeKeyConfig; use sp_core::H256; use std::{path::PathBuf, str::FromStr}; @@ -30,7 +31,7 @@ const NODE_KEY_ED25519_FILE: &str = "secret_ed25519"; /// Parameters used to create the `NodeKeyConfig`, which determines the keypair /// used for libp2p networking. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct NodeKeyParams { /// The secret key to use for libp2p networking. /// diff --git a/client/cli/src/params/offchain_worker_params.rs b/client/cli/src/params/offchain_worker_params.rs index 87f6e2375227b11cee381d784dc401aff948cebd..f8d48edc4729d5f97e4f389fe78dd3cd794423b9 100644 --- a/client/cli/src/params/offchain_worker_params.rs +++ b/client/cli/src/params/offchain_worker_params.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -32,7 +32,7 @@ use crate::OffchainWorkerEnabled; /// Offchain worker related parameters. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct OffchainWorkerParams { /// Should execute offchain workers on every block. /// diff --git a/client/cli/src/params/pruning_params.rs b/client/cli/src/params/pruning_params.rs index aa86d939e652a0fc2558f7668116c0b036aca4ad..7db808e6d8f2f8e249ff0e057a05297bfcf92a72 100644 --- a/client/cli/src/params/pruning_params.rs +++ b/client/cli/src/params/pruning_params.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,12 +15,13 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::error; use sc_service::{PruningMode, Role}; use structopt::StructOpt; /// Parameters to define the pruning mode -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct PruningParams { /// Specify the state pruning mode, a number of blocks to keep or 'archive'. /// diff --git a/client/cli/src/params/shared_params.rs b/client/cli/src/params/shared_params.rs index d29bd104ad0d761d6427c58971644ce80bed8832..ad9ab04070563ac5c7ea5939688f410a3257f232 100644 --- a/client/cli/src/params/shared_params.rs +++ b/client/cli/src/params/shared_params.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,41 +15,38 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + +use sc_service::config::BasePath; use std::path::PathBuf; use structopt::StructOpt; /// Shared parameters used by all `CoreParams`. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct SharedParams { /// Specify the chain specification (one of dev, local, or staging). - #[structopt(long = "chain", value_name = "CHAIN_SPEC")] + #[structopt(long, value_name = "CHAIN_SPEC")] pub chain: Option, /// Specify the development chain. - #[structopt(long = "dev")] + #[structopt(long, conflicts_with_all = &["chain"])] pub dev: bool, /// Specify custom base path. - #[structopt( - long = "base-path", - short = "d", - value_name = "PATH", - parse(from_os_str) - )] + #[structopt(long, short = "d", value_name = "PATH", parse(from_os_str))] pub base_path: Option, /// Sets a custom logging filter. Syntax is =, e.g. -lsync=debug. /// /// Log levels (least to most verbose) are error, warn, info, debug, and trace. /// By default, all targets log `info`. The global log level can be set with -l. - #[structopt(short = "l", long = "log", value_name = "LOG_PATTERN")] + #[structopt(short = "l", long, value_name = "LOG_PATTERN")] pub log: Vec, } impl SharedParams { /// Specify custom base path. - pub fn base_path(&self) -> Option { - self.base_path.clone() + pub fn base_path(&self) -> Option { + self.base_path.clone().map(Into::into) } /// Specify the development chain. diff --git a/client/cli/src/params/transaction_pool_params.rs b/client/cli/src/params/transaction_pool_params.rs index d5205b5c9f3e250a092c7d2be681adbda2f90baa..3ad278426922ec99ccc42e192cbda9db1d04207b 100644 --- a/client/cli/src/params/transaction_pool_params.rs +++ b/client/cli/src/params/transaction_pool_params.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,11 +15,12 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use sc_service::config::TransactionPoolOptions; use structopt::StructOpt; /// Parameters used to create the pool configuration. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, StructOpt)] pub struct TransactionPoolParams { /// Maximum number of transactions in the transaction pool. #[structopt(long = "pool-limit", value_name = "COUNT", default_value = "8192")] diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs index 042e21e4413e3ddf7ad25f171ee73a9d4895b943..fcc869dc870696ec5c9b9a66a6fb4f9fad7011c9 100644 --- a/client/cli/src/runner.rs +++ b/client/cli/src/runner.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,19 +15,20 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::CliConfiguration; use crate::Result; -use crate::SubstrateCli; use crate::Subcommand; +use crate::SubstrateCli; use chrono::prelude::*; use futures::pin_mut; use futures::select; use futures::{future, future::FutureExt, Future}; use log::info; -use sc_service::{AbstractService, Configuration, Role, ServiceBuilderCommand, TaskType}; +use sc_service::{Configuration, ServiceBuilderCommand, TaskType, TaskManager}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use sp_utils::metrics::{TOKIO_THREADS_ALIVE, TOKIO_THREADS_TOTAL}; -use std::{str::FromStr, fmt::Debug, marker::PhantomData, sync::Arc}; +use std::{fmt::Debug, marker::PhantomData, str::FromStr}; #[cfg(target_family = "unix")] async fn main(func: F) -> std::result::Result<(), Box> @@ -80,11 +81,11 @@ where pub fn build_runtime() -> std::result::Result { tokio::runtime::Builder::new() .threaded_scheduler() - .on_thread_start(||{ + .on_thread_start(|| { TOKIO_THREADS_ALIVE.inc(); TOKIO_THREADS_TOTAL.inc(); }) - .on_thread_stop(||{ + .on_thread_stop(|| { TOKIO_THREADS_ALIVE.dec(); }) .enable_all() @@ -117,41 +118,41 @@ impl Runner { let tokio_runtime = build_runtime()?; let runtime_handle = tokio_runtime.handle().clone(); - let task_executor = Arc::new( - move |fut, task_type| { - match task_type { - TaskType::Async => { runtime_handle.spawn(fut); } - TaskType::Blocking => { - runtime_handle.spawn( async move { - // `spawn_blocking` is looking for the current runtime, and as such has to be called - // from within `spawn`. - tokio::task::spawn_blocking(move || futures::executor::block_on(fut)) - }); - } + let task_executor = move |fut, task_type| { + match task_type { + TaskType::Async => { runtime_handle.spawn(fut); } + TaskType::Blocking => { + runtime_handle.spawn(async move { + // `spawn_blocking` is looking for the current runtime, and as such has to + // be called from within `spawn`. + tokio::task::spawn_blocking(move || futures::executor::block_on(fut)) + }); } } - ); + }; Ok(Runner { - config: command.create_configuration(cli, task_executor)?, + config: command.create_configuration(cli, task_executor.into())?, tokio_runtime, phantom: PhantomData, }) } - /// A helper function that runs an `AbstractService` with tokio and stops if the process receives - /// the signal `SIGTERM` or `SIGINT`. - pub fn run_node( - self, - new_light: FNL, - new_full: FNF, - runtime_version: sp_version::RuntimeVersion, - ) -> Result<()> where - FNL: FnOnce(Configuration) -> sc_service::error::Result, - FNF: FnOnce(Configuration) -> sc_service::error::Result, - SL: AbstractService + Unpin, - SF: AbstractService + Unpin, - { + /// Log information about the node itself. + /// + /// # Example: + /// + /// ```text + /// 2020-06-03 16:14:21 Substrate Node + /// 2020-06-03 16:14:21 ✌️ version 2.0.0-rc3-f4940588c-x86_64-linux-gnu + /// 2020-06-03 16:14:21 ❤️ by Parity Technologies , 2017-2020 + /// 2020-06-03 16:14:21 📋 Chain specification: Flaming Fir + /// 2020-06-03 16:14:21 🏷 Node name: jolly-rod-7462 + /// 2020-06-03 16:14:21 👤 Role: FULL + /// 2020-06-03 16:14:21 💾 Database: RocksDb at /tmp/c/chains/flamingfir7/db + /// 2020-06-03 16:14:21 ⛓ Native runtime: node-251 (substrate-node-1.tx1.au10) + /// ``` + fn print_node_infos(&self) { info!("{}", C::impl_name()); info!("✌️ version {}", C::impl_version()); info!( @@ -167,12 +168,7 @@ impl Runner { self.config.database, self.config.database.path().map_or_else(|| "".to_owned(), |p| p.display().to_string()) ); - info!("⛓ Native runtime: {}", runtime_version); - - match self.config.role { - Role::Light => self.run_service_until_exit(new_light), - _ => self.run_service_until_exit(new_full), - } + info!("⛓ Native runtime: {}", C::native_runtime_version(&self.config.chain_spec)); } /// A helper function that runs a future with tokio and stops if the process receives the signal @@ -203,33 +199,18 @@ impl Runner { } } - fn run_service_until_exit(mut self, service_builder: F) -> Result<()> - where - F: FnOnce(Configuration) -> std::result::Result, - T: AbstractService + Unpin, - { - let service = service_builder(self.config)?; - - let informant_future = sc_informant::build(&service, sc_informant::OutputFormat::Coloured); - let _informant_handle = self.tokio_runtime.spawn(informant_future); - - // we eagerly drop the service so that the internal exit future is fired, - // but we need to keep holding a reference to the global telemetry guard - // and drop the runtime first. - let _telemetry = service.telemetry(); - - { - let f = service.fuse(); - self.tokio_runtime - .block_on(main(f)) - .map_err(|e| e.to_string())?; - } - - // The `service` **must** have been destroyed here for the shutdown signal to propagate - // to all the tasks. Dropping `tokio_runtime` will block the thread until all tasks have - // shut down. - drop(self.tokio_runtime); - + /// A helper function that runs a node with tokio and stops if the process receives the signal + /// `SIGTERM` or `SIGINT`. + pub fn run_node_until_exit( + mut self, + initialise: impl FnOnce(Configuration) -> sc_service::error::Result, + ) -> Result<()> { + self.print_node_infos(); + let mut task_manager = initialise(self.config)?; + self.tokio_runtime.block_on(main(task_manager.future().fuse())) + .map_err(|e| e.to_string())?; + task_manager.terminate(); + drop(task_manager); Ok(()) } diff --git a/client/consensus/aura/Cargo.toml b/client/consensus/aura/Cargo.toml index a4f586126098bab67ffa789faf0d49e1274ec051..d080fd39d0e9bb56798b816359c751da04e0f833 100644 --- a/client/consensus/aura/Cargo.toml +++ b/client/consensus/aura/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-aura" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "Aura consensus algorithm for substrate" edition = "2018" @@ -12,37 +12,37 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.8", path = "../../../primitives/application-crypto" } -sp-consensus-aura = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/aura" } -sp-block-builder = { version = "2.0.0-alpha.8", path = "../../../primitives/block-builder" } -sc-block-builder = { version = "0.8.0-alpha.8", path = "../../../client/block-builder" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../../api" } -codec = { package = "parity-scale-codec", version = "1.3.0" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/common" } +sp-application-crypto = { version = "2.0.0-rc4", path = "../../../primitives/application-crypto" } +sp-consensus-aura = { version = "0.8.0-rc4", path = "../../../primitives/consensus/aura" } +sp-block-builder = { version = "2.0.0-rc4", path = "../../../primitives/block-builder" } +sc-block-builder = { version = "0.8.0-rc4", path = "../../../client/block-builder" } +sc-client-api = { version = "2.0.0-rc4", path = "../../api" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +sp-consensus = { version = "0.8.0-rc4", path = "../../../primitives/consensus/common" } derive_more = "0.99.2" futures = "0.3.4" futures-timer = "3.0.1" -sp-inherents = { version = "2.0.0-alpha.8", path = "../../../primitives/inherents" } -sc-keystore = { version = "2.0.0-alpha.8", path = "../../keystore" } +sp-inherents = { version = "2.0.0-rc4", path = "../../../primitives/inherents" } +sc-keystore = { version = "2.0.0-rc4", path = "../../keystore" } log = "0.4.8" parking_lot = "0.10.0" -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } -sp-io = { version = "2.0.0-alpha.8", path = "../../../primitives/io" } -sp-version = { version = "2.0.0-alpha.8", path = "../../../primitives/version" } -sc-consensus-slots = { version = "0.8.0-alpha.8", path = "../slots" } -sp-api = { version = "2.0.0-alpha.8", path = "../../../primitives/api" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-timestamp = { version = "2.0.0-alpha.8", path = "../../../primitives/timestamp" } -sc-telemetry = { version = "2.0.0-alpha.8", path = "../../telemetry" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.8.0-alpha.8"} +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } +sp-io = { version = "2.0.0-rc4", path = "../../../primitives/io" } +sp-version = { version = "2.0.0-rc4", path = "../../../primitives/version" } +sc-consensus-slots = { version = "0.8.0-rc4", path = "../slots" } +sp-api = { version = "2.0.0-rc4", path = "../../../primitives/api" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-timestamp = { version = "2.0.0-rc4", path = "../../../primitives/timestamp" } +sc-telemetry = { version = "2.0.0-rc4", path = "../../telemetry" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.8.0-rc4"} [dev-dependencies] -sp-keyring = { version = "2.0.0-alpha.8", path = "../../../primitives/keyring" } -sc-executor = { version = "0.8.0-alpha.8", path = "../../executor" } -sc-network = { version = "0.8.0-alpha.8", path = "../../network" } -sc-network-test = { version = "0.8.0-alpha.8", path = "../../network/test" } -sc-service = { version = "0.8.0-alpha.8", default-features = false, path = "../../service" } -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../../test-utils/runtime/client" } +sp-keyring = { version = "2.0.0-rc4", path = "../../../primitives/keyring" } +sc-executor = { version = "0.8.0-rc4", path = "../../executor" } +sc-network = { version = "0.8.0-rc4", path = "../../network" } +sc-network-test = { version = "0.8.0-rc4", path = "../../network/test" } +sc-service = { version = "0.8.0-rc4", default-features = false, path = "../../service" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../../test-utils/runtime/client" } env_logger = "0.7.0" tempfile = "3.1.0" diff --git a/client/consensus/aura/src/digests.rs b/client/consensus/aura/src/digests.rs index b415560e7687750ed5dd291a38d82aaeefd47354..3332e4c6a6dff99f2ee9ad367031fde15a33ada6 100644 --- a/client/consensus/aura/src/digests.rs +++ b/client/consensus/aura/src/digests.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Aura (Authority-Round) digests //! //! This implements the digests for AuRa, to allow the private diff --git a/client/consensus/aura/src/lib.rs b/client/consensus/aura/src/lib.rs index 7fc788e85b4e7a60eff29a5d1be6069c94953de4..8b30720d0b130433aa7a0237879d9a164c58dc6d 100644 --- a/client/consensus/aura/src/lib.rs +++ b/client/consensus/aura/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -246,7 +246,13 @@ impl sc_consensus_slots::SimpleSlotWorker for AuraW slot_number: u64, epoch_data: &Self::EpochData, ) -> Option { - slot_author::

(slot_number, epoch_data).cloned() + let expected_author = slot_author::

(slot_number, epoch_data); + expected_author.and_then(|p| { + self.keystore.read() + .key_pair_by_type::

(&p, sp_application_crypto::key_types::AURA).ok() + }).and_then(|p| { + Some(p.public()) + }) } fn pre_digest_data( @@ -827,7 +833,7 @@ pub fn import_queue( P: Pair + Send + Sync + 'static, P::Public: Clone + Eq + Send + Sync + Hash + Debug + Encode + Decode, P::Signature: Encode + Decode, - S: sp_core::traits::SpawnBlocking, + S: sp_core::traits::SpawnNamed, { register_aura_inherent_data_provider(&inherent_data_providers, slot_duration.get())?; initialize_authorities_cache(&*client)?; @@ -859,8 +865,11 @@ mod tests { use sp_keyring::sr25519::Keyring; use sc_client_api::BlockchainEvents; use sp_consensus_aura::sr25519::AuthorityPair; + use sc_consensus_slots::SimpleSlotWorker; use std::task::Poll; use sc_block_builder::BlockBuilderProvider; + use sp_runtime::traits::Header as _; + use substrate_test_runtime_client::runtime::{Header, H256}; type Error = sp_blockchain::Error; @@ -895,7 +904,7 @@ mod tests { type Proposal = future::Ready, Error>>; fn propose( - &mut self, + self, _: InherentData, digests: DigestFor, _: Duration, @@ -1044,4 +1053,55 @@ mod tests { Keyring::Charlie.public().into() ]); } + + #[test] + fn current_node_authority_should_claim_slot() { + let net = AuraTestNet::new(4); + + let mut authorities = vec![ + Keyring::Alice.public().into(), + Keyring::Bob.public().into(), + Keyring::Charlie.public().into() + ]; + + let keystore_path = tempfile::tempdir().expect("Creates keystore path"); + let keystore = sc_keystore::Store::open(keystore_path.path(), None).expect("Creates keystore."); + let my_key = keystore.write() + .generate_by_type::(AuthorityPair::ID) + .expect("Key should be created"); + authorities.push(my_key.public()); + + let net = Arc::new(Mutex::new(net)); + + let mut net = net.lock(); + let peer = net.peer(3); + let client = peer.client().as_full().expect("full clients are created").clone(); + let environ = DummyFactory(client.clone()); + + let worker = AuraWorker { + client: client.clone(), + block_import: Arc::new(Mutex::new(client)), + env: environ, + keystore, + sync_oracle: DummyOracle.clone(), + force_authoring: false, + _key_type: PhantomData::, + }; + + let head = Header::new( + 1, + H256::from_low_u64_be(0), + H256::from_low_u64_be(0), + Default::default(), + Default::default() + ); + assert!(worker.claim_slot(&head, 0, &authorities).is_none()); + assert!(worker.claim_slot(&head, 1, &authorities).is_none()); + assert!(worker.claim_slot(&head, 2, &authorities).is_none()); + assert!(worker.claim_slot(&head, 3, &authorities).is_some()); + assert!(worker.claim_slot(&head, 4, &authorities).is_none()); + assert!(worker.claim_slot(&head, 5, &authorities).is_none()); + assert!(worker.claim_slot(&head, 6, &authorities).is_none()); + assert!(worker.claim_slot(&head, 7, &authorities).is_some()); + } } diff --git a/client/consensus/babe/Cargo.toml b/client/consensus/babe/Cargo.toml index 033f3da62b79ae637c0982b48c28cd5dfc083b06..46c67e89171250a5d368d71b2f6f75fcdc230029 100644 --- a/client/consensus/babe/Cargo.toml +++ b/client/consensus/babe/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-babe" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "BABE consensus algorithm for substrate" edition = "2018" @@ -13,32 +13,32 @@ documentation = "https://docs.rs/sc-consensus-babe" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } -sp-consensus-babe = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/babe" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-application-crypto = { version = "2.0.0-alpha.8", path = "../../../primitives/application-crypto" } +codec = { package = "parity-scale-codec", version = "1.3.1", features = ["derive"] } +sp-consensus-babe = { version = "0.8.0-rc4", path = "../../../primitives/consensus/babe" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-application-crypto = { version = "2.0.0-rc4", path = "../../../primitives/application-crypto" } num-bigint = "0.2.3" num-rational = "0.2.2" num-traits = "0.2.8" serde = { version = "1.0.104", features = ["derive"] } -sp-version = { version = "2.0.0-alpha.8", path = "../../../primitives/version" } -sp-io = { version = "2.0.0-alpha.8", path = "../../../primitives/io" } -sp-inherents = { version = "2.0.0-alpha.8", path = "../../../primitives/inherents" } -sp-timestamp = { version = "2.0.0-alpha.8", path = "../../../primitives/timestamp" } -sc-telemetry = { version = "2.0.0-alpha.8", path = "../../telemetry" } -sc-keystore = { version = "2.0.0-alpha.8", path = "../../keystore" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../../api" } -sc-consensus-epochs = { version = "0.8.0-alpha.8", path = "../epochs" } -sp-api = { version = "2.0.0-alpha.8", path = "../../../primitives/api" } -sp-block-builder = { version = "2.0.0-alpha.8", path = "../../../primitives/block-builder" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/common" } -sp-consensus-vrf = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/vrf" } -sc-consensus-uncles = { version = "0.8.0-alpha.8", path = "../uncles" } -sc-consensus-slots = { version = "0.8.0-alpha.8", path = "../slots" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -fork-tree = { version = "2.0.0-alpha.8", path = "../../../utils/fork-tree" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.8.0-alpha.8"} +sp-version = { version = "2.0.0-rc4", path = "../../../primitives/version" } +sp-io = { version = "2.0.0-rc4", path = "../../../primitives/io" } +sp-inherents = { version = "2.0.0-rc4", path = "../../../primitives/inherents" } +sp-timestamp = { version = "2.0.0-rc4", path = "../../../primitives/timestamp" } +sc-telemetry = { version = "2.0.0-rc4", path = "../../telemetry" } +sc-keystore = { version = "2.0.0-rc4", path = "../../keystore" } +sc-client-api = { version = "2.0.0-rc4", path = "../../api" } +sc-consensus-epochs = { version = "0.8.0-rc4", path = "../epochs" } +sp-api = { version = "2.0.0-rc4", path = "../../../primitives/api" } +sp-block-builder = { version = "2.0.0-rc4", path = "../../../primitives/block-builder" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } +sp-consensus = { version = "0.8.0-rc4", path = "../../../primitives/consensus/common" } +sp-consensus-vrf = { version = "0.8.0-rc4", path = "../../../primitives/consensus/vrf" } +sc-consensus-uncles = { version = "0.8.0-rc4", path = "../uncles" } +sc-consensus-slots = { version = "0.8.0-rc4", path = "../slots" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +fork-tree = { version = "2.0.0-rc4", path = "../../../utils/fork-tree" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.8.0-rc4"} futures = "0.3.4" futures-timer = "3.0.1" parking_lot = "0.10.0" @@ -50,14 +50,15 @@ pdqselect = "0.1.0" derive_more = "0.99.2" [dev-dependencies] -sp-keyring = { version = "2.0.0-alpha.8", path = "../../../primitives/keyring" } -sc-executor = { version = "0.8.0-alpha.8", path = "../../executor" } -sc-network = { version = "0.8.0-alpha.8", path = "../../network" } -sc-network-test = { version = "0.8.0-alpha.8", path = "../../network/test" } -sc-service = { version = "0.8.0-alpha.8", default-features = false, path = "../../service" } -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../../test-utils/runtime/client" } -sc-block-builder = { version = "0.8.0-alpha.8", path = "../../block-builder" } +sp-keyring = { version = "2.0.0-rc4", path = "../../../primitives/keyring" } +sc-executor = { version = "0.8.0-rc4", path = "../../executor" } +sc-network = { version = "0.8.0-rc4", path = "../../network" } +sc-network-test = { version = "0.8.0-rc4", path = "../../network/test" } +sc-service = { version = "0.8.0-rc4", default-features = false, path = "../../service" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../../test-utils/runtime/client" } +sc-block-builder = { version = "0.8.0-rc4", path = "../../block-builder" } env_logger = "0.7.0" +rand_chacha = "0.2.2" tempfile = "3.1.0" [features] diff --git a/client/consensus/babe/rpc/Cargo.toml b/client/consensus/babe/rpc/Cargo.toml index 1f365a9e2ab2322bb096259ed0ba09a74ef179bc..03da64ff30193080a56491bf9ff46b44d9e050ca 100644 --- a/client/consensus/babe/rpc/Cargo.toml +++ b/client/consensus/babe/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-babe-rpc" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "RPC extensions for the BABE consensus algorithm" edition = "2018" @@ -12,24 +12,27 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sc-consensus-babe = { version = "0.8.0-alpha.8", path = "../" } -jsonrpc-core = "14.0.3" -jsonrpc-core-client = "14.0.5" -jsonrpc-derive = "14.0.3" -sp-consensus-babe = { version = "0.8.0-alpha.8", path = "../../../../primitives/consensus/babe" } +sc-consensus-babe = { version = "0.8.0-rc4", path = "../" } +sc-rpc-api = { version = "0.8.0-rc4", path = "../../../rpc-api" } +jsonrpc-core = "14.2.0" +jsonrpc-core-client = "14.2.0" +jsonrpc-derive = "14.2.1" +sp-consensus-babe = { version = "0.8.0-rc4", path = "../../../../primitives/consensus/babe" } serde = { version = "1.0.104", features=["derive"] } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../../primitives/runtime" } -sc-consensus-epochs = { version = "0.8.0-alpha.8", path = "../../epochs" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../../primitives/runtime" } +sc-consensus-epochs = { version = "0.8.0-rc4", path = "../../epochs" } futures = { version = "0.3.4", features = ["compat"] } derive_more = "0.99.2" -sp-api = { version = "2.0.0-alpha.8", path = "../../../../primitives/api" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../../../primitives/consensus/common" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../../primitives/core" } -sc-keystore = { version = "2.0.0-alpha.8", path = "../../../keystore" } +sp-api = { version = "2.0.0-rc4", path = "../../../../primitives/api" } +sp-consensus = { version = "0.8.0-rc4", path = "../../../../primitives/consensus/common" } +sp-core = { version = "2.0.0-rc4", path = "../../../../primitives/core" } +sp-application-crypto = { version = "2.0.0-rc4", path = "../../../../primitives/application-crypto" } +sc-keystore = { version = "2.0.0-rc4", path = "../../../keystore" } [dev-dependencies] -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../../../test-utils/runtime/client" } -sp-application-crypto = { version = "2.0.0-alpha.8", path = "../../../../primitives/application-crypto" } -sp-keyring = { version = "2.0.0-alpha.8", path = "../../../../primitives/keyring" } +sc-consensus = { version = "0.8.0-rc4", path = "../../../consensus/common" } +serde_json = "1.0.50" +sp-keyring = { version = "2.0.0-rc4", path = "../../../../primitives/keyring" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../../../test-utils/runtime/client" } tempfile = "3.1.0" diff --git a/client/consensus/babe/rpc/src/lib.rs b/client/consensus/babe/rpc/src/lib.rs index 395fa8dc4f3d904c05aec9b77d16064d87a2120e..652f4f00baac2ae582415142c540b95014d26894 100644 --- a/client/consensus/babe/rpc/src/lib.rs +++ b/client/consensus/babe/rpc/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! RPC api for babe. use sc_consensus_babe::{Epoch, authorship, Config}; @@ -31,12 +32,18 @@ use sp_consensus_babe::{ digests::PreDigest, }; use serde::{Deserialize, Serialize}; +use sp_core::{ + crypto::Public, + traits::BareCryptoStore, +}; +use sp_application_crypto::AppKey; use sc_keystore::KeyStorePtr; +use sc_rpc_api::DenyUnsafe; use sp_api::{ProvideRuntimeApi, BlockId}; use sp_runtime::traits::{Block as BlockT, Header as _}; use sp_consensus::{SelectChain, Error as ConsensusError}; use sp_blockchain::{HeaderBackend, HeaderMetadata, Error as BlockChainError}; -use std::{collections::HashMap, fmt, sync::Arc}; +use std::{collections::HashMap, sync::Arc}; type FutureResult = Box + Send>; @@ -49,8 +56,8 @@ pub trait BabeApi { fn epoch_authorship(&self) -> FutureResult>; } -/// Implements the BabeRPC trait for interacting with Babe. -pub struct BabeRPCHandler { +/// Implements the BabeRpc trait for interacting with Babe. +pub struct BabeRpcHandler { /// shared reference to the client. client: Arc, /// shared reference to EpochChanges @@ -61,9 +68,11 @@ pub struct BabeRPCHandler { babe_config: Config, /// The SelectChain strategy select_chain: SC, + /// Whether to deny unsafe calls + deny_unsafe: DenyUnsafe, } -impl BabeRPCHandler { +impl BabeRpcHandler { /// Creates a new instance of the BabeRpc handler. pub fn new( client: Arc, @@ -71,6 +80,7 @@ impl BabeRPCHandler { keystore: KeyStorePtr, babe_config: Config, select_chain: SC, + deny_unsafe: DenyUnsafe, ) -> Self { Self { client, @@ -78,19 +88,23 @@ impl BabeRPCHandler { keystore, babe_config, select_chain, + deny_unsafe, } } } -impl BabeApi for BabeRPCHandler +impl BabeApi for BabeRpcHandler where B: BlockT, C: ProvideRuntimeApi + HeaderBackend + HeaderMetadata + 'static, C::Api: BabeRuntimeApi, - ::Error: fmt::Debug, SC: SelectChain + Clone + 'static, { fn epoch_authorship(&self) -> FutureResult> { + if let Err(err) = self.deny_unsafe.check_if_safe() { + return Box::new(rpc_future::err(err.into())); + } + let ( babe_config, keystore, @@ -116,22 +130,23 @@ impl BabeApi for BabeRPCHandler let mut claims: HashMap = HashMap::new(); - let key_pairs = { - let keystore = keystore.read(); + let keys = { + let ks = keystore.read(); epoch.authorities.iter() .enumerate() - .flat_map(|(i, a)| { - keystore - .key_pair::(&a.0) - .ok() - .map(|kp| (kp, i)) + .filter_map(|(i, a)| { + if ks.has_keys(&[(a.0.to_raw_vec(), AuthorityId::ID)]) { + Some((a.0.clone(), i)) + } else { + None + } }) .collect::>() }; for slot_number in epoch_start..epoch_end { if let Some((claim, key)) = - authorship::claim_slot_using_key_pairs(slot_number, &epoch, &key_pairs) + authorship::claim_slot_using_keys(slot_number, &epoch, &keystore, &keys) { match claim { PreDigest::Primary { .. } => { @@ -213,7 +228,10 @@ fn epoch_data( mod tests { use super::*; use substrate_test_runtime_client::{ + runtime::Block, + Backend, DefaultTestClientBuilderExt, + TestClient, TestClientBuilderExt, TestClientBuilder, }; @@ -235,8 +253,9 @@ mod tests { (keystore, keystore_path) } - #[test] - fn rpc() { + fn test_babe_rpc_handler( + deny_unsafe: DenyUnsafe + ) -> BabeRpcHandler> { let builder = TestClientBuilder::new(); let (client, longest_chain) = builder.build_with_longest_chain(); let client = Arc::new(client); @@ -248,9 +267,21 @@ mod tests { ).expect("can initialize block-import"); let epoch_changes = link.epoch_changes().clone(); - let select_chain = longest_chain; let keystore = create_temp_keystore::(Ed25519Keyring::Alice).0; - let handler = BabeRPCHandler::new(client.clone(), epoch_changes, keystore, config, select_chain); + + BabeRpcHandler::new( + client.clone(), + epoch_changes, + keystore, + config, + longest_chain, + deny_unsafe, + ) + } + + #[test] + fn epoch_authorship_works() { + let handler = test_babe_rpc_handler(DenyUnsafe::No); let mut io = IoHandler::new(); io.extend_with(BabeApi::to_delegate(handler)); @@ -259,4 +290,19 @@ mod tests { assert_eq!(Some(response.into()), io.handle_request_sync(request)); } + + #[test] + fn epoch_authorship_is_unsafe() { + let handler = test_babe_rpc_handler(DenyUnsafe::Yes); + let mut io = IoHandler::new(); + + io.extend_with(BabeApi::to_delegate(handler)); + let request = r#"{"jsonrpc":"2.0","method":"babe_epochAuthorship","params": [],"id":1}"#; + + let response = io.handle_request_sync(request).unwrap(); + let mut response: serde_json::Value = serde_json::from_str(&response).unwrap(); + let error: RpcError = serde_json::from_value(response["error"].take()).unwrap(); + + assert_eq!(error, RpcError::method_not_found()) + } } diff --git a/client/consensus/babe/src/authorship.rs b/client/consensus/babe/src/authorship.rs index 1a6852c0c186dd35a38a9dadc94c70caea838013..682e04e380d7c32eeed887ee0db855b68e37039e 100644 --- a/client/consensus/babe/src/authorship.rs +++ b/client/consensus/babe/src/authorship.rs @@ -16,18 +16,24 @@ //! BABE authority selection and slot claiming. +use sp_application_crypto::AppKey; use sp_consensus_babe::{ - make_transcript, AuthorityId, BabeAuthorityWeight, BABE_VRF_PREFIX, - SlotNumber, AuthorityPair, + BABE_VRF_PREFIX, + AuthorityId, BabeAuthorityWeight, + SlotNumber, + make_transcript, + make_transcript_data, }; use sp_consensus_babe::digests::{ PreDigest, PrimaryPreDigest, SecondaryPlainPreDigest, SecondaryVRFPreDigest, }; use sp_consensus_vrf::schnorrkel::{VRFOutput, VRFProof}; -use sp_core::{U256, blake2_256}; +use sp_core::{U256, blake2_256, crypto::Public, traits::BareCryptoStore}; use codec::Encode; -use schnorrkel::vrf::VRFInOut; -use sp_core::Pair; +use schnorrkel::{ + keys::PublicKey, + vrf::VRFInOut, +}; use sc_keystore::KeyStorePtr; use super::Epoch; @@ -124,7 +130,8 @@ pub(super) fn secondary_slot_author( fn claim_secondary_slot( slot_number: SlotNumber, epoch: &Epoch, - key_pairs: &[(AuthorityPair, usize)], + keys: &[(AuthorityId, usize)], + keystore: &KeyStorePtr, author_secondary_vrf: bool, ) -> Option<(PreDigest, AuthorityId)> { let Epoch { authorities, randomness, epoch_index, .. } = epoch; @@ -139,31 +146,41 @@ fn claim_secondary_slot( *randomness, )?; - for (pair, authority_index) in key_pairs { - if pair.public() == *expected_author { + for (authority_id, authority_index) in keys { + if authority_id == expected_author { let pre_digest = if author_secondary_vrf { - let transcript = super::authorship::make_transcript( + let transcript_data = super::authorship::make_transcript_data( randomness, slot_number, *epoch_index, ); - - let s = get_keypair(&pair).vrf_sign(transcript); - - PreDigest::SecondaryVRF(SecondaryVRFPreDigest { + let result = keystore.read().sr25519_vrf_sign( + AuthorityId::ID, + authority_id.as_ref(), + transcript_data, + ); + if let Ok(signature) = result { + Some(PreDigest::SecondaryVRF(SecondaryVRFPreDigest { + slot_number, + vrf_output: VRFOutput(signature.output), + vrf_proof: VRFProof(signature.proof), + authority_index: *authority_index as u32, + })) + } else { + None + } + } else if keystore.read().has_keys(&[(authority_id.to_raw_vec(), AuthorityId::ID)]) { + Some(PreDigest::SecondaryPlain(SecondaryPlainPreDigest { slot_number, - vrf_output: VRFOutput(s.0.to_output()), - vrf_proof: VRFProof(s.1), authority_index: *authority_index as u32, - }) + })) } else { - PreDigest::SecondaryPlain(SecondaryPlainPreDigest { - slot_number, - authority_index: *authority_index as u32, - }) + None }; - return Some((pre_digest, pair.public())); + if let Some(pre_digest) = pre_digest { + return Some((pre_digest, authority_id.clone())); + } } } @@ -179,26 +196,22 @@ pub fn claim_slot( epoch: &Epoch, keystore: &KeyStorePtr, ) -> Option<(PreDigest, AuthorityId)> { - let key_pairs = { - let keystore = keystore.read(); - epoch.authorities.iter() - .enumerate() - .flat_map(|(i, a)| { - keystore.key_pair::(&a.0).ok().map(|kp| (kp, i)) - }) - .collect::>() - }; - claim_slot_using_key_pairs(slot_number, epoch, &key_pairs) + let authorities = epoch.authorities.iter() + .enumerate() + .map(|(index, a)| (a.0.clone(), index)) + .collect::>(); + claim_slot_using_keys(slot_number, epoch, keystore, &authorities) } /// Like `claim_slot`, but allows passing an explicit set of key pairs. Useful if we intend /// to make repeated calls for different slots using the same key pairs. -pub fn claim_slot_using_key_pairs( +pub fn claim_slot_using_keys( slot_number: SlotNumber, epoch: &Epoch, - key_pairs: &[(AuthorityPair, usize)], + keystore: &KeyStorePtr, + keys: &[(AuthorityId, usize)], ) -> Option<(PreDigest, AuthorityId)> { - claim_primary_slot(slot_number, epoch, epoch.config.c, &key_pairs) + claim_primary_slot(slot_number, epoch, epoch.config.c, keystore, &keys) .or_else(|| { if epoch.config.allowed_slots.is_secondary_plain_slots_allowed() || epoch.config.allowed_slots.is_secondary_vrf_slots_allowed() @@ -206,7 +219,8 @@ pub fn claim_slot_using_key_pairs( claim_secondary_slot( slot_number, &epoch, - &key_pairs, + keys, + keystore, epoch.config.allowed_slots.is_secondary_vrf_slots_allowed(), ) } else { @@ -215,11 +229,6 @@ pub fn claim_slot_using_key_pairs( }) } -fn get_keypair(q: &AuthorityPair) -> &schnorrkel::Keypair { - use sp_core::crypto::IsWrappedBy; - sp_core::sr25519::Pair::from_ref(q).as_ref() -} - /// Claim a primary slot if it is our turn. Returns `None` if it is not our turn. /// This hashes the slot number, epoch, genesis hash, and chain randomness into /// the VRF. If the VRF produces a value less than `threshold`, it is our turn, @@ -228,35 +237,89 @@ fn claim_primary_slot( slot_number: SlotNumber, epoch: &Epoch, c: (u64, u64), - key_pairs: &[(AuthorityPair, usize)], + keystore: &KeyStorePtr, + keys: &[(AuthorityId, usize)], ) -> Option<(PreDigest, AuthorityId)> { let Epoch { authorities, randomness, epoch_index, .. } = epoch; - for (pair, authority_index) in key_pairs { - let transcript = super::authorship::make_transcript(randomness, slot_number, *epoch_index); - + for (authority_id, authority_index) in keys { + let transcript = super::authorship::make_transcript( + randomness, + slot_number, + *epoch_index + ); + let transcript_data = super::authorship::make_transcript_data( + randomness, + slot_number, + *epoch_index + ); // Compute the threshold we will use. // // We already checked that authorities contains `key.public()`, so it can't // be empty. Therefore, this division in `calculate_threshold` is safe. let threshold = super::authorship::calculate_primary_threshold(c, authorities, *authority_index); - let pre_digest = get_keypair(pair) - .vrf_sign_after_check(transcript, |inout| super::authorship::check_primary_threshold(inout, threshold)) - .map(|s| { - PreDigest::Primary(PrimaryPreDigest { + let result = keystore.read().sr25519_vrf_sign( + AuthorityId::ID, + authority_id.as_ref(), + transcript_data, + ); + if let Ok(signature) = result { + let public = PublicKey::from_bytes(&authority_id.to_raw_vec()).ok()?; + let inout = match signature.output.attach_input_hash(&public, transcript) { + Ok(inout) => inout, + Err(_) => continue, + }; + if super::authorship::check_primary_threshold(&inout, threshold) { + let pre_digest = PreDigest::Primary(PrimaryPreDigest { slot_number, - vrf_output: VRFOutput(s.0.to_output()), - vrf_proof: VRFProof(s.1), + vrf_output: VRFOutput(signature.output), + vrf_proof: VRFProof(signature.proof), authority_index: *authority_index as u32, - }) - }); + }); - // early exit on first successful claim - if let Some(pre_digest) = pre_digest { - return Some((pre_digest, pair.public())); + return Some((pre_digest, authority_id.clone())); + } } } None } + +#[cfg(test)] +mod tests { + use super::*; + use sp_core::{sr25519::Pair, crypto::Pair as _}; + use sp_consensus_babe::{AuthorityId, BabeEpochConfiguration, AllowedSlots}; + + #[test] + fn claim_secondary_plain_slot_works() { + let keystore = sc_keystore::Store::new_in_memory(); + let valid_public_key = dbg!(keystore.write().sr25519_generate_new( + AuthorityId::ID, + Some(sp_core::crypto::DEV_PHRASE), + ).unwrap()); + + let authorities = vec![ + (AuthorityId::from(Pair::generate().0.public()), 5), + (AuthorityId::from(Pair::generate().0.public()), 7), + ]; + + let mut epoch = Epoch { + epoch_index: 10, + start_slot: 0, + duration: 20, + authorities: authorities.clone(), + randomness: Default::default(), + config: BabeEpochConfiguration { + c: (3, 10), + allowed_slots: AllowedSlots::PrimaryAndSecondaryPlainSlots, + }, + }; + + assert!(claim_slot(10, &epoch, &keystore).is_none()); + + epoch.authorities.push((valid_public_key.clone().into(), 10)); + assert_eq!(claim_slot(10, &epoch, &keystore).unwrap().1, valid_public_key.into()); + } +} diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index 3d14f0a7bf08d9a497ce0d8a7736ba0ce13c419d..961b0382c5858329422c7c66721b17e64031dfec 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -1291,7 +1291,7 @@ pub fn import_queue( finality_proof_import: Option>, client: Arc, inherent_data_providers: InherentDataProviders, - spawner: &impl sp_core::traits::SpawnBlocking, + spawner: &impl sp_core::traits::SpawnNamed, registry: Option<&Registry>, ) -> ClientResult>> where Inner: BlockImport> diff --git a/client/consensus/babe/src/tests.rs b/client/consensus/babe/src/tests.rs index 774cc5b7a4a413deae56bfeebed5c25064fddc07..1caed18c1781e44024d474230ecfdf979c35361a 100644 --- a/client/consensus/babe/src/tests.rs +++ b/client/consensus/babe/src/tests.rs @@ -21,8 +21,14 @@ #![allow(deprecated)] use super::*; use authorship::claim_slot; -use sp_core::crypto::Pair; -use sp_consensus_babe::{AuthorityPair, SlotNumber, AllowedSlots}; +use sp_core::{crypto::Pair, vrf::make_transcript as transcript_from_data}; +use sp_consensus_babe::{ + AuthorityPair, + SlotNumber, + AllowedSlots, + make_transcript, + make_transcript_data, +}; use sc_block_builder::{BlockBuilder, BlockBuilderProvider}; use sp_consensus::{ NoNetwork as DummyOracle, Proposal, RecordProof, @@ -35,6 +41,11 @@ use sp_runtime::{generic::DigestItem, traits::{Block as BlockT, DigestFor}}; use sc_client_api::{BlockchainEvents, backend::TransactionFor}; use log::debug; use std::{time::Duration, cell::RefCell, task::Poll}; +use rand::RngCore; +use rand_chacha::{ + rand_core::SeedableRng, + ChaChaRng, +}; type Item = DigestItem; @@ -159,7 +170,7 @@ impl Proposer for DummyProposer { type Proposal = future::Ready, Error>>; fn propose( - &mut self, + mut self, _: InherentData, pre_digests: DigestFor, _: Duration, @@ -796,3 +807,36 @@ fn verify_slots_are_strictly_increasing() { &mut block_import, ); } + +#[test] +fn babe_transcript_generation_match() { + let _ = env_logger::try_init(); + let keystore_path = tempfile::tempdir().expect("Creates keystore path"); + let keystore = sc_keystore::Store::open(keystore_path.path(), None).expect("Creates keystore"); + let pair = keystore.write().insert_ephemeral_from_seed::("//Alice") + .expect("Generates authority pair"); + + let epoch = Epoch { + start_slot: 0, + authorities: vec![(pair.public(), 1)], + randomness: [0; 32], + epoch_index: 1, + duration: 100, + config: BabeEpochConfiguration { + c: (3, 10), + allowed_slots: AllowedSlots::PrimaryAndSecondaryPlainSlots, + }, + }; + + let orig_transcript = make_transcript(&epoch.randomness.clone(), 1, epoch.epoch_index); + let new_transcript = make_transcript_data(&epoch.randomness, 1, epoch.epoch_index); + + let test = |t: merlin::Transcript| -> [u8; 16] { + let mut b = [0u8; 16]; + t.build_rng() + .finalize(&mut ChaChaRng::from_seed([0u8;32])) + .fill_bytes(&mut b); + b + }; + debug_assert!(test(orig_transcript) == test(transcript_from_data(new_transcript))); +} diff --git a/client/consensus/common/Cargo.toml b/client/consensus/common/Cargo.toml index 54fefb00fa93c906741a0d4c6a0fa113427b44fe..72bb051a0d0f53a524ed9fe8685857224c16cfc2 100644 --- a/client/consensus/common/Cargo.toml +++ b/client/consensus/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -12,7 +12,7 @@ description = "Collection of common consensus specific imlementations for Substr targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sc-client-api = { version = "2.0.0-alpha.8", path = "../../api" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/common" } +sc-client-api = { version = "2.0.0-rc4", path = "../../api" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-consensus = { version = "0.8.0-rc4", path = "../../../primitives/consensus/common" } diff --git a/client/consensus/epochs/Cargo.toml b/client/consensus/epochs/Cargo.toml index a55806ee38c36f54b1f5ea83f7a06cf6ce7ccb43..22f8794974953cce9d855a96bfb8a368c4608449 100644 --- a/client/consensus/epochs/Cargo.toml +++ b/client/consensus/epochs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-epochs" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "Generic epochs-based utilities for consensus" edition = "2018" @@ -12,9 +12,9 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", features = ["derive"] } parking_lot = "0.10.0" -fork-tree = { version = "2.0.0-alpha.8", path = "../../../utils/fork-tree" } -sp-runtime = { path = "../../../primitives/runtime" , version = "2.0.0-alpha.8"} -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } -sc-client-api = { path = "../../api" , version = "2.0.0-alpha.8"} +fork-tree = { version = "2.0.0-rc4", path = "../../../utils/fork-tree" } +sp-runtime = { path = "../../../primitives/runtime" , version = "2.0.0-rc4"} +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } +sc-client-api = { path = "../../api" , version = "2.0.0-rc4"} diff --git a/client/consensus/manual-seal/Cargo.toml b/client/consensus/manual-seal/Cargo.toml index 34b7d13cb7fdec2015dfcae9176458c80c913ad7..2da28b9ab9f997199604fa48b31579b6c63dc333 100644 --- a/client/consensus/manual-seal/Cargo.toml +++ b/client/consensus/manual-seal/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-manual-seal" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "Manual sealing engine for Substrate" edition = "2018" @@ -14,28 +14,28 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] derive_more = "0.99.2" futures = "0.3.4" -jsonrpc-core = "14.0.5" -jsonrpc-core-client = "14.0.5" -jsonrpc-derive = "14.0.5" +jsonrpc-core = "14.2.0" +jsonrpc-core-client = "14.2.0" +jsonrpc-derive = "14.2.1" log = "0.4.8" parking_lot = "0.10.0" serde = { version = "1.0", features=["derive"] } assert_matches = "1.3.0" -sc-client-api = { path = "../../../client/api", version = "2.0.0-alpha.8" } -sc-transaction-pool = { path = "../../transaction-pool", version = "2.0.0-alpha.8" } -sp-blockchain = { path = "../../../primitives/blockchain", version = "2.0.0-alpha.8" } -sp-consensus = { package = "sp-consensus", path = "../../../primitives/consensus/common", version = "0.8.0-alpha.8" } -sp-inherents = { path = "../../../primitives/inherents", version = "2.0.0-alpha.8" } -sp-runtime = { path = "../../../primitives/runtime", version = "2.0.0-alpha.8" } -sp-core = { path = "../../../primitives/core", version = "2.0.0-alpha.8" } -sp-transaction-pool = { path = "../../../primitives/transaction-pool", version = "2.0.0-alpha.8" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.8.0-alpha.8" } +sc-client-api = { path = "../../../client/api", version = "2.0.0-rc4" } +sc-transaction-pool = { path = "../../transaction-pool", version = "2.0.0-rc4" } +sp-blockchain = { path = "../../../primitives/blockchain", version = "2.0.0-rc4" } +sp-consensus = { package = "sp-consensus", path = "../../../primitives/consensus/common", version = "0.8.0-rc4" } +sp-inherents = { path = "../../../primitives/inherents", version = "2.0.0-rc4" } +sp-runtime = { path = "../../../primitives/runtime", version = "2.0.0-rc4" } +sp-core = { path = "../../../primitives/core", version = "2.0.0-rc4" } +sp-transaction-pool = { path = "../../../primitives/transaction-pool", version = "2.0.0-rc4" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.8.0-rc4" } [dev-dependencies] -sc-basic-authorship = { path = "../../basic-authorship", version = "0.8.0-alpha.8" } -substrate-test-runtime-client = { path = "../../../test-utils/runtime/client", version = "2.0.0-alpha.8" } -substrate-test-runtime-transaction-pool = { path = "../../../test-utils/runtime/transaction-pool", version = "2.0.0-alpha.8" } +sc-basic-authorship = { path = "../../basic-authorship", version = "0.8.0-rc4" } +substrate-test-runtime-client = { path = "../../../test-utils/runtime/client", version = "2.0.0-rc4" } +substrate-test-runtime-transaction-pool = { path = "../../../test-utils/runtime/transaction-pool", version = "2.0.0-rc4" } tokio = { version = "0.2", features = ["rt-core", "macros"] } env_logger = "0.7.0" tempfile = "3.1.0" diff --git a/client/consensus/manual-seal/src/error.rs b/client/consensus/manual-seal/src/error.rs index 343e50b291aee55560182398e4d1096fe77dbe01..2411a839b027ceb51b6e927f5c09ad030571231a 100644 --- a/client/consensus/manual-seal/src/error.rs +++ b/client/consensus/manual-seal/src/error.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! A manual sealing engine: the engine listens for rpc calls to seal blocks and create forks. //! This is suitable for a testing environment. use sp_consensus::{Error as ConsensusError, ImportResult}; diff --git a/client/consensus/manual-seal/src/lib.rs b/client/consensus/manual-seal/src/lib.rs index 6354e43ed3462f7dfebf276e80cf782d5f417dcb..53cc57ba6e8f2fd423ffbee4d3eb300d0ca01510 100644 --- a/client/consensus/manual-seal/src/lib.rs +++ b/client/consensus/manual-seal/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! A manual sealing engine: the engine listens for rpc calls to seal blocks and create forks. //! This is suitable for a testing environment. @@ -69,7 +70,7 @@ impl Verifier for ManualSealVerifier { /// Instantiate the import queue for the manual seal consensus engine. pub fn import_queue( block_import: BoxBlockImport, - spawner: &impl sp_core::traits::SpawnBlocking, + spawner: &impl sp_core::traits::SpawnNamed, registry: Option<&Registry>, ) -> BasicQueue where @@ -97,7 +98,7 @@ pub async fn run_manual_seal( inherent_data_providers: InherentDataProviders, ) where - A: txpool::ChainApi::Hash> + 'static, + A: txpool::ChainApi + 'static, B: BlockT + 'static, C: HeaderBackend + Finalizer + 'static, CB: ClientBackend + 'static, @@ -157,7 +158,7 @@ pub async fn run_instant_seal( inherent_data_providers: InherentDataProviders, ) where - A: txpool::ChainApi::Hash> + 'static, + A: txpool::ChainApi + 'static, B: BlockT + 'static, C: HeaderBackend + Finalizer + 'static, CB: ClientBackend + 'static, @@ -225,7 +226,8 @@ mod tests { let pool = Arc::new(BasicPool::new(Options::default(), api(), None).0); let env = ProposerFactory::new( client.clone(), - pool.clone() + pool.clone(), + None, ); // this test checks that blocks are created as soon as transactions are imported into the pool. let (sender, receiver) = futures::channel::oneshot::channel(); @@ -289,7 +291,8 @@ mod tests { let pool = Arc::new(BasicPool::new(Options::default(), api(), None).0); let env = ProposerFactory::new( client.clone(), - pool.clone() + pool.clone(), + None, ); // this test checks that blocks are created as soon as an engine command is sent over the stream. let (mut sink, stream) = futures::channel::mpsc::channel(1024); @@ -358,6 +361,7 @@ mod tests { let env = ProposerFactory::new( client.clone(), pool.clone(), + None, ); // this test checks that blocks are created as soon as an engine command is sent over the stream. let (mut sink, stream) = futures::channel::mpsc::channel(1024); @@ -409,11 +413,12 @@ mod tests { assert!(client.header(&BlockId::Number(0)).unwrap().is_some()); assert!(pool.submit_one(&BlockId::Number(1), SOURCE, uxt(Alice, 1)).await.is_ok()); + let header = client.header(&BlockId::Number(1)).expect("db error").expect("imported above"); pool.maintain(sp_transaction_pool::ChainEvent::NewBlock { - id: BlockId::Number(1), - header: client.header(&BlockId::Number(1)).expect("db error").expect("imported above"), + hash: header.hash(), + header, is_new_best: true, - retracted: vec![], + tree_route: None, }).await; let (tx1, rx1) = futures::channel::oneshot::channel(); diff --git a/client/consensus/manual-seal/src/seal_new_block.rs b/client/consensus/manual-seal/src/seal_new_block.rs index 88b58ef4cc2b39cbbe3af1209e54408390e074ef..c5aea11ced316bba9a329ed954a01afcd49263ff 100644 --- a/client/consensus/manual-seal/src/seal_new_block.rs +++ b/client/consensus/manual-seal/src/seal_new_block.rs @@ -87,7 +87,7 @@ pub async fn seal_new_block( E: Environment, >::Error: std::fmt::Display, >::Error: std::fmt::Display, - P: txpool::ChainApi::Hash>, + P: txpool::ChainApi, SC: SelectChain, { let future = async { @@ -108,7 +108,7 @@ pub async fn seal_new_block( None => select_chain.best_chain()? }; - let mut proposer = env.init(&header) + let proposer = env.init(&header) .map_err(|err| Error::StringError(format!("{}", err))).await?; let id = inherent_data_provider.create_inherent_data()?; let inherents_len = id.len(); diff --git a/client/consensus/pow/Cargo.toml b/client/consensus/pow/Cargo.toml index a41c1bea79d633561d3a5063f7e93503e0ad92c7..b0b142fd84c32d6974d36457011cd5fda3d19018 100644 --- a/client/consensus/pow/Cargo.toml +++ b/client/consensus/pow/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-pow" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "PoW consensus algorithm for substrate" edition = "2018" @@ -12,18 +12,18 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.8", path = "../../../primitives/api" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../../api" } -sp-block-builder = { version = "2.0.0-alpha.8", path = "../../../primitives/block-builder" } -sp-inherents = { version = "2.0.0-alpha.8", path = "../../../primitives/inherents" } -sp-consensus-pow = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/pow" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/common" } +codec = { package = "parity-scale-codec", version = "1.3.1", features = ["derive"] } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-rc4", path = "../../../primitives/api" } +sc-client-api = { version = "2.0.0-rc4", path = "../../api" } +sp-block-builder = { version = "2.0.0-rc4", path = "../../../primitives/block-builder" } +sp-inherents = { version = "2.0.0-rc4", path = "../../../primitives/inherents" } +sp-consensus-pow = { version = "0.8.0-rc4", path = "../../../primitives/consensus/pow" } +sp-consensus = { version = "0.8.0-rc4", path = "../../../primitives/consensus/common" } log = "0.4.8" futures = { version = "0.3.1", features = ["compat"] } -sp-timestamp = { version = "2.0.0-alpha.8", path = "../../../primitives/timestamp" } +sp-timestamp = { version = "2.0.0-rc4", path = "../../../primitives/timestamp" } derive_more = "0.99.2" -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.8.0-alpha.8"} +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.8.0-rc4"} diff --git a/client/consensus/pow/src/lib.rs b/client/consensus/pow/src/lib.rs index 846377b7bce8f93f32d97c8800ee1e131cbede6a..8c15528795c5534090ba3c959327a91274ce9c61 100644 --- a/client/consensus/pow/src/lib.rs +++ b/client/consensus/pow/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Proof of work consensus for Substrate. //! //! To use this engine, you can need to have a struct that implements @@ -465,7 +466,7 @@ pub fn import_queue( finality_proof_import: Option>, algorithm: Algorithm, inherent_data_providers: InherentDataProviders, - spawner: &impl sp_core::traits::SpawnBlocking, + spawner: &impl sp_core::traits::SpawnNamed, registry: Option<&Registry>, ) -> Result< PowImportQueue, @@ -609,7 +610,7 @@ fn mine_loop( continue 'outer } - let mut proposer = futures::executor::block_on(env.init(&best_header)) + let proposer = futures::executor::block_on(env.init(&best_header)) .map_err(|e| Error::Environment(format!("{:?}", e)))?; let inherent_data = inherent_data_providers diff --git a/client/consensus/slots/Cargo.toml b/client/consensus/slots/Cargo.toml index fdaa4871a17d17776ef5488f8f91a9718c7e7953..80eb83cca5653d17f588eec9058dc76d2eeba5c4 100644 --- a/client/consensus/slots/Cargo.toml +++ b/client/consensus/slots/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-slots" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "Generic slots-based utilities for consensus" edition = "2018" @@ -13,21 +13,21 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../../api" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-application-crypto = { version = "2.0.0-alpha.8", path = "../../../primitives/application-crypto" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../../primitives/state-machine" } -sp-api = { version = "2.0.0-alpha.8", path = "../../../primitives/api" } -sc-telemetry = { version = "2.0.0-alpha.8", path = "../../telemetry" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/common" } -sp-inherents = { version = "2.0.0-alpha.8", path = "../../../primitives/inherents" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +sc-client-api = { version = "2.0.0-rc4", path = "../../api" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-application-crypto = { version = "2.0.0-rc4", path = "../../../primitives/application-crypto" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../../primitives/state-machine" } +sp-api = { version = "2.0.0-rc4", path = "../../../primitives/api" } +sc-telemetry = { version = "2.0.0-rc4", path = "../../telemetry" } +sp-consensus = { version = "0.8.0-rc4", path = "../../../primitives/consensus/common" } +sp-inherents = { version = "2.0.0-rc4", path = "../../../primitives/inherents" } futures = "0.3.4" futures-timer = "3.0.1" parking_lot = "0.10.0" log = "0.4.8" [dev-dependencies] -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../../test-utils/runtime/client" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../../test-utils/runtime/client" } diff --git a/client/consensus/slots/src/lib.rs b/client/consensus/slots/src/lib.rs index f58e52da4121f57304b0f0c9084df943ffbf97ae..7687d3114b31dcd9dd49d6aa5c3d7db0b7d708c2 100644 --- a/client/consensus/slots/src/lib.rs +++ b/client/consensus/slots/src/lib.rs @@ -20,7 +20,6 @@ //! time during which certain events can and/or must occur. This crate //! provides generic functionality for slots. -#![deny(warnings)] #![forbid(unsafe_code, missing_docs)] mod slots; @@ -239,7 +238,7 @@ pub trait SimpleSlotWorker { let logs = self.pre_digest_data(slot_number, &claim); // deadline our production to approx. the end of the slot - let proposing = awaiting_proposer.and_then(move |mut proposer| proposer.propose( + let proposing = awaiting_proposer.and_then(move |proposer| proposer.propose( slot_info.inherent_data, sp_runtime::generic::Digest { logs, @@ -456,7 +455,7 @@ impl SlotDuration { CB: FnOnce(ApiRef, &BlockId) -> sp_blockchain::Result, T: SlotData + Encode + Decode + Debug, { - match client.get_aux(T::SLOT_KEY)? { + let slot_duration = match client.get_aux(T::SLOT_KEY)? { Some(v) => ::decode(&mut &v[..]) .map(SlotDuration) .map_err(|_| { @@ -472,7 +471,7 @@ impl SlotDuration { info!( "⏱ Loaded block-time = {:?} milliseconds from genesis on first-launch", - genesis_slot_duration + genesis_slot_duration.slot_duration() ); genesis_slot_duration @@ -480,7 +479,15 @@ impl SlotDuration { Ok(SlotDuration(genesis_slot_duration)) } + }?; + + if slot_duration.slot_duration() == 0 { + return Err(sp_blockchain::Error::Msg( + "Invalid value for slot_duration: the value must be greater than 0.".into(), + )) } + + Ok(slot_duration) } /// Returns slot data value. diff --git a/client/consensus/uncles/Cargo.toml b/client/consensus/uncles/Cargo.toml index e052e8b023b4d2f8d66767e186a810fc694054e7..957e8e3c0b8905d041992ac950718258739b12ce 100644 --- a/client/consensus/uncles/Cargo.toml +++ b/client/consensus/uncles/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-uncles" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "Generic uncle inclusion utilities for consensus" edition = "2018" @@ -12,10 +12,10 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sc-client-api = { version = "2.0.0-alpha.8", path = "../../api" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-authorship = { version = "2.0.0-alpha.8", path = "../../../primitives/authorship" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/common" } -sp-inherents = { version = "2.0.0-alpha.8", path = "../../../primitives/inherents" } +sc-client-api = { version = "2.0.0-rc4", path = "../../api" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-authorship = { version = "2.0.0-rc4", path = "../../../primitives/authorship" } +sp-consensus = { version = "0.8.0-rc4", path = "../../../primitives/consensus/common" } +sp-inherents = { version = "2.0.0-rc4", path = "../../../primitives/inherents" } log = "0.4.8" diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml index 9aaccf1da09c4dd1b30462acfddf7beeec3368b5..42cc60617a89353fd225c64c857ac73da8251e3f 100644 --- a/client/db/Cargo.toml +++ b/client/db/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-client-db" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -20,25 +20,25 @@ kvdb-memorydb = "0.6.0" linked-hash-map = "0.5.2" hash-db = "0.15.2" parity-util-mem = { version = "0.6.1", default-features = false, features = ["std"] } -codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", features = ["derive"] } blake2-rfc = "0.2.18" -sc-client-api = { version = "2.0.0-alpha.8", path = "../api" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../primitives/state-machine" } -sc-executor = { version = "0.8.0-alpha.8", path = "../executor" } -sc-state-db = { version = "0.8.0-alpha.8", path = "../state-db" } -sp-trie = { version = "2.0.0-alpha.8", path = "../../primitives/trie" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../primitives/consensus/common" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../primitives/blockchain" } -sp-database = { version = "2.0.0-alpha.8", path = "../../primitives/database" } +sc-client-api = { version = "2.0.0-rc4", path = "../api" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../primitives/state-machine" } +sc-executor = { version = "0.8.0-rc4", path = "../executor" } +sc-state-db = { version = "0.8.0-rc4", path = "../state-db" } +sp-trie = { version = "2.0.0-rc4", path = "../../primitives/trie" } +sp-consensus = { version = "0.8.0-rc4", path = "../../primitives/consensus/common" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../primitives/blockchain" } +sp-database = { version = "2.0.0-rc4", path = "../../primitives/database" } parity-db = { version = "0.1.2", optional = true } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.8.0-alpha.8", path = "../../utils/prometheus" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.8.0-rc4", path = "../../utils/prometheus" } [dev-dependencies] -sp-keyring = { version = "2.0.0-alpha.8", path = "../../primitives/keyring" } -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../test-utils/runtime/client" } +sp-keyring = { version = "2.0.0-rc4", path = "../../primitives/keyring" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../test-utils/runtime/client" } env_logger = "0.7.0" quickcheck = "0.9" kvdb-rocksdb = "0.8" @@ -47,3 +47,6 @@ tempfile = "3" [features] default = [] test-helpers = [] +with-kvdb-rocksdb = ["kvdb-rocksdb"] +with-parity-db = ["parity-db"] +with-subdb = [] diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 2d815bdebac19da53ccb3fff50bad335cff417b5..c3bed3e24f617477e838ae271b2452162c3cbad3 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! State backend that's useful for benchmarking use std::sync::Arc; @@ -23,10 +24,10 @@ use std::collections::HashMap; use hash_db::{Prefix, Hasher}; use sp_trie::{MemoryDB, prefixed_key}; -use sp_core::storage::ChildInfo; +use sp_core::{storage::ChildInfo, hexdisplay::HexDisplay}; use sp_runtime::traits::{Block as BlockT, HashFor}; use sp_runtime::Storage; -use sp_state_machine::{DBValue, backend::Backend as StateBackend}; +use sp_state_machine::{DBValue, backend::Backend as StateBackend, StorageCollection}; use kvdb::{KeyValueDB, DBTransaction}; use crate::storage_cache::{CachingState, SharedCache, new_shared_cache}; @@ -49,6 +50,40 @@ impl sp_state_machine::Storage> for StorageDb { root: Cell, @@ -58,6 +93,9 @@ pub struct BenchmarkingState { genesis: HashMap, (Vec, i32)>, record: Cell>>, shared_cache: SharedCache, // shared cache is always empty + key_tracker: RefCell, KeyTracker>>, + read_write_tracker: RefCell, + whitelist: RefCell>>, } impl BenchmarkingState { @@ -75,20 +113,25 @@ impl BenchmarkingState { genesis_root: Default::default(), record: Default::default(), shared_cache: new_shared_cache(0, (1, 10)), + key_tracker: Default::default(), + read_write_tracker: Default::default(), + whitelist: Default::default(), }; + state.add_whitelist_to_tracker(); + state.reopen()?; - let child_delta = genesis.children_default.into_iter().map(|(_storage_key, child_content)| ( - child_content.child_info, - child_content.data.into_iter().map(|(k, v)| (k, Some(v))), + let child_delta = genesis.children_default.iter().map(|(_storage_key, child_content)| ( + &child_content.child_info, + child_content.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), )); let (root, transaction): (B::Hash, _) = state.state.borrow_mut().as_mut().unwrap().full_storage_root( - genesis.top.into_iter().map(|(k, v)| (k, Some(v))), + genesis.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, ); state.genesis = transaction.clone().drain(); state.genesis_root = root.clone(); - state.commit(root, transaction)?; + state.commit(root, transaction, Vec::new())?; state.record.take(); Ok(state) } @@ -108,6 +151,86 @@ impl BenchmarkingState { )); Ok(()) } + + fn add_whitelist_to_tracker(&self) { + let mut key_tracker = self.key_tracker.borrow_mut(); + + let whitelisted = KeyTracker { + has_been_read: true, + has_been_written: true, + }; + + let whitelist = self.whitelist.borrow(); + + whitelist.iter().for_each(|key| { + key_tracker.insert(key.to_vec(), whitelisted); + }); + } + + fn wipe_tracker(&self) { + *self.key_tracker.borrow_mut() = HashMap::new(); + self.add_whitelist_to_tracker(); + *self.read_write_tracker.borrow_mut() = Default::default(); + } + + fn add_read_key(&self, key: &[u8]) { + log::trace!(target: "benchmark", "Read: {}", HexDisplay::from(&key)); + + let mut key_tracker = self.key_tracker.borrow_mut(); + let mut read_write_tracker = self.read_write_tracker.borrow_mut(); + + let maybe_tracker = key_tracker.get(key); + + let has_been_read = KeyTracker { + has_been_read: true, + has_been_written: false, + }; + + match maybe_tracker { + None => { + key_tracker.insert(key.to_vec(), has_been_read); + read_write_tracker.add_read(); + }, + Some(tracker) => { + if !tracker.has_been_read { + key_tracker.insert(key.to_vec(), has_been_read); + read_write_tracker.add_read(); + } else { + read_write_tracker.add_repeat_read(); + } + } + } + } + + fn add_write_key(&self, key: &[u8]) { + log::trace!(target: "benchmark", "Write: {}", HexDisplay::from(&key)); + + let mut key_tracker = self.key_tracker.borrow_mut(); + let mut read_write_tracker = self.read_write_tracker.borrow_mut(); + + let maybe_tracker = key_tracker.get(key); + + // If we have written to the key, we also consider that we have read from it. + let has_been_written = KeyTracker { + has_been_read: true, + has_been_written: true, + }; + + match maybe_tracker { + None => { + key_tracker.insert(key.to_vec(), has_been_written); + read_write_tracker.add_write(); + }, + Some(tracker) => { + if !tracker.has_been_written { + key_tracker.insert(key.to_vec(), has_been_written); + read_write_tracker.add_write(); + } else { + read_write_tracker.add_repeat_write(); + } + } + } + } } fn state_err() -> String { @@ -120,10 +243,12 @@ impl StateBackend> for BenchmarkingState { type TrieBackendStorage = as StateBackend>>::TrieBackendStorage; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { + self.add_read_key(key); self.state.borrow().as_ref().ok_or_else(state_err)?.storage(key) } fn storage_hash(&self, key: &[u8]) -> Result, Self::Error> { + self.add_read_key(key); self.state.borrow().as_ref().ok_or_else(state_err)?.storage_hash(key) } @@ -132,10 +257,12 @@ impl StateBackend> for BenchmarkingState { child_info: &ChildInfo, key: &[u8], ) -> Result>, Self::Error> { + self.add_read_key(key); self.state.borrow().as_ref().ok_or_else(state_err)?.child_storage(child_info, key) } fn exists_storage(&self, key: &[u8]) -> Result { + self.add_read_key(key); self.state.borrow().as_ref().ok_or_else(state_err)?.exists_storage(key) } @@ -144,10 +271,12 @@ impl StateBackend> for BenchmarkingState { child_info: &ChildInfo, key: &[u8], ) -> Result { + self.add_read_key(key); self.state.borrow().as_ref().ok_or_else(state_err)?.exists_child_storage(child_info, key) } fn next_storage_key(&self, key: &[u8]) -> Result>, Self::Error> { + self.add_read_key(key); self.state.borrow().as_ref().ok_or_else(state_err)?.next_storage_key(key) } @@ -156,6 +285,7 @@ impl StateBackend> for BenchmarkingState { child_info: &ChildInfo, key: &[u8], ) -> Result>, Self::Error> { + self.add_read_key(key); self.state.borrow().as_ref().ok_or_else(state_err)?.next_child_storage_key(child_info, key) } @@ -192,19 +322,18 @@ impl StateBackend> for BenchmarkingState { } } - fn storage_root(&self, delta: I) -> (B::Hash, Self::Transaction) where - I: IntoIterator, Option>)> - { + fn storage_root<'a>( + &self, + delta: impl Iterator)>, + ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { self.state.borrow().as_ref().map_or(Default::default(), |s| s.storage_root(delta)) } - fn child_storage_root( + fn child_storage_root<'a>( &self, child_info: &ChildInfo, - delta: I, - ) -> (B::Hash, bool, Self::Transaction) where - I: IntoIterator, Option>)>, - { + delta: impl Iterator)>, + ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_storage_root(child_info, delta)) } @@ -230,8 +359,11 @@ impl StateBackend> for BenchmarkingState { None } - fn commit(&self, storage_root: as Hasher>::Out, mut transaction: Self::Transaction) - -> Result<(), Self::Error> + fn commit(&self, + storage_root: as Hasher>::Out, + mut transaction: Self::Transaction, + storage_changes: StorageCollection, + ) -> Result<(), Self::Error> { if let Some(db) = self.db.take() { let mut db_transaction = DBTransaction::new(); @@ -245,10 +377,17 @@ impl StateBackend> for BenchmarkingState { } keys.push(key); } - self.record.set(keys); + let mut record = self.record.take(); + record.extend(keys); + self.record.set(record); db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?; self.root.set(storage_root); - self.db.set(Some(db)) + self.db.set(Some(db)); + + // Track DB Writes + storage_changes.iter().for_each(|(key, _)| { + self.add_write_key(key); + }); } else { return Err("Trying to commit to a closed db".into()) } @@ -272,9 +411,25 @@ impl StateBackend> for BenchmarkingState { self.root.set(self.genesis_root.clone()); self.reopen()?; + self.wipe_tracker(); Ok(()) } + /// Get the key tracking information for the state db. + fn read_write_count(&self) -> (u32, u32, u32, u32) { + let count = *self.read_write_tracker.borrow_mut(); + (count.reads, count.repeat_reads, count.writes, count.repeat_writes) + } + + /// Reset the key tracking information for the state db. + fn reset_read_write_count(&self) { + self.wipe_tracker() + } + + fn set_whitelist(&self, new: Vec>) { + *self.whitelist.borrow_mut() = new; + } + fn register_overlay_stats(&mut self, stats: &sp_state_machine::StateMachineStats) { self.state.borrow_mut().as_mut().map(|s| s.register_overlay_stats(stats)); } diff --git a/client/db/src/cache/list_cache.rs b/client/db/src/cache/list_cache.rs index c1d71cfb5d167634074ff8389dcb62884d7fd8b3..0856350fb091069ee5a33db055a6ba256ed3d26d 100644 --- a/client/db/src/cache/list_cache.rs +++ b/client/db/src/cache/list_cache.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! List-based cache. //! //! Maintains several lists, containing nodes that are inserted whenever @@ -1765,7 +1766,7 @@ pub mod tests { Some(Entry { valid_from: test_id(20), value: 20 }), vec![5, 6].into_iter().collect(), ))); - + assert_eq!( ops.operations, vec![CommitOperation::BlockFinalized( diff --git a/client/db/src/cache/list_entry.rs b/client/db/src/cache/list_entry.rs index 744cb226c8f330695cda6581cb346115bf12ab7d..565a62cff4f2d9643624d0836bb39e10f35b2027 100644 --- a/client/db/src/cache/list_entry.rs +++ b/client/db/src/cache/list_entry.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! List-cache storage entries. use sp_blockchain::Result as ClientResult; diff --git a/client/db/src/cache/list_storage.rs b/client/db/src/cache/list_storage.rs index 21917e6eb93b200c3596506ec1849b5bdb2765c2..377d744effa60faf57521d2a490852ef757ce1bf 100644 --- a/client/db/src/cache/list_storage.rs +++ b/client/db/src/cache/list_storage.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! List-cache storage definition and implementation. use std::sync::Arc; diff --git a/client/db/src/cache/mod.rs b/client/db/src/cache/mod.rs index a6cb93c6cec59c7024392655e3eb35cc45661aba..2b7cd2e62076e42e7534b7609be709687ae19962 100644 --- a/client/db/src/cache/mod.rs +++ b/client/db/src/cache/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! DB-backed cache of blockchain data. use std::{sync::Arc, collections::{HashMap, hash_map::Entry}}; diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index cbc637cf8415864d480d82d3b954296a4c9bd9fa..b4f4892a0497083484eb2b8251435a9f903df2a2 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Client backend that is backed by a database. //! //! # Canonicality vs. Finality @@ -30,20 +31,20 @@ pub mod light; pub mod offchain; -#[cfg(any(feature = "kvdb-rocksdb", test))] +#[cfg(any(feature = "with-kvdb-rocksdb", test))] pub mod bench; mod children; mod cache; mod changes_tries_storage; mod storage_cache; -#[cfg(any(feature = "kvdb-rocksdb", test))] +#[cfg(any(feature = "with-kvdb-rocksdb", test))] mod upgrade; mod utils; mod stats; -#[cfg(feature = "parity-db")] +#[cfg(feature = "with-parity-db")] mod parity_db; -#[cfg(feature = "subdb")] +#[cfg(feature = "with-subdb")] mod subdb; use std::sync::Arc; @@ -90,7 +91,7 @@ use log::{trace, debug, warn}; pub use sp_database::Database; pub use sc_state_db::PruningMode; -#[cfg(any(feature = "kvdb-rocksdb", test))] +#[cfg(any(feature = "with-kvdb-rocksdb", test))] pub use bench::BenchmarkingState; const MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR: u32 = 32768; @@ -211,21 +212,18 @@ impl StateBackend> for RefTrackingState { self.state.for_child_keys_with_prefix(child_info, prefix, f) } - fn storage_root(&self, delta: I) -> (B::Hash, Self::Transaction) - where - I: IntoIterator, Option>)> - { + fn storage_root<'a>( + &self, + delta: impl Iterator)>, + ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { self.state.storage_root(delta) } - fn child_storage_root( + fn child_storage_root<'a>( &self, child_info: &ChildInfo, - delta: I, - ) -> (B::Hash, bool, Self::Transaction) - where - I: IntoIterator, Option>)>, - { + delta: impl Iterator)>, + ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { self.state.child_storage_root(child_info, delta) } @@ -273,7 +271,7 @@ pub struct DatabaseSettings { } /// Where to find the database.. -#[derive(Clone)] +#[derive(Debug, Clone)] pub enum DatabaseSettingsSrc { /// Load a RocksDB database from a given path. Recommended for most uses. RocksDb { @@ -546,7 +544,12 @@ pub struct BlockImportOperation { impl BlockImportOperation { fn apply_offchain(&mut self, transaction: &mut Transaction) { - for (key, value_operation) in self.offchain_storage_updates.drain() { + for ((prefix, key), value_operation) in self.offchain_storage_updates.drain() { + let key: Vec = prefix + .into_iter() + .chain(sp_core::sp_std::iter::once(b'/')) + .chain(key.into_iter()) + .collect(); match value_operation { OffchainOverlayedChange::SetValue(val) => transaction.set_from_vec(columns::OFFCHAIN, &key, val), OffchainOverlayedChange::Remove => transaction.remove(columns::OFFCHAIN, &key), @@ -604,26 +607,25 @@ impl sc_client_api::backend::BlockImportOperation for Bloc &mut self, storage: Storage, ) -> ClientResult { - - if storage.top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) { + if storage.top.keys().any(|k| well_known_keys::is_child_storage_key(&k)) { return Err(sp_blockchain::Error::GenesisInvalid.into()); } - let child_delta = storage.children_default.into_iter().map(|(_storage_key, child_content)|( - child_content.child_info, - child_content.data.into_iter().map(|(k, v)| (k, Some(v))), + let child_delta = storage.children_default.iter().map(|(_storage_key, child_content)|( + &child_content.child_info, + child_content.data.iter().map(|(k, v)| (&k[..], Some(&v[..]))), )); let mut changes_trie_config: Option = None; let (root, transaction) = self.old_state.full_storage_root( - storage.top.into_iter().map(|(k, v)| { - if k == well_known_keys::CHANGES_TRIE_CONFIG { + storage.top.iter().map(|(k, v)| { + if &k[..] == well_known_keys::CHANGES_TRIE_CONFIG { changes_trie_config = Some( Decode::decode(&mut &v[..]) .expect("changes trie configuration is encoded properly at genesis") ); } - (k, Some(v)) + (&k[..], Some(&v[..])) }), child_delta ); @@ -1809,13 +1811,12 @@ pub(crate) mod tests { header.state_root = op.old_state.storage_root(storage .iter() - .cloned() - .map(|(x, y)| (x, Some(y))) + .map(|(x, y)| (&x[..], Some(&y[..]))) ).0.into(); let hash = header.hash(); op.reset_storage(Storage { - top: storage.iter().cloned().collect(), + top: storage.into_iter().collect(), children_default: Default::default(), }).unwrap(); op.set_block_data( @@ -1852,7 +1853,10 @@ pub(crate) mod tests { (vec![5, 5, 5], Some(vec![4, 5, 6])), ]; - let (root, overlay) = op.old_state.storage_root(storage.iter().cloned()); + let (root, overlay) = op.old_state.storage_root( + storage.iter() + .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))) + ); op.update_db_storage(overlay).unwrap(); header.state_root = root.into(); @@ -1891,17 +1895,11 @@ pub(crate) mod tests { extrinsics_root: Default::default(), }; - let storage: Vec<(_, _)> = vec![]; - - header.state_root = op.old_state.storage_root(storage - .iter() - .cloned() - .map(|(x, y)| (x, Some(y))) - ).0.into(); + header.state_root = op.old_state.storage_root(std::iter::empty()).0.into(); let hash = header.hash(); op.reset_storage(Storage { - top: storage.iter().cloned().collect(), + top: Default::default(), children_default: Default::default(), }).unwrap(); diff --git a/client/db/src/light.rs b/client/db/src/light.rs index 56c06ff406a33d7452b6d5c9acf76e45ca40631a..f115ac9599e8dca6871751af22e343441300bfe6 100644 --- a/client/db/src/light.rs +++ b/client/db/src/light.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! RocksDB-based light client blockchain storage. use std::{sync::Arc, collections::HashMap}; diff --git a/client/db/src/offchain.rs b/client/db/src/offchain.rs index 8cd94a159417f2a31f31bdbd1b8416f7a7c90e23..f6a0925a086171002e3479cc2206559be09a9250 100644 --- a/client/db/src/offchain.rs +++ b/client/db/src/offchain.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! RocksDB-based offchain workers local storage. use std::{ @@ -66,6 +67,14 @@ impl sp_core::offchain::OffchainStorage for LocalStorage { self.db.commit(tx); } + fn remove(&mut self, prefix: &[u8], key: &[u8]) { + let key: Vec = prefix.iter().chain(key).cloned().collect(); + let mut tx = Transaction::new(); + tx.remove(columns::OFFCHAIN, &key); + + self.db.commit(tx); + } + fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { let key: Vec = prefix.iter().chain(key).cloned().collect(); self.db.get(columns::OFFCHAIN, &key) diff --git a/client/db/src/parity_db.rs b/client/db/src/parity_db.rs index 1c8280e73059f16ff6719c5be14825f3413b6019..ad1c6c7656aaa605d6ebf38621fc451b28771135 100644 --- a/client/db/src/parity_db.rs +++ b/client/db/src/parity_db.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -27,7 +27,7 @@ fn handle_err(result: parity_db::Result) -> T { match result { Ok(r) => r, Err(e) => { - panic!("Critical database eror: {:?}", e); + panic!("Critical database error: {:?}", e); } } } diff --git a/client/db/src/stats.rs b/client/db/src/stats.rs index cb9c64c85080d5259390100624499373836bc2a5..8d208024b4bb29a1f064dfc00f3fa14b8df07f85 100644 --- a/client/db/src/stats.rs +++ b/client/db/src/stats.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Database usage statistics use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index 66ac74afa4f2a1d00458c58eeeda86e0b19e0f51..434b301ed6240e94539cbf5d3c06265231639039 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -621,21 +621,18 @@ impl>, B: BlockT> StateBackend> for Cachin self.state.for_child_keys_with_prefix(child_info, prefix, f) } - fn storage_root(&self, delta: I) -> (B::Hash, Self::Transaction) - where - I: IntoIterator, Option>)>, - { + fn storage_root<'a>( + &self, + delta: impl Iterator)>, + ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { self.state.storage_root(delta) } - fn child_storage_root( + fn child_storage_root<'a>( &self, child_info: &ChildInfo, - delta: I, - ) -> (B::Hash, bool, Self::Transaction) - where - I: IntoIterator, Option>)>, - { + delta: impl Iterator)>, + ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { self.state.child_storage_root(child_info, delta) } @@ -806,21 +803,18 @@ impl>, B: BlockT> StateBackend> for Syncin self.caching_state().for_child_keys_with_prefix(child_info, prefix, f) } - fn storage_root(&self, delta: I) -> (B::Hash, Self::Transaction) - where - I: IntoIterator, Option>)>, - { + fn storage_root<'a>( + &self, + delta: impl Iterator)>, + ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { self.caching_state().storage_root(delta) } - fn child_storage_root( + fn child_storage_root<'a>( &self, child_info: &ChildInfo, - delta: I, - ) -> (B::Hash, bool, Self::Transaction) - where - I: IntoIterator, Option>)>, - { + delta: impl Iterator)>, + ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { self.caching_state().child_storage_root(child_info, delta) } diff --git a/client/db/src/subdb.rs b/client/db/src/subdb.rs index 683c33dedd0f8e3983d6c3f33495e5ed0d082809..2f72632b045622fcd035e50f7560c2c4b0fe753c 100644 --- a/client/db/src/subdb.rs +++ b/client/db/src/subdb.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/client/db/src/utils.rs b/client/db/src/utils.rs index 64734ff7d21470c1535e75831ea2fa99e64a4977..d66d5abfea61adfe51fe243220f07d7b1d8d4f66 100644 --- a/client/db/src/utils.rs +++ b/client/db/src/utils.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Db-based backend utility structures and functions, used by both //! full and light storages. @@ -35,7 +36,7 @@ use crate::{DatabaseSettings, DatabaseSettingsSrc, Database, DbHash}; /// Number of columns in the db. Must be the same for both full && light dbs. /// Otherwise RocksDb will fail to open database && check its type. -#[cfg(any(feature = "kvdb-rocksdb", feature = "test-helpers", test))] +#[cfg(any(feature = "with-kvdb-rocksdb", feature = "test-helpers", test))] pub const NUM_COLUMNS: u32 = 11; /// Meta column. The set of keys in the column is shared by full && light storages. pub const COLUMN_META: u32 = 0; @@ -218,7 +219,7 @@ pub fn open_database( ); let db: Arc> = match &config.source { - #[cfg(any(feature = "kvdb-rocksdb", test))] + #[cfg(any(feature = "with-kvdb-rocksdb", test))] DatabaseSettingsSrc::RocksDb { path, cache_size } => { // first upgrade database to required version crate::upgrade::upgrade_db::(&path, db_type)?; @@ -254,27 +255,27 @@ pub fn open_database( .map_err(|err| sp_blockchain::Error::Backend(format!("{}", err)))?; sp_database::as_database(db) }, - #[cfg(not(any(feature = "kvdb-rocksdb", test)))] + #[cfg(not(any(feature = "with-kvdb-rocksdb", test)))] DatabaseSettingsSrc::RocksDb { .. } => { - return db_open_error("kvdb-rocksdb"); + return db_open_error("with-kvdb-rocksdb"); }, - #[cfg(feature = "subdb")] + #[cfg(feature = "with-subdb")] DatabaseSettingsSrc::SubDb { path } => { crate::subdb::open(&path, NUM_COLUMNS) .map_err(|e| sp_blockchain::Error::Backend(format!("{:?}", e)))? }, - #[cfg(not(feature = "subdb"))] + #[cfg(not(feature = "with-subdb"))] DatabaseSettingsSrc::SubDb { .. } => { - return db_open_error("subdb"); + return db_open_error("with-subdb"); }, - #[cfg(feature = "parity-db")] + #[cfg(feature = "with-parity-db")] DatabaseSettingsSrc::ParityDb { path } => { crate::parity_db::open(&path) .map_err(|e| sp_blockchain::Error::Backend(format!("{:?}", e)))? }, - #[cfg(not(feature = "parity-db"))] + #[cfg(not(feature = "with-parity-db"))] DatabaseSettingsSrc::ParityDb { .. } => { - return db_open_error("parity-db"); + return db_open_error("with-parity-db"); }, DatabaseSettingsSrc::Custom(db) => db.clone(), }; diff --git a/client/executor/Cargo.toml b/client/executor/Cargo.toml index cea11c9a0c6a697d3109695c1569f6e41bc3c2b0..b12156aeb19844802728bf08c31c39857715cc6e 100644 --- a/client/executor/Cargo.toml +++ b/client/executor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-executor" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -14,23 +14,23 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] derive_more = "0.99.2" -codec = { package = "parity-scale-codec", version = "1.3.0" } -sp-io = { version = "2.0.0-alpha.8", path = "../../primitives/io" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-trie = { version = "2.0.0-alpha.8", path = "../../primitives/trie" } -sp-serializer = { version = "2.0.0-alpha.8", path = "../../primitives/serializer" } -sp-version = { version = "2.0.0-alpha.8", path = "../../primitives/version" } -sp-panic-handler = { version = "2.0.0-alpha.8", path = "../../primitives/panic-handler" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +sp-io = { version = "2.0.0-rc4", path = "../../primitives/io" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-trie = { version = "2.0.0-rc4", path = "../../primitives/trie" } +sp-serializer = { version = "2.0.0-rc4", path = "../../primitives/serializer" } +sp-version = { version = "2.0.0-rc4", path = "../../primitives/version" } +sp-panic-handler = { version = "2.0.0-rc4", path = "../../primitives/panic-handler" } wasmi = "0.6.2" parity-wasm = "0.41.0" lazy_static = "1.4.0" -sp-api = { version = "2.0.0-alpha.8", path = "../../primitives/api" } -sp-wasm-interface = { version = "2.0.0-alpha.8", path = "../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0-alpha.8", path = "../../primitives/runtime-interface" } -sp-externalities = { version = "0.8.0-alpha.8", path = "../../primitives/externalities" } -sc-executor-common = { version = "0.8.0-alpha.8", path = "common" } -sc-executor-wasmi = { version = "0.8.0-alpha.8", path = "wasmi" } -sc-executor-wasmtime = { version = "0.8.0-alpha.8", path = "wasmtime", optional = true } +sp-api = { version = "2.0.0-rc4", path = "../../primitives/api" } +sp-wasm-interface = { version = "2.0.0-rc4", path = "../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-rc4", path = "../../primitives/runtime-interface" } +sp-externalities = { version = "0.8.0-rc4", path = "../../primitives/externalities" } +sc-executor-common = { version = "0.8.0-rc4", path = "common" } +sc-executor-wasmi = { version = "0.8.0-rc4", path = "wasmi" } +sc-executor-wasmtime = { version = "0.8.0-rc4", path = "wasmtime", optional = true } parking_lot = "0.10.0" log = "0.4.8" libsecp256k1 = "0.3.4" @@ -39,11 +39,14 @@ libsecp256k1 = "0.3.4" assert_matches = "1.3.0" wabt = "0.9.2" hex-literal = "0.2.1" -sc-runtime-test = { version = "2.0.0-alpha.8", path = "runtime-test" } -substrate-test-runtime = { version = "2.0.0-alpha.8", path = "../../test-utils/runtime" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../primitives/state-machine" } +sc-runtime-test = { version = "2.0.0-rc4", path = "runtime-test" } +substrate-test-runtime = { version = "2.0.0-rc4", path = "../../test-utils/runtime" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../primitives/state-machine" } test-case = "0.3.3" -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } +sp-tracing = { version = "2.0.0-rc4", path = "../../primitives/tracing" } +sc-tracing = { version = "2.0.0-rc4", path = "../tracing" } +tracing = "0.1.14" [features] default = [ "std" ] diff --git a/client/executor/common/Cargo.toml b/client/executor/common/Cargo.toml index 890010caf1d38ab5b05aa1345d4ff33a025de486..970fc2ded34d390b2b60ec1d9571d0aa794933f5 100644 --- a/client/executor/common/Cargo.toml +++ b/client/executor/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-executor-common" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -16,13 +16,13 @@ targets = ["x86_64-unknown-linux-gnu"] log = "0.4.8" derive_more = "0.99.2" parity-wasm = "0.41.0" -codec = { package = "parity-scale-codec", version = "1.3.0" } +codec = { package = "parity-scale-codec", version = "1.3.1" } wasmi = "0.6.2" -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-allocator = { version = "2.0.0-alpha.8", path = "../../../primitives/allocator" } -sp-wasm-interface = { version = "2.0.0-alpha.8", path = "../../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime-interface" } -sp-serializer = { version = "2.0.0-alpha.8", path = "../../../primitives/serializer" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-allocator = { version = "2.0.0-rc4", path = "../../../primitives/allocator" } +sp-wasm-interface = { version = "2.0.0-rc4", path = "../../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-rc4", path = "../../../primitives/runtime-interface" } +sp-serializer = { version = "2.0.0-rc4", path = "../../../primitives/serializer" } [features] default = [] diff --git a/client/executor/common/src/error.rs b/client/executor/common/src/error.rs index e2f482f23394e0a182c086084ee20c7237937d7b..04850e6f8dd7e34b1b438ec1a4c58f6ed026eb5b 100644 --- a/client/executor/common/src/error.rs +++ b/client/executor/common/src/error.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Rust executor possible errors. use sp_serializer; diff --git a/client/executor/common/src/sandbox.rs b/client/executor/common/src/sandbox.rs index 3500bf85755e7e5744a24da216ae68a1772daac1..b2c35b758271829c8771034da08aee46c52dd3db 100644 --- a/client/executor/common/src/sandbox.rs +++ b/client/executor/common/src/sandbox.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! This module implements sandboxing support in the runtime. //! //! Sandboxing is baked by wasmi at the moment. In future, however, we would like to add/switch to @@ -237,21 +238,33 @@ impl<'a, FE: SandboxCapabilities + 'a> Externals for GuestExternals<'a, FE> { .supervisor_externals .allocate_memory(invoke_args_len) .map_err(|_| trap("Can't allocate memory in supervisor for the arguments"))?; - self + + let deallocate = |this: &mut GuestExternals, ptr, fail_msg| { + this + .supervisor_externals + .deallocate_memory(ptr) + .map_err(|_| trap(fail_msg)) + }; + + if self .supervisor_externals .write_memory(invoke_args_ptr, &invoke_args_data) - .map_err(|_| trap("Can't write invoke args into memory"))?; + .is_err() + { + deallocate(self, invoke_args_ptr, "Failed dealloction after failed write of invoke arguments")?; + return Err(trap("Can't write invoke args into memory")); + } + let result = self.supervisor_externals.invoke( &self.sandbox_instance.dispatch_thunk, invoke_args_ptr, invoke_args_len, state, func_idx, - )?; - self - .supervisor_externals - .deallocate_memory(invoke_args_ptr) - .map_err(|_| trap("Can't deallocate memory for dispatch thunk's invoke arguments"))?; + ); + + deallocate(self, invoke_args_ptr, "Can't deallocate memory for dispatch thunk's invoke arguments")?; + let result = result?; // dispatch_thunk returns pointer to serialized arguments. // Unpack pointer and len of the serialized result data. @@ -265,12 +278,11 @@ impl<'a, FE: SandboxCapabilities + 'a> Externals for GuestExternals<'a, FE> { let serialized_result_val = self.supervisor_externals .read_memory(serialized_result_val_ptr, serialized_result_val_len) - .map_err(|_| trap("Can't read the serialized result from dispatch thunk"))?; - self.supervisor_externals - .deallocate_memory(serialized_result_val_ptr) - .map_err(|_| trap("Can't deallocate memory for dispatch thunk's result"))?; + .map_err(|_| trap("Can't read the serialized result from dispatch thunk")); - deserialize_result(&serialized_result_val) + deallocate(self, serialized_result_val_ptr, "Can't deallocate memory for dispatch thunk's result") + .and_then(|_| serialized_result_val) + .and_then(|serialized_result_val| deserialize_result(&serialized_result_val)) } } diff --git a/client/executor/common/src/util.rs b/client/executor/common/src/util.rs index 4734e8d6fd3461fad8d1268c7904ad473907f2bf..92a48e14018143944dae245abddb1db481ce4021 100644 --- a/client/executor/common/src/util.rs +++ b/client/executor/common/src/util.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! A set of utilities for resetting a wasm instance to its initial state. use crate::error::{self, Error}; diff --git a/client/executor/runtime-test/Cargo.toml b/client/executor/runtime-test/Cargo.toml index 54ba45921f506c3cf94eef39d3d1ccd2c6a9eac9..c01a9428f4fce7265e960ed9a725b4fea12d1539 100644 --- a/client/executor/runtime-test/Cargo.toml +++ b/client/executor/runtime-test/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-runtime-test" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" @@ -13,12 +13,12 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/io" } -sp-sandbox = { version = "0.8.0-alpha.8", default-features = false, path = "../../../primitives/sandbox" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/runtime" } -sp-allocator = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/allocator" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/io" } +sp-sandbox = { version = "0.8.0-rc4", default-features = false, path = "../../../primitives/sandbox" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/runtime" } +sp-allocator = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/allocator" } [build-dependencies] wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } diff --git a/client/executor/runtime-test/build.rs b/client/executor/runtime-test/build.rs index 647b4768141d2203155bf5325f9ff87f7d9c2446..c5f1f2402bd8b21d278c888044b94362c181672e 100644 --- a/client/executor/runtime-test/build.rs +++ b/client/executor/runtime-test/build.rs @@ -19,7 +19,7 @@ use wasm_builder_runner::WasmBuilder; fn main() { WasmBuilder::new() .with_current_project() - .with_wasm_builder_from_crates_or_path("1.0.9", "../../../utils/wasm-builder") + .with_wasm_builder_from_crates_or_path("1.0.11", "../../../utils/wasm-builder") .export_heap_base() .import_memory() .build() diff --git a/client/executor/runtime-test/src/lib.rs b/client/executor/runtime-test/src/lib.rs index 15a4177048a408b3b7f308f9d4ea63049f1b0769..4962c558eaa2dc7add290ad6d495e6ec061fb9f8 100644 --- a/client/executor/runtime-test/src/lib.rs +++ b/client/executor/runtime-test/src/lib.rs @@ -1,5 +1,4 @@ #![cfg_attr(not(feature = "std"), no_std)] -#![cfg_attr(feature = "strict", deny(warnings))] // Make the WASM binary available. #[cfg(feature = "std")] @@ -11,7 +10,7 @@ use sp_std::{vec::Vec, vec}; #[cfg(not(feature = "std"))] use sp_io::{ storage, hashing::{blake2_128, blake2_256, sha2_256, twox_128, twox_256}, - crypto::{ed25519_verify, sr25519_verify}, + crypto::{ed25519_verify, sr25519_verify}, wasm_tracing, }; #[cfg(not(feature = "std"))] use sp_runtime::{print, traits::{BlakeTwo256, Hash}}; @@ -247,6 +246,14 @@ sp_core::wasm_export_functions! { sp_allocator::FreeingBumpHeapAllocator::new(0); } + fn test_enter_span() -> u64 { + wasm_tracing::enter_span("integration_test_span_target", "integration_test_span_name") + } + + fn test_exit_span(span_id: u64) { + wasm_tracing::exit_span(span_id) + } + fn returns_mutable_static() -> u64 { unsafe { MUTABLE_STATIC += 1; diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 7ec0d22c6dfa541ad20e9f5c6bdfef968e621a50..21924270b8c1e00779db07b4f7b44a6766c1904e 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -497,9 +497,7 @@ fn offchain_http_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let (offchain, state) = testing::TestOffchainExt::new(); ext.register_extension(OffchainExt::new(offchain)); - state.write().expect_request( - 0, - testing::PendingRequest { + state.write().expect_request(testing::PendingRequest { method: "POST".into(), uri: "http://localhost:12345".into(), body: vec![1, 2, 3, 4], @@ -626,21 +624,134 @@ fn heap_is_reset_between_calls(wasm_method: WasmExecutionMethod) { #[test_case(WasmExecutionMethod::Interpreted)] #[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] fn parallel_execution(wasm_method: WasmExecutionMethod) { - let threads: Vec<_> = (0..8).map(|_| std::thread::spawn(move || { - let mut ext = TestExternalities::default(); - let mut ext = ext.ext(); - assert_eq!( - call_in_wasm( - "test_twox_128", - &[0], - wasm_method.clone(), - &mut ext, - ).unwrap(), - hex!("99e9d85137db46ef4bbea33613baafd5").to_vec().encode(), - ); - })).collect(); + let executor = std::sync::Arc::new(crate::WasmExecutor::new( + wasm_method, + Some(1024), + HostFunctions::host_functions(), + 8, + )); + let code_hash = blake2_256(WASM_BINARY).to_vec(); + let threads: Vec<_> = (0..8).map(|_| + { + let executor = executor.clone(); + let code_hash = code_hash.clone(); + std::thread::spawn(move || { + let mut ext = TestExternalities::default(); + let mut ext = ext.ext(); + assert_eq!( + executor.call_in_wasm( + &WASM_BINARY[..], + Some(code_hash.clone()), + "test_twox_128", + &[0], + &mut ext, + sp_core::traits::MissingHostFunctions::Allow, + ).unwrap(), + hex!("99e9d85137db46ef4bbea33613baafd5").to_vec().encode(), + ); + }) + }).collect(); for t in threads.into_iter() { t.join().unwrap(); } } + +#[test_case(WasmExecutionMethod::Interpreted)] +fn wasm_tracing_should_work(wasm_method: WasmExecutionMethod) { + + use std::sync::{Arc, Mutex}; + + use sc_tracing::SpanDatum; + + impl sc_tracing::TraceHandler for TestTraceHandler { + fn process_span(&self, sd: SpanDatum) { + self.0.lock().unwrap().push(sd); + } + } + + struct TestTraceHandler(Arc>>); + + let traces = Arc::new(Mutex::new(Vec::new())); + let handler = TestTraceHandler(traces.clone()); + + // Create subscriber with wasm_tracing disabled + let test_subscriber = sc_tracing::ProfilingSubscriber::new_with_handler( + Box::new(handler), "integration_test_span_target"); + + let _guard = tracing::subscriber::set_default(test_subscriber); + + let mut ext = TestExternalities::default(); + let mut ext = ext.ext(); + + // Test tracing disabled + assert!(!sp_tracing::wasm_tracing_enabled()); + + let span_id = call_in_wasm( + "test_enter_span", + &[], + wasm_method, + &mut ext, + ).unwrap(); + + assert_eq!( + 0u64.encode(), + span_id + ); + // Repeat to check span id always 0 when deactivated + let span_id = call_in_wasm( + "test_enter_span", + &[], + wasm_method, + &mut ext, + ).unwrap(); + + assert_eq!( + 0u64.encode(), + span_id + ); + + call_in_wasm( + "test_exit_span", + &span_id.encode(), + wasm_method, + &mut ext, + ).unwrap(); + // Check span has not been recorded + let len = traces.lock().unwrap().len(); + assert_eq!(len, 0); + + // Test tracing enabled + sp_tracing::set_wasm_tracing(true); + + let span_id = call_in_wasm( + "test_enter_span", + &[], + wasm_method, + &mut ext, + ).unwrap(); + + let span_id = u64::decode(&mut &span_id[..]).unwrap(); + + assert!( + span_id > 0 + ); + + call_in_wasm( + "test_exit_span", + &span_id.encode(), + wasm_method, + &mut ext, + ).unwrap(); + + // Check there is only the single trace + let len = traces.lock().unwrap().len(); + assert_eq!(len, 1); + + let span_datum = traces.lock().unwrap().pop().unwrap(); + let values = span_datum.values.into_inner(); + assert_eq!(span_datum.target, "integration_test_span_target"); + assert_eq!(span_datum.name, "integration_test_span_name"); + assert_eq!(values.get("wasm").unwrap(), "true"); + assert_eq!(values.get("is_valid_trace").unwrap(), "true"); +} diff --git a/client/executor/src/integration_tests/sandbox.rs b/client/executor/src/integration_tests/sandbox.rs index 35ece0a701f2de48821ab06195432e7d15fc2e35..f84e446b416c06c2f423d71103dbc1597ec85138 100644 --- a/client/executor/src/integration_tests/sandbox.rs +++ b/client/executor/src/integration_tests/sandbox.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use super::{TestExternalities, call_in_wasm}; use crate::WasmExecutionMethod; diff --git a/client/executor/src/lib.rs b/client/executor/src/lib.rs index 2630282b809ad7f0e24beccd3ec762c9870771fb..c02568c734b699fc0e4e4fb4f96852048a564d5e 100644 --- a/client/executor/src/lib.rs +++ b/client/executor/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! A crate that provides means of executing/dispatching calls into the runtime. //! //! There are a few responsibilities of this crate at the moment: diff --git a/client/executor/src/native_executor.rs b/client/executor/src/native_executor.rs index 61014bab3c58c1697e7bbbabf9033930cdcc5e9d..b1eb504d5a2b3f28fd154d724f265479eafa8f8a 100644 --- a/client/executor/src/native_executor.rs +++ b/client/executor/src/native_executor.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::{ RuntimeInfo, error::{Error, Result}, wasm_runtime::{RuntimeCache, WasmExecutionMethod}, diff --git a/client/executor/wasmi/Cargo.toml b/client/executor/wasmi/Cargo.toml index 4638727742ae1e77bbd37103aa60fa87aa057377..6f5486a5781f2d344ad2a59cccf38d12780a13db 100644 --- a/client/executor/wasmi/Cargo.toml +++ b/client/executor/wasmi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-executor-wasmi" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -15,9 +15,9 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] log = "0.4.8" wasmi = "0.6.2" -codec = { package = "parity-scale-codec", version = "1.3.0" } -sc-executor-common = { version = "0.8.0-alpha.8", path = "../common" } -sp-wasm-interface = { version = "2.0.0-alpha.8", path = "../../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime-interface" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-allocator = { version = "2.0.0-alpha.8", path = "../../../primitives/allocator" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +sc-executor-common = { version = "0.8.0-rc4", path = "../common" } +sp-wasm-interface = { version = "2.0.0-rc4", path = "../../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-rc4", path = "../../../primitives/runtime-interface" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-allocator = { version = "2.0.0-rc4", path = "../../../primitives/allocator" } diff --git a/client/executor/wasmtime/Cargo.toml b/client/executor/wasmtime/Cargo.toml index 0a803d772a7f1b3907bb33bab206afb9fd542a5e..26eddd1da6d1bf547a0ccfe8007591e0362d0541 100644 --- a/client/executor/wasmtime/Cargo.toml +++ b/client/executor/wasmtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-executor-wasmtime" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -15,14 +15,14 @@ targets = ["x86_64-unknown-linux-gnu"] log = "0.4.8" scoped-tls = "1.0" parity-wasm = "0.41.0" -codec = { package = "parity-scale-codec", version = "1.3.0" } -sc-executor-common = { version = "0.8.0-alpha.8", path = "../common" } -sp-wasm-interface = { version = "2.0.0-alpha.8", path = "../../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime-interface" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-allocator = { version = "2.0.0-alpha.8", path = "../../../primitives/allocator" } -wasmtime = { package = "substrate-wasmtime", version = "0.16.0-threadsafe.2" } -wasmtime-runtime = { package = "substrate-wasmtime-runtime", version = "0.16.0-threadsafe.2" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +sc-executor-common = { version = "0.8.0-rc4", path = "../common" } +sp-wasm-interface = { version = "2.0.0-rc4", path = "../../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-rc4", path = "../../../primitives/runtime-interface" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-allocator = { version = "2.0.0-rc4", path = "../../../primitives/allocator" } +wasmtime = { package = "substrate-wasmtime", version = "0.16.0-threadsafe.4" } +wasmtime-runtime = { package = "substrate-wasmtime-runtime", version = "0.16.0-threadsafe.4" } wasmtime-environ = "0.16" cranelift-wasm = "0.63" cranelift-codegen = "0.63" diff --git a/client/executor/wasmtime/src/imports.rs b/client/executor/wasmtime/src/imports.rs index 1fc570c2005e548b325329a24d1573ac0b9bc321..36752d72fa02334efc6a99191005da7c9577ce10 100644 --- a/client/executor/wasmtime/src/imports.rs +++ b/client/executor/wasmtime/src/imports.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::state_holder; use sc_executor_common::error::WasmError; use sp_wasm_interface::{Function, Value, ValueType}; diff --git a/client/executor/wasmtime/src/instance_wrapper.rs b/client/executor/wasmtime/src/instance_wrapper.rs index 006eb5eb20961b511111fd895fbec1e4b55ef2d0..9026b8054e652604d41a7b626c4f09c38f0eebfe 100644 --- a/client/executor/wasmtime/src/instance_wrapper.rs +++ b/client/executor/wasmtime/src/instance_wrapper.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Defines data and logic needed for interaction with an WebAssembly instance of a substrate //! runtime module. diff --git a/client/executor/wasmtime/src/instance_wrapper/globals_snapshot.rs b/client/executor/wasmtime/src/instance_wrapper/globals_snapshot.rs index 8dceca70a6ccf28da62817b65170f0f3266efa93..01d82451fcbbb4884df321103a7ee8fc61f10ed0 100644 --- a/client/executor/wasmtime/src/instance_wrapper/globals_snapshot.rs +++ b/client/executor/wasmtime/src/instance_wrapper/globals_snapshot.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use super::InstanceWrapper; use sc_executor_common::{ error::{Error, Result}, diff --git a/client/executor/wasmtime/src/state_holder.rs b/client/executor/wasmtime/src/state_holder.rs index e44bfad12233bb8c5d0d272bda8e491caa65f423..711d3bb735d7c04306525ba451f1e4be84ed9d21 100644 --- a/client/executor/wasmtime/src/state_holder.rs +++ b/client/executor/wasmtime/src/state_holder.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::host::{HostContext, HostState}; scoped_tls::scoped_thread_local!(static HOST_STATE: HostState); diff --git a/client/finality-grandpa/Cargo.toml b/client/finality-grandpa/Cargo.toml index 179a938270fe8e178186ea20dcea351c6ebc9eb1..36b1d59b0c0b2e4aad1a26a7a500928a9f04c191 100644 --- a/client/finality-grandpa/Cargo.toml +++ b/client/finality-grandpa/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-finality-grandpa" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -15,45 +15,46 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] derive_more = "0.99.2" -fork-tree = { version = "2.0.0-alpha.8", path = "../../utils/fork-tree" } +fork-tree = { version = "2.0.0-rc4", path = "../../utils/fork-tree" } futures = "0.3.4" futures-timer = "3.0.1" log = "0.4.8" parking_lot = "0.10.0" rand = "0.7.2" assert_matches = "1.3.0" -parity-scale-codec = { version = "1.3.0", features = ["derive"] } -sp-arithmetic = { version = "2.0.0-alpha.8", path = "../../primitives/arithmetic" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } -sp-utils = { version = "2.0.0-alpha.8", path = "../../primitives/utils" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../primitives/consensus/common" } -sc-consensus = { version = "0.8.0-alpha.8", path = "../../client/consensus/common" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-api = { version = "2.0.0-alpha.8", path = "../../primitives/api" } -sc-telemetry = { version = "2.0.0-alpha.8", path = "../telemetry" } -sc-keystore = { version = "2.0.0-alpha.8", path = "../keystore" } +parity-scale-codec = { version = "1.3.1", features = ["derive"] } +sp-application-crypto = { version = "2.0.0-rc4", path = "../../primitives/application-crypto" } +sp-arithmetic = { version = "2.0.0-rc4", path = "../../primitives/arithmetic" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-rc4", path = "../../primitives/utils" } +sp-consensus = { version = "0.8.0-rc4", path = "../../primitives/consensus/common" } +sc-consensus = { version = "0.8.0-rc4", path = "../../client/consensus/common" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-api = { version = "2.0.0-rc4", path = "../../primitives/api" } +sc-telemetry = { version = "2.0.0-rc4", path = "../telemetry" } +sc-keystore = { version = "2.0.0-rc4", path = "../keystore" } serde_json = "1.0.41" -sc-client-api = { version = "2.0.0-alpha.8", path = "../api" } -sp-inherents = { version = "2.0.0-alpha.8", path = "../../primitives/inherents" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../primitives/blockchain" } -sc-network = { version = "0.8.0-alpha.8", path = "../network" } -sc-network-gossip = { version = "0.8.0-alpha.8", path = "../network-gossip" } -sp-finality-tracker = { version = "2.0.0-alpha.8", path = "../../primitives/finality-tracker" } -sp-finality-grandpa = { version = "2.0.0-alpha.8", path = "../../primitives/finality-grandpa" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-alpha.8"} -sc-block-builder = { version = "0.8.0-alpha.8", path = "../block-builder" } +sc-client-api = { version = "2.0.0-rc4", path = "../api" } +sp-inherents = { version = "2.0.0-rc4", path = "../../primitives/inherents" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../primitives/blockchain" } +sc-network = { version = "0.8.0-rc4", path = "../network" } +sc-network-gossip = { version = "0.8.0-rc4", path = "../network-gossip" } +sp-finality-tracker = { version = "2.0.0-rc4", path = "../../primitives/finality-tracker" } +sp-finality-grandpa = { version = "2.0.0-rc4", path = "../../primitives/finality-grandpa" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-rc4"} +sc-block-builder = { version = "0.8.0-rc4", path = "../block-builder" } finality-grandpa = { version = "0.12.3", features = ["derive-codec"] } pin-project = "0.4.6" [dev-dependencies] finality-grandpa = { version = "0.12.3", features = ["derive-codec", "test-helpers"] } -sc-network = { version = "0.8.0-alpha.8", path = "../network" } -sc-network-test = { version = "0.8.0-alpha.8", path = "../network/test" } -sp-keyring = { version = "2.0.0-alpha.8", path = "../../primitives/keyring" } -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../test-utils/runtime/client" } -sp-consensus-babe = { version = "0.8.0-alpha.8", path = "../../primitives/consensus/babe" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../primitives/state-machine" } +sc-network = { version = "0.8.0-rc4", path = "../network" } +sc-network-test = { version = "0.8.0-rc4", path = "../network/test" } +sp-keyring = { version = "2.0.0-rc4", path = "../../primitives/keyring" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../test-utils/runtime/client" } +sp-consensus-babe = { version = "0.8.0-rc4", path = "../../primitives/consensus/babe" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../primitives/state-machine" } env_logger = "0.7.0" tokio = { version = "0.2", features = ["rt-core"] } tempfile = "3.1.0" -sp-api = { version = "2.0.0-alpha.8", path = "../../primitives/api" } +sp-api = { version = "2.0.0-rc4", path = "../../primitives/api" } diff --git a/client/finality-grandpa/rpc/Cargo.toml b/client/finality-grandpa/rpc/Cargo.toml index 75d7497235497f4cd36feb418389d3bb83a4d74e..a7d8e64087414f69da97087f84bd8c81be11fc51 100644 --- a/client/finality-grandpa/rpc/Cargo.toml +++ b/client/finality-grandpa/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-finality-grandpa-rpc" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "RPC extensions for the GRANDPA finality gadget" repository = "https://github.com/paritytech/substrate/" @@ -8,11 +8,11 @@ edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" [dependencies] -sc-finality-grandpa = { version = "0.8.0-alpha.8", path = "../" } +sc-finality-grandpa = { version = "0.8.0-rc4", path = "../" } finality-grandpa = { version = "0.12.3", features = ["derive-codec"] } -jsonrpc-core = "14.0.3" -jsonrpc-core-client = "14.0.3" -jsonrpc-derive = "14.0.3" +jsonrpc-core = "14.2.0" +jsonrpc-core-client = "14.2.0" +jsonrpc-derive = "14.2.1" futures = { version = "0.3.4", features = ["compat"] } serde = { version = "1.0.105", features = ["derive"] } serde_json = "1.0.50" @@ -20,4 +20,4 @@ log = "0.4.8" derive_more = "0.99.2" [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } diff --git a/client/finality-grandpa/rpc/src/error.rs b/client/finality-grandpa/rpc/src/error.rs index eab319d3a7810e5e7a1a2d5d04ed67dda4729f4d..bfd0596fdf3205d3eda8003a3a0f109b83fe836f 100644 --- a/client/finality-grandpa/rpc/src/error.rs +++ b/client/finality-grandpa/rpc/src/error.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::NOT_READY_ERROR_CODE; #[derive(derive_more::Display, derive_more::From)] diff --git a/client/finality-grandpa/rpc/src/lib.rs b/client/finality-grandpa/rpc/src/lib.rs index c146aaf10c13ef0bb6446229fd6d2de2126cebd8..1af84b7a84413007be22a8f01be7701648c1d489 100644 --- a/client/finality-grandpa/rpc/src/lib.rs +++ b/client/finality-grandpa/rpc/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! RPC API for GRANDPA. #![warn(missing_docs)] diff --git a/client/finality-grandpa/rpc/src/report.rs b/client/finality-grandpa/rpc/src/report.rs index d8667f56099cdfc280e4cd1f4198ad45c7ac37d3..a635728cb938adf73d4ae94e1db1a63d5fcb9f50 100644 --- a/client/finality-grandpa/rpc/src/report.rs +++ b/client/finality-grandpa/rpc/src/report.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use std::{ collections::{BTreeSet, HashSet}, fmt::Debug, diff --git a/client/finality-grandpa/src/authorities.rs b/client/finality-grandpa/src/authorities.rs index a897ac0209e15a704b8fd11ae55d63d2bd52c639..b4cb254864df7f3150451939e5759e7122b68ca6 100644 --- a/client/finality-grandpa/src/authorities.rs +++ b/client/finality-grandpa/src/authorities.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Utilities for dealing with authorities, authority sets, and handoffs. use fork_tree::ForkTree; diff --git a/client/finality-grandpa/src/communication/gossip.rs b/client/finality-grandpa/src/communication/gossip.rs index 7fe17e974bbaa842e69cc819ee4e5b30b0671768..7d9fe4e7f2da16f95c849730132ce76aadd65189 100644 --- a/client/finality-grandpa/src/communication/gossip.rs +++ b/client/finality-grandpa/src/communication/gossip.rs @@ -110,7 +110,7 @@ const CATCH_UP_THRESHOLD: u64 = 2; const PROPAGATION_ALL: u32 = 4; //in rounds; const PROPAGATION_ALL_AUTHORITIES: u32 = 2; //in rounds; const PROPAGATION_SOME_NON_AUTHORITIES: u32 = 3; //in rounds; -const ROUND_DURATION: u32 = 4; // measured in gossip durations +const ROUND_DURATION: u32 = 2; // measured in gossip durations const MIN_LUCKY: usize = 5; @@ -181,15 +181,27 @@ impl View { } } -/// A local view of protocol state. Only differs from `View` in that we also -/// track the round and set id at which the last commit was observed. +/// A local view of protocol state. Similar to `View` but we additionally track +/// the round and set id at which the last commit was observed, and the instant +/// at which the current round started. struct LocalView { round: Round, set_id: SetId, last_commit: Option<(N, Round, SetId)>, + round_start: Instant, } impl LocalView { + /// Creates a new `LocalView` at the given set id and round. + fn new(set_id: SetId, round: Round) -> LocalView { + LocalView { + set_id, + round, + last_commit: None, + round_start: Instant::now(), + } + } + /// Converts the local view to a `View` discarding round and set id /// information about the last commit. fn as_view(&self) -> View<&N> { @@ -205,9 +217,16 @@ impl LocalView { if set_id != self.set_id { self.set_id = set_id; self.round = Round(1); + self.round_start = Instant::now(); } } + /// Updates the current round. + fn update_round(&mut self, round: Round) { + self.round = round; + self.round_start = Instant::now(); + } + /// Returns the height of the block that the last observed commit finalizes. fn last_commit_height(&self) -> Option<&N> { self.last_commit.as_ref().map(|(number, _, _)| number) @@ -656,7 +675,6 @@ struct Inner { local_view: Option>>, peers: Peers>, live_topics: KeepTopics, - round_start: Instant, authorities: Vec, config: crate::Config, next_rebroadcast: Instant, @@ -689,7 +707,6 @@ impl Inner { local_view: None, peers: Peers::default(), live_topics: KeepTopics::new(), - round_start: Instant::now(), next_rebroadcast: Instant::now() + REBROADCAST_AFTER, authorities: Vec::new(), pending_catch_up: PendingCatchUp::None, @@ -715,10 +732,9 @@ impl Inner { debug!(target: "afg", "Voter {} noting beginning of round {:?} to network.", self.config.name(), (round, set_id)); - local_view.round = round; + local_view.update_round(round); self.live_topics.push(round, set_id); - self.round_start = Instant::now(); self.peers.reshuffle(); } self.multicast_neighbor_packet() @@ -729,13 +745,16 @@ impl Inner { fn note_set(&mut self, set_id: SetId, authorities: Vec) -> MaybeMessage { { let local_view = match self.local_view { - ref mut x @ None => x.get_or_insert(LocalView { - round: Round(1), + ref mut x @ None => x.get_or_insert(LocalView::new( set_id, - last_commit: None, - }), + Round(1), + )), Some(ref mut v) => if v.set_id == set_id { - if self.authorities != authorities { + let diff_authorities = + self.authorities.iter().collect::>() != + authorities.iter().collect(); + + if diff_authorities { debug!(target: "afg", "Gossip validator noted set {:?} twice with different authorities. \ Was the authority set hard forked?", @@ -814,7 +833,7 @@ impl Inner { return Action::Discard(cost::UNKNOWN_VOTER); } - if let Err(()) = sp_finality_grandpa::check_message_signature( + if !sp_finality_grandpa::check_message_signature( &full.message.message, &full.message.id, &full.message.signature, @@ -1121,8 +1140,10 @@ impl Inner { /// underlying gossip layer, which should happen every 30 seconds. fn round_message_allowed(&self, who: &PeerId, peer: &PeerInfo) -> bool { let round_duration = self.config.gossip_duration * ROUND_DURATION; - let round_elapsed = self.round_start.elapsed(); - + let round_elapsed = match self.local_view { + Some(ref local_view) => local_view.round_start.elapsed(), + None => return false, + }; if !self.config.is_authority && round_elapsed < round_duration * PROPAGATION_ALL @@ -1185,7 +1206,10 @@ impl Inner { /// underlying gossip layer, which should happen every 30 seconds. fn global_message_allowed(&self, who: &PeerId, peer: &PeerInfo) -> bool { let round_duration = self.config.gossip_duration * ROUND_DURATION; - let round_elapsed = self.round_start.elapsed(); + let round_elapsed = match self.local_view { + Some(ref local_view) => local_view.round_start.elapsed(), + None => return false, + }; match peer.roles { ObservedRole::OurSentry | ObservedRole::OurGuardedAuthority => true, @@ -2320,7 +2344,8 @@ mod tests { let test = |num_round, peers| { // rewind n round durations - val.inner.write().round_start = Instant::now() - round_duration * num_round; + val.inner.write().local_view.as_mut().unwrap().round_start = + Instant::now() - round_duration * num_round; let mut message_allowed = val.message_allowed(); move || { @@ -2448,7 +2473,8 @@ mod tests { } { - val.inner.write().round_start = Instant::now() - round_duration * 4; + val.inner.write().local_view.as_mut().unwrap().round_start = + Instant::now() - round_duration * 4; let mut message_allowed = val.message_allowed(); // on the fourth round duration we should allow messages to authorities // (on the second we would do `sqrt(authorities)`) @@ -2598,12 +2624,12 @@ mod tests { fn allow_noting_different_authorities_for_same_set() { let (val, _) = GossipValidator::::new(config(), voter_set_state(), None); - let a1 = vec![AuthorityId::default()]; + let a1 = vec![AuthorityId::from_slice(&[0; 32])]; val.note_set(SetId(1), a1.clone(), |_, _| {}); assert_eq!(val.inner().read().authorities, a1); - let a2 = vec![AuthorityId::default(), AuthorityId::default()]; + let a2 = vec![AuthorityId::from_slice(&[1; 32]), AuthorityId::from_slice(&[2; 32])]; val.note_set(SetId(1), a2.clone(), |_, _| {}); assert_eq!(val.inner().read().authorities, a2); diff --git a/client/finality-grandpa/src/communication/mod.rs b/client/finality-grandpa/src/communication/mod.rs index 1bef06572b0f2681370e5713be8b130c245e6559..b7bbad9f8e7c47058b2f38294ea8c0d76901d478 100644 --- a/client/finality-grandpa/src/communication/mod.rs +++ b/client/finality-grandpa/src/communication/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Communication streams for the polite-grandpa networking protocol. //! //! GRANDPA nodes communicate over a gossip network, where messages are not sent to @@ -34,12 +35,12 @@ use parking_lot::Mutex; use prometheus_endpoint::Registry; use std::{pin::Pin, sync::Arc, task::{Context, Poll}}; +use sp_core::traits::BareCryptoStorePtr; use finality_grandpa::Message::{Prevote, Precommit, PrimaryPropose}; use finality_grandpa::{voter, voter_set::VoterSet}; use sc_network::{NetworkService, ReputationChange}; use sc_network_gossip::{GossipEngine, Network as GossipNetwork}; use parity_scale_codec::{Encode, Decode}; -use sp_core::Pair; use sp_runtime::traits::{Block as BlockT, Hash as HashT, Header as HeaderT, NumberFor}; use sc_telemetry::{telemetry, CONSENSUS_DEBUG, CONSENSUS_INFO}; @@ -57,7 +58,7 @@ use gossip::{ VoteMessage, }; use sp_finality_grandpa::{ - AuthorityPair, AuthorityId, AuthoritySignature, SetId as SetIdNumber, RoundNumber, + AuthorityId, AuthoritySignature, SetId as SetIdNumber, RoundNumber, }; use sp_utils::mpsc::TracingUnboundedReceiver; @@ -104,6 +105,34 @@ mod benefit { pub(super) const PER_EQUIVOCATION: i32 = 10; } +/// A type that ties together our local authority id and a keystore where it is +/// available for signing. +pub struct LocalIdKeystore((AuthorityId, BareCryptoStorePtr)); + +impl LocalIdKeystore { + /// Returns a reference to our local authority id. + fn local_id(&self) -> &AuthorityId { + &(self.0).0 + } + + /// Returns a reference to the keystore. + fn keystore(&self) -> &BareCryptoStorePtr { + &(self.0).1 + } +} + +impl AsRef for LocalIdKeystore { + fn as_ref(&self) -> &BareCryptoStorePtr { + self.keystore() + } +} + +impl From<(AuthorityId, BareCryptoStorePtr)> for LocalIdKeystore { + fn from(inner: (AuthorityId, BareCryptoStorePtr)) -> LocalIdKeystore { + LocalIdKeystore(inner) + } +} + /// If the voter set is larger than this value some telemetry events are not /// sent to avoid increasing usage resource on the node and flooding the /// telemetry server (e.g. received votes, received commits.) @@ -271,10 +300,10 @@ impl> NetworkBridge { /// network all within the current set. pub(crate) fn round_communication( &self, + keystore: Option, round: Round, set_id: SetId, voters: Arc>, - local_key: Option, has_voted: HasVoted, ) -> ( impl Stream> + Unpin, @@ -286,10 +315,10 @@ impl> NetworkBridge { &*voters, ); - let locals = local_key.and_then(|pair| { - let id = pair.public(); - if voters.contains(&id) { - Some((pair, id)) + let keystore = keystore.and_then(|ks| { + let id = ks.local_id(); + if voters.contains(id) { + Some(ks) } else { None } @@ -349,10 +378,10 @@ impl> NetworkBridge { let (tx, out_rx) = mpsc::channel(0); let outgoing = OutgoingMessages:: { + keystore, round: round.0, set_id: set_id.0, network: self.gossip_engine.clone(), - locals, sender: tx, has_voted, }; @@ -627,7 +656,7 @@ pub struct SetId(pub SetIdNumber); pub(crate) struct OutgoingMessages { round: RoundNumber, set_id: SetIdNumber, - locals: Option<(AuthorityPair, AuthorityId)>, + keystore: Option, sender: mpsc::Sender>, network: Arc>>, has_voted: HasVoted, @@ -664,14 +693,19 @@ impl Sink> for OutgoingMessages } // when locals exist, sign messages on import - if let Some((ref pair, _)) = self.locals { + if let Some(ref keystore) = self.keystore { let target_hash = *(msg.target().0); let signed = sp_finality_grandpa::sign_message( + keystore.as_ref(), msg, - pair, + keystore.local_id().clone(), self.round, self.set_id, - ); + ).ok_or( + Error::Signing(format!( + "Failed to sign GRANDPA vote for round {} targetting {:?}", self.round, target_hash + )) + )?; let message = GossipMessage::Vote(VoteMessage:: { message: signed.clone(), @@ -759,7 +793,7 @@ fn check_compact_commit( use crate::communication::gossip::Misbehavior; use finality_grandpa::Message as GrandpaMessage; - if let Err(()) = sp_finality_grandpa::check_message_signature_with_buffer( + if !sp_finality_grandpa::check_message_signature_with_buffer( &GrandpaMessage::Precommit(precommit.clone()), id, sig, @@ -847,7 +881,7 @@ fn check_catch_up( for (msg, id, sig) in messages { signatures_checked += 1; - if let Err(()) = sp_finality_grandpa::check_message_signature_with_buffer( + if !sp_finality_grandpa::check_message_signature_with_buffer( &msg, id, sig, diff --git a/client/finality-grandpa/src/environment.rs b/client/finality-grandpa/src/environment.rs index b8996df9d4384ddce81db45a91904ee65ff4b6e2..cc6497fc72462b5474c283c9e9694f0998f711d4 100644 --- a/client/finality-grandpa/src/environment.rs +++ b/client/finality-grandpa/src/environment.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use std::collections::BTreeMap; use std::iter::FromIterator; use std::pin::Pin; @@ -34,7 +35,6 @@ use finality_grandpa::{ voter, voter_set::VoterSet, }; use sp_blockchain::{HeaderBackend, HeaderMetadata, Error as ClientError}; -use sp_core::Pair; use sp_runtime::generic::BlockId; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, NumberFor, One, Zero, @@ -703,11 +703,11 @@ where let prevote_timer = Delay::new(self.config.gossip_duration * 2); let precommit_timer = Delay::new(self.config.gossip_duration * 4); - let local_key = crate::is_voter(&self.voters, &self.config.keystore); + let local_key = crate::is_voter(&self.voters, self.config.keystore.as_ref()); let has_voted = match self.voter_set_state.has_voted(round) { HasVoted::Yes(id, vote) => { - if local_key.as_ref().map(|k| k.public() == id).unwrap_or(false) { + if local_key.as_ref().map(|k| k == &id).unwrap_or(false) { HasVoted::Yes(id, vote) } else { HasVoted::No @@ -716,11 +716,18 @@ where HasVoted::No => HasVoted::No, }; + // we can only sign when we have a local key in the authority set + // and we have a reference to the keystore. + let keystore = match (local_key.as_ref(), self.config.keystore.as_ref()) { + (Some(id), Some(keystore)) => Some((id.clone(), keystore.clone()).into()), + _ => None, + }; + let (incoming, outgoing) = self.network.round_communication( + keystore, crate::communication::Round(round), crate::communication::SetId(self.set_id), self.voters.clone(), - local_key.clone(), has_voted, ); @@ -739,7 +746,7 @@ where let outgoing = Box::pin(outgoing.sink_err_into()); voter::RoundData { - voter_id: local_key.map(|pair| pair.public()), + voter_id: local_key, prevote_timer: Box::pin(prevote_timer.map(Ok)), precommit_timer: Box::pin(precommit_timer.map(Ok)), incoming, @@ -748,10 +755,10 @@ where } fn proposed(&self, round: RoundNumber, propose: PrimaryPropose) -> Result<(), Self::Error> { - let local_id = crate::is_voter(&self.voters, &self.config.keystore); + let local_id = crate::is_voter(&self.voters, self.config.keystore.as_ref()); let local_id = match local_id { - Some(id) => id.public(), + Some(id) => id, None => return Ok(()), }; @@ -787,10 +794,10 @@ where } fn prevoted(&self, round: RoundNumber, prevote: Prevote) -> Result<(), Self::Error> { - let local_id = crate::is_voter(&self.voters, &self.config.keystore); + let local_id = crate::is_voter(&self.voters, self.config.keystore.as_ref()); let local_id = match local_id { - Some(id) => id.public(), + Some(id) => id, None => return Ok(()), }; @@ -828,10 +835,10 @@ where } fn precommitted(&self, round: RoundNumber, precommit: Precommit) -> Result<(), Self::Error> { - let local_id = crate::is_voter(&self.voters, &self.config.keystore); + let local_id = crate::is_voter(&self.voters, self.config.keystore.as_ref()); let local_id = match local_id { - Some(id) => id.public(), + Some(id) => id, None => return Ok(()), }; diff --git a/client/finality-grandpa/src/finality_proof.rs b/client/finality-grandpa/src/finality_proof.rs index d50418128b3463cb1aacd5c2309810962a57ff4c..55f6376579db4ced0e554293d3fabf569d221b94 100644 --- a/client/finality-grandpa/src/finality_proof.rs +++ b/client/finality-grandpa/src/finality_proof.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! GRANDPA block finality proof generation and check. //! //! Finality of block B is proved by providing: diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index 71184924924b2d8fc8d7d8a137919679c53bc12a..b37ab7907a62e75b26d054666b9c1b7f114a69bd 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use std::{sync::Arc, collections::HashMap}; use log::{debug, trace}; @@ -668,6 +669,7 @@ where Error::Blockchain(error) => ConsensusError::ClientImport(error), Error::Client(error) => ConsensusError::ClientImport(error.to_string()), Error::Safety(error) => ConsensusError::ClientImport(error), + Error::Signing(error) => ConsensusError::ClientImport(error), Error::Timer(error) => ConsensusError::ClientImport(error.to_string()), }); }, diff --git a/client/finality-grandpa/src/justification.rs b/client/finality-grandpa/src/justification.rs index 595435e4e321cba6d61e609a424675551ed8b23e..0e51a230c5a81bce7fe608ef72cbc774a42172bc 100644 --- a/client/finality-grandpa/src/justification.rs +++ b/client/finality-grandpa/src/justification.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -132,14 +133,14 @@ impl GrandpaJustification { let mut buf = Vec::new(); let mut visited_hashes = HashSet::new(); for signed in self.commit.precommits.iter() { - if sp_finality_grandpa::check_message_signature_with_buffer( + if !sp_finality_grandpa::check_message_signature_with_buffer( &finality_grandpa::Message::Precommit(signed.precommit.clone()), &signed.id, &signed.signature, self.round, set_id, &mut buf, - ).is_err() { + ) { return Err(ClientError::BadJustification( "invalid signature for precommit in grandpa justification".to_string())); } diff --git a/client/finality-grandpa/src/lib.rs b/client/finality-grandpa/src/lib.rs index 3cd5c5791e8756d4035e22a4bbe0ba115ed8e2e3..fa2a6fedd8b05f9556afa308e5248ae97624a367 100644 --- a/client/finality-grandpa/src/lib.rs +++ b/client/finality-grandpa/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Integration of the GRANDPA finality gadget into substrate. //! //! This crate is unstable and the API and usage may change. @@ -55,8 +56,10 @@ #![warn(missing_docs)] -use futures::prelude::*; -use futures::StreamExt; +use futures::{ + prelude::*, + StreamExt, +}; use log::{debug, info}; use sc_client_api::{ backend::{AuxStore, Backend}, @@ -69,10 +72,13 @@ use sp_api::ProvideRuntimeApi; use sp_blockchain::{HeaderBackend, Error as ClientError, HeaderMetadata}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{NumberFor, Block as BlockT, DigestFor, Zero}; -use sc_keystore::KeyStorePtr; use sp_inherents::InherentDataProviders; use sp_consensus::{SelectChain, BlockImport}; -use sp_core::Pair; +use sp_core::{ + crypto::Public, + traits::BareCryptoStorePtr, +}; +use sp_application_crypto::AppKey; use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver}; use sc_telemetry::{telemetry, CONSENSUS_INFO, CONSENSUS_DEBUG}; use parking_lot::RwLock; @@ -118,6 +124,7 @@ mod voting_rule; pub use authorities::SharedAuthoritySet; pub use finality_proof::{FinalityProofProvider, StorageAndProofProvider}; +pub use import::GrandpaBlockImport; pub use justification::GrandpaJustification; pub use light_import::light_block_import; pub use voting_rule::{ @@ -127,13 +134,12 @@ pub use finality_grandpa::voter::report; use aux_schema::PersistentData; use environment::{Environment, VoterSetState}; -use import::GrandpaBlockImport; use until_imported::UntilGlobalMessageBlocksImported; use communication::{NetworkBridge, Network as NetworkT}; -use sp_finality_grandpa::{AuthorityList, AuthorityPair, AuthoritySignature, SetId}; +use sp_finality_grandpa::{AuthorityList, AuthoritySignature, SetId}; // Re-export these two because it's just so damn convenient. -pub use sp_finality_grandpa::{AuthorityId, GrandpaApi, ScheduledChange}; +pub use sp_finality_grandpa::{AuthorityId, AuthorityPair, GrandpaApi, ScheduledChange}; use std::marker::PhantomData; #[cfg(test)] @@ -263,7 +269,7 @@ pub struct Config { /// Some local identifier of the voter. pub name: Option, /// The keystore that manages the keys of this node. - pub keystore: Option, + pub keystore: Option, } impl Config { @@ -283,6 +289,8 @@ pub enum Error { Blockchain(String), /// Could not complete a round on disk. Client(ClientError), + /// Could not sign outgoing message + Signing(String), /// An invariant has been violated (e.g. not finalizing pending change blocks in-order) Safety(String), /// A timer failed to fire. @@ -585,7 +593,7 @@ fn global_communication( voters: &Arc>, client: Arc, network: &NetworkBridge, - keystore: &Option, + keystore: Option<&BareCryptoStorePtr>, metrics: Option, ) -> ( impl Stream< @@ -728,7 +736,7 @@ pub fn run_grandpa_voter( .for_each(move |_| { let curr = authorities.current_authorities(); let mut auths = curr.iter().map(|(p, _)| p); - let maybe_authority_id = authority_id(&mut auths, &conf.keystore) + let maybe_authority_id = authority_id(&mut auths, conf.keystore.as_ref()) .unwrap_or_default(); telemetry!(CONSENSUS_INFO; "afg.authority_set"; @@ -864,8 +872,7 @@ where fn rebuild_voter(&mut self) { debug!(target: "afg", "{}: Starting new voter with set ID {}", self.env.config.name(), self.env.set_id); - let authority_id = is_voter(&self.env.voters, &self.env.config.keystore) - .map(|ap| ap.public()) + let authority_id = is_voter(&self.env.voters, self.env.config.keystore.as_ref()) .unwrap_or_default(); telemetry!(CONSENSUS_DEBUG; "afg.starting_new_voter"; @@ -900,7 +907,7 @@ where &self.env.voters, self.env.client.clone(), &self.env.network, - &self.env.config.keystore, + self.env.config.keystore.as_ref(), self.metrics.as_ref().map(|m| m.until_imported.clone()), ); @@ -1088,12 +1095,16 @@ pub fn setup_disabled_grandpa( /// Returns the key pair of the node that is being used in the current voter set or `None`. fn is_voter( voters: &Arc>, - keystore: &Option, -) -> Option { + keystore: Option<&BareCryptoStorePtr>, +) -> Option { match keystore { Some(keystore) => voters .iter() - .find_map(|(p, _)| keystore.read().key_pair::(&p).ok()), + .find(|(p, _)| { + keystore.read() + .has_keys(&[(p.to_raw_vec(), AuthorityId::ID)]) + }) + .map(|(p, _)| p.clone()), None => None, } } @@ -1101,19 +1112,16 @@ fn is_voter( /// Returns the authority id of this node, if available. fn authority_id<'a, I>( authorities: &mut I, - keystore: &Option, + keystore: Option<&BareCryptoStorePtr>, ) -> Option where I: Iterator, { match keystore { Some(keystore) => { authorities - .find_map(|p| { - keystore.read().key_pair::(&p) - .ok() - .map(|ap| ap.public()) - }) - } + .find(|p| keystore.read().has_keys(&[(p.to_raw_vec(), AuthorityId::ID)])) + .cloned() + }, None => None, } } diff --git a/client/finality-grandpa/src/light_import.rs b/client/finality-grandpa/src/light_import.rs index b63c6f0bd7c1d1ad3033ba4ab849ea408369f043..a7c9a655467c799ed500e6186fd64cc9825b1a7e 100644 --- a/client/finality-grandpa/src/light_import.rs +++ b/client/finality-grandpa/src/light_import.rs @@ -573,7 +573,7 @@ pub mod tests { use sp_consensus::{import_queue::CacheKeyId, ForkChoiceStrategy, BlockImport}; use sp_finality_grandpa::AuthorityId; use sp_core::{H256, crypto::Public}; - use sc_client_api::{in_mem::Blockchain as InMemoryAuxStore, StorageProof}; + use sc_client_api::{in_mem::Blockchain as InMemoryAuxStore, StorageProof, BlockBackend}; use substrate_test_runtime_client::runtime::{Block, Header}; use crate::tests::TestApi; use crate::finality_proof::{ diff --git a/client/finality-grandpa/src/observer.rs b/client/finality-grandpa/src/observer.rs index cab2b894ececeb71e3ff84137b9268b6197df0ee..6a7a1f07b05125cc08e4aa976c36f3e08387b87a 100644 --- a/client/finality-grandpa/src/observer.rs +++ b/client/finality-grandpa/src/observer.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; @@ -25,7 +26,7 @@ use finality_grandpa::{ BlockNumberOps, Error as GrandpaError, voter, voter_set::VoterSet }; use log::{debug, info, warn}; - +use sp_core::traits::BareCryptoStorePtr; use sp_consensus::SelectChain; use sc_client_api::backend::Backend; use sp_utils::mpsc::TracingUnboundedReceiver; @@ -210,7 +211,7 @@ struct ObserverWork> { client: Arc, network: NetworkBridge, persistent_data: PersistentData, - keystore: Option, + keystore: Option, voter_commands_rx: TracingUnboundedReceiver>>, _phantom: PhantomData, } @@ -227,7 +228,7 @@ where client: Arc, network: NetworkBridge, persistent_data: PersistentData, - keystore: Option, + keystore: Option, voter_commands_rx: TracingUnboundedReceiver>>, ) -> Self { @@ -238,7 +239,7 @@ where client, network, persistent_data, - keystore, + keystore: keystore.clone(), voter_commands_rx, _phantom: PhantomData, }; @@ -259,7 +260,7 @@ where &voters, self.client.clone(), &self.network, - &self.keystore, + self.keystore.as_ref(), None, ); diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index dd746edb429d4ecb92a9f820d58ce8decbca2767..50f9e8eba2357bdf85f08face7c43cc46ba81ea5 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Tests and test helpers for GRANDPA. use super::*; @@ -105,7 +106,7 @@ impl TestNetFactory for GrandpaTestNet { _cfg: &ProtocolConfig, _: &PeerData, ) -> Self::Verifier { - PassThroughVerifier(false) // use non-instant finality. + PassThroughVerifier::new(false) // use non-instant finality. } fn make_block_import(&self, client: PeersClient) @@ -281,7 +282,7 @@ fn make_ids(keys: &[Ed25519Keyring]) -> AuthorityList { keys.iter().map(|key| key.clone().public().into()).map(|id| (id, 1)).collect() } -fn create_keystore(authority: Ed25519Keyring) -> (KeyStorePtr, tempfile::TempDir) { +fn create_keystore(authority: Ed25519Keyring) -> (BareCryptoStorePtr, tempfile::TempDir) { let keystore_path = tempfile::tempdir().expect("Creates keystore path"); let keystore = sc_keystore::Store::open(keystore_path.path(), None).expect("Creates keystore"); keystore.write().insert_ephemeral_from_seed::(&authority.to_seed()) @@ -1049,7 +1050,7 @@ fn voter_persists_its_votes() { voter_rx: TracingUnboundedReceiver<()>, net: Arc>, client: PeersClient, - keystore: KeyStorePtr, + keystore: BareCryptoStorePtr, } impl Future for ResettableVoter { @@ -1135,7 +1136,7 @@ fn voter_persists_its_votes() { let config = Config { gossip_duration: TEST_GOSSIP_DURATION, justification_period: 32, - keystore: Some(keystore), + keystore: Some(keystore.clone()), name: Some(format!("peer#{}", 1)), is_authority: true, observer_enabled: true, @@ -1159,10 +1160,10 @@ fn voter_persists_its_votes() { ); let (round_rx, round_tx) = network.round_communication( + Some((peers[1].public().into(), keystore).into()), communication::Round(1), communication::SetId(0), Arc::new(VoterSet::new(voters).unwrap()), - Some(peers[1].pair().into()), HasVoted::No, ); diff --git a/client/finality-grandpa/src/until_imported.rs b/client/finality-grandpa/src/until_imported.rs index ab182004a9766240589c0d266ba210727c8c56b7..3ac94f3b062f0273f975e714c7318b9539243dae 100644 --- a/client/finality-grandpa/src/until_imported.rs +++ b/client/finality-grandpa/src/until_imported.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Helper stream for waiting until one or more blocks are imported before //! passing through inner items. This is done in a generic way to support //! many different kinds of items. @@ -584,7 +585,7 @@ mod tests { origin: BlockOrigin::File, header, is_new_best: false, - retracted: vec![], + tree_route: None, }).unwrap(); } } diff --git a/client/finality-grandpa/src/voting_rule.rs b/client/finality-grandpa/src/voting_rule.rs index 9e234328a30e99448e26dc58b01c044f9094f6ac..60493867ce1f4c4e9c50e662857ef1ffcd842cba 100644 --- a/client/finality-grandpa/src/voting_rule.rs +++ b/client/finality-grandpa/src/voting_rule.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Handling custom voting rules for GRANDPA. //! //! This exposes the `VotingRule` trait used to implement arbitrary voting diff --git a/client/informant/Cargo.toml b/client/informant/Cargo.toml index 695f480915d6e78d99cce34eaa9e99378618b1d1..d2df78537d8d2320ab741edb8890b975e845025f 100644 --- a/client/informant/Cargo.toml +++ b/client/informant/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-informant" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "Substrate informant." edition = "2018" @@ -17,8 +17,10 @@ futures = "0.3.4" log = "0.4.8" parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } wasm-timer = "0.2" -sc-client-api = { version = "2.0.0-alpha.8", path = "../api" } -sc-network = { version = "0.8.0-alpha.8", path = "../network" } -sc-service = { version = "0.8.0-alpha.8", default-features = false, path = "../service" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../primitives/blockchain" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } +sc-client-api = { version = "2.0.0-rc4", path = "../api" } +sc-network = { version = "0.8.0-rc4", path = "../network" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-rc2", path = "../../primitives/utils" } +sp-transaction-pool = { version = "2.0.0-rc2", path = "../../primitives/transaction-pool" } +parking_lot = "0.10.2" diff --git a/client/informant/src/display.rs b/client/informant/src/display.rs index 42f498998362e6eb1a6d02f4af1fc98ce285030f..4491eb61d69cafcbdc7813e680c024c844db03a5 100644 --- a/client/informant/src/display.rs +++ b/client/informant/src/display.rs @@ -14,15 +14,17 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +use crate::OutputFormat; use ansi_term::Colour; -use sc_client_api::ClientInfo; use log::info; -use sc_network::SyncState; -use sp_runtime::traits::{Block as BlockT, CheckedDiv, NumberFor, Zero, Saturating}; -use sc_service::NetworkStatus; -use std::{convert::{TryFrom, TryInto}, fmt}; +use sc_client_api::ClientInfo; +use sc_network::{NetworkStatus, SyncState}; +use sp_runtime::traits::{Block as BlockT, CheckedDiv, NumberFor, Saturating, Zero}; +use std::{ + convert::{TryFrom, TryInto}, + fmt, +}; use wasm_timer::Instant; -use crate::OutputFormat; /// State of the informant display system. /// @@ -67,16 +69,22 @@ impl InformantDisplay { self.last_update = Instant::now(); self.last_number = Some(best_number); - let (status, target) = match (net_status.sync_state, net_status.best_seen_block) { - (SyncState::Idle, _) => ("💤 Idle".into(), "".into()), - (SyncState::Downloading, None) => (format!("⚙️ Preparing{}", speed), "".into()), - (SyncState::Downloading, Some(n)) => (format!("⚙️ Syncing{}", speed), format!(", target=#{}", n)), + let (level, status, target) = match (net_status.sync_state, net_status.best_seen_block) { + (SyncState::Idle, _) => ("💤", "Idle".into(), "".into()), + (SyncState::Downloading, None) => ("⚙️ ", format!("Preparing{}", speed), "".into()), + (SyncState::Downloading, Some(n)) => ( + "⚙️ ", + format!("Syncing{}", speed), + format!(", target=#{}", n), + ), }; - if self.format == OutputFormat::Coloured { + if self.format.enable_color { info!( target: "substrate", - "{}{} ({} peers), best: #{} ({}), finalized #{} ({}), {} {}", + "{} {}{}{} ({} peers), best: #{} ({}), finalized #{} ({}), {} {}", + level, + self.format.prefix, Colour::White.bold().paint(&status), target, Colour::White.bold().paint(format!("{}", num_connected_peers)), @@ -86,11 +94,13 @@ impl InformantDisplay { info.chain.finalized_hash, Colour::Green.paint(format!("⬇ {}", TransferRateFormat(net_status.average_download_per_sec))), Colour::Red.paint(format!("⬆ {}", TransferRateFormat(net_status.average_upload_per_sec))), - ); + ) } else { info!( target: "substrate", - "{}{} ({} peers), best: #{} ({}), finalized #{} ({}), ⬇ {} ⬆ {}", + "{} {}{}{} ({} peers), best: #{} ({}), finalized #{} ({}), ⬇ {} ⬆ {}", + level, + self.format.prefix, status, target, num_connected_peers, @@ -100,7 +110,7 @@ impl InformantDisplay { info.chain.finalized_hash, TransferRateFormat(net_status.average_download_per_sec), TransferRateFormat(net_status.average_upload_per_sec), - ); + ) } } } diff --git a/client/informant/src/lib.rs b/client/informant/src/lib.rs index 3f4724824d4ebc3cad4ffbd024323fa428b900bc..d56afcf335917e7debf35d23e3ab2966de0c91f8 100644 --- a/client/informant/src/lib.rs +++ b/client/informant/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,36 +15,90 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Console informant. Prints sync progress and block events. Runs on the calling thread. use ansi_term::Colour; -use sc_client_api::{BlockchainEvents, UsageProvider}; use futures::prelude::*; -use log::{info, warn, trace}; -use sp_runtime::traits::Header; -use sc_service::AbstractService; -use std::time::Duration; +use log::{info, trace, warn}; +use parity_util_mem::MallocSizeOf; +use sc_client_api::{BlockchainEvents, UsageProvider}; +use sc_network::{network_state::NetworkState, NetworkStatus}; +use sp_blockchain::HeaderMetadata; +use sp_runtime::traits::{Block as BlockT, Header}; +use sp_transaction_pool::TransactionPool; +use sp_utils::{status_sinks, mpsc::tracing_unbounded}; +use std::{fmt::Display, sync::Arc, time::Duration, collections::VecDeque}; +use parking_lot::Mutex; mod display; /// The format to print telemetry output in. -#[derive(PartialEq)] -pub enum OutputFormat { - Coloured, - Plain, +#[derive(Clone, Debug)] +pub struct OutputFormat { + /// Enable color output in logs. True by default. + pub enable_color: bool, + /// Defines the informant's prefix for the logs. An empty string by default. + /// + /// By default substrate will show logs without a prefix. Example: + /// + /// ```text + /// 2020-05-28 15:11:06 ✨ Imported #2 (0xc21c…2ca8) + /// 2020-05-28 15:11:07 💤 Idle (0 peers), best: #2 (0xc21c…2ca8), finalized #0 (0x7299…e6df), ⬇ 0 ⬆ 0 + /// ``` + /// + /// But you can define a prefix by setting this string. This will output: + /// + /// ```text + /// 2020-05-28 15:11:06 ✨ [Prefix] Imported #2 (0xc21c…2ca8) + /// 2020-05-28 15:11:07 💤 [Prefix] Idle (0 peers), best: #2 (0xc21c…2ca8), finalized #0 (0x7299…e6df), ⬇ 0 ⬆ 0 + /// ``` + pub prefix: String, } -/// Creates an informant in the form of a `Future` that must be polled regularly. -pub fn build(service: &impl AbstractService, format: OutputFormat) -> impl futures::Future { - let client = service.client(); - let pool = service.transaction_pool(); - - let mut display = display::InformantDisplay::new(format); +impl Default for OutputFormat { + fn default() -> Self { + Self { + enable_color: true, + prefix: String::new(), + } + } +} - let display_notifications = service - .network_status(Duration::from_millis(5000)) +/// Marker trait for a type that implements `TransactionPool` and `MallocSizeOf` on `not(target_os = "unknown")`. +#[cfg(target_os = "unknown")] +pub trait TransactionPoolAndMaybeMallogSizeOf: TransactionPool {} + +/// Marker trait for a type that implements `TransactionPool` and `MallocSizeOf` on `not(target_os = "unknown")`. +#[cfg(not(target_os = "unknown"))] +pub trait TransactionPoolAndMaybeMallogSizeOf: TransactionPool + MallocSizeOf {} + +#[cfg(target_os = "unknown")] +impl TransactionPoolAndMaybeMallogSizeOf for T {} + +#[cfg(not(target_os = "unknown"))] +impl TransactionPoolAndMaybeMallogSizeOf for T {} + +/// Builds the informant and returns a `Future` that drives the informant. +pub fn build( + client: Arc, + network_status_sinks: Arc, NetworkState)>>>, + pool: Arc, + format: OutputFormat, +) -> impl futures::Future +where + C: UsageProvider + HeaderMetadata + BlockchainEvents, + >::Error: Display, +{ + let mut display = display::InformantDisplay::new(format.clone()); + + let client_1 = client.clone(); + let (network_status_sink, network_status_stream) = tracing_unbounded("mpsc_network_status"); + network_status_sinks.lock().push(Duration::from_millis(5000), network_status_sink); + + let display_notifications = network_status_stream .for_each(move |(net_status, _)| { - let info = client.usage_info(); + let info = client_1.usage_info(); if let Some(ref usage) = info.usage { trace!(target: "usage", "Usage statistics: {}", usage); } else { @@ -63,13 +117,30 @@ pub fn build(service: &impl AbstractService, format: OutputFormat) -> impl futur future::ready(()) }); - let client = service.client(); + future::join( + display_notifications, + display_block_import(client, format.prefix), + ).map(|_| ()) +} + +fn display_block_import( + client: Arc, + prefix: String, +) -> impl Future +where + C: UsageProvider + HeaderMetadata + BlockchainEvents, + >::Error: Display, +{ let mut last_best = { let info = client.usage_info(); Some((info.chain.best_number, info.chain.best_hash)) }; - let display_block_import = client.import_notification_stream().for_each(move |n| { + // Hashes of the last blocks we have seen at import. + let mut last_blocks = VecDeque::new(); + let max_blocks_to_track = 100; + + client.import_notification_stream().for_each(move |n| { // detect and log reorganizations. if let Some((ref last_num, ref last_hash)) = last_best { if n.header.parent_hash() != last_hash && n.is_new_best { @@ -81,7 +152,8 @@ pub fn build(service: &impl AbstractService, format: OutputFormat) -> impl futur match maybe_ancestor { Ok(ref ancestor) if ancestor.hash != *last_hash => info!( - "♻️ Reorg on #{},{} to #{},{}, common ancestor #{},{}", + "♻️ {}Reorg on #{},{} to #{},{}, common ancestor #{},{}", + prefix, Colour::Red.bold().paint(format!("{}", last_num)), last_hash, Colour::Green.bold().paint(format!("{}", n.header.number())), n.hash, Colour::White.bold().paint(format!("{}", ancestor.number)), ancestor.hash, @@ -96,12 +168,25 @@ pub fn build(service: &impl AbstractService, format: OutputFormat) -> impl futur last_best = Some((n.header.number().clone(), n.hash.clone())); } - info!(target: "substrate", "✨ Imported #{} ({})", Colour::White.bold().paint(format!("{}", n.header.number())), n.hash); - future::ready(()) - }); - future::join( - display_notifications, - display_block_import - ).map(|_| ()) + // If we already printed a message for a given block recently, + // we should not print it again. + if !last_blocks.contains(&n.hash) { + last_blocks.push_back(n.hash.clone()); + + if last_blocks.len() > max_blocks_to_track { + last_blocks.pop_front(); + } + + info!( + target: "substrate", + "✨ {}Imported #{} ({})", + prefix, + Colour::White.bold().paint(format!("{}", n.header.number())), + n.hash, + ); + } + + future::ready(()) + }) } diff --git a/client/keystore/Cargo.toml b/client/keystore/Cargo.toml index f81251bf6b0080528805b850c50f98085c4ed91e..585d3af5215d12d4ca0fcc5600c7458b1f3eaeca 100644 --- a/client/keystore/Cargo.toml +++ b/client/keystore/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-keystore" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -15,13 +15,14 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] derive_more = "0.99.2" -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-application-crypto = { version = "2.0.0-alpha.8", path = "../../primitives/application-crypto" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-application-crypto = { version = "2.0.0-rc4", path = "../../primitives/application-crypto" } hex = "0.4.0" +merlin = { version = "2.0", default-features = false } +parking_lot = "0.10.0" rand = "0.7.2" serde_json = "1.0.41" subtle = "2.1.1" -parking_lot = "0.10.0" [dev-dependencies] tempfile = "3.1.0" diff --git a/client/keystore/src/lib.rs b/client/keystore/src/lib.rs index 6510bb82327b1ef87a6b1de350382536f0184653..7fec32bae246ace852d042a6f1a9c47c1cb1ee3f 100644 --- a/client/keystore/src/lib.rs +++ b/client/keystore/src/lib.rs @@ -19,8 +19,10 @@ #![warn(missing_docs)] use std::{collections::{HashMap, HashSet}, path::PathBuf, fs::{self, File}, io::{self, Write}, sync::Arc}; use sp_core::{ - crypto::{IsWrappedBy, CryptoTypePublicPair, KeyTypeId, Pair as PairT, Protected, Public}, - traits::{BareCryptoStore, BareCryptoStoreError as TraitError}, + crypto::{IsWrappedBy, CryptoTypePublicPair, KeyTypeId, Pair as PairT, ExposeSecret, SecretString, Public}, + traits::{BareCryptoStore, Error as TraitError}, + sr25519::{Public as Sr25519Public, Pair as Sr25519Pair}, + vrf::{VRFTranscriptData, VRFSignature, make_transcript}, Encode, }; use sp_application_crypto::{AppKey, AppPublic, AppPair, ed25519, sr25519, ecdsa}; @@ -93,14 +95,14 @@ pub struct Store { path: Option, /// Map over `(KeyTypeId, Raw public key)` -> `Key phrase/seed` additional: HashMap<(KeyTypeId, Vec), String>, - password: Option>, + password: Option, } impl Store { /// Open the store at the given path. /// /// Optionally takes a password that will be used to encrypt/decrypt the keys. - pub fn open>(path: T, password: Option>) -> Result { + pub fn open>(path: T, password: Option) -> Result { let path = path.into(); fs::create_dir_all(&path)?; @@ -153,7 +155,7 @@ impl Store { pub fn insert_by_type(&self, key_type: KeyTypeId, suri: &str) -> Result { let pair = Pair::from_string( suri, - self.password.as_ref().map(|p| &***p) + self.password() ).map_err(|_| Error::InvalidSeed)?; self.insert_unknown(key_type, suri, pair.public().as_slice()) .map_err(|_| Error::Unavailable)?; @@ -171,7 +173,7 @@ impl Store { /// /// Places it into the file system store. pub fn generate_by_type(&self, key_type: KeyTypeId) -> Result { - let (pair, phrase, _) = Pair::generate_with_phrase(self.password.as_ref().map(|p| &***p)); + let (pair, phrase, _) = Pair::generate_with_phrase(self.password()); if let Some(path) = self.key_file_path(pair.public().as_slice(), key_type) { let mut file = File::create(path)?; serde_json::to_writer(&file, &phrase)?; @@ -227,7 +229,7 @@ impl Store { let phrase = self.key_phrase_by_type(public.as_slice(), key_type)?; let pair = Pair::from_string( &phrase, - self.password.as_ref().map(|p| &***p), + self.password(), ).map_err(|_| Error::InvalidPhrase)?; if &pair.public() == public { @@ -270,7 +272,7 @@ impl Store { fn raw_public_keys(&self, id: KeyTypeId) -> Result>> { let mut public_keys: Vec> = self.additional.keys() .into_iter() - .filter_map(|k| if k.0 == id { Some(k.1.clone()) } else { None }) + .filter_map(|k| if k.0 == id { Some(k.1.clone()) } else { None }) .collect(); if let Some(path) = &self.path { @@ -363,7 +365,7 @@ impl BareCryptoStore for Store { .map(|k| sr25519::Public::from_slice(k.as_slice())) .collect() }) - .unwrap_or_default() + .unwrap_or_default() } fn sr25519_generate_new( @@ -432,12 +434,31 @@ impl BareCryptoStore for Store { } fn password(&self) -> Option<&str> { - self.password.as_ref().map(|x| x.as_str()) + self.password.as_ref() + .map(|p| p.expose_secret()) + .map(|p| p.as_str()) } fn has_keys(&self, public_keys: &[(Vec, KeyTypeId)]) -> bool { public_keys.iter().all(|(p, t)| self.key_phrase_by_type(&p, *t).is_ok()) } + + fn sr25519_vrf_sign( + &self, + key_type: KeyTypeId, + public: &Sr25519Public, + transcript_data: VRFTranscriptData, + ) -> std::result::Result { + let transcript = make_transcript(transcript_data); + let pair = self.key_pair_by_type::(public, key_type) + .map_err(|e| TraitError::PairNotFound(e.to_string()))?; + + let (inout, proof, _) = pair.as_ref().vrf_sign(transcript); + Ok(VRFSignature { + output: inout.to_output(), + proof, + }) + } } #[cfg(test)] @@ -445,6 +466,7 @@ mod tests { use super::*; use tempfile::TempDir; use sp_core::{testing::SR25519, crypto::Ss58Codec}; + use std::str::FromStr; #[test] fn basic_store() { @@ -485,7 +507,10 @@ mod tests { fn password_being_used() { let password = String::from("password"); let temp_dir = TempDir::new().unwrap(); - let store = Store::open(temp_dir.path(), Some(password.clone().into())).unwrap(); + let store = Store::open( + temp_dir.path(), + Some(FromStr::from_str(password.as_str()).unwrap()), + ).unwrap(); let pair: ed25519::AppPair = store.write().generate().unwrap(); assert_eq!( @@ -497,7 +522,10 @@ mod tests { let store = Store::open(temp_dir.path(), None).unwrap(); assert!(store.read().key_pair::(&pair.public()).is_err()); - let store = Store::open(temp_dir.path(), Some(password.into())).unwrap(); + let store = Store::open( + temp_dir.path(), + Some(FromStr::from_str(password.as_str()).unwrap()), + ).unwrap(); assert_eq!( pair.public(), store.read().key_pair::(&pair.public()).unwrap().public(), diff --git a/client/light/Cargo.toml b/client/light/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..ced9989c9e9092244c1693c86c7d8bf3357e5a47 --- /dev/null +++ b/client/light/Cargo.toml @@ -0,0 +1,27 @@ +[package] +description = "components for a light client" +name = "sc-light" +version = "2.0.0-rc4" +license = "GPL-3.0-or-later WITH Classpath-exception-2.0" +authors = ["Parity Technologies "] +edition = "2018" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +documentation = "https://docs.rs/sc-light" + +[dependencies] +parking_lot = "0.10.0" +lazy_static = "1.4.0" +hash-db = "0.15.2" +sp-runtime = { version = "2.0.0-rc2", path = "../../primitives/runtime" } +sp-externalities = { version = "0.8.0-rc2", path = "../../primitives/externalities" } +sp-blockchain = { version = "2.0.0-rc2", path = "../../primitives/blockchain" } +sp-core = { version = "2.0.0-rc2", path = "../../primitives/core" } +sp-state-machine = { version = "0.8.0-rc2", path = "../../primitives/state-machine" } +sc-client-api = { version = "2.0.0-rc2", path = "../api" } +sp-api = { version = "2.0.0-rc2", path = "../../primitives/api" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +sc-executor = { version = "0.8.0-rc2", path = "../executor" } + +[features] +default = [] diff --git a/client/service/src/client/light/backend.rs b/client/light/src/backend.rs similarity index 95% rename from client/service/src/client/light/backend.rs rename to client/light/src/backend.rs index 7ba60567ab5cd2c417a7a19328fa7ce65b201eb5..2cf994d3f5993c3f2f0681c0db8cee026234bdf3 100644 --- a/client/service/src/client/light/backend.rs +++ b/client/light/src/backend.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -318,12 +318,12 @@ impl BlockImportOperation for ImportOperation storage.insert(None, input.top); // create a list of children keys to re-compute roots for - let child_delta = input.children_default.iter() - .map(|(_storage_key, storage_child)| (storage_child.child_info.clone(), None)) - .collect::>(); + let child_delta = input.children_default + .iter() + .map(|(_storage_key, storage_child)| (&storage_child.child_info, std::iter::empty())); // make sure to persist the child storage - for (_child_key, storage_child) in input.children_default { + for (_child_key, storage_child) in input.children_default.clone() { storage.insert(Some(storage_child.child_info), storage_child.data); } @@ -350,7 +350,11 @@ impl BlockImportOperation for ImportOperation Ok(()) } - fn mark_finalized(&mut self, block: BlockId, _justification: Option) -> ClientResult<()> { + fn mark_finalized( + &mut self, + block: BlockId, + _justification: Option, + ) -> ClientResult<()> { self.finalized_blocks.push(block); Ok(()) } @@ -459,10 +463,10 @@ impl StateBackend for GenesisOrUnavailableState } } - fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) - where - I: IntoIterator, Option>)> - { + fn storage_root<'a>( + &self, + delta: impl Iterator)>, + ) -> (H::Out, Self::Transaction) where H::Out: Ord { match *self { GenesisOrUnavailableState::Genesis(ref state) => state.storage_root(delta), @@ -470,14 +474,11 @@ impl StateBackend for GenesisOrUnavailableState } } - fn child_storage_root( + fn child_storage_root<'a>( &self, child_info: &ChildInfo, - delta: I, - ) -> (H::Out, bool, Self::Transaction) - where - I: IntoIterator, Option>)> - { + delta: impl Iterator)>, + ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { match *self { GenesisOrUnavailableState::Genesis(ref state) => { let (root, is_equal, _) = state.child_storage_root(child_info, delta); diff --git a/client/service/src/client/light/blockchain.rs b/client/light/src/blockchain.rs similarity index 97% rename from client/service/src/client/light/blockchain.rs rename to client/light/src/blockchain.rs index 02212a02636c47140443b79882c0751d4f5d1d2e..9d557db887d2990549106b5436d0e043ca06a66b 100644 --- a/client/service/src/client/light/blockchain.rs +++ b/client/light/src/blockchain.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Light client blockchain backend. Only stores headers and justifications of recent //! blocks. CHT roots are stored for headers of ancient blocks. @@ -24,8 +25,7 @@ use sp_runtime::{Justification, generic::BlockId}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero}; use sp_blockchain::{ - HeaderMetadata, CachedHeaderMetadata, - Error as ClientError, Result as ClientResult, + HeaderMetadata, CachedHeaderMetadata, Error as ClientError, Result as ClientResult, }; pub use sc_client_api::{ backend::{ @@ -41,7 +41,7 @@ pub use sc_client_api::{ }, cht, }; -use super::fetcher::RemoteHeaderRequest; +use crate::fetcher::RemoteHeaderRequest; /// Light client blockchain. pub struct Blockchain { diff --git a/client/service/src/client/light/call_executor.rs b/client/light/src/call_executor.rs similarity index 99% rename from client/service/src/client/light/call_executor.rs rename to client/light/src/call_executor.rs index 76551f2bfd8e5b78b3dbffec919047fa9de9c0ad..81be65339b696cbf3f4b39cf33425a53a8a78bf9 100644 --- a/client/service/src/client/light/call_executor.rs +++ b/client/light/src/call_executor.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Methods that light client could use to execute runtime calls. use std::{ diff --git a/client/service/src/client/light/fetcher.rs b/client/light/src/fetcher.rs similarity index 99% rename from client/service/src/client/light/fetcher.rs rename to client/light/src/fetcher.rs index e3b82bd2b2ce86c88dd118f5597375c49bb35f80..88d20cafc903f38d559926872dc50ab9305515fc 100644 --- a/client/service/src/client/light/fetcher.rs +++ b/client/light/src/fetcher.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Light client data fetcher. Fetches requested data from remote full nodes. use std::sync::Arc; @@ -45,8 +46,8 @@ pub use sc_client_api::{ }, cht, }; -use super::blockchain::{Blockchain}; -use super::call_executor::check_execution_proof; +use crate::blockchain::Blockchain; +use crate::call_executor::check_execution_proof; /// Remote data checker. pub struct LightDataChecker> { diff --git a/client/light/src/lib.rs b/client/light/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..deea642bd39d0720d23c4ede0cd5061fff08b3b5 --- /dev/null +++ b/client/light/src/lib.rs @@ -0,0 +1,57 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Light client components. + +use sp_runtime::traits::{Block as BlockT, HashFor}; +use sc_client_api::CloneableSpawn; +use std::sync::Arc; +use sp_core::traits::CodeExecutor; + +pub mod backend; +pub mod blockchain; +pub mod call_executor; +pub mod fetcher; + +pub use {backend::*, blockchain::*, call_executor::*, fetcher::*}; + +/// Create an instance of fetch data checker. +pub fn new_fetch_checker>( + blockchain: Arc>, + executor: E, + spawn_handle: Box, +) -> LightDataChecker, B, S> + where + E: CodeExecutor, +{ + LightDataChecker::new(blockchain, executor, spawn_handle) +} + +/// Create an instance of light client blockchain backend. +pub fn new_light_blockchain>(storage: S) -> Arc> { + Arc::new(Blockchain::new(storage)) +} + +/// Create an instance of light client backend. +pub fn new_light_backend(blockchain: Arc>) -> Arc>> + where + B: BlockT, + S: BlockchainStorage, +{ + Arc::new(Backend::new(blockchain)) +} diff --git a/client/network-gossip/Cargo.toml b/client/network-gossip/Cargo.toml index 78b80f1033687db3c9b0c500086a3a0082ff6e0e..aba5b49563f65f62e19ac700481b1555a2f16a58 100644 --- a/client/network-gossip/Cargo.toml +++ b/client/network-gossip/Cargo.toml @@ -1,7 +1,7 @@ [package] description = "Gossiping for the Substrate network protocol" name = "sc-network-gossip" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" authors = ["Parity Technologies "] edition = "2018" @@ -16,15 +16,15 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] futures = "0.3.4" futures-timer = "3.0.1" -libp2p = { version = "0.18.1", default-features = false, features = ["websocket"] } +libp2p = { version = "0.20.1", default-features = false } log = "0.4.8" lru = "0.4.3" -sc-network = { version = "0.8.0-alpha.8", path = "../network" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } +sc-network = { version = "0.8.0-rc4", path = "../network" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } wasm-timer = "0.2" [dev-dependencies] async-std = "1.5" quickcheck = "0.9.0" rand = "0.7.2" -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../test-utils/runtime/client" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../test-utils/runtime/client" } diff --git a/client/network-gossip/src/state_machine.rs b/client/network-gossip/src/state_machine.rs index 693e14f8137c0f8c1f247e97066d245785808acd..da07bde3e7d1ad97f10f65a8c1204e2fc291c9c5 100644 --- a/client/network-gossip/src/state_machine.rs +++ b/client/network-gossip/src/state_machine.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::{Network, MessageIntent, Validator, ValidatorContext, ValidationResult}; use std::collections::{HashMap, HashSet}; diff --git a/client/network-gossip/src/validator.rs b/client/network-gossip/src/validator.rs index 2fb9282c48a3e9c0a8417d0721622374e439e925..fd29aaddafe6dcd89f2e722bdcfa07854fc04225 100644 --- a/client/network-gossip/src/validator.rs +++ b/client/network-gossip/src/validator.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use sc_network::{ObservedRole, PeerId}; use sp_runtime::traits::Block as BlockT; diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index e2bdea235d6ca1331ad77425a83291d146c85e22..495895c7401d0d039217922645fe9a4639e30b33 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -1,7 +1,7 @@ [package] description = "Substrate network protocol" name = "sc-network" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" authors = ["Parity Technologies "] edition = "2018" @@ -19,13 +19,14 @@ prost-build = "0.6.1" [dependencies] bitflags = "1.2.0" +bs58 = "0.3.1" bytes = "0.5.0" -codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", features = ["derive"] } derive_more = "0.99.2" either = "1.5.3" erased-serde = "0.3.9" fnv = "1.0.6" -fork-tree = { version = "2.0.0-alpha.8", path = "../../utils/fork-tree" } +fork-tree = { version = "2.0.0-rc4", path = "../../utils/fork-tree" } futures = "0.3.4" futures-timer = "3.0.1" futures_codec = "0.3.3" @@ -38,23 +39,23 @@ lru = "0.4.0" nohash-hasher = "0.2.0" parking_lot = "0.10.0" pin-project = "0.4.6" -prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.8.0-alpha.8", path = "../../utils/prometheus" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.8.0-rc4", path = "../../utils/prometheus" } prost = "0.6.1" rand = "0.7.2" -sc-block-builder = { version = "0.8.0-alpha.8", path = "../block-builder" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../api" } -sc-peerset = { version = "2.0.0-alpha.8", path = "../peerset" } +sc-block-builder = { version = "0.8.0-rc4", path = "../block-builder" } +sc-client-api = { version = "2.0.0-rc4", path = "../api" } +sc-peerset = { version = "2.0.0-rc4", path = "../peerset" } serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" slog = { version = "2.5.2", features = ["nested-values"] } slog_derive = "0.2.0" smallvec = "0.6.10" -sp-arithmetic = { version = "2.0.0-alpha.8", path = "../../primitives/arithmetic" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../primitives/blockchain" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../primitives/consensus/common" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } -sp-utils = { version = "2.0.0-alpha.8", path = "../../primitives/utils" } +sp-arithmetic = { version = "2.0.0-rc4", path = "../../primitives/arithmetic" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../primitives/blockchain" } +sp-consensus = { version = "0.8.0-rc4", path = "../../primitives/consensus/common" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-rc4", path = "../../primitives/utils" } thiserror = "1" unsigned-varint = { version = "0.3.1", features = ["futures", "futures-codec"] } void = "1.0.2" @@ -62,21 +63,21 @@ wasm-timer = "0.2" zeroize = "1.0.0" [dependencies.libp2p] -version = "0.18.1" +version = "0.20.1" default-features = false -features = ["websocket", "kad", "mdns", "ping", "identify", "mplex", "yamux", "noise"] +features = ["identify", "kad", "mdns", "mplex", "noise", "ping", "tcp-async-std", "websocket", "yamux"] [dev-dependencies] async-std = "1.5" assert_matches = "1.3" env_logger = "0.7.0" -libp2p = { version = "0.18.1", default-features = false, features = ["secio"] } +libp2p = { version = "0.20.1", default-features = false, features = ["secio"] } quickcheck = "0.9.0" rand = "0.7.2" -sp-keyring = { version = "2.0.0-alpha.8", path = "../../primitives/keyring" } -sp-test-primitives = { version = "2.0.0-alpha.8", path = "../../primitives/test-primitives" } -substrate-test-runtime = { version = "2.0.0-alpha.8", path = "../../test-utils/runtime" } -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../test-utils/runtime/client" } +sp-keyring = { version = "2.0.0-rc4", path = "../../primitives/keyring" } +sp-test-primitives = { version = "2.0.0-rc4", path = "../../primitives/test-primitives" } +substrate-test-runtime = { version = "2.0.0-rc4", path = "../../test-utils/runtime" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../test-utils/runtime/client" } tempfile = "3.1.0" [features] diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 2394d22319e256192062f658cc624a749a92ae83..dec8788f3f4dc570413fcbf429edfb703fd5230b 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -311,13 +311,13 @@ impl NetworkBehaviourEventProcess { + block_requests::Event::Response { peer, original_request: _, response, request_duration } => { self.events.push_back(BehaviourOut::RequestFinished { peer: peer.clone(), protocol: self.block_requests.protocol_name().to_vec(), request_duration, }); - let ev = self.substrate.on_block_response(peer, original_request, response); + let ev = self.substrate.on_block_response(peer, response); self.inject_event(ev); } block_requests::Event::RequestCancelled { peer, request_duration, .. } | diff --git a/client/network/src/block_requests.rs b/client/network/src/block_requests.rs index ae5a3a0b4e8944479cded8ce58c33aa9fc5a9201..6d698a73001263394cc781edcf86c78150a6e58f 100644 --- a/client/network/src/block_requests.rs +++ b/client/network/src/block_requests.rs @@ -277,21 +277,13 @@ where return SendRequestOutcome::NotConnected; }; - let protobuf_rq = schema::v1::BlockRequest { - fields: u32::from_be_bytes([req.fields.bits(), 0, 0, 0]), - from_block: match req.from { - message::FromBlock::Hash(h) => - Some(schema::v1::block_request::FromBlock::Hash(h.encode())), - message::FromBlock::Number(n) => - Some(schema::v1::block_request::FromBlock::Number(n.encode())), - }, - to_block: req.to.map(|h| h.encode()).unwrap_or_default(), - direction: match req.direction { - message::Direction::Ascending => schema::v1::Direction::Ascending as i32, - message::Direction::Descending => schema::v1::Direction::Descending as i32, - }, - max_blocks: req.max.unwrap_or(0), - }; + let protobuf_rq = build_protobuf_block_request( + req.fields, + req.from.clone(), + req.to.clone(), + req.direction, + req.max, + ); let mut buf = Vec::with_capacity(protobuf_rq.encoded_len()); if let Err(err) = protobuf_rq.encode(&mut buf) { @@ -386,7 +378,7 @@ where return Err(io::Error::new(io::ErrorKind::Other, msg).into()) }; - let attributes = BlockAttributes::decode(&mut request.fields.to_be_bytes().as_ref())?; + let attributes = BlockAttributes::from_be_u32(request.fields)?; let get_header = attributes.contains(BlockAttributes::HEADER); let get_body = attributes.contains(BlockAttributes::BODY); let get_justification = attributes.contains(BlockAttributes::JUSTIFICATION); @@ -826,3 +818,28 @@ where }.boxed() } } + +/// Build protobuf block request message. +pub(crate) fn build_protobuf_block_request( + attributes: BlockAttributes, + from_block: message::FromBlock, + to_block: Option, + direction: message::Direction, + max_blocks: Option, +) -> schema::v1::BlockRequest { + schema::v1::BlockRequest { + fields: attributes.to_be_u32(), + from_block: match from_block { + message::FromBlock::Hash(h) => + Some(schema::v1::block_request::FromBlock::Hash(h.encode())), + message::FromBlock::Number(n) => + Some(schema::v1::block_request::FromBlock::Number(n.encode())), + }, + to_block: to_block.map(|h| h.encode()).unwrap_or_default(), + direction: match direction { + message::Direction::Ascending => schema::v1::Direction::Ascending as i32, + message::Direction::Descending => schema::v1::Direction::Descending as i32, + }, + max_blocks: max_blocks.unwrap_or(0), + } +} diff --git a/client/network/src/chain.rs b/client/network/src/chain.rs index e3deda0dc92dd02f5c4701959b4f94aabaea4a6b..20fbe0284397d513a909cbcf72733a265c060214 100644 --- a/client/network/src/chain.rs +++ b/client/network/src/chain.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Blockchain access trait use sp_blockchain::{Error, HeaderBackend, HeaderMetadata}; diff --git a/client/network/src/config.rs b/client/network/src/config.rs index f03b2169e9b4b0bd540265765eeb367d27c7202c..94b2993b4e6bd255e332de67d0219ecf90ff8a0a 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Configuration of the networking layer. //! //! The [`Params`] struct is the struct that must be passed in order to initialize the networking. @@ -271,8 +272,21 @@ pub fn parse_str_addr(addr_str: &str) -> Result<(PeerId, Multiaddr), ParseErr> { /// Splits a Multiaddress into a Multiaddress and PeerId. pub fn parse_addr(mut addr: Multiaddr)-> Result<(PeerId, Multiaddr), ParseErr> { let who = match addr.pop() { - Some(multiaddr::Protocol::P2p(key)) => PeerId::from_multihash(key) - .map_err(|_| ParseErr::InvalidPeerId)?, + Some(multiaddr::Protocol::P2p(key)) => { + if !matches!(key.algorithm(), multiaddr::multihash::Code::Identity) { + // (note: this is the "person bowing" emoji) + log::warn!( + "🙇 You are using the peer ID {}. This peer ID uses a legacy, deprecated \ + representation that will no longer be supported in the future. \ + Please refresh it by performing a RPC query to the appropriate node, \ + by looking at its logs, or by using `subkey inspect-node-key` on its \ + private key.", + bs58::encode(key.as_bytes()).into_string() + ); + } + + PeerId::from_multihash(key).map_err(|_| ParseErr::InvalidPeerId)? + }, _ => return Err(ParseErr::PeerIdMissing), }; @@ -411,9 +425,6 @@ pub struct NetworkConfiguration { pub max_parallel_downloads: u32, /// Should we insert non-global addresses into the DHT? pub allow_non_globals_in_dht: bool, - /// If true, uses the `//block-requests/` experimental protocol rather than - /// the legacy substream. This option is meant to be hard-wired to `true` in the future. - pub use_new_block_requests_protocol: bool, } impl NetworkConfiguration { @@ -445,7 +456,6 @@ impl NetworkConfiguration { }, max_parallel_downloads: 5, allow_non_globals_in_dht: false, - use_new_block_requests_protocol: true, } } } diff --git a/client/network/src/discovery.rs b/client/network/src/discovery.rs index 00adc56ec591de3c0038626f724c391ac3b7a400..c48722c0f796862a8cda9c3d93bd94d8fdfb4fbf 100644 --- a/client/network/src/discovery.rs +++ b/client/network/src/discovery.rs @@ -52,7 +52,7 @@ use ip_network::IpNetwork; use libp2p::core::{connection::{ConnectionId, ListenerId}, ConnectedPoint, Multiaddr, PeerId, PublicKey}; use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters, ProtocolsHandler}; use libp2p::swarm::protocols_handler::multi::MultiHandler; -use libp2p::kad::{Kademlia, KademliaConfig, KademliaEvent, Quorum, Record}; +use libp2p::kad::{Kademlia, KademliaConfig, KademliaEvent, QueryResult, Quorum, Record}; use libp2p::kad::GetClosestPeersError; use libp2p::kad::handler::KademliaHandler; use libp2p::kad::QueryId; @@ -62,7 +62,7 @@ use libp2p::swarm::toggle::Toggle; #[cfg(not(target_os = "unknown"))] use libp2p::mdns::{Mdns, MdnsEvent}; use libp2p::multiaddr::Protocol; -use log::{debug, info, trace, warn, error}; +use log::{debug, info, trace, warn}; use std::{cmp, collections::{HashMap, HashSet, VecDeque}, io, time::Duration}; use std::task::{Context, Poll}; use sp_core::hexdisplay::HexDisplay; @@ -177,7 +177,7 @@ impl DiscoveryConfig { kademlias: self.kademlias, next_kad_random_query: Delay::new(Duration::new(0, 0)), duration_to_next_kad: Duration::from_secs(1), - discoveries: VecDeque::new(), + pending_events: VecDeque::new(), local_peer_id: self.local_peer_id, num_connections: 0, allow_private_ipv4: self.allow_private_ipv4, @@ -213,8 +213,8 @@ pub struct DiscoveryBehaviour { next_kad_random_query: Delay, /// After `next_kad_random_query` triggers, the next one triggers after this duration. duration_to_next_kad: Duration, - /// Discovered nodes to return. - discoveries: VecDeque, + /// Events to return in priority when polled. + pending_events: VecDeque, /// Identity of our local node. local_peer_id: PeerId, /// Number of nodes we're currently connected to. @@ -248,7 +248,7 @@ impl DiscoveryBehaviour { for k in self.kademlias.values_mut() { k.add_address(&peer_id, addr.clone()) } - self.discoveries.push_back(peer_id.clone()); + self.pending_events.push_back(DiscoveryOut::Discovered(peer_id.clone())); self.user_defined.push((peer_id, addr)); } } @@ -272,7 +272,7 @@ impl DiscoveryBehaviour { /// A corresponding `ValueFound` or `ValueNotFound` event will later be generated. pub fn get_value(&mut self, key: &record::Key) { for k in self.kademlias.values_mut() { - k.get_record(key, Quorum::One) + k.get_record(key, Quorum::One); } } @@ -282,7 +282,10 @@ impl DiscoveryBehaviour { /// A corresponding `ValuePut` or `ValuePutFailed` event will later be generated. pub fn put_value(&mut self, key: record::Key, value: Vec) { for k in self.kademlias.values_mut() { - k.put_record(Record::new(key.clone(), value.clone()), Quorum::All) + if let Err(e) = k.put_record(Record::new(key.clone(), value.clone()), Quorum::All) { + warn!(target: "sub-libp2p", "Libp2p => Failed to put record: {:?}", e); + self.pending_events.push_back(DiscoveryOut::ValuePutFailed(key.clone())); + } } } @@ -321,7 +324,8 @@ impl DiscoveryBehaviour { let ip = match addr.iter().next() { Some(Protocol::Ip4(ip)) => IpNetwork::from(ip), Some(Protocol::Ip6(ip)) => IpNetwork::from(ip), - Some(Protocol::Dns4(_)) | Some(Protocol::Dns6(_)) => return true, + Some(Protocol::Dns(_)) | Some(Protocol::Dns4(_)) | Some(Protocol::Dns6(_)) + => return true, _ => return false }; ip.is_global() @@ -485,7 +489,6 @@ impl NetworkBehaviour for DiscoveryBehaviour { } fn inject_expired_listen_addr(&mut self, addr: &Multiaddr) { - info!(target: "sub-libp2p", "No longer listening on {}", addr); for k in self.kademlias.values_mut() { NetworkBehaviour::inject_expired_listen_addr(k, addr) } @@ -504,14 +507,12 @@ impl NetworkBehaviour for DiscoveryBehaviour { } fn inject_listener_error(&mut self, id: ListenerId, err: &(dyn std::error::Error + 'static)) { - error!(target: "sub-libp2p", "Error on libp2p listener {:?}: {}", id, err); for k in self.kademlias.values_mut() { NetworkBehaviour::inject_listener_error(k, id, err) } } fn inject_listener_closed(&mut self, id: ListenerId, reason: Result<(), &io::Error>) { - error!(target: "sub-libp2p", "Libp2p listener {:?} closed", id); for k in self.kademlias.values_mut() { NetworkBehaviour::inject_listener_closed(k, id, reason) } @@ -528,8 +529,7 @@ impl NetworkBehaviour for DiscoveryBehaviour { >, > { // Immediately process the content of `discovered`. - if let Some(peer_id) = self.discoveries.pop_front() { - let ev = DiscoveryOut::Discovered(peer_id); + if let Some(ev) = self.pending_events.pop_front() { return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); } @@ -541,7 +541,7 @@ impl NetworkBehaviour for DiscoveryBehaviour { "Libp2p <= Starting random Kademlia request for {:?}", random_peer_id); for k in self.kademlias.values_mut() { - k.get_closest_peers(random_peer_id.clone()) + k.get_closest_peers(random_peer_id.clone()); } true } else { @@ -578,7 +578,7 @@ impl NetworkBehaviour for DiscoveryBehaviour { let ev = DiscoveryOut::Discovered(peer); return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); } - KademliaEvent::GetClosestPeersResult(res) => { + KademliaEvent::QueryResult { result: QueryResult::GetClosestPeers(res), .. } => { match res { Err(GetClosestPeersError::Timeout { key, peers }) => { debug!(target: "sub-libp2p", @@ -596,12 +596,12 @@ impl NetworkBehaviour for DiscoveryBehaviour { } } } - KademliaEvent::GetRecordResult(res) => { + KademliaEvent::QueryResult { result: QueryResult::GetRecord(res), .. } => { let ev = match res { Ok(ok) => { let results = ok.records .into_iter() - .map(|r| (r.key, r.value)) + .map(|r| (r.record.key, r.record.value)) .collect(); DiscoveryOut::ValueFound(results) @@ -619,7 +619,7 @@ impl NetworkBehaviour for DiscoveryBehaviour { }; return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); } - KademliaEvent::PutRecordResult(res) => { + KademliaEvent::QueryResult { result: QueryResult::PutRecord(res), .. } => { let ev = match res { Ok(ok) => DiscoveryOut::ValuePut(ok.key), Err(e) => { @@ -630,7 +630,7 @@ impl NetworkBehaviour for DiscoveryBehaviour { }; return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); } - KademliaEvent::RepublishRecordResult(res) => { + KademliaEvent::QueryResult { result: QueryResult::RepublishRecord(res), .. } => { match res { Ok(ok) => debug!(target: "sub-libp2p", "Libp2p => Record republished: {:?}", @@ -675,9 +675,8 @@ impl NetworkBehaviour for DiscoveryBehaviour { continue; } - self.discoveries.extend(list.map(|(peer_id, _)| peer_id)); - if let Some(peer_id) = self.discoveries.pop_front() { - let ev = DiscoveryOut::Discovered(peer_id); + self.pending_events.extend(list.map(|(peer_id, _)| DiscoveryOut::Discovered(peer_id))); + if let Some(ev) = self.pending_events.pop_front() { return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); } }, diff --git a/client/network/src/error.rs b/client/network/src/error.rs index 270c0e7b8d0bb61e8a29ec32e32073459ae72c03..b87e495983eaf9c5d229c203599d1fccb3b8ac70 100644 --- a/client/network/src/error.rs +++ b/client/network/src/error.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,8 +15,10 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate network possible errors. +use crate::config::TransportConfig; use libp2p::{PeerId, Multiaddr}; use std::fmt; @@ -47,7 +49,18 @@ pub enum Error { second_id: PeerId, }, /// Prometheus metrics error. - Prometheus(prometheus_endpoint::PrometheusError) + Prometheus(prometheus_endpoint::PrometheusError), + /// The network addresses are invalid because they don't match the transport. + #[display( + fmt = "The following addresses are invalid because they don't match the transport: {:?}", + addresses, + )] + AddressesForAnotherTransport { + /// Transport used. + transport: TransportConfig, + /// The invalid addresses. + addresses: Vec, + }, } // Make `Debug` use the `Display` implementation. @@ -64,6 +77,7 @@ impl std::error::Error for Error { Error::Client(ref err) => Some(err), Error::DuplicateBootnode { .. } => None, Error::Prometheus(ref err) => Some(err), + Error::AddressesForAnotherTransport { .. } => None, } } } diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index 8174666ab7052b7e87db5f2af4273a800d44f81a..6106616d99d1018727ca30c5742fcc736d041064 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -77,7 +77,7 @@ //! - WebSockets for addresses of the form `/ip4/1.2.3.4/tcp/5/ws`. A TCP/IP connection is open and //! the WebSockets protocol is negotiated on top. Communications then happen inside WebSockets data //! frames. Encryption and multiplexing are additionally negotiated again inside this channel. -//! - DNS for addresses of the form `/dns4/example.com/tcp/5` or `/dns4/example.com/tcp/5/ws`. A +//! - DNS for addresses of the form `/dns/example.com/tcp/5` or `/dns/example.com/tcp/5/ws`. A //! node's address can contain a domain name. //! - (All of the above using IPv6 instead of IPv4.) //! @@ -269,6 +269,7 @@ pub use libp2p::{Multiaddr, PeerId}; pub use libp2p::multiaddr; pub use sc_peerset::ReputationChange; +use sp_runtime::traits::{Block as BlockT, NumberFor}; /// The maximum allowed number of established connections per peer. /// @@ -285,26 +286,6 @@ pub trait ExHashT: std::hash::Hash + Eq + std::fmt::Debug + Clone + Send + Sync impl ExHashT for T where T: std::hash::Hash + Eq + std::fmt::Debug + Clone + Send + Sync + 'static {} -/// A cloneable handle for reporting cost/benefits of peers. -#[derive(Clone)] -pub struct ReportHandle { - inner: sc_peerset::PeersetHandle, // wraps it so we don't have to worry about breaking API. -} - -impl From for ReportHandle { - fn from(peerset_handle: sc_peerset::PeersetHandle) -> Self { - ReportHandle { inner: peerset_handle } - } -} - -impl ReportHandle { - /// Report a given peer as either beneficial (+) or costly (-) according to the - /// given scalar. - pub fn report_peer(&self, who: PeerId, cost_benefit: ReputationChange) { - self.inner.report_peer(who, cost_benefit); - } -} - /// Trait for providing information about the local network state pub trait NetworkStateInfo { /// Returns the local external addresses. @@ -313,3 +294,22 @@ pub trait NetworkStateInfo { /// Returns the local Peer ID. fn local_peer_id(&self) -> PeerId; } + +/// Overview status of the network. +#[derive(Clone)] +pub struct NetworkStatus { + /// Current global sync state. + pub sync_state: SyncState, + /// Target sync block number. + pub best_seen_block: Option>, + /// Number of peers participating in syncing. + pub num_sync_peers: u32, + /// Total number of connected peers + pub num_connected_peers: usize, + /// Total number of active peers. + pub num_active_peers: usize, + /// Downloaded bytes per second averaged over the past few seconds. + pub average_download_per_sec: u64, + /// Uploaded bytes per second averaged over the past few seconds. + pub average_upload_per_sec: u64, +} diff --git a/client/network/src/light_client_handler.rs b/client/network/src/light_client_handler.rs index 236ae817474ab943812a107ce1059191ad8df7d0..ab6bea8761b95f6978faef999cd6fb6d63e3e914 100644 --- a/client/network/src/light_client_handler.rs +++ b/client/network/src/light_client_handler.rs @@ -27,9 +27,10 @@ use bytes::Bytes; use codec::{self, Encode, Decode}; use crate::{ + block_requests::build_protobuf_block_request, chain::Client, config::ProtocolId, - protocol::message::BlockAttributes, + protocol::message::{BlockAttributes, Direction, FromBlock}, schema, }; use futures::{channel::oneshot, future::BoxFuture, prelude::*, stream::FuturesUnordered}; @@ -1062,13 +1063,13 @@ fn retries(request: &Request) -> usize { fn serialize_request(request: &Request) -> Result, prost::EncodeError> { let request = match request { Request::Body { request, .. } => { - let rq = schema::v1::BlockRequest { - fields: u32::from(BlockAttributes::BODY.bits()), - from_block: Some(schema::v1::block_request::FromBlock::Hash(request.header.hash().encode())), - to_block: Vec::new(), - direction: schema::v1::Direction::Ascending as i32, - max_blocks: 1, - }; + let rq = build_protobuf_block_request::<_, NumberFor>( + BlockAttributes::BODY, + FromBlock::Hash(request.header.hash()), + None, + Direction::Ascending, + Some(1), + ); let mut buf = Vec::with_capacity(rq.encoded_len()); rq.encode(&mut buf)?; return Ok(buf); @@ -2036,4 +2037,22 @@ mod tests { assert_eq!(vec![(100, 2)], task::block_on(chan.1).unwrap().unwrap()); // ^--- from `DummyFetchChecker::check_changes_proof` } + + #[test] + fn body_request_fields_encoded_properly() { + let (sender, _) = oneshot::channel(); + let serialized_request = serialize_request::(&Request::Body { + request: RemoteBodyRequest { + header: dummy_header(), + retry_count: None, + }, + sender, + }).unwrap(); + let deserialized_request = schema::v1::BlockRequest::decode(&serialized_request[..]).unwrap(); + assert!( + BlockAttributes::from_be_u32(deserialized_request.fields) + .unwrap() + .contains(BlockAttributes::BODY) + ); + } } diff --git a/client/network/src/network_state.rs b/client/network/src/network_state.rs index d8920951bc5b02aa734561e50ceb1f91c406403b..970a63faed4ea7ac8167bb8b4d987380f39f3861 100644 --- a/client/network/src/network_state.rs +++ b/client/network/src/network_state.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Information about the networking, for diagnostic purposes. //! //! **Warning**: These APIs are not stable. diff --git a/client/network/src/on_demand_layer.rs b/client/network/src/on_demand_layer.rs index 8cf77174076042a87595366bb03314b29631d47e..084172ee57c4f002c66c27e22bbe6dfc42f750fa 100644 --- a/client/network/src/on_demand_layer.rs +++ b/client/network/src/on_demand_layer.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! On-demand requests service. use crate::light_client_handler; diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 7662a476e9a6fc9a45e48c0643cc4fcfaa7f5be8..ff3748bd55cf2ddbfd9ceaac9257c06fc905f7dd 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::{ ExHashT, chain::{Client, FinalityProofProvider}, @@ -47,7 +48,7 @@ use sp_runtime::traits::{ use sp_arithmetic::traits::SaturatedConversion; use message::{BlockAnnounce, Message}; use message::generic::{Message as GenericMessage, ConsensusMessage, Roles}; -use prometheus_endpoint::{Registry, Gauge, GaugeVec, HistogramVec, PrometheusError, Opts, register, U64}; +use prometheus_endpoint::{Registry, Gauge, Counter, GaugeVec, HistogramVec, PrometheusError, Opts, register, U64}; use sync::{ChainSync, SyncState}; use std::borrow::Cow; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; @@ -71,13 +72,15 @@ pub use generic_proto::LegacyConnectionKillError; const REQUEST_TIMEOUT_SEC: u64 = 40; /// Interval at which we perform time based maintenance const TICK_TIMEOUT: time::Duration = time::Duration::from_millis(1100); -/// Interval at which we propagate extrinsics; +/// Interval at which we propagate transactions; const PROPAGATE_TIMEOUT: time::Duration = time::Duration::from_millis(2900); /// Maximim number of known block hashes to keep for a peer. const MAX_KNOWN_BLOCKS: usize = 1024; // ~32kb per peer + LruHashSet overhead -/// Maximim number of known extrinsic hashes to keep for a peer. -const MAX_KNOWN_EXTRINSICS: usize = 4096; // ~128kb per peer + overhead +/// Maximim number of known transaction hashes to keep for a peer. +/// +/// This should be approx. 2 blocks full of transactions for the network to function properly. +const MAX_KNOWN_TRANSACTIONS: usize = 10240; // ~300kb per peer + overhead. /// Maximim number of transaction validation request we keep at any moment. const MAX_PENDING_TRANSACTIONS: usize = 8192; @@ -101,29 +104,27 @@ mod rep { pub const CLOGGED_PEER: Rep = Rep::new(-(1 << 12), "Clogged message queue"); /// Reputation change when a peer doesn't respond in time to our messages. pub const TIMEOUT: Rep = Rep::new(-(1 << 10), "Request timeout"); - /// Reputation change when a peer sends us a status message while we already received one. - pub const UNEXPECTED_STATUS: Rep = Rep::new(-(1 << 20), "Unexpected status message"); /// Reputation change when we are a light client and a peer is behind us. pub const PEER_BEHIND_US_LIGHT: Rep = Rep::new(-(1 << 8), "Useless for a light peer"); - /// Reputation change when a peer sends us any extrinsic. + /// Reputation change when a peer sends us any transaction. /// - /// This forces node to verify it, thus the negative value here. Once extrinsic is verified, - /// reputation change should be refunded with `ANY_EXTRINSIC_REFUND` - pub const ANY_EXTRINSIC: Rep = Rep::new(-(1 << 4), "Any extrinsic"); - /// Reputation change when a peer sends us any extrinsic that is not invalid. - pub const ANY_EXTRINSIC_REFUND: Rep = Rep::new(1 << 4, "Any extrinsic (refund)"); - /// Reputation change when a peer sends us an extrinsic that we didn't know about. - pub const GOOD_EXTRINSIC: Rep = Rep::new(1 << 7, "Good extrinsic"); - /// Reputation change when a peer sends us a bad extrinsic. - pub const BAD_EXTRINSIC: Rep = Rep::new(-(1 << 12), "Bad extrinsic"); + /// This forces node to verify it, thus the negative value here. Once transaction is verified, + /// reputation change should be refunded with `ANY_TRANSACTION_REFUND` + pub const ANY_TRANSACTION: Rep = Rep::new(-(1 << 4), "Any transaction"); + /// Reputation change when a peer sends us any transaction that is not invalid. + pub const ANY_TRANSACTION_REFUND: Rep = Rep::new(1 << 4, "Any transaction (refund)"); + /// Reputation change when a peer sends us an transaction that we didn't know about. + pub const GOOD_TRANSACTION: Rep = Rep::new(1 << 7, "Good transaction"); + /// Reputation change when a peer sends us a bad transaction. + pub const BAD_TRANSACTION: Rep = Rep::new(-(1 << 12), "Bad transaction"); /// We sent an RPC query to the given node, but it failed. pub const RPC_FAILED: Rep = Rep::new(-(1 << 12), "Remote call failed"); /// We received a message that failed to decode. pub const BAD_MESSAGE: Rep = Rep::new(-(1 << 12), "Bad message"); /// We received an unexpected response. pub const UNEXPECTED_RESPONSE: Rep = Rep::new_fatal("Unexpected response packet"); - /// We received an unexpected extrinsic packet. - pub const UNEXPECTED_EXTRINSICS: Rep = Rep::new_fatal("Unexpected extrinsics packet"); + /// We received an unexpected transaction packet. + pub const UNEXPECTED_TRANSACTIONS: Rep = Rep::new_fatal("Unexpected transactions packet"); /// We received an unexpected light node request. pub const UNEXPECTED_REQUEST: Rep = Rep::new_fatal("Unexpected block request packet"); /// Peer has different genesis. @@ -144,6 +145,7 @@ struct Metrics { fork_targets: Gauge, finality_proofs: GaugeVec, justifications: GaugeVec, + propagated_transactions: Counter, } impl Metrics { @@ -189,6 +191,10 @@ impl Metrics { )?; register(g, r)? }, + propagated_transactions: register(Counter::new( + "sync_propagated_transactions", + "Number of transactions propagated to at least one peer", + )?, r)?, }) } } @@ -215,11 +221,11 @@ impl Future for PendingTransaction { pub struct Protocol { /// Interval at which we call `tick`. tick_timeout: Pin + Send>>, - /// Interval at which we call `propagate_extrinsics`. + /// Interval at which we call `propagate_transactions`. propagate_timeout: Pin + Send>>, /// Pending list of messages to return from `poll` as a priority. pending_messages: VecDeque>, - /// Pending extrinsic verification tasks. + /// Pending transactions verification tasks. pending_transactions: FuturesUnordered, config: ProtocolConfig, genesis_hash: B::Hash, @@ -249,9 +255,6 @@ pub struct Protocol { metrics: Option, /// The `PeerId`'s of all boot nodes. boot_node_ids: Arc>, - /// If true, we send back requests as `CustomMessageOutcome` events. If false, we directly - /// dispatch requests using the legacy substream. - use_new_block_requests_protocol: bool, } #[derive(Default)] @@ -277,7 +280,7 @@ struct Peer { /// Requests we are no longer interested in. obsolete_requests: HashMap, /// Holds a set of transactions known to this peer. - known_extrinsics: LruHashSet, + known_transactions: LruHashSet, /// Holds a set of blocks known to this peer. known_blocks: LruHashSet, /// Request counter, @@ -363,6 +366,7 @@ impl Protocol { /// Create a new instance. pub fn new( config: ProtocolConfig, + local_peer_id: PeerId, chain: Arc>, transaction_pool: Arc>, finality_proof_provider: Option>>, @@ -372,7 +376,6 @@ impl Protocol { block_announce_validator: Box + Send>, metrics_registry: Option<&Registry>, boot_node_ids: Arc>, - use_new_block_requests_protocol: bool, queue_size_report: Option, ) -> error::Result<(Protocol, sc_peerset::PeersetHandle)> { let info = chain.info(); @@ -396,7 +399,13 @@ impl Protocol { let (peerset, peerset_handle) = sc_peerset::Peerset::from_config(peerset_config); let versions = &((MIN_VERSION as u8)..=(CURRENT_VERSION as u8)).collect::>(); - let mut behaviour = GenericProto::new(protocol_id.clone(), versions, peerset, queue_size_report); + let mut behaviour = GenericProto::new( + local_peer_id, + protocol_id.clone(), + versions, + peerset, + queue_size_report + ); let mut legacy_equiv_by_name = HashMap::new(); @@ -450,7 +459,6 @@ impl Protocol { None }, boot_node_ids, - use_new_block_requests_protocol, }; Ok((protocol, peerset_handle)) @@ -525,9 +533,9 @@ impl Protocol { self.sync.status().queued_blocks } - /// Number of processed blocks. - pub fn num_processed_blocks(&self) -> usize { - self.sync.num_processed_blocks() + /// Number of downloaded blocks. + pub fn num_downloaded_blocks(&self) -> usize { + self.sync.num_downloaded_blocks() } /// Number of active sync requests. @@ -535,28 +543,15 @@ impl Protocol { self.sync.num_sync_requests() } - /// Accepts a response from the legacy substream and determines what the corresponding - /// request was. - fn handle_response( - &mut self, - who: PeerId, - response: &message::BlockResponse - ) -> Option> { - if let Some(ref mut peer) = self.context_data.peers.get_mut(&who) { - if peer.obsolete_requests.remove(&response.id).is_some() { - trace!(target: "sync", "Ignoring obsolete block response packet from {} ({})", who, response.id); - return None; - } - // Clear the request. If the response is invalid peer will be disconnected anyway. - let request = peer.block_request.take(); - if request.as_ref().map_or(false, |(_, r)| r.id == response.id) { - return request.map(|(_, r)| r) - } - trace!(target: "sync", "Unexpected response packet from {} ({})", who, response.id); - self.peerset_handle.report_peer(who.clone(), rep::UNEXPECTED_RESPONSE); - self.behaviour.disconnect_peer(&who); - } - None + /// Sync local state with the blockchain state. + pub fn update_chain(&mut self) { + let info = self.context_data.chain.info(); + self.sync.update_chain_info(&info.best_hash, info.best_number); + } + + /// Inform sync about an own imported block. + pub fn own_block_imported(&mut self, hash: B::Hash, number: NumberFor) { + self.sync.update_chain_info(&hash, number); } fn update_peer_info(&mut self, who: &PeerId) { @@ -596,11 +591,9 @@ impl Protocol { GenericMessage::Status(s) => return self.on_status_message(who, s), GenericMessage::BlockRequest(r) => self.on_block_request(who, r), GenericMessage::BlockResponse(r) => { - if let Some(request) = self.handle_response(who.clone(), &r) { - let outcome = self.on_block_response(who.clone(), request, r); - self.update_peer_info(&who); - return outcome - } + let outcome = self.on_block_response(who.clone(), r); + self.update_peer_info(&who); + return outcome }, GenericMessage::BlockAnnounce(announce) => { let outcome = self.on_block_announce(who.clone(), announce); @@ -608,7 +601,7 @@ impl Protocol { return outcome; }, GenericMessage::Transactions(m) => - self.on_extrinsics(who, m), + self.on_transactions(who, m), GenericMessage::RemoteCallRequest(request) => self.on_remote_call_request(who, request), GenericMessage::RemoteCallResponse(_) => warn!(target: "sub-libp2p", "Received unexpected RemoteCallResponse"), @@ -667,16 +660,6 @@ impl Protocol { CustomMessageOutcome::None } - fn send_request(&mut self, who: &PeerId, message: Message) { - send_request::( - &mut self.behaviour, - &mut self.context_data.stats, - &mut self.context_data.peers, - who, - message, - ); - } - fn send_message( &mut self, who: &PeerId, @@ -692,6 +675,10 @@ impl Protocol { ); } + fn update_peer_request(&mut self, who: &PeerId, request: &mut message::BlockRequest) { + update_peer_request::(&mut self.context_data.peers, who, request) + } + /// Called when a new peer is connected pub fn on_peer_connected(&mut self, who: PeerId) { trace!(target: "sync", "Connecting {}", who); @@ -733,8 +720,8 @@ impl Protocol { // Print some diagnostics. if let Some(peer) = self.context_data.peers.get(&who) { debug!(target: "sync", "Clogged peer {} (protocol_version: {:?}; roles: {:?}; \ - known_extrinsics: {:?}; known_blocks: {:?}; best_hash: {:?}; best_number: {:?})", - who, peer.info.protocol_version, peer.info.roles, peer.known_extrinsics, peer.known_blocks, + known_transactions: {:?}; known_blocks: {:?}; best_hash: {:?}; best_number: {:?})", + who, peer.info.protocol_version, peer.info.roles, peer.known_transactions, peer.known_blocks, peer.info.best_hash, peer.info.best_number); } else { debug!(target: "sync", "Peer clogged before being properly connected"); @@ -831,9 +818,34 @@ impl Protocol { pub fn on_block_response( &mut self, peer: PeerId, - request: message::BlockRequest, response: message::BlockResponse, ) -> CustomMessageOutcome { + let request = if let Some(ref mut p) = self.context_data.peers.get_mut(&peer) { + if p.obsolete_requests.remove(&response.id).is_some() { + trace!(target: "sync", "Ignoring obsolete block response packet from {} ({})", peer, response.id); + return CustomMessageOutcome::None; + } + // Clear the request. If the response is invalid peer will be disconnected anyway. + match p.block_request.take() { + Some((_, request)) if request.id == response.id => request, + Some(_) => { + trace!(target: "sync", "Ignoring obsolete block response packet from {} ({})", peer, response.id); + return CustomMessageOutcome::None; + } + None => { + trace!(target: "sync", "Unexpected response packet from unknown peer {}", peer); + self.behaviour.disconnect_peer(&peer); + self.peerset_handle.report_peer(peer, rep::UNEXPECTED_RESPONSE); + return CustomMessageOutcome::None; + } + } + } else { + trace!(target: "sync", "Unexpected response packet from unknown peer {}", peer); + self.behaviour.disconnect_peer(&peer); + self.peerset_handle.report_peer(peer, rep::UNEXPECTED_RESPONSE); + return CustomMessageOutcome::None; + }; + let blocks_range = || match ( response.blocks.first().and_then(|b| b.header.as_ref().map(|h| h.number())), response.blocks.last().and_then(|b| b.header.as_ref().map(|h| h.number())), @@ -878,15 +890,11 @@ impl Protocol { match self.sync.on_block_data(&peer, Some(request), response) { Ok(sync::OnBlockData::Import(origin, blocks)) => CustomMessageOutcome::BlockImport(origin, blocks), - Ok(sync::OnBlockData::Request(peer, req)) => { - if self.use_new_block_requests_protocol { - CustomMessageOutcome::BlockRequest { - target: peer, - request: req, - } - } else { - self.send_request(&peer, GenericMessage::BlockRequest(req)); - CustomMessageOutcome::None + Ok(sync::OnBlockData::Request(peer, mut req)) => { + self.update_peer_request(&peer, &mut req); + CustomMessageOutcome::BlockRequest { + target: peer, + request: req, } } Err(sync::BadPeer(id, repu)) => { @@ -959,12 +967,7 @@ impl Protocol { trace!(target: "sync", "New peer {} {:?}", who, status); let _protocol_version = { if self.context_data.peers.contains_key(&who) { - log!( - target: "sync", - if self.important_peers.contains(&who) { Level::Warn } else { Level::Debug }, - "Unexpected status packet from {}", who - ); - self.peerset_handle.report_peer(who, rep::UNEXPECTED_STATUS); + debug!(target: "sync", "Ignoring duplicate status packet from {}", who); return CustomMessageOutcome::None; } if status.genesis_hash != self.genesis_hash { @@ -1045,7 +1048,7 @@ impl Protocol { let peer = Peer { info, block_request: None, - known_extrinsics: LruHashSet::new(NonZeroUsize::new(MAX_KNOWN_EXTRINSICS) + known_transactions: LruHashSet::new(NonZeroUsize::new(MAX_KNOWN_TRANSACTIONS) .expect("Constant is nonzero")), known_blocks: LruHashSet::new(NonZeroUsize::new(MAX_KNOWN_BLOCKS) .expect("Constant is nonzero")), @@ -1063,15 +1066,12 @@ impl Protocol { if info.roles.is_full() { match self.sync.new_peer(who.clone(), info.best_hash, info.best_number) { Ok(None) => (), - Ok(Some(req)) => { - if self.use_new_block_requests_protocol { - self.pending_messages.push_back(CustomMessageOutcome::BlockRequest { - target: who.clone(), - request: req, - }); - } else { - self.send_request(&who, GenericMessage::BlockRequest(req)) - } + Ok(Some(mut req)) => { + self.update_peer_request(&who, &mut req); + self.pending_messages.push_back(CustomMessageOutcome::BlockRequest { + target: who.clone(), + request: req, + }); }, Err(sync::BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id); @@ -1137,28 +1137,29 @@ impl Protocol { .map(|(peer_id, peer)| (peer_id, peer.info.roles)) } - /// Called when peer sends us new extrinsics - fn on_extrinsics( + /// Called when peer sends us new transactions + fn on_transactions( &mut self, who: PeerId, - extrinsics: message::Transactions + transactions: message::Transactions ) { - // sending extrinsic to light node is considered a bad behavior + // sending transaction to light node is considered a bad behavior if !self.config.roles.is_full() { - trace!(target: "sync", "Peer {} is trying to send extrinsic to the light node", who); + trace!(target: "sync", "Peer {} is trying to send transactions to the light node", who); self.behaviour.disconnect_peer(&who); - self.peerset_handle.report_peer(who, rep::UNEXPECTED_EXTRINSICS); + self.peerset_handle.report_peer(who, rep::UNEXPECTED_TRANSACTIONS); return; } - // Accept extrinsics only when fully synced + // Accept transactions only when fully synced if self.sync.status().state != SyncState::Idle { - trace!(target: "sync", "{} Ignoring extrinsics while syncing", who); + trace!(target: "sync", "{} Ignoring transactions while syncing", who); return; } - trace!(target: "sync", "Received {} extrinsics from {}", extrinsics.len(), who); + + trace!(target: "sync", "Received {} transactions from {}", transactions.len(), who); if let Some(ref mut peer) = self.context_data.peers.get_mut(&who) { - for t in extrinsics { + for t in transactions { if self.pending_transactions.len() > MAX_PENDING_TRANSACTIONS { debug!( target: "sync", @@ -1169,9 +1170,9 @@ impl Protocol { } let hash = self.transaction_pool.hash_of(&t); - peer.known_extrinsics.insert(hash); + peer.known_transactions.insert(hash); - self.peerset_handle.report_peer(who.clone(), rep::ANY_EXTRINSIC); + self.peerset_handle.report_peer(who.clone(), rep::ANY_TRANSACTION); self.pending_transactions.push(PendingTransaction { peer_id: who.clone(), @@ -1181,45 +1182,45 @@ impl Protocol { } } - fn on_handle_extrinsic_import(&mut self, who: PeerId, import: TransactionImport) { + fn on_handle_transaction_import(&mut self, who: PeerId, import: TransactionImport) { match import { - TransactionImport::KnownGood => self.peerset_handle.report_peer(who, rep::ANY_EXTRINSIC_REFUND), - TransactionImport::NewGood => self.peerset_handle.report_peer(who, rep::GOOD_EXTRINSIC), - TransactionImport::Bad => self.peerset_handle.report_peer(who, rep::BAD_EXTRINSIC), + TransactionImport::KnownGood => self.peerset_handle.report_peer(who, rep::ANY_TRANSACTION_REFUND), + TransactionImport::NewGood => self.peerset_handle.report_peer(who, rep::GOOD_TRANSACTION), + TransactionImport::Bad => self.peerset_handle.report_peer(who, rep::BAD_TRANSACTION), TransactionImport::None => {}, } } - /// Propagate one extrinsic. - pub fn propagate_extrinsic( + /// Propagate one transaction. + pub fn propagate_transaction( &mut self, hash: &H, ) { - debug!(target: "sync", "Propagating extrinsic [{:?}]", hash); + debug!(target: "sync", "Propagating transaction [{:?}]", hash); // Accept transactions only when fully synced if self.sync.status().state != SyncState::Idle { return; } - if let Some(extrinsic) = self.transaction_pool.transaction(hash) { - let propagated_to = self.do_propagate_extrinsics(&[(hash.clone(), extrinsic)]); + if let Some(transaction) = self.transaction_pool.transaction(hash) { + let propagated_to = self.do_propagate_transactions(&[(hash.clone(), transaction)]); self.transaction_pool.on_broadcasted(propagated_to); } } - fn do_propagate_extrinsics( + fn do_propagate_transactions( &mut self, - extrinsics: &[(H, B::Extrinsic)], + transactions: &[(H, B::Extrinsic)], ) -> HashMap> { let mut propagated_to = HashMap::new(); for (who, peer) in self.context_data.peers.iter_mut() { - // never send extrinsics to the light node + // never send transactions to the light node if !peer.info.roles.is_full() { continue; } - let (hashes, to_send): (Vec<_>, Vec<_>) = extrinsics + let (hashes, to_send): (Vec<_>, Vec<_>) = transactions .iter() - .filter(|&(ref hash, _)| peer.known_extrinsics.insert(hash.clone())) + .filter(|&(ref hash, _)| peer.known_transactions.insert(hash.clone())) .cloned() .unzip(); @@ -1242,18 +1243,24 @@ impl Protocol { } } + if propagated_to.len() > 0 { + if let Some(ref metrics) = self.metrics { + metrics.propagated_transactions.inc(); + } + } + propagated_to } - /// Call when we must propagate ready extrinsics to peers. - pub fn propagate_extrinsics(&mut self) { - debug!(target: "sync", "Propagating extrinsics"); + /// Call when we must propagate ready transactions to peers. + pub fn propagate_transactions(&mut self) { + debug!(target: "sync", "Propagating transactions"); // Accept transactions only when fully synced if self.sync.status().state != SyncState::Idle { return; } - let extrinsics = self.transaction_pool.transactions(); - let propagated_to = self.do_propagate_extrinsics(&extrinsics); + let transactions = self.transaction_pool.transactions(); + let propagated_to = self.do_propagate_transactions(&transactions); self.transaction_pool.on_broadcasted(propagated_to); } @@ -1280,7 +1287,7 @@ impl Protocol { } let is_best = self.context_data.chain.info().best_hash == hash; - debug!(target: "sync", "Reannouncing block {:?}", hash); + debug!(target: "sync", "Reannouncing block {:?} is_best: {}", hash, is_best); self.send_announcement(&header, data, is_best, true) } @@ -1400,15 +1407,11 @@ impl Protocol { Ok(sync::OnBlockData::Import(origin, blocks)) => { CustomMessageOutcome::BlockImport(origin, blocks) }, - Ok(sync::OnBlockData::Request(peer, req)) => { - if self.use_new_block_requests_protocol { - CustomMessageOutcome::BlockRequest { - target: peer, - request: req, - } - } else { - self.send_request(&peer, GenericMessage::BlockRequest(req)); - CustomMessageOutcome::None + Ok(sync::OnBlockData::Request(peer, mut req)) => { + self.update_peer_request(&peer, &mut req); + CustomMessageOutcome::BlockRequest { + target: peer, + request: req, } } Err(sync::BadPeer(id, repu)) => { @@ -1419,17 +1422,6 @@ impl Protocol { } } - /// Call this when a block has been imported in the import queue - pub fn on_block_imported(&mut self, header: &B::Header, is_best: bool) { - if is_best { - self.sync.update_chain_info(header); - self.behaviour.set_notif_protocol_handshake( - &self.block_announces_protocol, - BlockAnnouncesHandshake::build(&self.config, &self.context_data.chain).encode() - ); - } - } - /// Call this when a block has been finalized. The sync layer may have some additional /// requesting to perform. pub fn on_block_finalized(&mut self, hash: B::Hash, header: &B::Header) { @@ -1494,12 +1486,23 @@ impl Protocol { /// A batch of blocks have been processed, with or without errors. /// Call this when a batch of blocks have been processed by the importqueue, with or without /// errors. - pub fn blocks_processed( + pub fn on_blocks_processed( &mut self, imported: usize, count: usize, results: Vec<(Result>, BlockImportError>, B::Hash)> ) { + let new_best = results.iter().rev().find_map(|r| match r { + (Ok(BlockImportResult::ImportedUnknown(n, aux, _)), hash) if aux.is_new_best => Some((*n, hash.clone())), + _ => None, + }); + if let Some((best_num, best_hash)) = new_best { + self.sync.update_chain_info(&best_hash, best_num); + self.behaviour.set_notif_protocol_handshake( + &self.block_announces_protocol, + BlockAnnouncesHandshake::build(&self.config, &self.context_data.chain).encode() + ); + } let results = self.sync.on_blocks_processed( imported, count, @@ -1507,22 +1510,12 @@ impl Protocol { ); for result in results { match result { - Ok((id, req)) => { - if self.use_new_block_requests_protocol { - self.pending_messages.push_back(CustomMessageOutcome::BlockRequest { - target: id, - request: req, - }); - } else { - let msg = GenericMessage::BlockRequest(req); - send_request( - &mut self.behaviour, - &mut self.context_data.stats, - &mut self.context_data.peers, - &id, - msg - ) - } + Ok((id, mut req)) => { + update_peer_request(&mut self.context_data.peers, &id, &mut req); + self.pending_messages.push_back(CustomMessageOutcome::BlockRequest { + target: id, + request: req, + }); } Err(sync::BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id); @@ -1901,25 +1894,20 @@ pub enum CustomMessageOutcome { None, } -fn send_request( - behaviour: &mut GenericProto, - stats: &mut HashMap<&'static str, PacketStats>, +fn update_peer_request( peers: &mut HashMap>, who: &PeerId, - mut message: Message, + request: &mut message::BlockRequest, ) { - if let GenericMessage::BlockRequest(ref mut r) = message { - if let Some(ref mut peer) = peers.get_mut(who) { - r.id = peer.next_request_id; - peer.next_request_id += 1; - if let Some((timestamp, request)) = peer.block_request.take() { - trace!(target: "sync", "Request {} for {} is now obsolete.", request.id, who); - peer.obsolete_requests.insert(request.id, timestamp); - } - peer.block_request = Some((Instant::now(), r.clone())); + if let Some(ref mut peer) = peers.get_mut(who) { + request.id = peer.next_request_id; + peer.next_request_id += 1; + if let Some((timestamp, request)) = peer.block_request.take() { + trace!(target: "sync", "Request {} for {} is now obsolete.", request.id, who); + peer.obsolete_requests.insert(request.id, timestamp); } + peer.block_request = Some((Instant::now(), request.clone())); } - send_message::(behaviour, stats, who, None, message) } fn send_message( @@ -1996,63 +1984,35 @@ impl NetworkBehaviour for Protocol { } while let Poll::Ready(Some(())) = self.propagate_timeout.poll_next_unpin(cx) { - self.propagate_extrinsics(); + self.propagate_transactions(); } - for (id, r) in self.sync.block_requests() { - if self.use_new_block_requests_protocol { - let event = CustomMessageOutcome::BlockRequest { - target: id.clone(), - request: r, - }; - self.pending_messages.push_back(event); - } else { - send_request( - &mut self.behaviour, - &mut self.context_data.stats, - &mut self.context_data.peers, - &id, - GenericMessage::BlockRequest(r), - ) - } + for (id, mut r) in self.sync.block_requests() { + update_peer_request(&mut self.context_data.peers, &id, &mut r); + let event = CustomMessageOutcome::BlockRequest { + target: id.clone(), + request: r, + }; + self.pending_messages.push_back(event); } - for (id, r) in self.sync.justification_requests() { - if self.use_new_block_requests_protocol { - let event = CustomMessageOutcome::BlockRequest { - target: id, - request: r, - }; - self.pending_messages.push_back(event); - } else { - send_request( - &mut self.behaviour, - &mut self.context_data.stats, - &mut self.context_data.peers, - &id, - GenericMessage::BlockRequest(r), - ) - } + for (id, mut r) in self.sync.justification_requests() { + update_peer_request(&mut self.context_data.peers, &id, &mut r); + let event = CustomMessageOutcome::BlockRequest { + target: id, + request: r, + }; + self.pending_messages.push_back(event); } for (id, r) in self.sync.finality_proof_requests() { - if self.use_new_block_requests_protocol { - let event = CustomMessageOutcome::FinalityProofRequest { - target: id, - block_hash: r.block, - request: r.request, - }; - self.pending_messages.push_back(event); - } else { - send_request( - &mut self.behaviour, - &mut self.context_data.stats, - &mut self.context_data.peers, - &id, - GenericMessage::FinalityProofRequest(r), - ) - } + let event = CustomMessageOutcome::FinalityProofRequest { + target: id, + block_hash: r.block, + request: r.request, + }; + self.pending_messages.push_back(event); } if let Poll::Ready(Some((peer_id, result))) = self.pending_transactions.poll_next_unpin(cx) { - self.on_handle_extrinsic_import(peer_id, result); + self.on_handle_transaction_import(peer_id, result); } if let Some(message) = self.pending_messages.pop_front() { return Poll::Ready(NetworkBehaviourAction::GenerateEvent(message)); @@ -2091,7 +2051,7 @@ impl NetworkBehaviour for Protocol { } Some(Fallback::Transactions) => { if let Ok(m) = message::Transactions::decode(&mut message.as_ref()) { - self.on_extrinsics(peer_id, m); + self.on_transactions(peer_id, m); } else { warn!(target: "sub-libp2p", "Failed to decode transactions list"); } @@ -2187,6 +2147,7 @@ mod tests { let (mut protocol, _) = Protocol::::new( ProtocolConfig::default(), + PeerId::random(), client.clone(), Arc::new(EmptyTransactionPool), None, @@ -2199,10 +2160,9 @@ mod tests { reserved_only: false, priority_groups: Vec::new(), }, - Box::new(DefaultBlockAnnounceValidator::new(client.clone())), + Box::new(DefaultBlockAnnounceValidator), None, Default::default(), - true, None, ).unwrap(); diff --git a/client/network/src/protocol/generic_proto/behaviour.rs b/client/network/src/protocol/generic_proto/behaviour.rs index 4984c0d86d9c7b6fc611e258042c81e0cea5c855..be2451c3f4a03bf2d4231a7b03a0d0c3d9dd07f6 100644 --- a/client/network/src/protocol/generic_proto/behaviour.rs +++ b/client/network/src/protocol/generic_proto/behaviour.rs @@ -34,7 +34,7 @@ use prometheus_endpoint::HistogramVec; use rand::distributions::{Distribution as _, Uniform}; use smallvec::SmallVec; use std::task::{Context, Poll}; -use std::{borrow::Cow, cmp, collections::hash_map::Entry}; +use std::{borrow::Cow, cmp, collections::{hash_map::Entry, VecDeque}}; use std::{error, mem, pin::Pin, str, time::Duration}; use wasm_timer::Instant; @@ -109,6 +109,9 @@ use wasm_timer::Instant; /// tries to connect, the connection is accepted. A ban only delays dialing attempts. /// pub struct GenericProto { + /// `PeerId` of the local node. + local_peer_id: PeerId, + /// Legacy protocol to open with peers. Never modified. legacy_protocol: RegisteredProtocol, @@ -123,6 +126,18 @@ pub struct GenericProto { /// List of peers in our state. peers: FnvHashMap, + /// The elements in `peers` occasionally contain `Delay` objects that we would normally have + /// to be polled one by one. In order to avoid doing so, as an optimization, every `Delay` is + /// instead put inside of `delays` and reference by a [`DelayId`]. This stream + /// yields `PeerId`s whose `DelayId` is potentially ready. + /// + /// By design, we never remove elements from this list. Elements are removed only when the + /// `Delay` triggers. As such, this stream may produce obsolete elements. + delays: stream::FuturesUnordered + Send>>>, + + /// [`DelayId`] to assign to the next delay. + next_delay_id: DelayId, + /// List of incoming messages we have sent to the peer set manager and that are waiting for an /// answer. incoming: SmallVec<[IncomingPeer; 6]>, @@ -132,12 +147,16 @@ pub struct GenericProto { next_incoming_index: sc_peerset::IncomingIndex, /// Events to produce from `poll()`. - events: SmallVec<[NetworkBehaviourAction; 4]>, + events: VecDeque>, /// If `Some`, report the message queue sizes on this `Histogram`. queue_size_report: Option, } +/// Identifier for a delay firing. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +struct DelayId(u64); + /// State of a peer we're connected to. #[derive(Debug)] enum PeerState { @@ -155,8 +174,8 @@ enum PeerState { /// The peerset requested that we connect to this peer. We are currently not connected. PendingRequest { - /// When to actually start dialing. - timer: futures_timer::Delay, + /// When to actually start dialing. References an entry in `delays`. + timer: DelayId, /// When the `timer` will trigger. timer_deadline: Instant, }, @@ -180,8 +199,8 @@ enum PeerState { DisabledPendingEnable { /// The connections that are currently open for custom protocol traffic. open: SmallVec<[ConnectionId; crate::MAX_CONNECTIONS_PER_PEER]>, - /// When to enable this remote. - timer: futures_timer::Delay, + /// When to enable this remote. References an entry in `delays`. + timer: DelayId, /// When the `timer` will trigger. timer_deadline: Instant, }, @@ -321,6 +340,7 @@ impl GenericProto { /// The `queue_size_report` is an optional Prometheus metric that can report the size of the /// messages queue. If passed, it must have one label for the protocol name. pub fn new( + local_peer_id: PeerId, protocol: impl Into, versions: &[u8], peerset: sc_peerset::Peerset, @@ -329,13 +349,16 @@ impl GenericProto { let legacy_protocol = RegisteredProtocol::new(protocol, versions); GenericProto { + local_peer_id, legacy_protocol, notif_protocols: Vec::new(), peerset, peers: FnvHashMap::default(), + delays: Default::default(), + next_delay_id: DelayId(0), incoming: SmallVec::new(), next_incoming_index: sc_peerset::IncomingIndex(0), - events: SmallVec::new(), + events: VecDeque::new(), queue_size_report, } } @@ -369,7 +392,7 @@ impl GenericProto { // Send an event to all the peers we're connected to, updating the handshake message. for (peer_id, _) in self.peers.iter().filter(|(_, state)| state.is_connected()) { - self.events.push(NetworkBehaviourAction::NotifyHandler { + self.events.push_back(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), handler: NotifyHandler::All, event: NotifsHandlerIn::UpdateHandshake { @@ -441,7 +464,7 @@ impl GenericProto { debug!(target: "sub-libp2p", "PSM <= Dropped({:?})", peer_id); self.peerset.dropped(peer_id.clone()); debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", peer_id); - self.events.push(NetworkBehaviourAction::NotifyHandler { + self.events.push_back(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), handler: NotifyHandler::All, event: NotifsHandlerIn::Disable, @@ -466,7 +489,7 @@ impl GenericProto { inc.alive = false; debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", peer_id); - self.events.push(NetworkBehaviourAction::NotifyHandler { + self.events.push_back(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), handler: NotifyHandler::All, event: NotifsHandlerIn::Disable, @@ -507,9 +530,18 @@ impl GenericProto { /// /// Can be called multiple times with the same `PeerId`s. pub fn add_discovered_nodes(&mut self, peer_ids: impl Iterator) { - self.peerset.discovered(peer_ids.map(|peer_id| { + let local_peer_id = &self.local_peer_id; + self.peerset.discovered(peer_ids.filter_map(|peer_id| { + if peer_id == *local_peer_id { + error!( + target: "sub-libp2p", + "Discovered our own identity. This is a minor inconsequential bug." + ); + return None; + } + debug!(target: "sub-libp2p", "PSM <= Discovered({:?})", peer_id); - peer_id + Some(peer_id) })); } @@ -548,7 +580,7 @@ impl GenericProto { ); trace!(target: "sub-libp2p", "Handler({:?}) <= Packet", target); - self.events.push(NetworkBehaviourAction::NotifyHandler { + self.events.push_back(NetworkBehaviourAction::NotifyHandler { peer_id: target.clone(), handler: NotifyHandler::One(conn), event: NotifsHandlerIn::SendNotification { @@ -578,7 +610,7 @@ impl GenericProto { trace!(target: "sub-libp2p", "External API => Packet for {:?}", target); trace!(target: "sub-libp2p", "Handler({:?}) <= Packet", target); - self.events.push(NetworkBehaviourAction::NotifyHandler { + self.events.push_back(NetworkBehaviourAction::NotifyHandler { peer_id: target.clone(), handler: NotifyHandler::One(conn), event: NotifsHandlerIn::SendLegacy { @@ -600,7 +632,7 @@ impl GenericProto { // If there's no entry in `self.peers`, start dialing. debug!(target: "sub-libp2p", "PSM => Connect({:?}): Starting to connect", entry.key()); debug!(target: "sub-libp2p", "Libp2p <= Dial {:?}", entry.key()); - self.events.push(NetworkBehaviourAction::DialPeer { + self.events.push_back(NetworkBehaviourAction::DialPeer { peer_id: entry.key().clone(), condition: DialPeerCondition::Disconnected }); @@ -613,10 +645,20 @@ impl GenericProto { match mem::replace(occ_entry.get_mut(), PeerState::Poisoned) { PeerState::Banned { ref until } if *until > now => { + let peer_id = occ_entry.key().clone(); debug!(target: "sub-libp2p", "PSM => Connect({:?}): Will start to connect at \ - until {:?}", occ_entry.key(), until); + until {:?}", peer_id, until); + + let delay_id = self.next_delay_id; + self.next_delay_id.0 += 1; + let delay = futures_timer::Delay::new(*until - now); + self.delays.push(async move { + delay.await; + (delay_id, peer_id) + }.boxed()); + *occ_entry.into_mut() = PeerState::PendingRequest { - timer: futures_timer::Delay::new(*until - now), + timer: delay_id, timer_deadline: *until, }; }, @@ -624,7 +666,7 @@ impl GenericProto { PeerState::Banned { .. } => { debug!(target: "sub-libp2p", "PSM => Connect({:?}): Starting to connect", occ_entry.key()); debug!(target: "sub-libp2p", "Libp2p <= Dial {:?}", occ_entry.key()); - self.events.push(NetworkBehaviourAction::DialPeer { + self.events.push_back(NetworkBehaviourAction::DialPeer { peer_id: occ_entry.key().clone(), condition: DialPeerCondition::Disconnected }); @@ -635,11 +677,21 @@ impl GenericProto { open, banned_until: Some(ref banned) } if *banned > now => { + let peer_id = occ_entry.key().clone(); debug!(target: "sub-libp2p", "PSM => Connect({:?}): But peer is banned until {:?}", - occ_entry.key(), banned); + peer_id, banned); + + let delay_id = self.next_delay_id; + self.next_delay_id.0 += 1; + let delay = futures_timer::Delay::new(*banned - now); + self.delays.push(async move { + delay.await; + (delay_id, peer_id) + }.boxed()); + *occ_entry.into_mut() = PeerState::DisabledPendingEnable { open, - timer: futures_timer::Delay::new(*banned - now), + timer: delay_id, timer_deadline: *banned, }; }, @@ -648,7 +700,7 @@ impl GenericProto { debug!(target: "sub-libp2p", "PSM => Connect({:?}): Enabling connections.", occ_entry.key()); debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", occ_entry.key()); - self.events.push(NetworkBehaviourAction::NotifyHandler { + self.events.push_back(NetworkBehaviourAction::NotifyHandler { peer_id: occ_entry.key().clone(), handler: NotifyHandler::All, event: NotifsHandlerIn::Enable, @@ -667,7 +719,7 @@ impl GenericProto { incoming for incoming peer") } debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", occ_entry.key()); - self.events.push(NetworkBehaviourAction::NotifyHandler { + self.events.push_back(NetworkBehaviourAction::NotifyHandler { peer_id: occ_entry.key().clone(), handler: NotifyHandler::All, event: NotifsHandlerIn::Enable, @@ -732,7 +784,7 @@ impl GenericProto { PeerState::Enabled { open } => { debug!(target: "sub-libp2p", "PSM => Drop({:?}): Disabling connections.", entry.key()); debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", entry.key()); - self.events.push(NetworkBehaviourAction::NotifyHandler { + self.events.push_back(NetworkBehaviourAction::NotifyHandler { peer_id: entry.key().clone(), handler: NotifyHandler::All, event: NotifsHandlerIn::Disable, @@ -787,7 +839,7 @@ impl GenericProto { debug!(target: "sub-libp2p", "PSM => Accept({:?}, {:?}): Enabling connections.", index, incoming.peer_id); debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", incoming.peer_id); - self.events.push(NetworkBehaviourAction::NotifyHandler { + self.events.push_back(NetworkBehaviourAction::NotifyHandler { peer_id: incoming.peer_id, handler: NotifyHandler::All, event: NotifsHandlerIn::Enable, @@ -820,7 +872,7 @@ impl GenericProto { debug!(target: "sub-libp2p", "PSM => Reject({:?}, {:?}): Rejecting connections.", index, incoming.peer_id); debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", incoming.peer_id); - self.events.push(NetworkBehaviourAction::NotifyHandler { + self.events.push_back(NetworkBehaviourAction::NotifyHandler { peer_id: incoming.peer_id, handler: NotifyHandler::All, event: NotifsHandlerIn::Disable, @@ -867,7 +919,7 @@ impl NetworkBehaviour for GenericProto { peer_id, endpoint ); *st = PeerState::Enabled { open: SmallVec::new() }; - self.events.push(NetworkBehaviourAction::NotifyHandler { + self.events.push_back(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), handler: NotifyHandler::One(*conn), event: NotifsHandlerIn::Enable @@ -911,7 +963,7 @@ impl NetworkBehaviour for GenericProto { "Libp2p => Connected({},{:?}): Not requested by PSM, disabling.", peer_id, endpoint); *st = PeerState::Disabled { open: SmallVec::new(), banned_until }; - self.events.push(NetworkBehaviourAction::NotifyHandler { + self.events.push_back(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), handler: NotifyHandler::One(*conn), event: NotifsHandlerIn::Disable @@ -927,7 +979,7 @@ impl NetworkBehaviour for GenericProto { (PeerState::Enabled { .. }, _) => { debug!(target: "sub-libp2p", "Handler({},{:?}) <= Enable secondary connection", peer_id, conn); - self.events.push(NetworkBehaviourAction::NotifyHandler { + self.events.push_back(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), handler: NotifyHandler::One(*conn), event: NotifsHandlerIn::Enable @@ -937,7 +989,7 @@ impl NetworkBehaviour for GenericProto { (PeerState::Disabled { .. }, _) | (PeerState::DisabledPendingEnable { .. }, _) => { debug!(target: "sub-libp2p", "Handler({},{:?}) <= Disable secondary connection", peer_id, conn); - self.events.push(NetworkBehaviourAction::NotifyHandler { + self.events.push_back(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), handler: NotifyHandler::One(*conn), event: NotifsHandlerIn::Disable @@ -965,7 +1017,7 @@ impl NetworkBehaviour for GenericProto { reason: "Disconnected by libp2p".into(), }; - self.events.push(NetworkBehaviourAction::GenerateEvent(event)); + self.events.push_back(NetworkBehaviourAction::GenerateEvent(event)); } } _ => {} @@ -1034,7 +1086,9 @@ impl NetworkBehaviour for GenericProto { // In the incoming state, we don't report "Dropped". Instead we will just ignore the // corresponding Accept/Reject. Some(PeerState::Incoming { }) => { - if let Some(state) = self.incoming.iter_mut().find(|i| i.peer_id == *peer_id) { + if let Some(state) = self.incoming.iter_mut() + .find(|i| i.alive && i.peer_id == *peer_id) + { debug!(target: "sub-libp2p", "Libp2p => Disconnected({}): Was in incoming mode with id {:?}.", peer_id, state.incoming_id); @@ -1130,7 +1184,7 @@ impl NetworkBehaviour for GenericProto { debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", source); debug!(target: "sub-libp2p", "PSM <= Dropped({:?})", source); self.peerset.dropped(source.clone()); - self.events.push(NetworkBehaviourAction::NotifyHandler { + self.events.push_back(NetworkBehaviourAction::NotifyHandler { peer_id: source.clone(), handler: NotifyHandler::All, event: NotifsHandlerIn::Disable, @@ -1202,7 +1256,7 @@ impl NetworkBehaviour for GenericProto { reason, peer_id: source, }; - self.events.push(NetworkBehaviourAction::GenerateEvent(event)); + self.events.push_back(NetworkBehaviourAction::GenerateEvent(event)); } else { debug!(target: "sub-libp2p", "Secondary connection closed custom protocol."); } @@ -1240,7 +1294,7 @@ impl NetworkBehaviour for GenericProto { if first { debug!(target: "sub-libp2p", "External API <= Open({:?})", source); let event = GenericProtoOut::CustomProtocolOpen { peer_id: source }; - self.events.push(NetworkBehaviourAction::GenerateEvent(event)); + self.events.push_back(NetworkBehaviourAction::GenerateEvent(event)); } else { debug!(target: "sub-libp2p", "Secondary connection opened custom protocol."); } @@ -1255,7 +1309,7 @@ impl NetworkBehaviour for GenericProto { message, }; - self.events.push(NetworkBehaviourAction::GenerateEvent(event)); + self.events.push_back(NetworkBehaviourAction::GenerateEvent(event)); } NotifsHandlerOut::Notification { protocol_name, message } => { @@ -1273,7 +1327,7 @@ impl NetworkBehaviour for GenericProto { message, }; - self.events.push(NetworkBehaviourAction::GenerateEvent(event)); + self.events.push_back(NetworkBehaviourAction::GenerateEvent(event)); } NotifsHandlerOut::Clogged { messages } => { @@ -1282,7 +1336,7 @@ impl NetworkBehaviour for GenericProto { trace!(target: "sub-libp2p", "External API <= Clogged({:?})", source); warn!(target: "sub-libp2p", "Queue of packets to send to {:?} is \ pretty large", source); - self.events.push(NetworkBehaviourAction::GenerateEvent(GenericProtoOut::Clogged { + self.events.push_back(NetworkBehaviourAction::GenerateEvent(GenericProtoOut::Clogged { peer_id: source, messages, })); @@ -1321,6 +1375,10 @@ impl NetworkBehaviour for GenericProto { Self::OutEvent, >, > { + if let Some(event) = self.events.pop_front() { + return Poll::Ready(event); + } + // Poll for instructions from the peerset. // Note that the peerset is a *best effort* crate, and we have to use defensive programming. loop { @@ -1345,51 +1403,43 @@ impl NetworkBehaviour for GenericProto { } } - for (peer_id, peer_state) in self.peers.iter_mut() { - match mem::replace(peer_state, PeerState::Poisoned) { - PeerState::PendingRequest { mut timer, timer_deadline } => { - if let Poll::Pending = Pin::new(&mut timer).poll(cx) { - *peer_state = PeerState::PendingRequest { timer, timer_deadline }; - continue; - } - + while let Poll::Ready(Some((delay_id, peer_id))) = + Pin::new(&mut self.delays).poll_next(cx) { + let peer_state = match self.peers.get_mut(&peer_id) { + Some(s) => s, + // We intentionally never remove elements from `delays`, and it may + // thus contain peers which are now gone. This is a normal situation. + None => continue, + }; + + match peer_state { + PeerState::PendingRequest { timer, .. } if *timer == delay_id => { debug!(target: "sub-libp2p", "Libp2p <= Dial {:?} now that ban has expired", peer_id); - self.events.push(NetworkBehaviourAction::DialPeer { - peer_id: peer_id.clone(), + self.events.push_back(NetworkBehaviourAction::DialPeer { + peer_id, condition: DialPeerCondition::Disconnected }); *peer_state = PeerState::Requested; } - PeerState::DisabledPendingEnable { - mut timer, - open, - timer_deadline - } => { - if let Poll::Pending = Pin::new(&mut timer).poll(cx) { - *peer_state = PeerState::DisabledPendingEnable { - timer, - open, - timer_deadline - }; - continue; - } - + PeerState::DisabledPendingEnable { timer, open, .. } if *timer == delay_id => { debug!(target: "sub-libp2p", "Handler({:?}) <= Enable (ban expired)", peer_id); - self.events.push(NetworkBehaviourAction::NotifyHandler { - peer_id: peer_id.clone(), + self.events.push_back(NetworkBehaviourAction::NotifyHandler { + peer_id, handler: NotifyHandler::All, event: NotifsHandlerIn::Enable, }); - *peer_state = PeerState::Enabled { open }; + *peer_state = PeerState::Enabled { open: mem::replace(open, Default::default()) }; } - st => *peer_state = st, + // We intentionally never remove elements from `delays`, and it may + // thus contain obsolete entries. This is a normal situation. + _ => {}, } } - if !self.events.is_empty() { - return Poll::Ready(self.events.remove(0)) + if let Some(event) = self.events.pop_front() { + return Poll::Ready(event); } Poll::Pending diff --git a/client/network/src/protocol/generic_proto/handler.rs b/client/network/src/protocol/generic_proto/handler.rs index a013b933c26a44d4dacb61b6c1b7408c120ac613..3b4469a872598f6f0b9c26b7647020a70ab5acc8 100644 --- a/client/network/src/protocol/generic_proto/handler.rs +++ b/client/network/src/protocol/generic_proto/handler.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/client/network/src/protocol/generic_proto/handler/legacy.rs b/client/network/src/protocol/generic_proto/handler/legacy.rs index e51b37139b82bec6bad05e14bd82a5b7e4966e06..c7de2d265e96fdf66f567366bd10fd6cf8ece265 100644 --- a/client/network/src/protocol/generic_proto/handler/legacy.rs +++ b/client/network/src/protocol/generic_proto/handler/legacy.rs @@ -30,7 +30,7 @@ use libp2p::swarm::{ }; use log::{debug, error}; use smallvec::{smallvec, SmallVec}; -use std::{borrow::Cow, error, fmt, io, mem, time::Duration}; +use std::{borrow::Cow, collections::VecDeque, error, fmt, io, mem, time::Duration}; use std::{pin::Pin, task::{Context, Poll}}; /// Implements the `IntoProtocolsHandler` trait of libp2p. @@ -117,7 +117,7 @@ impl IntoProtocolsHandler for LegacyProtoHandlerProto { substreams: SmallVec::new(), init_deadline: Delay::new(Duration::from_secs(20)) }, - events_queue: SmallVec::new(), + events_queue: VecDeque::new(), } } } @@ -142,7 +142,7 @@ pub struct LegacyProtoHandler { /// /// This queue must only ever be modified to insert elements at the back, or remove the first /// element. - events_queue: SmallVec<[ProtocolsHandlerEvent; 16]>, + events_queue: VecDeque>, } /// State of the handler. @@ -277,7 +277,7 @@ impl LegacyProtoHandler { ProtocolState::Init { substreams: incoming, .. } => { if incoming.is_empty() { if let ConnectedPoint::Dialer { .. } = self.endpoint { - self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest { + self.events_queue.push_back(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol: SubstreamProtocol::new(self.protocol.clone()), info: (), }); @@ -290,7 +290,7 @@ impl LegacyProtoHandler { version: incoming[0].protocol_version(), endpoint: self.endpoint.clone() }; - self.events_queue.push(ProtocolsHandlerEvent::Custom(event)); + self.events_queue.push_back(ProtocolsHandlerEvent::Custom(event)); ProtocolState::Normal { substreams: incoming.into_iter().collect(), shutdown: SmallVec::new() @@ -488,7 +488,7 @@ impl LegacyProtoHandler { version: substream.protocol_version(), endpoint: self.endpoint.clone() }; - self.events_queue.push(ProtocolsHandlerEvent::Custom(event)); + self.events_queue.push_back(ProtocolsHandlerEvent::Custom(event)); ProtocolState::Normal { substreams: smallvec![substream], shutdown: SmallVec::new() @@ -565,7 +565,7 @@ impl ProtocolsHandler for LegacyProtoHandler { _ => false, }; - self.events_queue.push(ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::ProtocolError { + self.events_queue.push_back(ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::ProtocolError { is_severe, error: Box::new(err), })); @@ -587,8 +587,7 @@ impl ProtocolsHandler for LegacyProtoHandler { ProtocolsHandlerEvent > { // Flush the events queue if necessary. - if !self.events_queue.is_empty() { - let event = self.events_queue.remove(0); + if let Some(event) = self.events_queue.pop_front() { return Poll::Ready(event) } diff --git a/client/network/src/protocol/generic_proto/handler/notif_in.rs b/client/network/src/protocol/generic_proto/handler/notif_in.rs index a979b8f9cd34a7e4dacbc5c61cc43dd444f2b00a..be78fb970e90b2524fd1478f48462e159dd1b28d 100644 --- a/client/network/src/protocol/generic_proto/handler/notif_in.rs +++ b/client/network/src/protocol/generic_proto/handler/notif_in.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Implementations of the `IntoProtocolsHandler` and `ProtocolsHandler` traits for ingoing //! substreams for a single gossiping protocol. //! @@ -36,8 +37,7 @@ use libp2p::swarm::{ NegotiatedSubstream, }; use log::{error, warn}; -use smallvec::SmallVec; -use std::{borrow::Cow, fmt, pin::Pin, task::{Context, Poll}}; +use std::{borrow::Cow, collections::VecDeque, fmt, pin::Pin, task::{Context, Poll}}; /// Implements the `IntoProtocolsHandler` trait of libp2p. /// @@ -69,7 +69,7 @@ pub struct NotifsInHandler { /// /// This queue is only ever modified to insert elements at the back, or remove the first /// element. - events_queue: SmallVec<[ProtocolsHandlerEvent; 16]>, + events_queue: VecDeque>, } /// Event that can be received by a `NotifsInHandler`. @@ -129,7 +129,7 @@ impl IntoProtocolsHandler for NotifsInHandlerProto { in_protocol: self.in_protocol, substream: None, pending_accept_refuses: 0, - events_queue: SmallVec::new(), + events_queue: VecDeque::new(), } } } @@ -159,7 +159,7 @@ impl ProtocolsHandler for NotifsInHandler { ) { // If a substream already exists, we drop it and replace it with the new incoming one. if self.substream.is_some() { - self.events_queue.push(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Closed)); + self.events_queue.push_back(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::Closed)); } // Note that we drop the existing substream, which will send an equivalent to a TCP "RST" @@ -170,7 +170,7 @@ impl ProtocolsHandler for NotifsInHandler { // and we can't close "more" than that anyway. self.substream = Some(proto); - self.events_queue.push(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::OpenRequest(msg))); + self.events_queue.push_back(ProtocolsHandlerEvent::Custom(NotifsInHandlerOut::OpenRequest(msg))); self.pending_accept_refuses = self.pending_accept_refuses .checked_add(1) .unwrap_or_else(|| { @@ -232,8 +232,7 @@ impl ProtocolsHandler for NotifsInHandler { ProtocolsHandlerEvent > { // Flush the events queue if necessary. - if !self.events_queue.is_empty() { - let event = self.events_queue.remove(0); + if let Some(event) = self.events_queue.pop_front() { return Poll::Ready(event) } diff --git a/client/network/src/protocol/generic_proto/handler/notif_out.rs b/client/network/src/protocol/generic_proto/handler/notif_out.rs index b5d6cd61ada2a9e54bfa5873e0f7992ab1dfcfcb..6b97ad67e34c696f58996a8c13f983f0486b84fa 100644 --- a/client/network/src/protocol/generic_proto/handler/notif_out.rs +++ b/client/network/src/protocol/generic_proto/handler/notif_out.rs @@ -35,8 +35,7 @@ use libp2p::swarm::{ }; use log::{debug, warn, error}; use prometheus_endpoint::Histogram; -use smallvec::SmallVec; -use std::{borrow::Cow, fmt, mem, pin::Pin, task::{Context, Poll}, time::Duration}; +use std::{borrow::Cow, collections::VecDeque, fmt, mem, pin::Pin, task::{Context, Poll}, time::Duration}; use wasm_timer::Instant; /// Maximum duration to open a substream and receive the handshake message. After that, we @@ -85,7 +84,7 @@ impl IntoProtocolsHandler for NotifsOutHandlerProto { when_connection_open: Instant::now(), queue_size_report: self.queue_size_report, state: State::Disabled, - events_queue: SmallVec::new(), + events_queue: VecDeque::new(), peer_id: peer_id.clone(), } } @@ -116,7 +115,7 @@ pub struct NotifsOutHandler { /// /// This queue must only ever be modified to insert elements at the back, or remove the first /// element. - events_queue: SmallVec<[ProtocolsHandlerEvent; 16]>, + events_queue: VecDeque>, /// Who we are connected to. peer_id: PeerId, @@ -247,7 +246,7 @@ impl ProtocolsHandler for NotifsOutHandler { match mem::replace(&mut self.state, State::Poisoned) { State::Opening { initial_message } => { let ev = NotifsOutHandlerOut::Open { handshake: handshake_msg }; - self.events_queue.push(ProtocolsHandlerEvent::Custom(ev)); + self.events_queue.push_back(ProtocolsHandlerEvent::Custom(ev)); self.state = State::Open { substream, initial_message }; }, // If the handler was disabled while we were negotiating the protocol, immediately @@ -267,7 +266,7 @@ impl ProtocolsHandler for NotifsOutHandler { match mem::replace(&mut self.state, State::Poisoned) { State::Disabled => { let proto = NotificationsOut::new(self.protocol_name.clone(), initial_message.clone()); - self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest { + self.events_queue.push_back(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol: SubstreamProtocol::new(proto).with_timeout(OPEN_TIMEOUT), info: (), }); @@ -287,7 +286,7 @@ impl ProtocolsHandler for NotifsOutHandler { } let proto = NotificationsOut::new(self.protocol_name.clone(), initial_message.clone()); - self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest { + self.events_queue.push_back(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol: SubstreamProtocol::new(proto).with_timeout(OPEN_TIMEOUT), info: (), }); @@ -347,7 +346,7 @@ impl ProtocolsHandler for NotifsOutHandler { State::Opening { .. } => { self.state = State::Refused; let ev = NotifsOutHandlerOut::Refused; - self.events_queue.push(ProtocolsHandlerEvent::Custom(ev)); + self.events_queue.push_back(ProtocolsHandlerEvent::Custom(ev)); }, State::DisabledOpening => self.state = State::Disabled, State::Poisoned => error!("☎️ Notifications handler in a poisoned state"), @@ -371,9 +370,8 @@ impl ProtocolsHandler for NotifsOutHandler { cx: &mut Context, ) -> Poll> { // Flush the events queue if necessary. - if !self.events_queue.is_empty() { - let event = self.events_queue.remove(0); - return Poll::Ready(event); + if let Some(event) = self.events_queue.pop_front() { + return Poll::Ready(event) } match &mut self.state { @@ -385,7 +383,7 @@ impl ProtocolsHandler for NotifsOutHandler { let initial_message = mem::replace(initial_message, Vec::new()); self.state = State::Opening { initial_message: initial_message.clone() }; let proto = NotificationsOut::new(self.protocol_name.clone(), initial_message); - self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest { + self.events_queue.push_back(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol: SubstreamProtocol::new(proto).with_timeout(OPEN_TIMEOUT), info: (), }); diff --git a/client/network/src/protocol/generic_proto/tests.rs b/client/network/src/protocol/generic_proto/tests.rs index 1bc6e745f887685af41a6c9cfdfc05da84af7918..de02ac5f3468d20bd3e9bfabd13597c5b33614a1 100644 --- a/client/network/src/protocol/generic_proto/tests.rs +++ b/client/network/src/protocol/generic_proto/tests.rs @@ -42,6 +42,7 @@ fn build_nodes() -> (Swarm, Swarm) { for index in 0 .. 2 { let keypair = keypairs[index].clone(); + let local_peer_id = keypair.public().into_peer_id(); let transport = libp2p::core::transport::MemoryTransport .and_then(move |out, endpoint| { let secio = libp2p::secio::SecioConfig::new(keypair); @@ -82,7 +83,7 @@ fn build_nodes() -> (Swarm, Swarm) { }); let behaviour = CustomProtoWithAddr { - inner: GenericProto::new(&b"test"[..], &[1], peerset, None), + inner: GenericProto::new(local_peer_id, &b"test"[..], &[1], peerset, None), addrs: addrs .iter() .enumerate() diff --git a/client/network/src/protocol/generic_proto/upgrade.rs b/client/network/src/protocol/generic_proto/upgrade.rs index 7172451cee59aa68f7a315c918e921d05c499513..6322a10b572a9fa1e9d60566343e513fc2a6f15a 100644 --- a/client/network/src/protocol/generic_proto/upgrade.rs +++ b/client/network/src/protocol/generic_proto/upgrade.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/client/network/src/protocol/generic_proto/upgrade/legacy.rs b/client/network/src/protocol/generic_proto/upgrade/legacy.rs index 15e34197d4418ed69f3fd674dccdaaad325fa1b3..13560113bb1a22bc75344c29bc76712a57d4cf99 100644 --- a/client/network/src/protocol/generic_proto/upgrade/legacy.rs +++ b/client/network/src/protocol/generic_proto/upgrade/legacy.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::config::ProtocolId; use bytes::BytesMut; use futures::prelude::*; diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index 155e945f4eec545f8cec080b713ea5041fb3b0e9..a7fbb92387cf6022e894accbdf4855520a912f15 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Network packet message types. These get serialized and put into the lower level protocol payload. use bitflags::bitflags; @@ -86,6 +87,20 @@ bitflags! { } } +impl BlockAttributes { + /// Encodes attributes as big endian u32, compatible with SCALE-encoding (i.e the + /// significant byte has zero index). + pub fn to_be_u32(&self) -> u32 { + u32::from_be_bytes([self.bits(), 0, 0, 0]) + } + + /// Decodes attributes, encoded with the `encode_to_be_u32()` call. + pub fn from_be_u32(encoded: u32) -> Result { + BlockAttributes::from_bits(encoded.to_be_bytes()[0]) + .ok_or_else(|| Error::from("Invalid BlockAttributes")) + } +} + impl Encode for BlockAttributes { fn encode_to(&self, dest: &mut T) { dest.push_byte(self.bits()) diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index f4c935de5d3af5d02a7e1c71a8b8b788caca6779..bfd8c4fe218dec5116e1c2d9a49230af1b96fb61 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -48,6 +48,7 @@ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, Header, NumberFor, Zero, One, CheckedSub, SaturatedConversion, Hash, HashFor} }; +use sp_arithmetic::traits::Saturating; use std::{fmt, ops::Range, collections::{HashMap, HashSet, VecDeque}, sync::Arc}; mod blocks; @@ -189,8 +190,8 @@ pub struct ChainSync { block_announce_validator: Box + Send>, /// Maximum number of peers to ask the same blocks in parallel. max_parallel_downloads: u32, - /// Total number of processed blocks (imported or failed). - processed_blocks: usize, + /// Total number of downloaded blocks. + downloaded_blocks: usize, } /// All the data we have about a Peer that we are trying to sync with @@ -375,7 +376,7 @@ impl ChainSync { pending_requests: Default::default(), block_announce_validator, max_parallel_downloads, - processed_blocks: 0, + downloaded_blocks: 0, } } @@ -388,7 +389,7 @@ impl ChainSync { /// Returns the current sync status. pub fn status(&self) -> Status { - let best_seen = self.peers.values().max_by_key(|p| p.best_number).map(|p| p.best_number); + let best_seen = self.peers.values().map(|p| p.best_number).max(); let sync_state = if let Some(n) = best_seen { // A chain is classified as downloading if the provided best block is @@ -415,9 +416,9 @@ impl ChainSync { self.fork_targets.len() } - /// Number of processed blocks. - pub fn num_processed_blocks(&self) -> usize { - self.processed_blocks + /// Number of downloaded blocks. + pub fn num_downloaded_blocks(&self) -> usize { + self.downloaded_blocks } /// Handle a new connected peer. @@ -513,10 +514,10 @@ impl ChainSync { } } - /// Signal that `best_header` has been queued for import and update the + /// Signal that a new best block has been imported. /// `ChainSync` state with that information. - pub fn update_chain_info(&mut self, best_header: &B::Header) { - self.on_block_queued(&best_header.hash(), *best_header.number()) + pub fn update_chain_info(&mut self, best_hash: &B::Hash, best_number: NumberFor) { + self.on_block_queued(best_hash, best_number); } /// Schedule a justification request for the given block. @@ -651,7 +652,6 @@ impl ChainSync { let blocks = &mut self.blocks; let attrs = &self.required_block_attributes; let fork_targets = &mut self.fork_targets; - let mut have_requests = false; let last_finalized = self.client.info().finalized_number; let best_queued = self.best_queued_number; let client = &self.client; @@ -681,7 +681,6 @@ impl ChainSync { peer.common_number, req, ); - have_requests = true; Some((id, req)) } else if let Some((hash, req)) = fork_sync_request( id, @@ -697,7 +696,6 @@ impl ChainSync { ) { trace!(target: "sync", "Downloading fork {:?} from {}", hash, id); peer.state = PeerSyncState::DownloadingStale(hash); - have_requests = true; Some((id, req)) } else { None @@ -719,6 +717,7 @@ impl ChainSync { request: Option>, response: BlockResponse ) -> Result, BadPeer> { + self.downloaded_blocks += response.blocks.len(); let mut new_blocks: Vec> = if let Some(peer) = self.peers.get_mut(who) { let mut blocks = response.blocks; @@ -1004,8 +1003,6 @@ impl ChainSync { for (_, hash) in &results { self.queue_blocks.remove(&hash); } - self.processed_blocks += results.len(); - for (result, hash) in results { if has_error { continue; @@ -1190,6 +1187,21 @@ impl ChainSync { peer.recently_announced.pop_front(); } peer.recently_announced.push_back(hash.clone()); + + // Let external validator check the block announcement. + let assoc_data = announce.data.as_ref().map_or(&[][..], |v| v.as_slice()); + let is_best = match self.block_announce_validator.validate(&header, assoc_data) { + Ok(Validation::Success { is_new_best }) => is_new_best || is_best, + Ok(Validation::Failure) => { + debug!(target: "sync", "Block announcement validation of block {} from {} failed", hash, who); + return OnBlockAnnounce::Nothing + } + Err(e) => { + error!(target: "sync", "💔 Block announcement validation errored: {}", e); + return OnBlockAnnounce::Nothing + } + }; + if is_best { // update their best block peer.best_number = number; @@ -1220,20 +1232,6 @@ impl ChainSync { return OnBlockAnnounce::Nothing } - // Let external validator check the block announcement. - let assoc_data = announce.data.as_ref().map_or(&[][..], |v| v.as_slice()); - match self.block_announce_validator.validate(&header, assoc_data) { - Ok(Validation::Success) => (), - Ok(Validation::Failure) => { - debug!(target: "sync", "Block announcement validation of block {} from {} failed", hash, who); - return OnBlockAnnounce::Nothing - } - Err(e) => { - error!(target: "sync", "💔 Block announcement validation errored: {}", e); - return OnBlockAnnounce::Nothing - } - } - if ancient_parent { trace!(target: "sync", "Ignored ancient block announced from {}: {} {:?}", who, hash, header); return OnBlockAnnounce::Nothing @@ -1274,7 +1272,6 @@ impl ChainSync { /// Restart the sync process. fn restart<'a>(&'a mut self) -> impl Iterator), BadPeer>> + 'a { - self.processed_blocks = 0; self.blocks.clear(); let info = self.client.info(); self.best_queued_hash = info.best_hash; @@ -1433,14 +1430,24 @@ fn peer_block_request( max_parallel_downloads, MAX_DOWNLOAD_AHEAD, ) { + // The end is not part of the range. + let last = range.end.saturating_sub(One::one()); + + let from = if peer.best_number == last { + message::FromBlock::Hash(peer.best_hash) + } else { + message::FromBlock::Number(last) + }; + let request = message::generic::BlockRequest { id: 0, fields: attrs.clone(), - from: message::FromBlock::Number(range.start), + from, to: None, - direction: message::Direction::Ascending, + direction: message::Direction::Descending, max: Some((range.end - range.start).saturated_into::()) }; + Some((range, request)) } else { None @@ -1563,7 +1570,7 @@ mod test { let client = Arc::new(TestClientBuilder::new().build()); let info = client.info(); - let block_announce_validator = Box::new(DefaultBlockAnnounceValidator::new(client.clone())); + let block_announce_validator = Box::new(DefaultBlockAnnounceValidator); let peer_id = PeerId::random(); let mut sync = ChainSync::new( diff --git a/client/network/src/protocol/sync/blocks.rs b/client/network/src/protocol/sync/blocks.rs index c79c0d7f51ecd6a11239a25ffd153bea0286e278..b64c9e053e97b3018562beca4f439d71f17c8edb 100644 --- a/client/network/src/protocol/sync/blocks.rs +++ b/client/network/src/protocol/sync/blocks.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use std::cmp; use std::ops::Range; use std::collections::{HashMap, BTreeMap}; diff --git a/client/network/src/protocol/sync/extra_requests.rs b/client/network/src/protocol/sync/extra_requests.rs index 1e63749c25a9f1b501c1b6f7ed912ba580283fc1..6d688c130fafdec413e3ff905f3e33802cb85aeb 100644 --- a/client/network/src/protocol/sync/extra_requests.rs +++ b/client/network/src/protocol/sync/extra_requests.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use sp_blockchain::Error as ClientError; use crate::protocol::sync::{PeerSync, PeerSyncState}; use fork_tree::ForkTree; diff --git a/client/network/src/schema.rs b/client/network/src/schema.rs index dbc85dd97b390f83fc4e3a7c5be1841d4508d33f..44fbbffd25406d5b985707f46f8c2e9123ca992a 100644 --- a/client/network/src/schema.rs +++ b/client/network/src/schema.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Include sources generated from protobuf definitions. pub mod v1 { diff --git a/client/network/src/service.rs b/client/network/src/service.rs index ba6c7a39b4b2125c5ac3c70c28e43804f605b9ca..2ef6b7bc214573a1873ceb87a70f31110b7016fa 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Main entry point of the sc-network crate. //! //! There are two main structs in this module: [`NetworkWorker`] and [`NetworkService`]. @@ -59,10 +60,11 @@ use sp_runtime::{ }; use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use std::{ - borrow::Cow, + borrow::{Borrow, Cow}, collections::HashSet, fs, io, marker::PhantomData, + num:: NonZeroUsize, pin::Pin, str, sync::{ @@ -105,6 +107,24 @@ impl NetworkWorker { /// for the network processing to advance. From it, you can extract a `NetworkService` using /// `worker.service()`. The `NetworkService` can be shared through the codebase. pub fn new(params: Params) -> Result, Error> { + // Ensure the listen addresses are consistent with the transport. + ensure_addresses_consistent_with_transport( + params.network_config.listen_addresses.iter(), + ¶ms.network_config.transport, + )?; + ensure_addresses_consistent_with_transport( + params.network_config.boot_nodes.iter().map(|x| &x.multiaddr), + ¶ms.network_config.transport, + )?; + ensure_addresses_consistent_with_transport( + params.network_config.reserved_nodes.iter().map(|x| &x.multiaddr), + ¶ms.network_config.transport, + )?; + ensure_addresses_consistent_with_transport( + params.network_config.public_addresses.iter(), + ¶ms.network_config.transport, + )?; + let (to_worker, from_worker) = tracing_unbounded("mpsc_network_worker"); if let Some(path) = params.network_config.net_config_path { @@ -155,12 +175,14 @@ impl NetworkWorker { Role::Sentry { validators } => { for validator in validators { sentries_and_validators.insert(validator.peer_id.clone()); + reserved_nodes.insert(validator.peer_id.clone()); known_addresses.push((validator.peer_id.clone(), validator.multiaddr.clone())); } } Role::Authority { sentry_nodes } => { for sentry_node in sentry_nodes { sentries_and_validators.insert(sentry_node.peer_id.clone()); + reserved_nodes.insert(sentry_node.peer_id.clone()); known_addresses.push((sentry_node.peer_id.clone(), sentry_node.multiaddr.clone())); } } @@ -185,7 +207,13 @@ impl NetworkWorker { let local_identity = params.network_config.node_key.clone().into_keypair()?; let local_public = local_identity.public(); let local_peer_id = local_public.clone().into_peer_id(); - info!(target: "sub-libp2p", "🏷 Local node identity is: {}", local_peer_id.to_base58()); + let local_peer_id_legacy = bs58::encode(Borrow::<[u8]>::borrow(&local_peer_id)).into_string(); + info!( + target: "sub-libp2p", + "🏷 Local node identity is: {} (legacy representation: {})", + local_peer_id.to_base58(), + local_peer_id_legacy + ); // Initialize the metrics. let metrics = match ¶ms.metrics_registry { @@ -204,6 +232,7 @@ impl NetworkWorker { roles: From::from(¶ms.role), max_parallel_downloads: params.network_config.max_parallel_downloads, }, + local_peer_id.clone(), params.chain.clone(), params.transaction_pool, params.finality_proof_provider.clone(), @@ -213,7 +242,6 @@ impl NetworkWorker { params.block_announce_validator, params.metrics_registry.as_ref(), boot_node_ids.clone(), - params.network_config.use_new_block_requests_protocol, metrics.as_ref().map(|m| m.notifications_queues_size.clone()), )?; @@ -286,7 +314,9 @@ impl NetworkWorker { transport::build_transport(local_identity, config_mem, config_wasm, flowctrl) }; let mut builder = SwarmBuilder::new(transport, behaviour, local_peer_id.clone()) - .peer_connection_limit(crate::MAX_CONNECTIONS_PER_PEER); + .peer_connection_limit(crate::MAX_CONNECTIONS_PER_PEER) + .notify_handler_buffer_size(NonZeroUsize::new(32).expect("32 != 0; qed")) + .connection_event_buffer_size(1024); if let Some(spawner) = params.executor { struct SpawnImpl(F); impl + Send>>)> Executor for SpawnImpl { @@ -379,9 +409,9 @@ impl NetworkWorker { self.network_service.user_protocol().num_queued_blocks() } - /// Returns the number of processed blocks. - pub fn num_processed_blocks(&self) -> usize { - self.network_service.user_protocol().num_processed_blocks() + /// Returns the number of downloaded blocks. + pub fn num_downloaded_blocks(&self) -> usize { + self.network_service.user_protocol().num_downloaded_blocks() } /// Number of active sync requests. @@ -400,16 +430,18 @@ impl NetworkWorker { &self.service } - /// You must call this when a new block is imported by the client. - pub fn on_block_imported(&mut self, header: B::Header, is_best: bool) { - self.network_service.user_protocol_mut().on_block_imported(&header, is_best); - } - /// You must call this when a new block is finalized by the client. pub fn on_block_finalized(&mut self, hash: B::Hash, header: B::Header) { self.network_service.user_protocol_mut().on_block_finalized(hash, &header); } + /// This should be called when blocks are added to the + /// chain by something other than the import queue. + /// Currently this is only useful for tests. + pub fn update_chain(&mut self) { + self.network_service.user_protocol_mut().update_chain(); + } + /// Returns the local `PeerId`. pub fn local_peer_id(&self) -> &PeerId { Swarm::::local_peer_id(&self.network_service) @@ -546,13 +578,17 @@ impl NetworkService { /// Registers a new notifications protocol. /// - /// After that, you can call `write_notifications`. + /// After a protocol has been registered, you can call `write_notifications`. + /// + /// **Important**: This method is a work-around, and you are instead strongly encouraged to + /// pass the protocol in the `NetworkConfiguration::notifications_protocols` list instead. + /// If you have no other choice but to use this method, you are very strongly encouraged to + /// call it very early on. Any connection open will retain the protocols that were registered + /// then, and not any new one. /// /// Please call `event_stream` before registering a protocol, otherwise you may miss events /// about the protocol that you have registered. - /// - /// You are very strongly encouraged to call this method very early on. Any connection open - /// will retain the protocols that were registered then, and not any new one. + // TODO: remove this method after https://github.com/paritytech/substrate/issues/4587 pub fn register_notifications_protocol( &self, engine_id: ConsensusEngineId, @@ -569,15 +605,15 @@ impl NetworkService { /// All transactions will be fetched from the `TransactionPool` that was passed at /// initialization as part of the configuration and propagated to peers. pub fn trigger_repropagate(&self) { - let _ = self.to_worker.unbounded_send(ServiceToWorkerMsg::PropagateExtrinsics); + let _ = self.to_worker.unbounded_send(ServiceToWorkerMsg::PropagateTransactions); } /// You must call when new transaction is imported by the transaction pool. /// /// This transaction will be fetched from the `TransactionPool` that was passed at /// initialization as part of the configuration and propagated to peers. - pub fn propagate_extrinsic(&self, hash: H) { - let _ = self.to_worker.unbounded_send(ServiceToWorkerMsg::PropagateExtrinsic(hash)); + pub fn propagate_transaction(&self, hash: H) { + let _ = self.to_worker.unbounded_send(ServiceToWorkerMsg::PropagateTransaction(hash)); } /// Make sure an important block is propagated to peers. @@ -653,8 +689,15 @@ impl NetworkService { /// Adds a `PeerId` and its address as reserved. The string should encode the address /// and peer ID of the remote node. + /// + /// Returns an `Err` if the given string is not a valid multiaddress + /// or contains an invalid peer ID (which includes the local peer ID). pub fn add_reserved_peer(&self, peer: String) -> Result<(), String> { let (peer_id, addr) = parse_str_addr(&peer).map_err(|e| format!("{:?}", e))?; + // Make sure the local peer ID is never added to the PSM. + if peer_id == self.local_peer_id { + return Err("Local peer ID cannot be added as a reserved peer.".to_string()) + } self.peerset.add_reserved_peer(peer_id.clone()); let _ = self .to_worker @@ -675,12 +718,26 @@ impl NetworkService { } /// Modify a peerset priority group. + /// + /// Returns an `Err` if one of the given addresses contains an invalid + /// peer ID (which includes the local peer ID). pub fn set_priority_group(&self, group_id: String, peers: HashSet) -> Result<(), String> { - let peers = peers.into_iter().map(|p| { - parse_addr(p).map_err(|e| format!("{:?}", e)) - }).collect::, String>>()?; + let peers = peers.into_iter() + .map(|p| match parse_addr(p) { + Err(e) => Err(format!("{:?}", e)), + Ok((peer, addr)) => + // Make sure the local peer ID is never added to the PSM + // or added as a "known address", even if given. + if peer == self.local_peer_id { + Err("Local peer ID in priority group.".to_string()) + } else { + Ok((peer, addr)) + } + }) + .collect::, String>>()?; let peer_ids = peers.iter().map(|(peer_id, _addr)| peer_id.clone()).collect(); + self.peerset.set_priority_group(group_id, peer_ids); for (peer_id, addr) in peers.into_iter() { @@ -696,6 +753,23 @@ impl NetworkService { pub fn num_connected(&self) -> usize { self.num_connected.load(Ordering::Relaxed) } + + /// This function should be called when blocks are added to the chain by something other + /// than the import queue. + /// + /// > **Important**: This function is a hack and can be removed at any time. Do **not** use it. + pub fn update_chain(&self) { + let _ = self + .to_worker + .unbounded_send(ServiceToWorkerMsg::UpdateChain); + } + + /// Inform the network service about an own imported block. + pub fn own_block_imported(&self, hash: B::Hash, number: NumberFor) { + let _ = self + .to_worker + .unbounded_send(ServiceToWorkerMsg::OwnBlockImported(hash, number)); + } } impl sp_consensus::SyncOracle @@ -742,8 +816,8 @@ impl NetworkStateInfo for NetworkService /// /// Each entry corresponds to a method of `NetworkService`. enum ServiceToWorkerMsg { - PropagateExtrinsic(H), - PropagateExtrinsics, + PropagateTransaction(H), + PropagateTransactions, RequestJustification(B::Hash, NumberFor), AnnounceBlock(B::Hash, Vec), GetValue(record::Key), @@ -761,6 +835,8 @@ enum ServiceToWorkerMsg { protocol_name: Cow<'static, [u8]>, }, DisconnectPeer(PeerId), + UpdateChain, + OwnBlockImported(B::Hash, NumberFor), } /// Main network worker. Must be polled in order for the network to advance. @@ -796,6 +872,8 @@ struct Metrics { // This list is ordered alphabetically connections_closed_total: CounterVec, connections_opened_total: CounterVec, + distinct_peers_connections_closed_total: Counter, + distinct_peers_connections_opened_total: Counter, import_queue_blocks_submitted: Counter, import_queue_finality_proofs_submitted: Counter, import_queue_justifications_submitted: Counter, @@ -831,17 +909,25 @@ impl Metrics { connections_closed_total: register(CounterVec::new( Opts::new( "sub_libp2p_connections_closed_total", - "Total number of connections closed, by reason and direction" + "Total number of connections closed, by direction and reason" ), &["direction", "reason"] )?, registry)?, connections_opened_total: register(CounterVec::new( Opts::new( "sub_libp2p_connections_opened_total", - "Total number of connections opened" + "Total number of connections opened by direction" ), &["direction"] )?, registry)?, + distinct_peers_connections_closed_total: register(Counter::new( + "sub_libp2p_distinct_peers_connections_closed_total", + "Total number of connections closed with distinct peers" + )?, registry)?, + distinct_peers_connections_opened_total: register(Counter::new( + "sub_libp2p_distinct_peers_connections_opened_total", + "Total number of connections opened with distinct peers" + )?, registry)?, import_queue_blocks_submitted: register(Counter::new( "import_queue_blocks_submitted", "Number of blocks submitted to the import queue.", @@ -1061,10 +1147,10 @@ impl Future for NetworkWorker { this.network_service.user_protocol_mut().announce_block(hash, data), ServiceToWorkerMsg::RequestJustification(hash, number) => this.network_service.user_protocol_mut().request_justification(&hash, number), - ServiceToWorkerMsg::PropagateExtrinsic(hash) => - this.network_service.user_protocol_mut().propagate_extrinsic(&hash), - ServiceToWorkerMsg::PropagateExtrinsics => - this.network_service.user_protocol_mut().propagate_extrinsics(), + ServiceToWorkerMsg::PropagateTransaction(hash) => + this.network_service.user_protocol_mut().propagate_transaction(&hash), + ServiceToWorkerMsg::PropagateTransactions => + this.network_service.user_protocol_mut().propagate_transactions(), ServiceToWorkerMsg::GetValue(key) => this.network_service.get_value(&key), ServiceToWorkerMsg::PutValue(key, value) => @@ -1089,6 +1175,10 @@ impl Future for NetworkWorker { }, ServiceToWorkerMsg::DisconnectPeer(who) => this.network_service.user_protocol_mut().disconnect_peer(&who), + ServiceToWorkerMsg::UpdateChain => + this.network_service.user_protocol_mut().update_chain(), + ServiceToWorkerMsg::OwnBlockImported(hash, number) => + this.network_service.user_protocol_mut().own_block_imported(hash, number), } } @@ -1152,40 +1242,44 @@ impl Future for NetworkWorker { } this.event_streams.send(ev); }, - Poll::Ready(SwarmEvent::ConnectionEstablished { peer_id, endpoint, .. }) => { + Poll::Ready(SwarmEvent::ConnectionEstablished { peer_id, endpoint, num_established }) => { trace!(target: "sub-libp2p", "Libp2p => Connected({:?})", peer_id); + if let Some(metrics) = this.metrics.as_ref() { - match endpoint { - ConnectedPoint::Dialer { .. } => - metrics.connections_opened_total.with_label_values(&["out"]).inc(), - ConnectedPoint::Listener { .. } => - metrics.connections_opened_total.with_label_values(&["in"]).inc(), + let direction = match endpoint { + ConnectedPoint::Dialer { .. } => "out", + ConnectedPoint::Listener { .. } => "in", + }; + metrics.connections_opened_total.with_label_values(&[direction]).inc(); + + if num_established.get() == 1 { + metrics.distinct_peers_connections_opened_total.inc(); } } }, - Poll::Ready(SwarmEvent::ConnectionClosed { peer_id, cause, endpoint, .. }) => { + Poll::Ready(SwarmEvent::ConnectionClosed { peer_id, cause, endpoint, num_established }) => { trace!(target: "sub-libp2p", "Libp2p => Disconnected({:?}, {:?})", peer_id, cause); if let Some(metrics) = this.metrics.as_ref() { - let dir = match endpoint { + let direction = match endpoint { ConnectedPoint::Dialer { .. } => "out", ConnectedPoint::Listener { .. } => "in", }; - - match cause { - ConnectionError::IO(_) => - metrics.connections_closed_total.with_label_values(&[dir, "transport-error"]).inc(), + let reason = match cause { + ConnectionError::IO(_) => "transport-error", ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A( EitherError::A(EitherError::A(EitherError::B( - EitherError::A(PingFailure::Timeout)))))))) => - metrics.connections_closed_total.with_label_values(&[dir, "ping-timeout"]).inc(), + EitherError::A(PingFailure::Timeout)))))))) => "ping-timeout", ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A( EitherError::A(EitherError::A(EitherError::A( - EitherError::B(LegacyConnectionKillError)))))))) => - metrics.connections_closed_total.with_label_values(&[dir, "force-closed"]).inc(), - ConnectionError::Handler(NodeHandlerWrapperError::Handler(_)) => - metrics.connections_closed_total.with_label_values(&[dir, "protocol-error"]).inc(), - ConnectionError::Handler(NodeHandlerWrapperError::KeepAliveTimeout) => - metrics.connections_closed_total.with_label_values(&[dir, "keep-alive-timeout"]).inc(), + EitherError::B(LegacyConnectionKillError)))))))) => "force-closed", + ConnectionError::Handler(NodeHandlerWrapperError::Handler(_)) => "protocol-error", + ConnectionError::Handler(NodeHandlerWrapperError::KeepAliveTimeout) => "keep-alive-timeout", + }; + metrics.connections_closed_total.with_label_values(&[direction, reason]).inc(); + + // `num_established` represents the number of *remaining* connections. + if num_established == 0 { + metrics.distinct_peers_connections_closed_total.inc(); } } }, @@ -1196,7 +1290,7 @@ impl Future for NetworkWorker { } }, Poll::Ready(SwarmEvent::ExpiredListenAddr(addr)) => { - trace!(target: "sub-libp2p", "Libp2p => ExpiredListenAddr({})", addr); + info!(target: "sub-libp2p", "📪 No longer listening on {}", addr); if let Some(metrics) = this.metrics.as_ref() { metrics.listeners_local_addresses.dec(); } @@ -1264,10 +1358,23 @@ impl Future for NetworkWorker { trace!(target: "sub-libp2p", "Libp2p => UnknownPeerUnreachableAddr({}): {}", address, error), Poll::Ready(SwarmEvent::ListenerClosed { reason, addresses }) => { - warn!(target: "sub-libp2p", "Libp2p => ListenerClosed: {:?}", reason); if let Some(metrics) = this.metrics.as_ref() { metrics.listeners_local_addresses.sub(addresses.len() as u64); } + let addrs = addresses.into_iter().map(|a| a.to_string()) + .collect::>().join(", "); + match reason { + Ok(()) => error!( + target: "sub-libp2p", + "📪 Libp2p listener ({}) closed gracefully", + addrs + ), + Err(e) => error!( + target: "sub-libp2p", + "📪 Libp2p listener ({}) closed: {}", + addrs, e + ), + } }, Poll::Ready(SwarmEvent::ListenerError { error }) => { trace!(target: "sub-libp2p", "Libp2p => ListenerError: {}", error); @@ -1349,7 +1456,7 @@ impl<'a, B: BlockT, H: ExHashT> Link for NetworkLink<'a, B, H> { count: usize, results: Vec<(Result>, BlockImportError>, B::Hash)> ) { - self.protocol.user_protocol_mut().blocks_processed(imported, count, results) + self.protocol.user_protocol_mut().on_blocks_processed(imported, count, results) } fn justification_imported(&mut self, who: PeerId, hash: &B::Hash, number: NumberFor, success: bool) { self.protocol.user_protocol_mut().justification_import_result(hash.clone(), number, success); @@ -1380,3 +1487,40 @@ impl<'a, B: BlockT, H: ExHashT> Link for NetworkLink<'a, B, H> { } } } + +fn ensure_addresses_consistent_with_transport<'a>( + addresses: impl Iterator, + transport: &TransportConfig, +) -> Result<(), Error> { + if matches!(transport, TransportConfig::MemoryOnly) { + let addresses: Vec<_> = addresses + .filter(|x| x.iter() + .any(|y| !matches!(y, libp2p::core::multiaddr::Protocol::Memory(_))) + ) + .cloned() + .collect(); + + if !addresses.is_empty() { + return Err(Error::AddressesForAnotherTransport { + transport: transport.clone(), + addresses, + }); + } + } else { + let addresses: Vec<_> = addresses + .filter(|x| x.iter() + .any(|y| matches!(y, libp2p::core::multiaddr::Protocol::Memory(_))) + ) + .cloned() + .collect(); + + if !addresses.is_empty() { + return Err(Error::AddressesForAnotherTransport { + transport: transport.clone(), + addresses, + }); + } + } + + Ok(()) +} diff --git a/client/network/src/service/out_events.rs b/client/network/src/service/out_events.rs index 08c0968cf94a00bc01f35454a0473853d3b0fe71..1b86a5fa4317d633c70f3dd4d97bf8146d9c81dd 100644 --- a/client/network/src/service/out_events.rs +++ b/client/network/src/service/out_events.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Registering events streams. //! //! This code holds the logic that is used for the network service to inform other parts of @@ -34,7 +35,7 @@ use crate::Event; use super::maybe_utf8_bytes_to_string; -use futures::{prelude::*, channel::mpsc, ready}; +use futures::{prelude::*, channel::mpsc, ready, stream::FusedStream}; use parking_lot::Mutex; use prometheus_endpoint::{register, CounterVec, GaugeVec, Opts, PrometheusError, Registry, U64}; use std::{ @@ -118,8 +119,10 @@ impl fmt::Debug for Receiver { impl Drop for Receiver { fn drop(&mut self) { - // Empty the list to properly decrease the metrics. - while let Some(Some(_)) = self.next().now_or_never() {} + if !self.inner.is_terminated() { + // Empty the list to properly decrease the metrics. + while let Some(Some(_)) = self.next().now_or_never() {} + } } } diff --git a/client/network/src/service/tests.rs b/client/network/src/service/tests.rs index eada49d74161c6db34782c4cbdc9928f1e5c6d01..17d9553fa66bb2e527fe260561effe14e073ba36 100644 --- a/client/network/src/service/tests.rs +++ b/client/network/src/service/tests.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,8 +15,10 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::{config, Event, NetworkService, NetworkWorker}; +use libp2p::PeerId; use futures::prelude::*; use sp_runtime::traits::{Block as BlockT, Header as _}; use std::{sync::Arc, time::Duration}; @@ -102,7 +104,7 @@ fn build_test_full_node(config: config::NetworkConfiguration) protocol_id: config::ProtocolId::from(&b"/test-protocol-name"[..]), import_queue, block_announce_validator: Box::new( - sp_consensus::block_validation::DefaultBlockAnnounceValidator::new(client.clone()), + sp_consensus::block_validation::DefaultBlockAnnounceValidator, ), metrics_registry: None, }) @@ -137,6 +139,7 @@ fn build_nodes_one_proto() let (node2, events_stream2) = build_test_full_node(config::NetworkConfiguration { notifications_protocols: vec![(ENGINE_ID, From::from(&b"/foo"[..]))], + listen_addresses: vec![], reserved_nodes: vec![config::MultiaddrWithPeerId { multiaddr: listen_addr, peer_id: node1.local_peer_id().clone(), @@ -271,3 +274,189 @@ fn notifications_state_consistent() { } }); } + +#[test] +fn lots_of_incoming_peers_works() { + let listen_addr = config::build_multiaddr![Memory(rand::random::())]; + + let (main_node, _) = build_test_full_node(config::NetworkConfiguration { + notifications_protocols: vec![(ENGINE_ID, From::from(&b"/foo"[..]))], + listen_addresses: vec![listen_addr.clone()], + in_peers: u32::max_value(), + transport: config::TransportConfig::MemoryOnly, + .. config::NetworkConfiguration::new_local() + }); + + let main_node_peer_id = main_node.local_peer_id().clone(); + + // We spawn background tasks and push them in this `Vec`. They will all be waited upon before + // this test ends. + let mut background_tasks_to_wait = Vec::new(); + + for _ in 0..32 { + let main_node_peer_id = main_node_peer_id.clone(); + + let (_dialing_node, event_stream) = build_test_full_node(config::NetworkConfiguration { + notifications_protocols: vec![(ENGINE_ID, From::from(&b"/foo"[..]))], + listen_addresses: vec![], + reserved_nodes: vec![config::MultiaddrWithPeerId { + multiaddr: listen_addr.clone(), + peer_id: main_node_peer_id.clone(), + }], + transport: config::TransportConfig::MemoryOnly, + .. config::NetworkConfiguration::new_local() + }); + + background_tasks_to_wait.push(async_std::task::spawn(async move { + // Create a dummy timer that will "never" fire, and that will be overwritten when we + // actually need the timer. Using an Option would be technically cleaner, but it would + // make the code below way more complicated. + let mut timer = futures_timer::Delay::new(Duration::from_secs(3600 * 24 * 7)).fuse(); + + let mut event_stream = event_stream.fuse(); + loop { + futures::select! { + _ = timer => { + // Test succeeds when timer fires. + return; + } + ev = event_stream.next() => { + match ev.unwrap() { + Event::NotificationStreamOpened { remote, .. } => { + assert_eq!(remote, main_node_peer_id); + // Test succeeds after 5 seconds. This timer is here in order to + // detect a potential problem after opening. + timer = futures_timer::Delay::new(Duration::from_secs(5)).fuse(); + } + Event::NotificationStreamClosed { .. } => { + // Test failed. + panic!(); + } + _ => {} + } + } + } + } + })); + } + + futures::executor::block_on(async move { + future::join_all(background_tasks_to_wait).await + }); +} + +#[test] +#[should_panic(expected = "don't match the transport")] +fn ensure_listen_addresses_consistent_with_transport_memory() { + let listen_addr = config::build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(0_u16)]; + + let _ = build_test_full_node(config::NetworkConfiguration { + listen_addresses: vec![listen_addr.clone()], + transport: config::TransportConfig::MemoryOnly, + .. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None) + }); +} + +#[test] +#[should_panic(expected = "don't match the transport")] +fn ensure_listen_addresses_consistent_with_transport_not_memory() { + let listen_addr = config::build_multiaddr![Memory(rand::random::())]; + + let _ = build_test_full_node(config::NetworkConfiguration { + listen_addresses: vec![listen_addr.clone()], + .. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None) + }); +} + +#[test] +#[should_panic(expected = "don't match the transport")] +fn ensure_boot_node_addresses_consistent_with_transport_memory() { + let listen_addr = config::build_multiaddr![Memory(rand::random::())]; + let boot_node = config::MultiaddrWithPeerId { + multiaddr: config::build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(0_u16)], + peer_id: PeerId::random(), + }; + + let _ = build_test_full_node(config::NetworkConfiguration { + listen_addresses: vec![listen_addr.clone()], + transport: config::TransportConfig::MemoryOnly, + boot_nodes: vec![boot_node], + .. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None) + }); +} + +#[test] +#[should_panic(expected = "don't match the transport")] +fn ensure_boot_node_addresses_consistent_with_transport_not_memory() { + let listen_addr = config::build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(0_u16)]; + let boot_node = config::MultiaddrWithPeerId { + multiaddr: config::build_multiaddr![Memory(rand::random::())], + peer_id: PeerId::random(), + }; + + let _ = build_test_full_node(config::NetworkConfiguration { + listen_addresses: vec![listen_addr.clone()], + boot_nodes: vec![boot_node], + .. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None) + }); +} + +#[test] +#[should_panic(expected = "don't match the transport")] +fn ensure_reserved_node_addresses_consistent_with_transport_memory() { + let listen_addr = config::build_multiaddr![Memory(rand::random::())]; + let reserved_node = config::MultiaddrWithPeerId { + multiaddr: config::build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(0_u16)], + peer_id: PeerId::random(), + }; + + let _ = build_test_full_node(config::NetworkConfiguration { + listen_addresses: vec![listen_addr.clone()], + transport: config::TransportConfig::MemoryOnly, + reserved_nodes: vec![reserved_node], + .. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None) + }); +} + +#[test] +#[should_panic(expected = "don't match the transport")] +fn ensure_reserved_node_addresses_consistent_with_transport_not_memory() { + let listen_addr = config::build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(0_u16)]; + let reserved_node = config::MultiaddrWithPeerId { + multiaddr: config::build_multiaddr![Memory(rand::random::())], + peer_id: PeerId::random(), + }; + + let _ = build_test_full_node(config::NetworkConfiguration { + listen_addresses: vec![listen_addr.clone()], + reserved_nodes: vec![reserved_node], + .. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None) + }); +} + +#[test] +#[should_panic(expected = "don't match the transport")] +fn ensure_public_addresses_consistent_with_transport_memory() { + let listen_addr = config::build_multiaddr![Memory(rand::random::())]; + let public_address = config::build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(0_u16)]; + + let _ = build_test_full_node(config::NetworkConfiguration { + listen_addresses: vec![listen_addr.clone()], + transport: config::TransportConfig::MemoryOnly, + public_addresses: vec![public_address], + .. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None) + }); +} + +#[test] +#[should_panic(expected = "don't match the transport")] +fn ensure_public_addresses_consistent_with_transport_not_memory() { + let listen_addr = config::build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(0_u16)]; + let public_address = config::build_multiaddr![Memory(rand::random::())]; + + let _ = build_test_full_node(config::NetworkConfiguration { + listen_addresses: vec![listen_addr.clone()], + public_addresses: vec![public_address], + .. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None) + }); +} diff --git a/client/network/src/transport.rs b/client/network/src/transport.rs index 5dca5ff43bdd64749a9e143caea6748ec627264f..0c9a809384e4cf6dc8aaf218cbfc610973d67fe5 100644 --- a/client/network/src/transport.rs +++ b/client/network/src/transport.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,14 +15,18 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use futures::prelude::*; use libp2p::{ InboundUpgradeExt, OutboundUpgradeExt, PeerId, Transport, + core::{ + self, either::{EitherError, EitherOutput}, muxing::StreamMuxerBox, + transport::{boxed::Boxed, OptionalTransport}, upgrade + }, mplex, identity, bandwidth, wasm_ext, noise }; #[cfg(not(target_os = "unknown"))] use libp2p::{tcp, dns, websocket}; -use libp2p::core::{self, upgrade, transport::boxed::Boxed, transport::OptionalTransport, muxing::StreamMuxerBox}; use std::{io, sync::Arc, time::Duration, usize}; pub use self::bandwidth::BandwidthSinks; @@ -42,14 +46,22 @@ pub fn build_transport( ) -> (Boxed<(PeerId, StreamMuxerBox), io::Error>, Arc) { // Build configuration objects for encryption mechanisms. let noise_config = { - let noise_keypair = noise::Keypair::new().into_authentic(&keypair) - // For more information about this panic, see in "On the Importance of Checking - // Cryptographic Protocols for Faults" by Dan Boneh, Richard A. DeMillo, - // and Richard J. Lipton. + // For more information about these two panics, see in "On the Importance of + // Checking Cryptographic Protocols for Faults" by Dan Boneh, Richard A. DeMillo, + // and Richard J. Lipton. + let noise_keypair_legacy = noise::Keypair::::new().into_authentic(&keypair) + .expect("can only fail in case of a hardware bug; since this signing is performed only \ + once and at initialization, we're taking the bet that the inconvenience of a very \ + rare panic here is basically zero"); + let noise_keypair_spec = noise::Keypair::::new().into_authentic(&keypair) .expect("can only fail in case of a hardware bug; since this signing is performed only \ once and at initialization, we're taking the bet that the inconvenience of a very \ rare panic here is basically zero"); - noise::NoiseConfig::ix(noise_keypair) + + core::upgrade::SelectUpgrade::new( + noise::NoiseConfig::xx(noise_keypair_spec), + noise::NoiseConfig::ix(noise_keypair_legacy) + ) }; // Build configuration objects for multiplexing mechanisms. @@ -97,11 +109,22 @@ pub fn build_transport( // Encryption let transport = transport.and_then(move |stream, endpoint| { core::upgrade::apply(stream, noise_config, endpoint, upgrade::Version::V1) - .and_then(|(remote_id, out)| async move { - let remote_key = match remote_id { - noise::RemoteIdentity::IdentityKey(key) => key, + .map_err(|err| + err.map_err(|err| match err { + EitherError::A(err) => err, + EitherError::B(err) => err, + }) + ) + .and_then(|result| async move { + let remote_key = match &result { + EitherOutput::First((noise::RemoteIdentity::IdentityKey(key), _)) => key.clone(), + EitherOutput::Second((noise::RemoteIdentity::IdentityKey(key), _)) => key.clone(), _ => return Err(upgrade::UpgradeError::Apply(noise::NoiseError::InvalidKey)) }; + let out = match result { + EitherOutput::First((_, o)) => o, + EitherOutput::Second((_, o)) => o, + }; Ok((out, remote_key.into_peer_id())) }) }); diff --git a/client/network/test/Cargo.toml b/client/network/test/Cargo.toml index 38e1ceb8ea7dd1ec226ece0c710e762f08a8d9d9..6527d093bd6fed3cb69faedc744bd5868bfbf4d6 100644 --- a/client/network/test/Cargo.toml +++ b/client/network/test/Cargo.toml @@ -1,7 +1,7 @@ [package] description = "Integration tests for Substrate network protocol" name = "sc-network-test" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" authors = ["Parity Technologies "] edition = "2018" @@ -13,23 +13,23 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sc-network = { version = "0.8.0-alpha.8", path = "../" } +sc-network = { version = "0.8.0-rc4", path = "../" } log = "0.4.8" parking_lot = "0.10.0" futures = "0.3.4" futures-timer = "3.0.1" rand = "0.7.2" -libp2p = { version = "0.18.1", default-features = false, features = ["libp2p-websocket"] } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/common" } -sc-consensus = { version = "0.8.0-alpha.8", path = "../../../client/consensus/common" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../../api" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sc-block-builder = { version = "0.8.0-alpha.8", path = "../../block-builder" } -sp-consensus-babe = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/babe" } +libp2p = { version = "0.20.1", default-features = false } +sp-consensus = { version = "0.8.0-rc4", path = "../../../primitives/consensus/common" } +sc-consensus = { version = "0.8.0-rc4", path = "../../../client/consensus/common" } +sc-client-api = { version = "2.0.0-rc4", path = "../../api" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sc-block-builder = { version = "0.8.0-rc4", path = "../../block-builder" } +sp-consensus-babe = { version = "0.8.0-rc4", path = "../../../primitives/consensus/babe" } env_logger = "0.7.0" -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../../test-utils/runtime/client" } -substrate-test-runtime = { version = "2.0.0-alpha.8", path = "../../../test-utils/runtime" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../../test-utils/runtime/client" } +substrate-test-runtime = { version = "2.0.0-rc4", path = "../../../test-utils/runtime" } tempfile = "3.1.0" -sc-service = { version = "0.8.0-alpha.8", default-features = false, features = ["test-helpers"], path = "../../service" } +sc-service = { version = "0.8.0-rc4", default-features = false, features = ["test-helpers"], path = "../../service" } diff --git a/client/network/test/src/block_import.rs b/client/network/test/src/block_import.rs index 4e66ff879f1206b143b95c0bd52377a887b260e8..6762b74b6b8bd825c9c200015162bfd9274d845e 100644 --- a/client/network/test/src/block_import.rs +++ b/client/network/test/src/block_import.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Testing block import logic. use sp_consensus::ImportedAux; @@ -54,7 +55,12 @@ fn import_single_good_block_works() { let mut expected_aux = ImportedAux::default(); expected_aux.is_new_best = true; - match import_single_block(&mut substrate_test_runtime_client::new(), BlockOrigin::File, block, &mut PassThroughVerifier(true)) { + match import_single_block( + &mut substrate_test_runtime_client::new(), + BlockOrigin::File, + block, + &mut PassThroughVerifier::new(true) + ) { Ok(BlockImportResult::ImportedUnknown(ref num, ref aux, ref org)) if *num == number && *aux == expected_aux && *org == Some(peer_id) => {} r @ _ => panic!("{:?}", r) @@ -64,7 +70,12 @@ fn import_single_good_block_works() { #[test] fn import_single_good_known_block_is_ignored() { let (mut client, _hash, number, _, block) = prepare_good_block(); - match import_single_block(&mut client, BlockOrigin::File, block, &mut PassThroughVerifier(true)) { + match import_single_block( + &mut client, + BlockOrigin::File, + block, + &mut PassThroughVerifier::new(true) + ) { Ok(BlockImportResult::ImportedKnown(ref n)) if *n == number => {} _ => panic!() } @@ -74,7 +85,12 @@ fn import_single_good_known_block_is_ignored() { fn import_single_good_block_without_header_fails() { let (_, _, _, peer_id, mut block) = prepare_good_block(); block.header = None; - match import_single_block(&mut substrate_test_runtime_client::new(), BlockOrigin::File, block, &mut PassThroughVerifier(true)) { + match import_single_block( + &mut substrate_test_runtime_client::new(), + BlockOrigin::File, + block, + &mut PassThroughVerifier::new(true) + ) { Err(BlockImportError::IncompleteHeader(ref org)) if *org == Some(peer_id) => {} _ => panic!() } @@ -85,7 +101,7 @@ fn async_import_queue_drops() { let executor = sp_core::testing::SpawnBlockingExecutor::new(); // Perform this test multiple times since it exhibits non-deterministic behavior. for _ in 0..100 { - let verifier = PassThroughVerifier(true); + let verifier = PassThroughVerifier::new(true); let queue = BasicQueue::new( verifier, diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 76bc2afa6953cf09b39048aba733869077b80b61..2896c4e3e1853fde5da5a0cc3cf8f13c3c14e042 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -39,7 +39,7 @@ use sc_client_api::{ use sc_consensus::LongestChain; use sc_block_builder::{BlockBuilder, BlockBuilderProvider}; use sc_network::config::Role; -use sp_consensus::block_validation::DefaultBlockAnnounceValidator; +use sp_consensus::block_validation::{DefaultBlockAnnounceValidator, BlockAnnounceValidator}; use sp_consensus::import_queue::{ BasicQueue, BoxJustificationImport, Verifier, BoxFinalityProofImport, }; @@ -67,7 +67,33 @@ type AuthorityId = sp_consensus_babe::AuthorityId; /// A Verifier that accepts all blocks and passes them on with the configured /// finality to be imported. #[derive(Clone)] -pub struct PassThroughVerifier(pub bool); +pub struct PassThroughVerifier { + finalized: bool, + fork_choice: ForkChoiceStrategy, +} + +impl PassThroughVerifier { + /// Create a new instance. + /// + /// Every verified block will use `finalized` for the `BlockImportParams`. + pub fn new(finalized: bool) -> Self { + Self { + finalized, + fork_choice: ForkChoiceStrategy::LongestChain, + } + } + + /// Create a new instance. + /// + /// Every verified block will use `finalized` for the `BlockImportParams` and + /// the given [`ForkChoiceStrategy`]. + pub fn new_with_fork_choice(finalized: bool, fork_choice: ForkChoiceStrategy) -> Self { + Self { + finalized, + fork_choice, + } + } +} /// This `Verifier` accepts all data as valid. impl Verifier for PassThroughVerifier { @@ -85,9 +111,9 @@ impl Verifier for PassThroughVerifier { .map(|blob| vec![(well_known_cache_keys::AUTHORITIES, blob.to_vec())]); let mut import = BlockImportParams::new(origin, header); import.body = body; - import.finalized = self.0; + import.finalized = self.finalized; import.justification = justification; - import.fork_choice = Some(ForkChoiceStrategy::LongestChain); + import.fork_choice = Some(self.fork_choice.clone()); Ok((import, maybe_keys)) } @@ -221,9 +247,9 @@ impl Peer { self.network.num_connected_peers() } - /// Returns the number of processed blocks. - pub fn num_processed_blocks(&self) -> usize { - self.network.num_processed_blocks() + /// Returns the number of downloaded blocks. + pub fn num_downloaded_blocks(&self) -> usize { + self.network.num_downloaded_blocks() } /// Returns true if we have no peer. @@ -294,12 +320,13 @@ impl Peer { } else { Default::default() }; + self.block_import.import_block(import_block, cache).expect("block_import failed"); - self.network.on_block_imported(header, true); self.network.service().announce_block(hash, Vec::new()); at = hash; } + self.network.update_chain(); self.network.service().announce_block(at.clone(), Vec::new()); at } @@ -519,6 +546,15 @@ impl VerifierAdapter { } } +/// Configuration for a full peer. +#[derive(Default)] +pub struct FullPeerConfig { + /// Pruning window size. + pub keep_blocks: Option, + /// Block announce validator. + pub block_announce_validator: Option + Send + Sync>>, +} + pub trait TestNetFactory: Sized { type Verifier: 'static + Verifier; type PeerData: Default; @@ -579,12 +615,12 @@ pub trait TestNetFactory: Sized { } fn add_full_peer(&mut self) { - self.add_full_peer_with_states(None) + self.add_full_peer_with_config(Default::default()) } /// Add a full peer. - fn add_full_peer_with_states(&mut self, keep_blocks: Option) { - let test_client_builder = match keep_blocks { + fn add_full_peer_with_config(&mut self, config: FullPeerConfig) { + let test_client_builder = match config.keep_blocks { Some(keep_blocks) => TestClientBuilder::with_pruning_window(keep_blocks), None => TestClientBuilder::with_default_backend(), }; @@ -641,7 +677,8 @@ pub trait TestNetFactory: Sized { transaction_pool: Arc::new(EmptyTransactionPool), protocol_id: ProtocolId::from(&b"test-protocol-name"[..]), import_queue, - block_announce_validator: Box::new(DefaultBlockAnnounceValidator::new(client.clone())), + block_announce_validator: config.block_announce_validator + .unwrap_or(Box::new(DefaultBlockAnnounceValidator)), metrics_registry: None, }).unwrap(); @@ -720,7 +757,7 @@ pub trait TestNetFactory: Sized { transaction_pool: Arc::new(EmptyTransactionPool), protocol_id: ProtocolId::from(&b"test-protocol-name"[..]), import_queue, - block_announce_validator: Box::new(DefaultBlockAnnounceValidator::new(client.clone())), + block_announce_validator: Box::new(DefaultBlockAnnounceValidator), metrics_registry: None, }).unwrap(); @@ -787,6 +824,20 @@ pub trait TestNetFactory: Sized { Poll::Ready(()) } + /// Polls the testnet until all peers are connected to each other. + /// + /// Must be executed in a task context. + fn poll_until_connected(&mut self, cx: &mut FutureContext) -> Poll<()> { + self.poll(cx); + + let num_peers = self.peers().len(); + if self.peers().iter().all(|p| p.num_peers() == num_peers - 1) { + return Poll::Ready(()) + } + + Poll::Pending + } + /// Blocks the current thread until we are sync'ed. /// /// Calls `poll_until_sync` repeatedly. @@ -801,6 +852,15 @@ pub trait TestNetFactory: Sized { futures::executor::block_on(futures::future::poll_fn::<(), _>(|cx| self.poll_until_idle(cx))); } + /// Blocks the current thread until all peers are connected to each other. + /// + /// Calls `poll_until_connected` repeatedly with the runtime passed as parameter. + fn block_until_connected(&mut self) { + futures::executor::block_on( + futures::future::poll_fn::<(), _>(|cx| self.poll_until_connected(cx)), + ); + } + /// Polls the testnet. Processes all the pending actions and returns `NotReady`. fn poll(&mut self, cx: &mut FutureContext) { self.mut_peers(|peers| { @@ -813,10 +873,6 @@ pub trait TestNetFactory: Sized { // We poll `imported_blocks_stream`. while let Poll::Ready(Some(notification)) = peer.imported_blocks_stream.as_mut().poll_next(cx) { - peer.network.on_block_imported( - notification.header, - true, - ); peer.network.service().announce_block(notification.hash, Vec::new()); } @@ -835,6 +891,17 @@ pub trait TestNetFactory: Sized { pub struct TestNet { peers: Vec>, + fork_choice: ForkChoiceStrategy, +} + +impl TestNet { + /// Create a `TestNet` that used the given fork choice rule. + pub fn with_fork_choice(fork_choice: ForkChoiceStrategy) -> Self { + Self { + peers: Vec::new(), + fork_choice, + } + } } impl TestNetFactory for TestNet { @@ -845,13 +912,14 @@ impl TestNetFactory for TestNet { fn from_config(_config: &ProtocolConfig) -> Self { TestNet { peers: Vec::new(), + fork_choice: ForkChoiceStrategy::LongestChain, } } fn make_verifier(&self, _client: PeersClient, _config: &ProtocolConfig, _peer_data: &()) -> Self::Verifier { - PassThroughVerifier(false) + PassThroughVerifier::new_with_fork_choice(false, self.fork_choice.clone()) } fn peer(&mut self, i: usize) -> &mut Peer<()> { diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 50051540d29b6ca3fbbb2325a16f181d9a264f84..1cf2a8fee37982495811a2bd464bb0eb5644b728 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,10 +15,13 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use sp_consensus::BlockOrigin; use std::time::Duration; use futures::executor::block_on; use super::*; +use sp_consensus::block_validation::Validation; +use substrate_test_runtime::Header; fn test_ancestor_search_when_common_is(n: usize) { let _ = ::env_logger::try_init(); @@ -581,10 +584,10 @@ fn can_sync_explicit_forks() { #[test] fn syncs_header_only_forks() { - let _ = ::env_logger::try_init(); + let _ = env_logger::try_init(); let mut net = TestNet::new(0); - net.add_full_peer_with_states(None); - net.add_full_peer_with_states(Some(3)); + net.add_full_peer_with_config(Default::default()); + net.add_full_peer_with_config(FullPeerConfig { keep_blocks: Some(3), ..Default::default() }); net.peer(0).push_blocks(2, false); net.peer(1).push_blocks(2, false); @@ -672,17 +675,17 @@ fn imports_stale_once() { // check that NEW block is imported from announce message let new_hash = net.peer(0).push_blocks(1, false); import_with_announce(&mut net, new_hash); - assert_eq!(net.peer(1).num_processed_blocks(), 1); + assert_eq!(net.peer(1).num_downloaded_blocks(), 1); // check that KNOWN STALE block is imported from announce message let known_stale_hash = net.peer(0).push_blocks_at(BlockId::Number(0), 1, true); import_with_announce(&mut net, known_stale_hash); - assert_eq!(net.peer(1).num_processed_blocks(), 2); + assert_eq!(net.peer(1).num_downloaded_blocks(), 2); } #[test] fn can_sync_to_peers_with_wrong_common_block() { - let _ = ::env_logger::try_init(); + let _ = env_logger::try_init(); let mut net = TestNet::new(2); net.peer(0).push_blocks(2, true); @@ -709,3 +712,41 @@ fn can_sync_to_peers_with_wrong_common_block() { assert!(net.peer(1).client().header(&BlockId::Hash(final_hash)).unwrap().is_some()); } +/// Returns `is_new_best = true` for each validated announcement. +struct NewBestBlockAnnounceValidator; + +impl BlockAnnounceValidator for NewBestBlockAnnounceValidator { + fn validate( + &mut self, + _: &Header, + _: &[u8], + ) -> Result> { + Ok(Validation::Success { is_new_best: true }) + } +} + +#[test] +fn sync_blocks_when_block_announce_validator_says_it_is_new_best() { + let _ = env_logger::try_init(); + log::trace!(target: "sync", "Test"); + let mut net = TestNet::with_fork_choice(ForkChoiceStrategy::Custom(false)); + net.add_full_peer_with_config(Default::default()); + net.add_full_peer_with_config(Default::default()); + net.add_full_peer_with_config(FullPeerConfig { + block_announce_validator: Some(Box::new(NewBestBlockAnnounceValidator)), + ..Default::default() + }); + + net.block_until_connected(); + + let block_hash = net.peer(0).push_blocks(1, false); + + while !net.peer(2).has_block(&block_hash) { + net.block_until_idle(); + } + + // Peer1 should not have the block, because peer 0 did not reported the block + // as new best. However, peer2 has a special block announcement validator + // that flags all blocks as `is_new_best` and thus, it should have synced the blocks. + assert!(!net.peer(1).has_block(&block_hash)); +} diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml index 29c36331df03374ab16a84108877f3b29787eb62..cd5a63a75c8f964c7cf66c0eb326772d353eaf3e 100644 --- a/client/offchain/Cargo.toml +++ b/client/offchain/Cargo.toml @@ -1,7 +1,7 @@ [package] description = "Substrate offchain workers" name = "sc-offchain" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" authors = ["Parity Technologies "] edition = "2018" @@ -13,23 +13,23 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] bytes = "0.5" -sc-client-api = { version = "2.0.0-alpha.8", path = "../api" } -sp-api = { version = "2.0.0-alpha.8", path = "../../primitives/api" } +sc-client-api = { version = "2.0.0-rc4", path = "../api" } +sp-api = { version = "2.0.0-rc4", path = "../../primitives/api" } fnv = "1.0.6" futures = "0.3.4" futures-timer = "3.0.1" log = "0.4.8" threadpool = "1.7" num_cpus = "1.10" -sp-offchain = { version = "2.0.0-alpha.8", path = "../../primitives/offchain" } -codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } +sp-offchain = { version = "2.0.0-rc4", path = "../../primitives/offchain" } +codec = { package = "parity-scale-codec", version = "1.3.1", features = ["derive"] } parking_lot = "0.10.0" -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } rand = "0.7.2" -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } -sp-utils = { version = "2.0.0-alpha.8", path = "../../primitives/utils" } -sc-network = { version = "0.8.0-alpha.8", path = "../network" } -sc-keystore = { version = "2.0.0-alpha.8", path = "../keystore" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-rc4", path = "../../primitives/utils" } +sc-network = { version = "0.8.0-rc4", path = "../network" } +sc-keystore = { version = "2.0.0-rc4", path = "../keystore" } [target.'cfg(not(target_os = "unknown"))'.dependencies] hyper = "0.13.2" @@ -37,12 +37,12 @@ hyper-rustls = "0.20" [dev-dependencies] env_logger = "0.7.0" -fdlimit = "0.1.4" -sc-client-db = { version = "0.8.0-alpha.8", default-features = true, path = "../db/" } -sc-transaction-pool = { version = "2.0.0-alpha.8", path = "../../client/transaction-pool" } -sp-transaction-pool = { version = "2.0.0-alpha.8", path = "../../primitives/transaction-pool" } -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../test-utils/runtime/client" } +sc-client-db = { version = "0.8.0-rc4", default-features = true, path = "../db/" } +sc-transaction-pool = { version = "2.0.0-rc4", path = "../../client/transaction-pool" } +sp-transaction-pool = { version = "2.0.0-rc4", path = "../../primitives/transaction-pool" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../test-utils/runtime/client" } tokio = "0.2" +lazy_static = "1.4.0" [features] default = [] diff --git a/client/offchain/src/api.rs b/client/offchain/src/api.rs index 45a82d230c11a6dd6d6c8ce03635d1c49c6d7ce7..0aa5d4ad788abe1fdbf8c0525ac375cd66a68375 100644 --- a/client/offchain/src/api.rs +++ b/client/offchain/src/api.rs @@ -31,6 +31,7 @@ use sp_core::offchain::{ OpaqueNetworkState, OpaquePeerId, OpaqueMultiaddr, StorageKind, }; pub use sp_offchain::STORAGE_PREFIX; +pub use http::SharedClient; #[cfg(not(target_os = "unknown"))] mod http; @@ -100,6 +101,13 @@ impl OffchainExt for Api { } } + fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { + match kind { + StorageKind::PERSISTENT => self.db.remove(STORAGE_PREFIX, key), + StorageKind::LOCAL => unavailable_yet(LOCAL_DB), + } + } + fn local_storage_compare_and_set( &mut self, kind: StorageKind, @@ -253,8 +261,9 @@ impl AsyncApi { db: S, network_state: Arc, is_validator: bool, - ) -> (Api, AsyncApi) { - let (http_api, http_worker) = http::http(); + shared_client: SharedClient, + ) -> (Api, Self) { + let (http_api, http_worker) = http::http(shared_client); let api = Api { db, @@ -263,7 +272,7 @@ impl AsyncApi { http: http_api, }; - let async_api = AsyncApi { + let async_api = Self { http: Some(http_worker), }; @@ -301,11 +310,14 @@ mod tests { let _ = env_logger::try_init(); let db = LocalStorage::new_test(); let mock = Arc::new(MockNetworkStateInfo()); + let shared_client = SharedClient::new(); + AsyncApi::new( db, mock, false, + shared_client, ) } diff --git a/client/offchain/src/api/http.rs b/client/offchain/src/api/http.rs index a64fe0389706c1098eda9f1c2fbd2294843d7597..1f542b7c11e19b8bd76a39ec3c6555b5c716a866 100644 --- a/client/offchain/src/api/http.rs +++ b/client/offchain/src/api/http.rs @@ -31,11 +31,24 @@ use fnv::FnvHashMap; use futures::{prelude::*, future, channel::mpsc}; use log::error; use sp_core::offchain::{HttpRequestId, Timestamp, HttpRequestStatus, HttpError}; -use std::{fmt, io::Read as _, mem, pin::Pin, task::Context, task::Poll}; +use std::{convert::TryFrom, fmt, io::Read as _, pin::Pin, task::{Context, Poll}}; use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver}; +use std::sync::Arc; +use hyper::{Client as HyperClient, Body, client}; +use hyper_rustls::HttpsConnector; + +/// Wrapper struct used for keeping the hyper_rustls client running. +#[derive(Clone)] +pub struct SharedClient(Arc, Body>>); + +impl SharedClient { + pub fn new() -> Self { + Self(Arc::new(HyperClient::builder().build(HttpsConnector::new()))) + } +} /// Creates a pair of [`HttpApi`] and [`HttpWorker`]. -pub fn http() -> (HttpApi, HttpWorker) { +pub fn http(shared_client: SharedClient) -> (HttpApi, HttpWorker) { let (to_worker, from_api) = tracing_unbounded("mpsc_ocw_to_worker"); let (to_api, from_worker) = tracing_unbounded("mpsc_ocw_to_api"); @@ -51,7 +64,7 @@ pub fn http() -> (HttpApi, HttpWorker) { let engine = HttpWorker { to_api, from_api, - http_client: hyper::Client::builder().build(hyper_rustls::HttpsConnector::new()), + http_client: shared_client.0, requests: Vec::new(), }; @@ -151,8 +164,8 @@ impl HttpApi { _ => return Err(()) }; - let name = hyper::header::HeaderName::from_bytes(name.as_bytes()).map_err(|_| ())?; - let value = hyper::header::HeaderValue::from_str(value).map_err(|_| ())?; + let name = hyper::header::HeaderName::try_from(name).map_err(drop)?; + let value = hyper::header::HeaderValue::try_from(value).map_err(drop)?; // Note that we're always appending headers and never replacing old values. // We assume here that the user knows what they're doing. request.headers_mut().append(name, value); @@ -185,7 +198,7 @@ impl HttpApi { future::MaybeDone::Done(Err(_)) => return Err(HttpError::IoError), future::MaybeDone::Future(_) | future::MaybeDone::Gone => { - debug_assert!(if let future::MaybeDone::Done(_) = deadline { true } else { false }); + debug_assert!(matches!(deadline, future::MaybeDone::Done(..))); return Err(HttpError::DeadlineReached) } }; @@ -347,7 +360,7 @@ impl HttpApi { if let future::MaybeDone::Done(msg) = next_msg { msg } else { - debug_assert!(if let future::MaybeDone::Done(_) = deadline { true } else { false }); + debug_assert!(matches!(deadline, future::MaybeDone::Done(..))); continue } }; @@ -551,7 +564,7 @@ pub struct HttpWorker { /// Used to receive messages from the `HttpApi`. from_api: TracingUnboundedReceiver, /// The engine that runs HTTP requests. - http_client: hyper::Client, hyper::Body>, + http_client: Arc, Body>>, /// HTTP requests that are being worked on by the engine. requests: Vec<(HttpRequestId, HttpWorkerRequest)>, } @@ -585,25 +598,21 @@ impl Future for HttpWorker { match request { HttpWorkerRequest::Dispatched(mut future) => { // Check for an HTTP response from the Internet. - let mut response = match Future::poll(Pin::new(&mut future), cx) { + let response = match Future::poll(Pin::new(&mut future), cx) { Poll::Pending => { me.requests.push((id, HttpWorkerRequest::Dispatched(future))); continue }, Poll::Ready(Ok(response)) => response, - Poll::Ready(Err(err)) => { - let _ = me.to_api.unbounded_send(WorkerToApi::Fail { - id, - error: err, - }); + Poll::Ready(Err(error)) => { + let _ = me.to_api.unbounded_send(WorkerToApi::Fail { id, error }); continue; // don't insert the request back } }; // We received a response! Decompose it into its parts. - let status_code = response.status(); - let headers = mem::replace(response.headers_mut(), hyper::HeaderMap::new()); - let body = response.into_body(); + let (head, body) = response.into_parts(); + let (status_code, headers) = (head.status, head.headers); let (body_tx, body_rx) = mpsc::channel(3); let _ = me.to_api.unbounded_send(WorkerToApi::Response { @@ -689,29 +698,29 @@ impl fmt::Debug for HttpWorkerRequest { mod tests { use core::convert::Infallible; use crate::api::timestamp; - use super::http; + use super::{http, SharedClient}; use sp_core::offchain::{HttpError, HttpRequestId, HttpRequestStatus, Duration}; + use futures::future; + use lazy_static::lazy_static; + + // Using lazy_static to avoid spawning lots of different SharedClients, + // as spawning a SharedClient is CPU-intensive and opens lots of fds. + lazy_static! { + static ref SHARED_CLIENT: SharedClient = SharedClient::new(); + } // Returns an `HttpApi` whose worker is ran in the background, and a `SocketAddr` to an HTTP // server that runs in the background as well. macro_rules! build_api_server { () => {{ - fn tokio_run(future: impl std::future::Future) { - let _ = tokio::runtime::Runtime::new().unwrap().block_on(future); - } - - // We spawn quite a bit of HTTP servers here due to how async API - // works for offchain workers, so be sure to raise the FD limit - // (particularly useful for macOS where the default soft limit may - // not be enough). - fdlimit::raise_fd_limit(); - - let (api, worker) = http(); - std::thread::spawn(move || tokio_run(worker)); + let hyper_client = SHARED_CLIENT.clone(); + let (api, worker) = http(hyper_client.clone()); let (addr_tx, addr_rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { - tokio_run(async move { + let mut rt = tokio::runtime::Runtime::new().unwrap(); + let worker = rt.spawn(worker); + let server = rt.spawn(async move { let server = hyper::Server::bind(&"127.0.0.1:0".parse().unwrap()) .serve(hyper::service::make_service_fn(|_| { async move { Ok::<_, Infallible>(hyper::service::service_fn(move |_req| async move { @@ -721,8 +730,9 @@ mod tests { })) }})); let _ = addr_tx.send(server.local_addr()); - server.await + server.await.map_err(drop) }); + let _ = rt.block_on(future::join(worker, server)); }); (api, addr_rx.recv().unwrap()) }}; @@ -891,10 +901,10 @@ mod tests { #[test] fn response_headers_invalid_call() { let (mut api, addr) = build_api_server!(); - assert!(api.response_headers(HttpRequestId(0xdead)).is_empty()); + assert_eq!(api.response_headers(HttpRequestId(0xdead)), &[]); let id = api.request_start("POST", &format!("http://{}", addr)).unwrap(); - assert!(api.response_headers(id).is_empty()); + assert_eq!(api.response_headers(id), &[]); let id = api.request_start("POST", &format!("http://{}", addr)).unwrap(); api.request_write_body(id, &[], None).unwrap(); @@ -904,12 +914,12 @@ mod tests { let id = api.request_start("GET", &format!("http://{}", addr)).unwrap(); api.response_wait(&[id], None); - assert!(!api.response_headers(id).is_empty()); + assert_ne!(api.response_headers(id), &[]); let id = api.request_start("GET", &format!("http://{}", addr)).unwrap(); let mut buf = [0; 128]; while api.response_read_body(id, &mut buf, None).unwrap() != 0 {} - assert!(api.response_headers(id).is_empty()); + assert_eq!(api.response_headers(id), &[]); } #[test] @@ -917,11 +927,11 @@ mod tests { let (mut api, addr) = build_api_server!(); let id = api.request_start("POST", &format!("http://{}", addr)).unwrap(); - assert!(api.response_headers(id).is_empty()); + assert_eq!(api.response_headers(id), &[]); let id = api.request_start("POST", &format!("http://{}", addr)).unwrap(); api.request_add_header(id, "Foo", "Bar").unwrap(); - assert!(api.response_headers(id).is_empty()); + assert_eq!(api.response_headers(id), &[]); let id = api.request_start("GET", &format!("http://{}", addr)).unwrap(); api.request_add_header(id, "Foo", "Bar").unwrap(); @@ -930,7 +940,7 @@ mod tests { // where we haven't received any response yet. This test can theoretically fail if the // HTTP response comes back faster than the kernel schedules our thread, but that is highly // unlikely. - assert!(api.response_headers(id).is_empty()); + assert_eq!(api.response_headers(id), &[]); } #[test] diff --git a/client/offchain/src/api/http_dummy.rs b/client/offchain/src/api/http_dummy.rs index 5ff77a1068312bbc5da2f71d9c15a9dee7bd8380..1c83325c93b2095fe4bc5fca9c7dbfb4e48e3b2a 100644 --- a/client/offchain/src/api/http_dummy.rs +++ b/client/offchain/src/api/http_dummy.rs @@ -19,8 +19,18 @@ use sp_core::offchain::{HttpRequestId, Timestamp, HttpRequestStatus, HttpError}; use std::{future::Future, pin::Pin, task::Context, task::Poll}; +/// Wrapper struct (wrapping nothing in case of http_dummy) used for keeping the hyper_rustls client running. +#[derive(Clone)] +pub struct SharedClient; + +impl SharedClient { + pub fn new() -> Self { + Self + } +} + /// Creates a pair of [`HttpApi`] and [`HttpWorker`]. -pub fn http() -> (HttpApi, HttpWorker) { +pub fn http(_: SharedClient) -> (HttpApi, HttpWorker) { (HttpApi, HttpWorker) } diff --git a/client/offchain/src/api/timestamp.rs b/client/offchain/src/api/timestamp.rs index e5494fe70d784bca20917a38a530ef37c06b3393..222d3273cb355fa64ac4f52f8dcb568b69be25b8 100644 --- a/client/offchain/src/api/timestamp.rs +++ b/client/offchain/src/api/timestamp.rs @@ -51,12 +51,14 @@ pub fn timestamp_from_now(timestamp: Timestamp) -> Duration { pub fn deadline_to_future( deadline: Option, ) -> futures::future::MaybeDone { - use futures::future; + use futures::future::{self, Either}; - future::maybe_done(match deadline { - Some(deadline) => future::Either::Left( - futures_timer::Delay::new(timestamp_from_now(deadline)) - ), - None => future::Either::Right(future::pending()) + future::maybe_done(match deadline.map(timestamp_from_now) { + None => Either::Left(future::pending()), + // Only apply delay if we need to wait a non-zero duration + Some(duration) if duration <= Duration::from_secs(0) => + Either::Right(Either::Left(future::ready(()))), + Some(duration) => + Either::Right(Either::Right(futures_timer::Delay::new(duration))), }) } diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 332e9f779a8e27f3e655238ed5a70510db21605a..7c90065746aa3f2be72966da799a89f7d96af846 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -19,7 +19,7 @@ //! The offchain workers is a special function of the runtime that //! gets executed after block is imported. During execution //! it's able to asynchronously submit extrinsics that will either -//! be propagated to other nodes added to the next block +//! be propagated to other nodes or added to the next block //! produced by the node as unsigned transactions. //! //! Offchain workers can be used for computation-heavy tasks @@ -41,10 +41,12 @@ use sp_api::{ApiExt, ProvideRuntimeApi}; use futures::future::Future; use log::{debug, warn}; use sc_network::NetworkStateInfo; -use sp_core::{offchain::{self, OffchainStorage}, ExecutionContext}; +use sp_core::{offchain::{self, OffchainStorage}, ExecutionContext, traits::SpawnNamed}; use sp_runtime::{generic::BlockId, traits::{self, Header}}; +use futures::{prelude::*, future::ready}; mod api; +use api::SharedClient; pub use sp_offchain::{OffchainWorkerApi, STORAGE_PREFIX}; @@ -54,16 +56,19 @@ pub struct OffchainWorkers { db: Storage, _block: PhantomData, thread_pool: Mutex, + shared_client: SharedClient, } impl OffchainWorkers { /// Creates new `OffchainWorkers`. pub fn new(client: Arc, db: Storage) -> Self { + let shared_client = SharedClient::new(); Self { client, db, _block: PhantomData, thread_pool: Mutex::new(ThreadPool::new(num_cpus::get())), + shared_client, } } } @@ -119,6 +124,7 @@ impl OffchainWorkers< self.db.clone(), network_state.clone(), is_validator, + self.shared_client.clone(), ); debug!("Spawning offchain workers at {:?}", at); let header = header.clone(); @@ -161,12 +167,49 @@ impl OffchainWorkers< } } +/// Inform the offchain worker about new imported blocks +pub async fn notification_future( + is_validator: bool, + client: Arc, + offchain: Arc>, + spawner: Spawner, + network_state_info: Arc, +) + where + Block: traits::Block, + Client: ProvideRuntimeApi + sc_client_api::BlockchainEvents + Send + Sync + 'static, + Client::Api: OffchainWorkerApi, + Storage: OffchainStorage + 'static, + Spawner: SpawnNamed +{ + client.import_notification_stream().for_each(move |n| { + if n.is_new_best { + spawner.spawn( + "offchain-on-block", + offchain.on_block_imported( + &n.header, + network_state_info.clone(), + is_validator, + ).boxed(), + ); + } else { + log::debug!( + target: "sc_offchain", + "Skipping offchain workers for non-canon block: {:?}", + n.header, + ) + } + + ready(()) + }).await; +} + #[cfg(test)] mod tests { use super::*; use std::sync::Arc; use sc_network::{Multiaddr, PeerId}; - use substrate_test_runtime_client::runtime::Block; + use substrate_test_runtime_client::{TestClient, runtime::Block}; use sc_transaction_pool::{BasicPool, FullChainApi}; use sp_transaction_pool::{TransactionPool, InPoolTransaction}; use sc_client_api::ExecutorProvider; @@ -183,7 +226,9 @@ mod tests { } } - struct TestPool(BasicPool, Block>); + struct TestPool( + BasicPool, Block> + ); impl sp_transaction_pool::OffchainSubmitTransaction for TestPool { fn submit_at( @@ -200,8 +245,8 @@ mod tests { #[test] fn should_call_into_runtime_and_produce_extrinsic() { - // given let _ = env_logger::try_init(); + let client = Arc::new(substrate_test_runtime_client::new()); let pool = Arc::new(TestPool(BasicPool::new( Default::default(), diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index f23e89e0f547726ed7e0df501371825a94e60a7f..bdec765eda730d8729e65f82508dfc71713221e8 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -3,7 +3,7 @@ description = "Connectivity manager based on reputation" homepage = "http://parity.io" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" name = "sc-peerset" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" repository = "https://github.com/paritytech/substrate/" @@ -15,8 +15,8 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] futures = "0.3.4" -libp2p = { version = "0.18.1", default-features = false } -sp-utils = { version = "2.0.0-alpha.8", path = "../../primitives/utils"} +libp2p = { version = "0.20.1", default-features = false } +sp-utils = { version = "2.0.0-rc4", path = "../../primitives/utils"} log = "0.4.8" serde_json = "1.0.41" wasm-timer = "0.2" diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 6a6f3d09059418d3bf547bfc5858da3dbff10ce5..6f28dd036a3cc53408583be8cbe04e24af9a8448 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Peer Set Manager (PSM). Contains the strategy for choosing which nodes the network should be //! connected to. @@ -181,9 +182,13 @@ pub struct PeersetConfig { /// errors. #[derive(Debug)] pub struct Peerset { + /// Underlying data structure for the nodes's states. data: peersstate::PeersState, /// If true, we only accept reserved nodes. reserved_only: bool, + /// Lists of nodes that don't occupy slots and that we should try to always be connected to. + /// Is kept in sync with the list of reserved nodes in [`Peerset::data`]. + priority_groups: HashMap>, /// Receiver for messages from the `PeersetHandle` and from `tx`. rx: TracingUnboundedReceiver, /// Sending side of `rx`. @@ -208,17 +213,18 @@ impl Peerset { let now = Instant::now(); let mut peerset = Peerset { - data: peersstate::PeersState::new(config.in_peers, config.out_peers, config.reserved_only), + data: peersstate::PeersState::new(config.in_peers, config.out_peers), tx, rx, reserved_only: config.reserved_only, + priority_groups: config.priority_groups.clone().into_iter().collect(), message_queue: VecDeque::new(), created: now, latest_time_update: now, }; - for (group, nodes) in config.priority_groups { - peerset.data.set_priority_group(&group, nodes); + for node in config.priority_groups.into_iter().flat_map(|(_, l)| l) { + peerset.data.add_no_slot_node(node); } for peer_id in config.bootnodes { @@ -234,61 +240,92 @@ impl Peerset { } fn on_add_reserved_peer(&mut self, peer_id: PeerId) { - let mut reserved = self.data.get_priority_group(RESERVED_NODES).unwrap_or_default(); - reserved.insert(peer_id); - self.data.set_priority_group(RESERVED_NODES, reserved); - self.alloc_slots(); + self.on_add_to_priority_group(RESERVED_NODES, peer_id); } fn on_remove_reserved_peer(&mut self, peer_id: PeerId) { - let mut reserved = self.data.get_priority_group(RESERVED_NODES).unwrap_or_default(); - reserved.remove(&peer_id); - self.data.set_priority_group(RESERVED_NODES, reserved); - match self.data.peer(&peer_id) { - peersstate::Peer::Connected(peer) => { - if self.reserved_only { - peer.disconnect(); - self.message_queue.push_back(Message::Drop(peer_id)); - } - } - peersstate::Peer::NotConnected(_) => {}, - peersstate::Peer::Unknown(_) => {}, - } + self.on_remove_from_priority_group(RESERVED_NODES, peer_id); } fn on_set_reserved_only(&mut self, reserved_only: bool) { self.reserved_only = reserved_only; - self.data.set_priority_only(reserved_only); if self.reserved_only { - // Disconnect non-reserved nodes. - let reserved = self.data.get_priority_group(RESERVED_NODES).unwrap_or_default(); + // Disconnect all the nodes that aren't reserved. for peer_id in self.data.connected_peers().cloned().collect::>().into_iter() { + if self.priority_groups.get(RESERVED_NODES).map_or(false, |g| g.contains(&peer_id)) { + continue; + } + let peer = self.data.peer(&peer_id).into_connected() .expect("We are enumerating connected peers, therefore the peer is connected; qed"); - if !reserved.contains(&peer_id) { - peer.disconnect(); - self.message_queue.push_back(Message::Drop(peer_id)); - } + peer.disconnect(); + self.message_queue.push_back(Message::Drop(peer_id)); } + } else { self.alloc_slots(); } } fn on_set_priority_group(&mut self, group_id: &str, peers: HashSet) { - self.data.set_priority_group(group_id, peers); - self.alloc_slots(); + // Determine the difference between the current group and the new list. + let (to_insert, to_remove) = { + let current_group = self.priority_groups.entry(group_id.to_owned()).or_default(); + let to_insert = peers.difference(current_group) + .cloned().collect::>(); + let to_remove = current_group.difference(&peers) + .cloned().collect::>(); + (to_insert, to_remove) + }; + + // Enumerate elements in `peers` not in `current_group`. + for peer_id in &to_insert { + // We don't call `on_add_to_priority_group` here in order to avoid calling + // `alloc_slots` all the time. + self.priority_groups.entry(group_id.to_owned()).or_default().insert(peer_id.clone()); + self.data.add_no_slot_node(peer_id.clone()); + } + + // Enumerate elements in `current_group` not in `peers`. + for peer in to_remove { + self.on_remove_from_priority_group(group_id, peer); + } + + if !to_insert.is_empty() { + self.alloc_slots(); + } } fn on_add_to_priority_group(&mut self, group_id: &str, peer_id: PeerId) { - self.data.add_to_priority_group(group_id, peer_id); + self.priority_groups.entry(group_id.to_owned()).or_default().insert(peer_id.clone()); + self.data.add_no_slot_node(peer_id); self.alloc_slots(); } fn on_remove_from_priority_group(&mut self, group_id: &str, peer_id: PeerId) { - self.data.remove_from_priority_group(group_id, &peer_id); - self.alloc_slots(); + if let Some(priority_group) = self.priority_groups.get_mut(group_id) { + if !priority_group.remove(&peer_id) { + // `PeerId` wasn't in the group in the first place. + return; + } + } else { + // Group doesn't exist, so the `PeerId` can't be in it. + return; + } + + // If that `PeerId` isn't in any other group, then it is no longer no-slot-occupying. + if !self.priority_groups.values().any(|l| l.contains(&peer_id)) { + self.data.remove_no_slot_node(&peer_id); + } + + // Disconnect the peer if necessary. + if group_id != RESERVED_NODES && self.reserved_only { + if let peersstate::Peer::Connected(peer) = self.data.peer(&peer_id) { + peer.disconnect(); + self.message_queue.push_back(Message::Drop(peer_id)); + } + } } fn on_report_peer(&mut self, peer_id: PeerId, change: ReputationChange) { @@ -375,25 +412,82 @@ impl Peerset { fn alloc_slots(&mut self) { self.update_time(); - // Try to grab the next node to attempt to connect to. - while let Some(next) = { - if self.reserved_only { - self.data.priority_not_connected_peer_from_group(RESERVED_NODES) - } else { - self.data.priority_not_connected_peer() - } - } { + // Try to connect to all the reserved nodes that we are not connected to. + loop { + let next = { + let data = &mut self.data; + self.priority_groups + .get(RESERVED_NODES) + .into_iter() + .flatten() + .filter(move |n| { + data.peer(n).into_connected().is_none() + }) + .next() + .cloned() + }; + + let next = match next { + Some(n) => n, + None => break, + }; + + let next = match self.data.peer(&next) { + peersstate::Peer::Unknown(n) => n.discover(), + peersstate::Peer::NotConnected(n) => n, + peersstate::Peer::Connected(_) => { + debug_assert!(false, "State inconsistency: not connected state"); + break; + } + }; + match next.try_outgoing() { Ok(conn) => self.message_queue.push_back(Message::Connect(conn.into_peer_id())), Err(_) => break, // No more slots available. } } + // Nothing more to do if we're in reserved mode. + if self.reserved_only { + return; + } + + // Try to connect to all the nodes in priority groups and that we are not connected to. loop { - if self.reserved_only { - break + let next = { + let data = &mut self.data; + self.priority_groups + .values() + .flatten() + .filter(move |n| { + data.peer(n).into_connected().is_none() + }) + .next() + .cloned() + }; + + let next = match next { + Some(n) => n, + None => break, + }; + + let next = match self.data.peer(&next) { + peersstate::Peer::Unknown(n) => n.discover(), + peersstate::Peer::NotConnected(n) => n, + peersstate::Peer::Connected(_) => { + debug_assert!(false, "State inconsistency: not connected state"); + break; + } + }; + + match next.try_outgoing() { + Ok(conn) => self.message_queue.push_back(Message::Connect(conn.into_peer_id())), + Err(_) => break, // No more slots available. } + } + // Now, we try to connect to non-priority nodes. + loop { // Try to grab the next node to attempt to connect to. let next = match self.data.highest_not_connected_peer() { Some(p) => p, @@ -426,6 +520,13 @@ impl Peerset { trace!(target: "peerset", "Incoming {:?}", peer_id); self.update_time(); + if self.reserved_only { + if !self.priority_groups.get(RESERVED_NODES).map_or(false, |n| n.contains(&peer_id)) { + self.message_queue.push_back(Message::Reject(index)); + return; + } + } + let not_connected = match self.data.peer(&peer_id) { // If we're already connected, don't answer, as the docs mention. peersstate::Peer::Connected(_) => return, @@ -528,9 +629,9 @@ impl Peerset { self.data.peers().len() } - /// Returns priority group by id. - pub fn get_priority_group(&self, group_id: &str) -> Option> { - self.data.get_priority_group(group_id) + /// Returns the content of a priority group. + pub fn priority_group(&self, group_id: &str) -> Option> { + self.priority_groups.get(group_id).map(|l| l.iter()) } } @@ -582,7 +683,6 @@ mod tests { assert_eq!(message, expected_message); peerset = p; } - assert!(peerset.message_queue.is_empty(), peerset.message_queue); peerset } @@ -647,6 +747,26 @@ mod tests { ]); } + #[test] + fn test_peerset_reject_incoming_in_reserved_only() { + let incoming = PeerId::random(); + let ii = IncomingIndex(1); + let config = PeersetConfig { + in_peers: 50, + out_peers: 50, + bootnodes: vec![], + reserved_only: true, + priority_groups: vec![], + }; + + let (mut peerset, _) = Peerset::from_config(config); + peerset.incoming(incoming.clone(), ii); + + assert_messages(peerset, vec![ + Message::Reject(ii), + ]); + } + #[test] fn test_peerset_discovered() { let bootnode = PeerId::random(); @@ -712,4 +832,3 @@ mod tests { futures::executor::block_on(fut); } } - diff --git a/client/peerset/src/peersstate.rs b/client/peerset/src/peersstate.rs index 843ec0a36006f436e3d9b2b9c83175ec632566e5..59879f629e31ecdbfbcb224a8a7dfb437f150746 100644 --- a/client/peerset/src/peersstate.rs +++ b/client/peerset/src/peersstate.rs @@ -14,10 +14,19 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Contains the state storage behind the peerset. +//! Reputation and slots allocation system behind the peerset. +//! +//! The [`PeersState`] state machine is responsible for managing the reputation and allocating +//! slots. It holds a list of nodes, each associated with a reputation value and whether we are +//! connected or not to this node. Thanks to this list, it knows how many slots are occupied. It +//! also holds a list of nodes which don't occupy slots. +//! +//! > Note: This module is purely dedicated to managing slots and reputations. Features such as +//! > for example connecting to some nodes in priority should be added outside of this +//! > module, rather than inside. use libp2p::PeerId; -use log::{error, warn}; +use log::error; use std::{borrow::Cow, collections::{HashSet, HashMap}}; use wasm_timer::Instant; @@ -37,23 +46,24 @@ pub struct PeersState { /// sort, to make the logic easier. nodes: HashMap, - /// Number of non-priority nodes for which the `ConnectionState` is `In`. + /// Number of slot-occupying nodes for which the `ConnectionState` is `In`. num_in: u32, - /// Number of non-priority nodes for which the `ConnectionState` is `In`. + /// Number of slot-occupying nodes for which the `ConnectionState` is `In`. num_out: u32, - /// Maximum allowed number of non-priority nodes for which the `ConnectionState` is `In`. + /// Maximum allowed number of slot-occupying nodes for which the `ConnectionState` is `In`. max_in: u32, - /// Maximum allowed number of non-priority nodes for which the `ConnectionState` is `Out`. + /// Maximum allowed number of slot-occupying nodes for which the `ConnectionState` is `Out`. max_out: u32, - /// Priority groups. Each group is identified by a string ID and contains a set of peer IDs. - priority_nodes: HashMap>, - - /// Only allow connections to/from peers in a priority group. - priority_only: bool, + /// List of node identities (discovered or not) that don't occupy slots. + /// + /// Note for future readers: this module is purely dedicated to managing slots. If you are + /// considering adding more features, please consider doing so outside of this module rather + /// than inside. + no_slot_nodes: HashSet, } /// State of a single node that we know about. @@ -106,15 +116,14 @@ impl ConnectionState { impl PeersState { /// Builds a new empty `PeersState`. - pub fn new(in_peers: u32, out_peers: u32, priority_only: bool) -> Self { + pub fn new(in_peers: u32, out_peers: u32) -> Self { PeersState { nodes: HashMap::new(), num_in: 0, num_out: 0, max_in: in_peers, max_out: out_peers, - priority_nodes: HashMap::new(), - priority_only, + no_slot_nodes: HashSet::new(), } } @@ -157,34 +166,6 @@ impl PeersState { .map(|(p, _)| p) } - /// Returns the first priority peer that we are not connected to. - /// - /// If multiple nodes are prioritized, which one is returned is unspecified. - pub fn priority_not_connected_peer(&mut self) -> Option { - let id = self.priority_nodes.values() - .flatten() - .find(|&id| self.nodes.get(id).map_or(false, |node| !node.connection_state.is_connected())) - .cloned(); - id.map(move |id| NotConnectedPeer { - state: self, - peer_id: Cow::Owned(id), - }) - } - - /// Returns the first priority peer that we are not connected to. - /// - /// If multiple nodes are prioritized, which one is returned is unspecified. - pub fn priority_not_connected_peer_from_group(&mut self, group_id: &str) -> Option { - let id = self.priority_nodes.get(group_id) - .and_then(|group| group.iter() - .find(|&id| self.nodes.get(id).map_or(false, |node| !node.connection_state.is_connected())) - .cloned()); - id.map(move |id| NotConnectedPeer { - state: self, - peer_id: Cow::Owned(id), - }) - } - /// Returns the peer with the highest reputation and that we are not connected to. /// /// If multiple nodes have the same reputation, which one is returned is unspecified. @@ -212,170 +193,40 @@ impl PeersState { } } - fn disconnect(&mut self, peer_id: &PeerId) { - let is_priority = self.is_priority(peer_id); - if let Some(mut node) = self.nodes.get_mut(peer_id) { - if !is_priority { - match node.connection_state { - ConnectionState::In => self.num_in -= 1, - ConnectionState::Out => self.num_out -= 1, - ConnectionState::NotConnected { .. } => - debug_assert!(false, "State inconsistency: disconnecting a disconnected node") - } - } - node.connection_state = ConnectionState::NotConnected { - last_connected: Instant::now(), - }; - } else { - warn!(target: "peerset", "Attempting to disconnect unknown peer {}", peer_id); - } - } - - /// Sets the peer as connected with an outgoing connection. - fn try_outgoing(&mut self, peer_id: &PeerId) -> bool { - let is_priority = self.is_priority(peer_id); - - // We are only accepting connections from priority nodes. - if !is_priority && self.priority_only { - return false; - } - - // Note that it is possible for num_out to be strictly superior to the max, in case we were - // connected to reserved node then marked them as not reserved. - if self.num_out >= self.max_out && !is_priority { - return false; + /// Add a node to the list of nodes that don't occupy slots. + /// + /// Has no effect if the peer was already in the group. + pub fn add_no_slot_node(&mut self, peer_id: PeerId) { + // Reminder: `HashSet::insert` returns false if the node was already in the set + if !self.no_slot_nodes.insert(peer_id.clone()) { + return; } - if let Some(mut peer) = self.nodes.get_mut(peer_id) { - peer.connection_state = ConnectionState::Out; - if !is_priority { - self.num_out += 1; + if let Some(peer) = self.nodes.get_mut(&peer_id) { + match peer.connection_state { + ConnectionState::In => self.num_in -= 1, + ConnectionState::Out => self.num_out -= 1, + ConnectionState::NotConnected { .. } => {}, } - return true; } - false } - /// Tries to accept the peer as an incoming connection. - /// - /// If there are enough slots available, switches the node to "connected" and returns `true`. If - /// the slots are full, the node stays "not connected" and we return `false`. + /// Removes a node from the list of nodes that don't occupy slots. /// - /// Note that reserved nodes don't count towards the number of slots. - fn try_accept_incoming(&mut self, peer_id: &PeerId) -> bool { - let is_priority = self.is_priority(peer_id); - - // We are only accepting connections from priority nodes. - if !is_priority && self.priority_only { - return false; + /// Has no effect if the peer was not in the group. + pub fn remove_no_slot_node(&mut self, peer_id: &PeerId) { + // Reminder: `HashSet::remove` returns false if the node was already not in the set + if !self.no_slot_nodes.remove(peer_id) { + return; } - // Note that it is possible for num_in to be strictly superior to the max, in case we were - // connected to reserved node then marked them as not reserved. - if self.num_in >= self.max_in && !is_priority { - return false; - } - if let Some(mut peer) = self.nodes.get_mut(peer_id) { - peer.connection_state = ConnectionState::In; - if !is_priority { - self.num_in += 1; + if let Some(peer) = self.nodes.get_mut(peer_id) { + match peer.connection_state { + ConnectionState::In => self.num_in += 1, + ConnectionState::Out => self.num_out += 1, + ConnectionState::NotConnected { .. } => {}, } - return true; } - false - } - - /// Sets priority group - pub fn set_priority_group(&mut self, group_id: &str, peers: HashSet) { - // update slot counters - let all_other_groups: HashSet<_> = self.priority_nodes - .iter() - .filter(|(g, _)| *g != group_id) - .flat_map(|(_, id)| id.clone()) - .collect(); - let existing_group = self.priority_nodes.remove(group_id).unwrap_or_default(); - for id in existing_group { - // update slots for nodes that are no longer priority - if !all_other_groups.contains(&id) { - if let Some(peer) = self.nodes.get_mut(&id) { - match peer.connection_state { - ConnectionState::In => self.num_in += 1, - ConnectionState::Out => self.num_out += 1, - ConnectionState::NotConnected { .. } => {}, - } - } - } - } - - for id in &peers { - // update slots for nodes that become priority - if !all_other_groups.contains(id) { - let peer = self.nodes.entry(id.clone()).or_default(); - match peer.connection_state { - ConnectionState::In => self.num_in -= 1, - ConnectionState::Out => self.num_out -= 1, - ConnectionState::NotConnected { .. } => {}, - } - } - } - self.priority_nodes.insert(group_id.into(), peers); - } - - /// Add a peer to a priority group. - pub fn add_to_priority_group(&mut self, group_id: &str, peer_id: PeerId) { - let mut peers = self.priority_nodes.get(group_id).cloned().unwrap_or_default(); - peers.insert(peer_id); - self.set_priority_group(group_id, peers); - } - - /// Remove a peer from a priority group. - pub fn remove_from_priority_group(&mut self, group_id: &str, peer_id: &PeerId) { - let mut peers = self.priority_nodes.get(group_id).cloned().unwrap_or_default(); - peers.remove(peer_id); - self.set_priority_group(group_id, peers); - } - - /// Get priority group content. - pub fn get_priority_group(&self, group_id: &str) -> Option> { - self.priority_nodes.get(group_id).cloned() - } - - /// Set whether to only allow connections to/from peers in a priority group. - /// Calling this method does not affect any existing connection, e.g. - /// enabling priority only will not disconnect from any non-priority peers - /// we are already connected to, only future incoming/outgoing connection - /// attempts will be affected. - pub fn set_priority_only(&mut self, priority: bool) { - self.priority_only = priority; - } - - /// Check that node is any priority group. - fn is_priority(&self, peer_id: &PeerId) -> bool { - self.priority_nodes.iter().any(|(_, group)| group.contains(peer_id)) - } - - /// Returns the reputation value of the node. - fn reputation(&self, peer_id: &PeerId) -> i32 { - self.nodes.get(peer_id).map_or(0, |p| p.reputation) - } - - /// Sets the reputation of the peer. - fn set_reputation(&mut self, peer_id: &PeerId, value: i32) { - let node = self.nodes - .entry(peer_id.clone()) - .or_default(); - node.reputation = value; - } - - /// Performs an arithmetic addition on the reputation score of that peer. - /// - /// In case of overflow, the value will be capped. - /// If the peer is unknown to us, we insert it and consider that it has a reputation of 0. - fn add_reputation(&mut self, peer_id: &PeerId, modifier: i32) { - let node = self.nodes - .entry(peer_id.clone()) - .or_default(); - node.reputation = node.reputation.saturating_add(modifier); } } @@ -437,7 +288,23 @@ impl<'a> ConnectedPeer<'a> { /// Switches the peer to "not connected". pub fn disconnect(self) -> NotConnectedPeer<'a> { - self.state.disconnect(&self.peer_id); + let is_no_slot_occupy = self.state.no_slot_nodes.contains(&*self.peer_id); + if let Some(mut node) = self.state.nodes.get_mut(&*self.peer_id) { + if !is_no_slot_occupy { + match node.connection_state { + ConnectionState::In => self.state.num_in -= 1, + ConnectionState::Out => self.state.num_out -= 1, + ConnectionState::NotConnected { .. } => + debug_assert!(false, "State inconsistency: disconnecting a disconnected node") + } + } + node.connection_state = ConnectionState::NotConnected { + last_connected: Instant::now(), + }; + } else { + debug_assert!(false, "State inconsistency: disconnecting a disconnected node"); + } + NotConnectedPeer { state: self.state, peer_id: self.peer_id, @@ -446,19 +313,27 @@ impl<'a> ConnectedPeer<'a> { /// Returns the reputation value of the node. pub fn reputation(&self) -> i32 { - self.state.reputation(&self.peer_id) + self.state.nodes.get(&*self.peer_id).map_or(0, |p| p.reputation) } /// Sets the reputation of the peer. pub fn set_reputation(&mut self, value: i32) { - self.state.set_reputation(&self.peer_id, value) + if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) { + node.reputation = value; + } else { + debug_assert!(false, "State inconsistency: set_reputation on an unknown node"); + } } /// Performs an arithmetic addition on the reputation score of that peer. /// /// In case of overflow, the value will be capped. pub fn add_reputation(&mut self, modifier: i32) { - self.state.add_reputation(&self.peer_id, modifier) + if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) { + node.reputation = node.reputation.saturating_add(modifier); + } else { + debug_assert!(false, "State inconsistency: add_reputation on an unknown node"); + } } } @@ -520,16 +395,29 @@ impl<'a> NotConnectedPeer<'a> { /// If there are enough slots available, switches the node to "connected" and returns `Ok`. If /// the slots are full, the node stays "not connected" and we return `Err`. /// - /// Note that priority nodes don't count towards the number of slots. + /// Non-slot-occupying nodes don't count towards the number of slots. pub fn try_outgoing(self) -> Result, NotConnectedPeer<'a>> { - if self.state.try_outgoing(&self.peer_id) { - Ok(ConnectedPeer { - state: self.state, - peer_id: self.peer_id, - }) + let is_no_slot_occupy = self.state.no_slot_nodes.contains(&*self.peer_id); + + // Note that it is possible for num_out to be strictly superior to the max, in case we were + // connected to reserved node then marked them as not reserved. + if self.state.num_out >= self.state.max_out && !is_no_slot_occupy { + return Err(self); + } + + if let Some(mut peer) = self.state.nodes.get_mut(&*self.peer_id) { + peer.connection_state = ConnectionState::Out; + if !is_no_slot_occupy { + self.state.num_out += 1; + } } else { - Err(self) + debug_assert!(false, "State inconsistency: try_outgoing on an unknown node"); } + + Ok(ConnectedPeer { + state: self.state, + peer_id: self.peer_id, + }) } /// Tries to accept the peer as an incoming connection. @@ -537,34 +425,54 @@ impl<'a> NotConnectedPeer<'a> { /// If there are enough slots available, switches the node to "connected" and returns `Ok`. If /// the slots are full, the node stays "not connected" and we return `Err`. /// - /// Note that priority nodes don't count towards the number of slots. + /// Non-slot-occupying nodes don't count towards the number of slots. pub fn try_accept_incoming(self) -> Result, NotConnectedPeer<'a>> { - if self.state.try_accept_incoming(&self.peer_id) { - Ok(ConnectedPeer { - state: self.state, - peer_id: self.peer_id, - }) + let is_no_slot_occupy = self.state.no_slot_nodes.contains(&*self.peer_id); + + // Note that it is possible for num_in to be strictly superior to the max, in case we were + // connected to reserved node then marked them as not reserved. + if self.state.num_in >= self.state.max_in && !is_no_slot_occupy { + return Err(self); + } + + if let Some(mut peer) = self.state.nodes.get_mut(&*self.peer_id) { + peer.connection_state = ConnectionState::In; + if !is_no_slot_occupy { + self.state.num_in += 1; + } } else { - Err(self) + debug_assert!(false, "State inconsistency: try_accept_incoming on an unknown node"); } + + Ok(ConnectedPeer { + state: self.state, + peer_id: self.peer_id, + }) } /// Returns the reputation value of the node. pub fn reputation(&self) -> i32 { - self.state.reputation(&self.peer_id) + self.state.nodes.get(&*self.peer_id).map_or(0, |p| p.reputation) } /// Sets the reputation of the peer. pub fn set_reputation(&mut self, value: i32) { - self.state.set_reputation(&self.peer_id, value) + if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) { + node.reputation = value; + } else { + debug_assert!(false, "State inconsistency: set_reputation on an unknown node"); + } } /// Performs an arithmetic addition on the reputation score of that peer. /// /// In case of overflow, the value will be capped. - /// If the peer is unknown to us, we insert it and consider that it has a reputation of 0. pub fn add_reputation(&mut self, modifier: i32) { - self.state.add_reputation(&self.peer_id, modifier) + if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) { + node.reputation = node.reputation.saturating_add(modifier); + } else { + debug_assert!(false, "State inconsistency: add_reputation on an unknown node"); + } } /// Un-discovers the peer. Removes it from the list. @@ -618,7 +526,7 @@ mod tests { #[test] fn full_slots_in() { - let mut peers_state = PeersState::new(1, 1, false); + let mut peers_state = PeersState::new(1, 1); let id1 = PeerId::random(); let id2 = PeerId::random(); @@ -632,14 +540,14 @@ mod tests { } #[test] - fn priority_node_doesnt_use_slot() { - let mut peers_state = PeersState::new(1, 1, false); + fn no_slot_node_doesnt_use_slot() { + let mut peers_state = PeersState::new(1, 1); let id1 = PeerId::random(); let id2 = PeerId::random(); - peers_state.set_priority_group("test", vec![id1.clone()].into_iter().collect()); - if let Peer::NotConnected(p) = peers_state.peer(&id1) { - assert!(p.try_accept_incoming().is_ok()); + peers_state.add_no_slot_node(id1.clone()); + if let Peer::Unknown(p) = peers_state.peer(&id1) { + assert!(p.discover().try_accept_incoming().is_ok()); } else { panic!() } if let Peer::Unknown(e) = peers_state.peer(&id2) { @@ -649,7 +557,7 @@ mod tests { #[test] fn disconnecting_frees_slot() { - let mut peers_state = PeersState::new(1, 1, false); + let mut peers_state = PeersState::new(1, 1); let id1 = PeerId::random(); let id2 = PeerId::random(); @@ -659,28 +567,9 @@ mod tests { assert!(peers_state.peer(&id2).into_not_connected().unwrap().try_accept_incoming().is_ok()); } - #[test] - fn priority_not_connected_peer() { - let mut peers_state = PeersState::new(25, 25, false); - let id1 = PeerId::random(); - let id2 = PeerId::random(); - - assert!(peers_state.priority_not_connected_peer().is_none()); - peers_state.peer(&id1).into_unknown().unwrap().discover(); - peers_state.peer(&id2).into_unknown().unwrap().discover(); - - assert!(peers_state.priority_not_connected_peer().is_none()); - peers_state.set_priority_group("test", vec![id1.clone()].into_iter().collect()); - assert!(peers_state.priority_not_connected_peer().is_some()); - peers_state.set_priority_group("test", vec![id2.clone(), id2.clone()].into_iter().collect()); - assert!(peers_state.priority_not_connected_peer().is_some()); - peers_state.set_priority_group("test", vec![].into_iter().collect()); - assert!(peers_state.priority_not_connected_peer().is_none()); - } - #[test] fn highest_not_connected_peer() { - let mut peers_state = PeersState::new(25, 25, false); + let mut peers_state = PeersState::new(25, 25); let id1 = PeerId::random(); let id2 = PeerId::random(); @@ -700,87 +589,11 @@ mod tests { } #[test] - fn disconnect_priority_doesnt_panic() { - let mut peers_state = PeersState::new(1, 1, false); + fn disconnect_no_slot_doesnt_panic() { + let mut peers_state = PeersState::new(1, 1); let id = PeerId::random(); - peers_state.set_priority_group("test", vec![id.clone()].into_iter().collect()); - let peer = peers_state.peer(&id).into_not_connected().unwrap().try_outgoing().unwrap(); + peers_state.add_no_slot_node(id.clone()); + let peer = peers_state.peer(&id).into_unknown().unwrap().discover().try_outgoing().unwrap(); peer.disconnect(); } - - #[test] - fn multiple_priority_groups_slot_count() { - let mut peers_state = PeersState::new(1, 1, false); - let id = PeerId::random(); - - if let Peer::Unknown(p) = peers_state.peer(&id) { - assert!(p.discover().try_accept_incoming().is_ok()); - } else { panic!() } - - assert_eq!(peers_state.num_in, 1); - peers_state.set_priority_group("test1", vec![id.clone()].into_iter().collect()); - assert_eq!(peers_state.num_in, 0); - peers_state.set_priority_group("test2", vec![id.clone()].into_iter().collect()); - assert_eq!(peers_state.num_in, 0); - peers_state.set_priority_group("test1", vec![].into_iter().collect()); - assert_eq!(peers_state.num_in, 0); - peers_state.set_priority_group("test2", vec![].into_iter().collect()); - assert_eq!(peers_state.num_in, 1); - } - - #[test] - fn priority_only_mode_ignores_drops_unknown_nodes() { - // test whether connection to/from given peer is allowed - let test_connection = |peers_state: &mut PeersState, id| { - if let Peer::Unknown(p) = peers_state.peer(id) { - p.discover(); - } - - let incoming = if let Peer::NotConnected(p) = peers_state.peer(id) { - p.try_accept_incoming().is_ok() - } else { - panic!() - }; - - if incoming { - peers_state.peer(id).into_connected().map(|p| p.disconnect()); - } - - let outgoing = if let Peer::NotConnected(p) = peers_state.peer(id) { - p.try_outgoing().is_ok() - } else { - panic!() - }; - - if outgoing { - peers_state.peer(id).into_connected().map(|p| p.disconnect()); - } - - incoming || outgoing - }; - - let mut peers_state = PeersState::new(1, 1, true); - let id = PeerId::random(); - - // this is an unknown peer and our peer state is set to only allow - // priority peers so any connection attempt should be denied. - assert!(!test_connection(&mut peers_state, &id)); - - // disabling priority only mode should allow the connection to go - // through. - peers_state.set_priority_only(false); - assert!(test_connection(&mut peers_state, &id)); - - // re-enabling it we should again deny connections from the peer. - peers_state.set_priority_only(true); - assert!(!test_connection(&mut peers_state, &id)); - - // but if we add the peer to a priority group it should be accepted. - peers_state.set_priority_group("TEST_GROUP", vec![id.clone()].into_iter().collect()); - assert!(test_connection(&mut peers_state, &id)); - - // and removing it will cause the connection to once again be denied. - peers_state.remove_from_priority_group("TEST_GROUP", &id); - assert!(!test_connection(&mut peers_state, &id)); - } } diff --git a/client/peerset/tests/fuzz.rs b/client/peerset/tests/fuzz.rs index 12e20a04bec59bacbeeafd87a6d7842cd5636df1..6fa29e3d834cfcdaa0b035e03849432e20e7b3f3 100644 --- a/client/peerset/tests/fuzz.rs +++ b/client/peerset/tests/fuzz.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use futures::prelude::*; use libp2p::PeerId; use rand::distributions::{Distribution, Uniform, WeightedIndex}; @@ -45,13 +46,13 @@ fn test_once() { id }).collect(), priority_groups: { - let list = (0 .. Uniform::new_inclusive(0, 2).sample(&mut rng)).map(|_| { + let nodes = (0 .. Uniform::new_inclusive(0, 2).sample(&mut rng)).map(|_| { let id = PeerId::random(); known_nodes.insert(id.clone()); reserved_nodes.insert(id.clone()); id }).collect(); - vec![("reserved".to_owned(), list)] + vec![("foo".to_string(), nodes)] }, reserved_only: Uniform::new_inclusive(0, 10).sample(&mut rng) == 0, in_peers: Uniform::new_inclusive(0, 25).sample(&mut rng), diff --git a/client/proposer-metrics/Cargo.toml b/client/proposer-metrics/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..b10336f340c2af37a6c6303084d9058f94dd254f --- /dev/null +++ b/client/proposer-metrics/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "sc-proposer-metrics" +version = "0.8.0-rc4" +authors = ["Parity Technologies "] +edition = "2018" +license = "GPL-3.0-or-later WITH Classpath-exception-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Basic metrics for block production." + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +log = "0.4.8" +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-rc4"} diff --git a/client/proposer-metrics/src/lib.rs b/client/proposer-metrics/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..5cb749f4a260aa1964e230cb8edb3dd9515f595c --- /dev/null +++ b/client/proposer-metrics/src/lib.rs @@ -0,0 +1,67 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Prometheus basic proposer metrics. + +use prometheus_endpoint::{register, PrometheusError, Registry, Histogram, HistogramOpts, Gauge, U64}; + +/// Optional shareable link to basic authorship metrics. +#[derive(Clone, Default)] +pub struct MetricsLink(Option); + +impl MetricsLink { + pub fn new(registry: Option<&Registry>) -> Self { + Self( + registry.and_then(|registry| + Metrics::register(registry) + .map_err(|err| log::warn!("Failed to register proposer prometheus metrics: {}", err)) + .ok() + ) + ) + } + + pub fn report(&self, do_this: impl FnOnce(&Metrics) -> O) -> Option { + Some(do_this(self.0.as_ref()?)) + } +} + +/// Authorship metrics. +#[derive(Clone)] +pub struct Metrics { + pub block_constructed: Histogram, + pub number_of_transactions: Gauge, +} + +impl Metrics { + pub fn register(registry: &Registry) -> Result { + Ok(Self { + block_constructed: register( + Histogram::with_opts(HistogramOpts::new( + "proposer_block_constructed", + "Histogram of time taken to construct new block", + ))?, + registry, + )?, + number_of_transactions: register( + Gauge::new( + "proposer_number_of_transactions", + "Number of transactions included in block", + )?, + registry, + )?, + }) + } +} diff --git a/client/rpc-api/Cargo.toml b/client/rpc-api/Cargo.toml index 3f984a09ed227069eb23379d33297971f6a7f316..a991cf9afa1ab5983ad5bc26f963a901cf8afeed 100644 --- a/client/rpc-api/Cargo.toml +++ b/client/rpc-api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-rpc-api" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -12,20 +12,20 @@ description = "Substrate RPC interfaces." targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0" } +codec = { package = "parity-scale-codec", version = "1.3.1" } derive_more = "0.99.2" futures = { version = "0.3.1", features = ["compat"] } -jsonrpc-core = "14.0.3" -jsonrpc-core-client = "14.0.5" -jsonrpc-derive = "14.0.3" -jsonrpc-pubsub = "14.0.3" +jsonrpc-core = "14.2.0" +jsonrpc-core-client = "14.2.0" +jsonrpc-derive = "14.2.1" +jsonrpc-pubsub = "14.2.0" log = "0.4.8" parking_lot = "0.10.0" -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-version = { version = "2.0.0-alpha.8", path = "../../primitives/version" } -sp-runtime = { path = "../../primitives/runtime" , version = "2.0.0-alpha.8"} -sp-chain-spec = { path = "../../primitives/chain-spec" , version = "2.0.0-alpha.8"} +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-version = { version = "2.0.0-rc4", path = "../../primitives/version" } +sp-runtime = { path = "../../primitives/runtime" , version = "2.0.0-rc4"} +sp-chain-spec = { path = "../../primitives/chain-spec" , version = "2.0.0-rc4"} serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" -sp-transaction-pool = { version = "2.0.0-alpha.8", path = "../../primitives/transaction-pool" } -sp-rpc = { version = "2.0.0-alpha.8", path = "../../primitives/rpc" } +sp-transaction-pool = { version = "2.0.0-rc4", path = "../../primitives/transaction-pool" } +sp-rpc = { version = "2.0.0-rc4", path = "../../primitives/rpc" } diff --git a/client/rpc-api/src/author/error.rs b/client/rpc-api/src/author/error.rs index 0b7d02a9148a11a0c83d7508b1e1f6af85091f28..69c036be95fe085bcefeb1f38b8edbe594facaad 100644 --- a/client/rpc-api/src/author/error.rs +++ b/client/rpc-api/src/author/error.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,10 +15,12 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Authoring RPC module errors. use crate::errors; use jsonrpc_core as rpc; +use sp_runtime::transaction_validity::InvalidTransaction; /// Author RPC Result type. pub type Result = std::result::Result; @@ -113,10 +115,18 @@ impl From for rpc::Error { message: format!("Verification Error: {}", e).into(), data: Some(format!("{:?}", e).into()), }, - Error::Pool(PoolError::InvalidTransaction(e)) => rpc::Error { + Error::Pool(PoolError::InvalidTransaction(InvalidTransaction::Custom(e))) => rpc::Error { code: rpc::ErrorCode::ServerError(POOL_INVALID_TX), message: "Invalid Transaction".into(), - data: serde_json::to_value(e).ok(), + data: Some(format!("Custom error: {}", e).into()), + }, + Error::Pool(PoolError::InvalidTransaction(e)) => { + let msg: &str = e.into(); + rpc::Error { + code: rpc::ErrorCode::ServerError(POOL_INVALID_TX), + message: "Invalid Transaction".into(), + data: Some(msg.into()), + } }, Error::Pool(PoolError::UnknownTransaction(e)) => rpc::Error { code: rpc::ErrorCode::ServerError(POOL_UNKNOWN_VALIDITY), diff --git a/client/rpc-api/src/author/mod.rs b/client/rpc-api/src/author/mod.rs index aa3ec9bf5c6e3f78b282d385b8fb6797004c5fd7..29f5b1d26e84cb9c8644292da26c74e07dd23a39 100644 --- a/client/rpc-api/src/author/mod.rs +++ b/client/rpc-api/src/author/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate block-author/full-node API. pub mod error; diff --git a/client/rpc-api/src/chain/error.rs b/client/rpc-api/src/chain/error.rs index 6b3dd6341afe599e8a0d475b712f08404902a3bb..fd7bd0a43d778bf2a03be96baa5960e15f3dbd47 100644 --- a/client/rpc-api/src/chain/error.rs +++ b/client/rpc-api/src/chain/error.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/client/rpc-api/src/chain/mod.rs b/client/rpc-api/src/chain/mod.rs index 13afa3ffc6a8703804b35a00e2e2eb6e7aa530c9..753ac5617a29d70380e3502d95a2145af8a926a9 100644 --- a/client/rpc-api/src/chain/mod.rs +++ b/client/rpc-api/src/chain/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate blockchain API. pub mod error; @@ -48,7 +49,7 @@ pub trait ChainApi { #[rpc(name = "chain_getBlockHash", alias("chain_getHead"))] fn block_hash( &self, - hash: Option>>, + hash: Option>, ) -> Result>>; /// Get hash of the last finalized block in the canon chain. diff --git a/client/rpc-api/src/child_state/mod.rs b/client/rpc-api/src/child_state/mod.rs index 8d0d0049af8255a619e570a8a3cc87d1e295f49a..d956a7554f8ee798d2fa84c777ae33b591f257ce 100644 --- a/client/rpc-api/src/child_state/mod.rs +++ b/client/rpc-api/src/child_state/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate state API. use jsonrpc_derive::rpc; diff --git a/client/rpc-api/src/errors.rs b/client/rpc-api/src/errors.rs index 96bbadc27d3f63eadd1d361b33d333f0cceddf0b..4e1a5b10fc5128fbb3e36dfb788a23c0575e965d 100644 --- a/client/rpc-api/src/errors.rs +++ b/client/rpc-api/src/errors.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use log::warn; pub fn internal(e: E) -> jsonrpc_core::Error { diff --git a/client/rpc-api/src/helpers.rs b/client/rpc-api/src/helpers.rs index 5eaa8d8d8f0932736aa2e40dbcc8805a192517aa..025fef1102c492adfb142a837d7e2f5e6a262e11 100644 --- a/client/rpc-api/src/helpers.rs +++ b/client/rpc-api/src/helpers.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use jsonrpc_core::futures::prelude::*; use futures::{channel::oneshot, compat::Compat}; diff --git a/client/rpc-api/src/lib.rs b/client/rpc-api/src/lib.rs index f742d73b692c00de347e8a8e6f7994b529889267..cd2608dda92be9bf9fed546e3a3ffaca27005ec9 100644 --- a/client/rpc-api/src/lib.rs +++ b/client/rpc-api/src/lib.rs @@ -23,10 +23,8 @@ mod errors; mod helpers; mod policy; -mod subscriptions; pub use jsonrpc_core::IoHandlerExtension as RpcExtension; -pub use subscriptions::{Subscriptions, TaskExecutor}; pub use helpers::Receiver; pub use policy::DenyUnsafe; diff --git a/client/rpc-api/src/offchain/error.rs b/client/rpc-api/src/offchain/error.rs index bfdd663121752927e7e9328dd4d24c0d6d235828..ea5223f1ce7f935aefa8b6629ee2e82eebb48c13 100644 --- a/client/rpc-api/src/offchain/error.rs +++ b/client/rpc-api/src/offchain/error.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Offchain RPC errors. use jsonrpc_core as rpc; diff --git a/client/rpc-api/src/offchain/mod.rs b/client/rpc-api/src/offchain/mod.rs index 9c60cf65b2c2f2989370ed461befa43e76fc7f73..427b6a1cc017bfeb09e52a43e14c14fa6ddf8669 100644 --- a/client/rpc-api/src/offchain/mod.rs +++ b/client/rpc-api/src/offchain/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate offchain API. pub mod error; diff --git a/client/rpc-api/src/policy.rs b/client/rpc-api/src/policy.rs index 74c2d0594b3a5e471094aa8e144a6640047013ed..141dcfbc415f8294d6f140db7ce74012afa200ac 100644 --- a/client/rpc-api/src/policy.rs +++ b/client/rpc-api/src/policy.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Policy-related types. //! //! Contains a `DenyUnsafe` type that can be used to deny potentially unsafe @@ -25,21 +26,21 @@ use jsonrpc_core as rpc; /// Signifies whether a potentially unsafe RPC should be denied. #[derive(Clone, Copy, Debug)] pub enum DenyUnsafe { - /// Denies only potentially unsafe RPCs. - Yes, - /// Allows calling every RPCs. - No + /// Denies only potentially unsafe RPCs. + Yes, + /// Allows calling every RPCs. + No, } impl DenyUnsafe { - /// Returns `Ok(())` if the RPCs considered unsafe are safe to call, - /// otherwise returns `Err(UnsafeRpcError)`. - pub fn check_if_safe(self) -> Result<(), UnsafeRpcError> { - match self { - DenyUnsafe::Yes => Err(UnsafeRpcError), - DenyUnsafe::No => Ok(()) - } - } + /// Returns `Ok(())` if the RPCs considered unsafe are safe to call, + /// otherwise returns `Err(UnsafeRpcError)`. + pub fn check_if_safe(self) -> Result<(), UnsafeRpcError> { + match self { + DenyUnsafe::Yes => Err(UnsafeRpcError), + DenyUnsafe::No => Ok(()), + } + } } /// Signifies whether an RPC considered unsafe is denied to be called externally. @@ -47,15 +48,15 @@ impl DenyUnsafe { pub struct UnsafeRpcError; impl std::fmt::Display for UnsafeRpcError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "RPC call is unsafe to be called externally") - } + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "RPC call is unsafe to be called externally") + } } impl std::error::Error for UnsafeRpcError {} impl From for rpc::Error { - fn from(_: UnsafeRpcError) -> rpc::Error { - rpc::Error::method_not_found() - } + fn from(_: UnsafeRpcError) -> rpc::Error { + rpc::Error::method_not_found() + } } diff --git a/client/rpc-api/src/state/error.rs b/client/rpc-api/src/state/error.rs index a33bbd3df418382efe4f542237d963cf257fcab0..2fcca3c343f09efdd2acb1ce767cb2f80a80e5bd 100644 --- a/client/rpc-api/src/state/error.rs +++ b/client/rpc-api/src/state/error.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! State RPC errors. use crate::errors; diff --git a/client/rpc-api/src/state/helpers.rs b/client/rpc-api/src/state/helpers.rs index af414a04983261a4dd659adce91ccae8afa54fe8..0d176ea67f35be0449bf8bb63c49401260e39070 100644 --- a/client/rpc-api/src/state/helpers.rs +++ b/client/rpc-api/src/state/helpers.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate state API helpers. use sp_core::Bytes; diff --git a/client/rpc-api/src/state/mod.rs b/client/rpc-api/src/state/mod.rs index a31ef598c81a6e5cad5505e68c047ff28c1e626b..1bfbb4786e677bb2eb89cb2b51346f9ae4d56890 100644 --- a/client/rpc-api/src/state/mod.rs +++ b/client/rpc-api/src/state/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate state API. pub mod error; diff --git a/client/rpc-api/src/subscriptions.rs b/client/rpc-api/src/subscriptions.rs deleted file mode 100644 index db5d1784504e5f9aaf553b2b43255a8c6c09e99c..0000000000000000000000000000000000000000 --- a/client/rpc-api/src/subscriptions.rs +++ /dev/null @@ -1,120 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -use std::collections::HashMap; -use std::sync::{Arc, atomic::{self, AtomicUsize}}; - -use log::{error, warn}; -use jsonrpc_pubsub::{SubscriptionId, typed::{Sink, Subscriber}}; -use parking_lot::Mutex; -use jsonrpc_core::futures::sync::oneshot; -use jsonrpc_core::futures::{Future, future}; - -type Id = u64; - -/// Alias for a an implementation of `futures::future::Executor`. -pub type TaskExecutor = Arc + Send>> + Send + Sync>; - -/// Generate unique ids for subscriptions. -#[derive(Clone, Debug)] -pub struct IdProvider { - next_id: Arc, -} -impl Default for IdProvider { - fn default() -> Self { - IdProvider { - next_id: Arc::new(AtomicUsize::new(1)), - } - } -} - -impl IdProvider { - /// Returns next id for the subscription. - pub fn next_id(&self) -> Id { - self.next_id.fetch_add(1, atomic::Ordering::AcqRel) as u64 - } -} - -/// Subscriptions manager. -/// -/// Takes care of assigning unique subscription ids and -/// driving the sinks into completion. -#[derive(Clone)] -pub struct Subscriptions { - next_id: IdProvider, - active_subscriptions: Arc>>>, - executor: TaskExecutor, -} - -impl Subscriptions { - /// Creates new `Subscriptions` object. - pub fn new(executor: TaskExecutor) -> Self { - Subscriptions { - next_id: Default::default(), - active_subscriptions: Default::default(), - executor, - } - } - - /// Borrows the internal task executor. - /// - /// This can be used to spawn additional tasks on the underlying event loop. - pub fn executor(&self) -> &TaskExecutor { - &self.executor - } - - /// Creates new subscription for given subscriber. - /// - /// Second parameter is a function that converts Subscriber sink into a future. - /// This future will be driven to completion by the underlying event loop - /// or will be cancelled in case #cancel is invoked. - pub fn add(&self, subscriber: Subscriber, into_future: G) -> SubscriptionId where - G: FnOnce(Sink) -> R, - R: future::IntoFuture, - F: future::Future + Send + 'static, - { - let id = self.next_id.next_id(); - let subscription_id: SubscriptionId = id.into(); - if let Ok(sink) = subscriber.assign_id(subscription_id.clone()) { - let (tx, rx) = oneshot::channel(); - let future = into_future(sink) - .into_future() - .select(rx.map_err(|e| warn!("Error timeing out: {:?}", e))) - .then(|_| Ok(())); - - self.active_subscriptions.lock().insert(id, tx); - if self.executor.execute(Box::new(future)).is_err() { - error!("Failed to spawn RPC subscription task"); - } - } - - subscription_id - } - - /// Cancel subscription. - /// - /// Returns true if subscription existed or false otherwise. - pub fn cancel(&self, id: SubscriptionId) -> bool { - if let SubscriptionId::Number(id) = id { - if let Some(tx) = self.active_subscriptions.lock().remove(&id) { - let _ = tx.send(()); - return true; - } - } - false - } -} diff --git a/client/rpc-api/src/system/error.rs b/client/rpc-api/src/system/error.rs index a4b7a1ca54ffed51855553440a95c0fcac97a1a4..4897aa485cbe47591b787e771fda59543d3644c4 100644 --- a/client/rpc-api/src/system/error.rs +++ b/client/rpc-api/src/system/error.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! System RPC module errors. use crate::system::helpers::Health; diff --git a/client/rpc-api/src/system/helpers.rs b/client/rpc-api/src/system/helpers.rs index 0b29011f21b21b8912f1d97bebc7f77f3b945f75..5dbe93543d8e5313a617a91ccfd3fa919ec661ee 100644 --- a/client/rpc-api/src/system/helpers.rs +++ b/client/rpc-api/src/system/helpers.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate system API helpers. use std::fmt; diff --git a/client/rpc-api/src/system/mod.rs b/client/rpc-api/src/system/mod.rs index 769a914ecbe40ce61a5e36337379a0402a1c47eb..a7b746ee1b114afa525b5c077d22bf34ec1d5f54 100644 --- a/client/rpc-api/src/system/mod.rs +++ b/client/rpc-api/src/system/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate system API. pub mod error; diff --git a/client/rpc-servers/Cargo.toml b/client/rpc-servers/Cargo.toml index 1f9172b908bb9ae0bc0cc526cc53dcba023431cd..155729817da32e69ddfd6b84552d729ddb16871d 100644 --- a/client/rpc-servers/Cargo.toml +++ b/client/rpc-servers/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-rpc-server" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -12,13 +12,14 @@ description = "Substrate RPC servers." targets = ["x86_64-unknown-linux-gnu"] [dependencies] -jsonrpc-core = "14.0.3" -pubsub = { package = "jsonrpc-pubsub", version = "14.0.3" } +jsonrpc-core = "14.2.0" +pubsub = { package = "jsonrpc-pubsub", version = "14.2.0" } log = "0.4.8" serde = "1.0.101" serde_json = "1.0.41" -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } [target.'cfg(not(target_os = "unknown"))'.dependencies] -http = { package = "jsonrpc-http-server", version = "14.0.3" } -ws = { package = "jsonrpc-ws-server", version = "14.0.3" } +http = { package = "jsonrpc-http-server", version = "14.2.0" } +ws = { package = "jsonrpc-ws-server", version = "14.2.0" } +ipc = { version = "14.2.0", package = "jsonrpc-ipc-server" } diff --git a/client/rpc-servers/src/lib.rs b/client/rpc-servers/src/lib.rs index cd1b33ad8b0b2ff09f37ebcc2b2eda9e466c7e25..1e476262acea8761e8a0e266189934b02ad0f20d 100644 --- a/client/rpc-servers/src/lib.rs +++ b/client/rpc-servers/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate RPC servers. #![warn(missing_docs)] @@ -61,6 +62,8 @@ pub fn rpc_handler( mod inner { use super::*; + /// Type alias for ipc server + pub type IpcServer = ipc::Server; /// Type alias for http server pub type HttpServer = http::Server; /// Type alias for ws server @@ -88,6 +91,23 @@ mod inner { .start_http(addr) } + /// Start IPC server listening on given path. + /// + /// **Note**: Only available if `not(target_os = "unknown")`. + pub fn start_ipc( + addr: &str, + io: RpcHandler, + ) -> io::Result { + let builder = ipc::ServerBuilder::new(io); + #[cfg(target_os = "unix")] + builder.set_security_attributes({ + let security_attributes = ipc::SecurityAttributes::empty(); + security_attributes.set_mode(0o600)?; + security_attributes + }); + builder.start(addr) + } + /// Start WS server listening on given address. /// /// **Note**: Only available if `not(target_os = "unknown")`. diff --git a/client/rpc/Cargo.toml b/client/rpc/Cargo.toml index a97a46fa29b6f3c684143d611826784c1531a000..9568a4d44ffef117a0595663c3c57fd433be6af9 100644 --- a/client/rpc/Cargo.toml +++ b/client/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-rpc" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -12,38 +12,38 @@ description = "Substrate Client RPC" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sc-rpc-api = { version = "0.8.0-alpha.8", path = "../rpc-api" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../api" } -sp-api = { version = "2.0.0-alpha.8", path = "../../primitives/api" } -codec = { package = "parity-scale-codec", version = "1.3.0" } +sc-rpc-api = { version = "0.8.0-rc4", path = "../rpc-api" } +sc-client-api = { version = "2.0.0-rc4", path = "../api" } +sp-api = { version = "2.0.0-rc4", path = "../../primitives/api" } +codec = { package = "parity-scale-codec", version = "1.3.1" } futures = { version = "0.3.1", features = ["compat"] } -jsonrpc-pubsub = "14.0.3" +jsonrpc-pubsub = "14.2.0" log = "0.4.8" -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -rpc = { package = "jsonrpc-core", version = "14.0.3" } -sp-version = { version = "2.0.0-alpha.8", path = "../../primitives/version" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +rpc = { package = "jsonrpc-core", version = "14.2.0" } +sp-version = { version = "2.0.0-rc4", path = "../../primitives/version" } serde_json = "1.0.41" -sp-session = { version = "2.0.0-alpha.8", path = "../../primitives/session" } -sp-offchain = { version = "2.0.0-alpha.8", path = "../../primitives/offchain" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } -sp-utils = { version = "2.0.0-alpha.8", path = "../../primitives/utils" } -sp-rpc = { version = "2.0.0-alpha.8", path = "../../primitives/rpc" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../primitives/state-machine" } -sp-chain-spec = { version = "2.0.0-alpha.8", path = "../../primitives/chain-spec" } -sc-executor = { version = "0.8.0-alpha.8", path = "../executor" } -sc-block-builder = { version = "0.8.0-alpha.8", path = "../../client/block-builder" } -sc-keystore = { version = "2.0.0-alpha.8", path = "../keystore" } -sp-transaction-pool = { version = "2.0.0-alpha.8", path = "../../primitives/transaction-pool" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../primitives/blockchain" } +sp-session = { version = "2.0.0-rc4", path = "../../primitives/session" } +sp-offchain = { version = "2.0.0-rc4", path = "../../primitives/offchain" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-rc4", path = "../../primitives/utils" } +sp-rpc = { version = "2.0.0-rc4", path = "../../primitives/rpc" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../primitives/state-machine" } +sp-chain-spec = { version = "2.0.0-rc4", path = "../../primitives/chain-spec" } +sc-executor = { version = "0.8.0-rc4", path = "../executor" } +sc-block-builder = { version = "0.8.0-rc4", path = "../../client/block-builder" } +sc-keystore = { version = "2.0.0-rc4", path = "../keystore" } +sp-transaction-pool = { version = "2.0.0-rc4", path = "../../primitives/transaction-pool" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../primitives/blockchain" } hash-db = { version = "0.15.2", default-features = false } parking_lot = "0.10.0" [dev-dependencies] assert_matches = "1.3.0" futures01 = { package = "futures", version = "0.1.29" } -sc-network = { version = "0.8.0-alpha.8", path = "../network" } -sp-io = { version = "2.0.0-alpha.8", path = "../../primitives/io" } -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../test-utils/runtime/client" } +sc-network = { version = "0.8.0-rc4", path = "../network" } +sp-io = { version = "2.0.0-rc4", path = "../../primitives/io" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../test-utils/runtime/client" } tokio = "0.1.22" -sc-transaction-pool = { version = "2.0.0-alpha.8", path = "../transaction-pool" } +sc-transaction-pool = { version = "2.0.0-rc4", path = "../transaction-pool" } lazy_static = "1.4.0" diff --git a/client/rpc/src/author/mod.rs b/client/rpc/src/author/mod.rs index 297fe2794b6e66c0e5339c7c622056e37fdaabd9..974c1b85e1b39816a9be584c6bfc56b10796c2c2 100644 --- a/client/rpc/src/author/mod.rs +++ b/client/rpc/src/author/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate block-author/full-node API. #[cfg(test)] @@ -31,8 +32,8 @@ use rpc::futures::{ }; use futures::{StreamExt as _, compat::Compat}; use futures::future::{ready, FutureExt, TryFutureExt}; -use sc_rpc_api::{DenyUnsafe, Subscriptions}; -use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId}; +use sc_rpc_api::DenyUnsafe; +use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId, manager::SubscriptionManager}; use codec::{Encode, Decode}; use sp_core::{Bytes, traits::BareCryptoStorePtr}; use sp_api::ProvideRuntimeApi; @@ -54,7 +55,7 @@ pub struct Author { /// Transactions pool pool: Arc

, /// Subscriptions manager - subscriptions: Subscriptions, + subscriptions: SubscriptionManager, /// The key store. keystore: BareCryptoStorePtr, /// Whether to deny unsafe calls @@ -66,7 +67,7 @@ impl Author { pub fn new( client: Arc, pool: Arc

, - subscriptions: Subscriptions, + subscriptions: SubscriptionManager, keystore: BareCryptoStorePtr, deny_unsafe: DenyUnsafe, ) -> Self { @@ -80,7 +81,6 @@ impl Author { } } - /// Currently we treat all RPC transactions as externals. /// /// Possibly in the future we could allow opt-in for special treatment diff --git a/client/rpc/src/author/tests.rs b/client/rpc/src/author/tests.rs index 2f944afc8f83aa89ed8e42a20afa52763432b8bc..f2f4ddebb2f1d3d272ceb3d00be3b1c8ff62e038 100644 --- a/client/rpc/src/author/tests.rs +++ b/client/rpc/src/author/tests.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use super::*; use std::{mem, sync::Arc}; @@ -57,11 +58,9 @@ struct TestSetup { impl Default for TestSetup { fn default() -> Self { let keystore = KeyStore::new(); - let client = Arc::new( - substrate_test_runtime_client::TestClientBuilder::new() - .set_keystore(keystore.clone()) - .build() - ); + let client_builder = substrate_test_runtime_client::TestClientBuilder::new(); + let client = Arc::new(client_builder.set_keystore(keystore.clone()).build()); + let pool = Arc::new(BasicPool::new( Default::default(), Arc::new(FullChainApi::new(client.clone())), @@ -80,7 +79,7 @@ impl TestSetup { Author { client: self.client.clone(), pool: self.pool.clone(), - subscriptions: Subscriptions::new(Arc::new(crate::testing::TaskExecutor)), + subscriptions: SubscriptionManager::new(Arc::new(crate::testing::TaskExecutor)), keystore: self.keystore.clone(), deny_unsafe: DenyUnsafe::No, } @@ -132,8 +131,14 @@ fn should_watch_extrinsic() { uxt(AccountKeyring::Alice, 0).encode().into(), ); - // then - assert_eq!(executor::block_on(id_rx.compat()), Ok(Ok(1.into()))); + let id = executor::block_on(id_rx.compat()).unwrap().unwrap(); + assert_matches!(id, SubscriptionId::String(_)); + + let id = match id { + SubscriptionId::String(id) => id, + _ => unreachable!(), + }; + // check notifications let replacement = { let tx = Transfer { @@ -146,15 +151,22 @@ fn should_watch_extrinsic() { }; AuthorApi::submit_extrinsic(&p, replacement.encode().into()).wait().unwrap(); let (res, data) = executor::block_on(data.into_future().compat()).unwrap(); - assert_eq!( - res, - Some(r#"{"jsonrpc":"2.0","method":"test","params":{"result":"ready","subscription":1}}"#.into()) - ); + + let expected = Some(format!( + r#"{{"jsonrpc":"2.0","method":"test","params":{{"result":"ready","subscription":"{}"}}}}"#, + id, + )); + assert_eq!(res, expected); + let h = blake2_256(&replacement.encode()); - assert_eq!( - executor::block_on(data.into_future().compat()).unwrap().0, - Some(format!(r#"{{"jsonrpc":"2.0","method":"test","params":{{"result":{{"usurped":"0x{}"}},"subscription":1}}}}"#, HexDisplay::from(&h))) - ); + let expected = Some(format!( + r#"{{"jsonrpc":"2.0","method":"test","params":{{"result":{{"usurped":"0x{}"}},"subscription":"{}"}}}}"#, + HexDisplay::from(&h), + id, + )); + + let res = executor::block_on(data.into_future().compat()).unwrap().0; + assert_eq!(res, expected); } #[test] diff --git a/client/rpc/src/chain/chain_full.rs b/client/rpc/src/chain/chain_full.rs index c1b062754bdac41eff2a2c3c04ad1375b09ce915..816dbba8664172a2cf467a5dc723386e689823d3 100644 --- a/client/rpc/src/chain/chain_full.rs +++ b/client/rpc/src/chain/chain_full.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use rpc::futures::future::result; +use jsonrpc_pubsub::manager::SubscriptionManager; -use sc_rpc_api::Subscriptions; use sc_client_api::{BlockchainEvents, BlockBackend}; use sp_runtime::{generic::{BlockId, SignedBlock}, traits::{Block as BlockT}}; @@ -32,14 +32,14 @@ pub struct FullChain { /// Substrate client. client: Arc, /// Current subscriptions. - subscriptions: Subscriptions, + subscriptions: SubscriptionManager, /// phantom member to pin the block type _phantom: PhantomData, } impl FullChain { /// Create new Chain API RPC handler. - pub fn new(client: Arc, subscriptions: Subscriptions) -> Self { + pub fn new(client: Arc, subscriptions: SubscriptionManager) -> Self { Self { client, subscriptions, @@ -56,7 +56,7 @@ impl ChainBackend for FullChain whe &self.client } - fn subscriptions(&self) -> &Subscriptions { + fn subscriptions(&self) -> &SubscriptionManager { &self.subscriptions } diff --git a/client/rpc/src/chain/chain_light.rs b/client/rpc/src/chain/chain_light.rs index 059233089d05d1559880c2ace86975d2d562fffd..8a4afbed71c165708fd5d4b0effca47abe6995f7 100644 --- a/client/rpc/src/chain/chain_light.rs +++ b/client/rpc/src/chain/chain_light.rs @@ -19,8 +19,8 @@ use std::sync::Arc; use futures::{future::ready, FutureExt, TryFutureExt}; use rpc::futures::future::{result, Future, Either}; +use jsonrpc_pubsub::manager::SubscriptionManager; -use sc_rpc_api::Subscriptions; use sc_client_api::light::{Fetcher, RemoteBodyRequest, RemoteBlockchain}; use sp_runtime::{ generic::{BlockId, SignedBlock}, @@ -37,7 +37,7 @@ pub struct LightChain { /// Substrate client. client: Arc, /// Current subscriptions. - subscriptions: Subscriptions, + subscriptions: SubscriptionManager, /// Remote blockchain reference remote_blockchain: Arc>, /// Remote fetcher reference. @@ -48,7 +48,7 @@ impl> LightChain { /// Create new Chain API RPC handler. pub fn new( client: Arc, - subscriptions: Subscriptions, + subscriptions: SubscriptionManager, remote_blockchain: Arc>, fetcher: Arc, ) -> Self { @@ -70,7 +70,7 @@ impl ChainBackend for LightChain &Subscriptions { + fn subscriptions(&self) -> &SubscriptionManager { &self.subscriptions } diff --git a/client/rpc/src/chain/mod.rs b/client/rpc/src/chain/mod.rs index 23e43e57fb53b7948d35f20cb586c20faa402b51..8b6bf19d235659b53d4da23551503d1e60f68467 100644 --- a/client/rpc/src/chain/mod.rs +++ b/client/rpc/src/chain/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate blockchain API. mod chain_full; @@ -31,9 +32,8 @@ use rpc::{ futures::{stream, Future, Sink, Stream}, }; -use sc_rpc_api::Subscriptions; use sc_client_api::{BlockchainEvents, light::{Fetcher, RemoteBlockchain}}; -use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId}; +use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId, manager::SubscriptionManager}; use sp_rpc::{number::NumberOrHex, list::ListOrValue}; use sp_runtime::{ generic::{BlockId, SignedBlock}, @@ -56,7 +56,7 @@ trait ChainBackend: Send + Sync + 'static fn client(&self) -> &Arc; /// Get subscriptions reference. - fn subscriptions(&self) -> &Subscriptions; + fn subscriptions(&self) -> &SubscriptionManager; /// Tries to unwrap passed block hash, or uses best block hash otherwise. fn unwrap_or_best(&self, hash: Option) -> Block::Hash { @@ -75,17 +75,27 @@ trait ChainBackend: Send + Sync + 'static /// Get hash of the n-th block in the canon chain. /// /// By default returns latest block hash. - fn block_hash( - &self, - number: Option>>, - ) -> Result> { - Ok(match number { - None => Some(self.client().info().best_hash), - Some(num_or_hex) => self.client() - .header(BlockId::number(num_or_hex.to_number()?)) - .map_err(client_err)? - .map(|h| h.hash()), - }) + fn block_hash(&self, number: Option) -> Result> { + match number { + None => Ok(Some(self.client().info().best_hash)), + Some(num_or_hex) => { + use std::convert::TryInto; + + // FIXME <2329>: Database seems to limit the block number to u32 for no reason + let block_num: u32 = num_or_hex.try_into().map_err(|_| { + Error::from(format!( + "`{:?}` > u32::max_value(), the max block number is u32.", + num_or_hex + )) + })?; + let block_num = >::from(block_num); + Ok(self + .client() + .header(BlockId::number(block_num)) + .map_err(client_err)? + .map(|h| h.hash())) + } + } } /// Get hash of the last finalized block in the canon chain. @@ -176,7 +186,7 @@ trait ChainBackend: Send + Sync + 'static /// Create new state API that works on full node. pub fn new_full( client: Arc, - subscriptions: Subscriptions, + subscriptions: SubscriptionManager, ) -> Chain where Block: BlockT + 'static, @@ -190,7 +200,7 @@ pub fn new_full( /// Create new state API that works on light node. pub fn new_light>( client: Arc, - subscriptions: Subscriptions, + subscriptions: SubscriptionManager, remote_blockchain: Arc>, fetcher: Arc, ) -> Chain @@ -233,7 +243,7 @@ impl ChainApi, Block::Hash, Block::Header, Signe fn block_hash( &self, - number: Option>>> + number: Option>, ) -> Result>> { match number { None => self.backend.block_hash(None).map(ListOrValue::Value), @@ -278,7 +288,7 @@ impl ChainApi, Block::Hash, Block::Header, Signe /// Subscribe to new headers. fn subscribe_headers( client: &Arc, - subscriptions: &Subscriptions, + subscriptions: &SubscriptionManager, subscriber: Subscriber, best_block_hash: G, stream: F, diff --git a/client/rpc/src/chain/tests.rs b/client/rpc/src/chain/tests.rs index 3c8ee5c817851dc575f3bd08f6e5c7d57de2bc95..b36fc4eab1d865f5a3c97e2d615d88cf33955274 100644 --- a/client/rpc/src/chain/tests.rs +++ b/client/rpc/src/chain/tests.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use super::*; use assert_matches::assert_matches; use substrate_test_runtime_client::{ @@ -30,7 +31,7 @@ use crate::testing::TaskExecutor; #[test] fn should_return_header() { let client = Arc::new(substrate_test_runtime_client::new()); - let api = new_full(client.clone(), Subscriptions::new(Arc::new(TaskExecutor))); + let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); assert_matches!( api.header(Some(client.genesis_hash()).into()).wait(), @@ -62,7 +63,7 @@ fn should_return_header() { #[test] fn should_return_a_block() { let mut client = Arc::new(substrate_test_runtime_client::new()); - let api = new_full(client.clone(), Subscriptions::new(Arc::new(TaskExecutor))); + let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); let block = client.new_block(Default::default()).unwrap().build().unwrap().block; let block_hash = block.hash(); @@ -113,7 +114,7 @@ fn should_return_a_block() { #[test] fn should_return_block_hash() { let mut client = Arc::new(substrate_test_runtime_client::new()); - let api = new_full(client.clone(), Subscriptions::new(Arc::new(TaskExecutor))); + let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); assert_matches!( api.block_hash(None.into()), @@ -148,7 +149,7 @@ fn should_return_block_hash() { ); assert_matches!( - api.block_hash(Some(vec![0u64.into(), 1.into(), 2.into()].into())), + api.block_hash(Some(vec![0u64.into(), 1u64.into(), 2u64.into()].into())), Ok(ListOrValue::List(list)) if list == &[client.genesis_hash().into(), block.hash().into(), None] ); } @@ -157,7 +158,7 @@ fn should_return_block_hash() { #[test] fn should_return_finalized_hash() { let mut client = Arc::new(substrate_test_runtime_client::new()); - let api = new_full(client.clone(), Subscriptions::new(Arc::new(TaskExecutor))); + let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); assert_matches!( api.finalized_head(), @@ -187,12 +188,15 @@ fn should_notify_about_latest_block() { { let mut client = Arc::new(substrate_test_runtime_client::new()); - let api = new_full(client.clone(), Subscriptions::new(Arc::new(TaskExecutor))); + let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); api.subscribe_all_heads(Default::default(), subscriber); // assert id assigned - assert_eq!(executor::block_on(id.compat()), Ok(Ok(SubscriptionId::Number(1)))); + assert!(matches!( + executor::block_on(id.compat()), + Ok(Ok(SubscriptionId::String(_))) + )); let block = client.new_block(Default::default()).unwrap().build().unwrap().block; client.import(BlockOrigin::Own, block).unwrap(); @@ -214,12 +218,15 @@ fn should_notify_about_best_block() { { let mut client = Arc::new(substrate_test_runtime_client::new()); - let api = new_full(client.clone(), Subscriptions::new(Arc::new(TaskExecutor))); + let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); api.subscribe_new_heads(Default::default(), subscriber); // assert id assigned - assert_eq!(executor::block_on(id.compat()), Ok(Ok(SubscriptionId::Number(1)))); + assert!(matches!( + executor::block_on(id.compat()), + Ok(Ok(SubscriptionId::String(_))) + )); let block = client.new_block(Default::default()).unwrap().build().unwrap().block; client.import(BlockOrigin::Own, block).unwrap(); @@ -241,12 +248,15 @@ fn should_notify_about_finalized_block() { { let mut client = Arc::new(substrate_test_runtime_client::new()); - let api = new_full(client.clone(), Subscriptions::new(Arc::new(TaskExecutor))); + let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); api.subscribe_finalized_heads(Default::default(), subscriber); // assert id assigned - assert_eq!(executor::block_on(id.compat()), Ok(Ok(SubscriptionId::Number(1)))); + assert!(matches!( + executor::block_on(id.compat()), + Ok(Ok(SubscriptionId::String(_))) + )); let block = client.new_block(Default::default()).unwrap().build().unwrap().block; client.import(BlockOrigin::Own, block).unwrap(); diff --git a/client/rpc/src/lib.rs b/client/rpc/src/lib.rs index defb7257a5d79f4bc4d776cac535bbaa2cd1cc4b..53a63b449c87e8b3382e3f7b3230539c6a749d15 100644 --- a/client/rpc/src/lib.rs +++ b/client/rpc/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate RPC implementation. //! //! A core implementation of Substrate RPC interfaces. @@ -23,7 +24,7 @@ mod metadata; -pub use sc_rpc_api::{DenyUnsafe, Subscriptions}; +pub use sc_rpc_api::DenyUnsafe; pub use self::metadata::Metadata; pub use rpc::IoHandlerExtension as RpcExtension; diff --git a/client/rpc/src/metadata.rs b/client/rpc/src/metadata.rs index cf795197f5048f81acb601d90bee2f30d9c04514..0416b07a6797bb21501bb71f343106efa595087f 100644 --- a/client/rpc/src/metadata.rs +++ b/client/rpc/src/metadata.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! RPC Metadata use std::sync::Arc; diff --git a/client/rpc/src/offchain/mod.rs b/client/rpc/src/offchain/mod.rs index 4ec11bfeb5676ed70a113155280c3023ea14db29..f8d2bb6a50f9c653491dc0536e019ad99ce20aa1 100644 --- a/client/rpc/src/offchain/mod.rs +++ b/client/rpc/src/offchain/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate offchain API. #[cfg(test)] diff --git a/client/rpc/src/offchain/tests.rs b/client/rpc/src/offchain/tests.rs index 8fa23715f79a700048684acb43b7f671844e55f1..f65971a7ffe8ae278495dfb80921e979806c187a 100644 --- a/client/rpc/src/offchain/tests.rs +++ b/client/rpc/src/offchain/tests.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use super::*; use assert_matches::assert_matches; use sp_core::{Bytes, offchain::storage::InMemOffchainStorage}; diff --git a/client/rpc/src/state/mod.rs b/client/rpc/src/state/mod.rs index 395da68a10c67b9e0bd15f982885b0c931aa84f0..921cc7efc699d9d902e114aed7586983411f77f3 100644 --- a/client/rpc/src/state/mod.rs +++ b/client/rpc/src/state/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate state API. mod state_full; @@ -24,10 +25,10 @@ mod state_light; mod tests; use std::sync::Arc; -use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId}; +use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId, manager::SubscriptionManager}; use rpc::{Result as RpcResult, futures::{Future, future::result}}; -use sc_rpc_api::{Subscriptions, state::ReadProof}; +use sc_rpc_api::state::ReadProof; use sc_client_api::light::{RemoteBlockchain, Fetcher}; use sp_core::{Bytes, storage::{StorageKey, PrefixedStorageKey, StorageData, StorageChangeSet}}; use sp_version::RuntimeVersion; @@ -169,7 +170,7 @@ pub trait StateBackend: Send + Sync + 'static /// Create new state API that works on full node. pub fn new_full( client: Arc, - subscriptions: Subscriptions, + subscriptions: SubscriptionManager, ) -> (State, ChildState) where Block: BlockT + 'static, @@ -190,7 +191,7 @@ pub fn new_full( /// Create new state API that works on light node. pub fn new_light>( client: Arc, - subscriptions: Subscriptions, + subscriptions: SubscriptionManager, remote_blockchain: Arc>, fetcher: Arc, ) -> (State, ChildState) diff --git a/client/rpc/src/state/state_full.rs b/client/rpc/src/state/state_full.rs index 82f87e9acf22352c122412ee9f7cba5522414786..f0ae79a033b58dfb4ae48e344b03ac58975906b3 100644 --- a/client/rpc/src/state/state_full.rs +++ b/client/rpc/src/state/state_full.rs @@ -21,10 +21,10 @@ use std::sync::Arc; use std::ops::Range; use futures::{future, StreamExt as _, TryStreamExt as _}; use log::warn; -use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId}; +use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId, manager::SubscriptionManager}; use rpc::{Result as RpcResult, futures::{stream, Future, Sink, Stream, future::result}}; -use sc_rpc_api::{Subscriptions, state::ReadProof}; +use sc_rpc_api::state::ReadProof; use sc_client_api::backend::Backend; use sp_blockchain::{Result as ClientResult, Error as ClientError, HeaderMetadata, CachedHeaderMetadata, HeaderBackend}; use sc_client_api::BlockchainEvents; @@ -60,7 +60,7 @@ struct QueryStorageRange { /// State API backend for full nodes. pub struct FullState { client: Arc, - subscriptions: Subscriptions, + subscriptions: SubscriptionManager, _phantom: PhantomData<(BE, Block)> } @@ -72,7 +72,7 @@ impl FullState Block: BlockT + 'static, { /// Create new state API backend for full nodes. - pub fn new(client: Arc, subscriptions: Subscriptions) -> Self { + pub fn new(client: Arc, subscriptions: SubscriptionManager) -> Self { Self { client, subscriptions, _phantom: PhantomData } } diff --git a/client/rpc/src/state/state_light.rs b/client/rpc/src/state/state_light.rs index af5d4248e3a422e2bfc759bc40c0b269ee68ccfc..ec275a2d78b79d9a071c6220d8013e39d0b31ccd 100644 --- a/client/rpc/src/state/state_light.rs +++ b/client/rpc/src/state/state_light.rs @@ -28,7 +28,7 @@ use futures::{ StreamExt as _, TryStreamExt as _, }; use hash_db::Hasher; -use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId}; +use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId, manager::SubscriptionManager}; use log::warn; use parking_lot::Mutex; use rpc::{ @@ -38,7 +38,7 @@ use rpc::{ futures::stream::Stream, }; -use sc_rpc_api::{Subscriptions, state::ReadProof}; +use sc_rpc_api::state::ReadProof; use sp_blockchain::{Error as ClientError, HeaderBackend}; use sc_client_api::{ BlockchainEvents, @@ -63,7 +63,7 @@ type StorageMap = HashMap>; #[derive(Clone)] pub struct LightState, Client> { client: Arc, - subscriptions: Subscriptions, + subscriptions: SubscriptionManager, version_subscriptions: SimpleSubscriptions, storage_subscriptions: Arc>>, remote_blockchain: Arc>, @@ -143,7 +143,7 @@ impl + 'static, Client> LightState, - subscriptions: Subscriptions, + subscriptions: SubscriptionManager, remote_blockchain: Arc>, fetcher: Arc, ) -> Self { diff --git a/client/rpc/src/state/tests.rs b/client/rpc/src/state/tests.rs index 85a12e11f959adf2f73d44615e5dea5077a8dc56..0cc16ce8d5e927e3b55b8f3b292de1483fed1730 100644 --- a/client/rpc/src/state/tests.rs +++ b/client/rpc/src/state/tests.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use super::*; use super::state_full::split_range; use self::error::Error; @@ -54,7 +55,7 @@ fn should_return_storage() { .add_extra_child_storage(&child_info, KEY.to_vec(), CHILD_VALUE.to_vec()) .build(); let genesis_hash = client.genesis_hash(); - let (client, child) = new_full(Arc::new(client), Subscriptions::new(Arc::new(TaskExecutor))); + let (client, child) = new_full(Arc::new(client), SubscriptionManager::new(Arc::new(TaskExecutor))); let key = StorageKey(KEY.to_vec()); assert_eq!( @@ -89,7 +90,7 @@ fn should_return_child_storage() { .add_child_storage(&child_info, "key", vec![42_u8]) .build()); let genesis_hash = client.genesis_hash(); - let (_client, child) = new_full(client, Subscriptions::new(Arc::new(TaskExecutor))); + let (_client, child) = new_full(client, SubscriptionManager::new(Arc::new(TaskExecutor))); let child_key = prefixed_storage_key(); let key = StorageKey(b"key".to_vec()); @@ -124,7 +125,7 @@ fn should_return_child_storage() { fn should_call_contract() { let client = Arc::new(substrate_test_runtime_client::new()); let genesis_hash = client.genesis_hash(); - let (client, _child) = new_full(client, Subscriptions::new(Arc::new(TaskExecutor))); + let (client, _child) = new_full(client, SubscriptionManager::new(Arc::new(TaskExecutor))); assert_matches!( client.call("balanceOf".into(), Bytes(vec![1,2,3]), Some(genesis_hash).into()).wait(), @@ -138,12 +139,15 @@ fn should_notify_about_storage_changes() { { let mut client = Arc::new(substrate_test_runtime_client::new()); - let (api, _child) = new_full(client.clone(), Subscriptions::new(Arc::new(TaskExecutor))); + let (api, _child) = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); api.subscribe_storage(Default::default(), subscriber, None.into()); // assert id assigned - assert_eq!(executor::block_on(id.compat()), Ok(Ok(SubscriptionId::Number(1)))); + assert!(matches!( + executor::block_on(id.compat()), + Ok(Ok(SubscriptionId::String(_))) + )); let mut builder = client.new_block(Default::default()).unwrap(); builder.push_transfer(runtime::Transfer { @@ -169,7 +173,7 @@ fn should_send_initial_storage_changes_and_notifications() { { let mut client = Arc::new(substrate_test_runtime_client::new()); - let (api, _child) = new_full(client.clone(), Subscriptions::new(Arc::new(TaskExecutor))); + let (api, _child) = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); let alice_balance_key = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Alice.into())); @@ -178,7 +182,10 @@ fn should_send_initial_storage_changes_and_notifications() { ]).into()); // assert id assigned - assert_eq!(executor::block_on(id.compat()), Ok(Ok(SubscriptionId::Number(1)))); + assert!(matches!( + executor::block_on(id.compat()), + Ok(Ok(SubscriptionId::String(_))) + )); let mut builder = client.new_block(Default::default()).unwrap(); builder.push_transfer(runtime::Transfer { @@ -204,7 +211,7 @@ fn should_send_initial_storage_changes_and_notifications() { #[test] fn should_query_storage() { fn run_tests(mut client: Arc, has_changes_trie_config: bool) { - let (api, _child) = new_full(client.clone(), Subscriptions::new(Arc::new(TaskExecutor))); + let (api, _child) = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); let mut add_block = |nonce| { let mut builder = client.new_block(Default::default()).unwrap(); @@ -421,7 +428,7 @@ fn should_split_ranges() { #[test] fn should_return_runtime_version() { let client = Arc::new(substrate_test_runtime_client::new()); - let (api, _child) = new_full(client.clone(), Subscriptions::new(Arc::new(TaskExecutor))); + let (api, _child) = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); let result = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\ \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",3],\ @@ -444,12 +451,16 @@ fn should_notify_on_runtime_version_initially() { { let client = Arc::new(substrate_test_runtime_client::new()); - let (api, _child) = new_full(client.clone(), Subscriptions::new(Arc::new(TaskExecutor))); + let (api, _child) = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); api.subscribe_runtime_version(Default::default(), subscriber); // assert id assigned - assert_eq!(executor::block_on(id.compat()), Ok(Ok(SubscriptionId::Number(1)))); + assert!(matches!( + executor::block_on(id.compat()), + Ok(Ok(SubscriptionId::String(_))) + )); + } // assert initial version sent. diff --git a/client/rpc/src/system/mod.rs b/client/rpc/src/system/mod.rs index b73a924c41dd27a04149b6664d69f507bb4fc920..4b8abbe1444620e23467edfef567d9fd0f71c570 100644 --- a/client/rpc/src/system/mod.rs +++ b/client/rpc/src/system/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate system API. #[cfg(test)] diff --git a/client/rpc/src/system/tests.rs b/client/rpc/src/system/tests.rs index 826e85f2773c8cb2323fd76cd78fa0689261a78b..7fe5cdc752a06bde155f071cc5e812237b8b7403 100644 --- a/client/rpc/src/system/tests.rs +++ b/client/rpc/src/system/tests.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,13 +15,15 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use super::*; use sc_network::{self, PeerId}; use sc_network::config::Role; use substrate_test_runtime_client::runtime::Block; use assert_matches::assert_matches; -use futures::{prelude::*, channel::mpsc}; +use futures::prelude::*; +use sp_utils::mpsc::tracing_unbounded; use std::thread; struct Status { @@ -45,7 +47,7 @@ impl Default for Status { fn api>>(sync: T) -> System { let status = sync.into().unwrap_or_default(); let should_have_peers = !status.is_dev; - let (tx, rx) = mpsc::unbounded(); + let (tx, rx) = tracing_unbounded("rpc_system_tests"); thread::spawn(move || { futures::executor::block_on(rx.for_each(move |request| { match request { diff --git a/client/rpc/src/testing.rs b/client/rpc/src/testing.rs index 48f9f90b9d9b2fad991f412efe327264c78d67f6..afca07a7fbe6a63048c9cfdc4d44cf2fcb933095 100644 --- a/client/rpc/src/testing.rs +++ b/client/rpc/src/testing.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml index 04a6f488b1dd136f5720e4be5e1fa675605fe41f..f63d3f183db1d8a97931528502a2e95e9ea65ddf 100644 --- a/client/service/Cargo.toml +++ b/client/service/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-service" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] default = ["db"] # The RocksDB feature activates the RocksDB database backend. If it is not activated, and you pass # a path to a database, an error will be produced at runtime. -db = ["sc-client-db/kvdb-rocksdb", "sc-client-db/parity-db"] +db = ["sc-client-db/with-kvdb-rocksdb", "sc-client-db/with-parity-db"] wasmtime = [ "sc-executor/wasmtime", ] @@ -26,6 +26,7 @@ test-helpers = [] derive_more = "0.99.2" futures01 = { package = "futures", version = "0.1.29" } futures = { version = "0.3.4", features = ["compat"] } +jsonrpc-pubsub = "14.2.0" rand = "0.7.3" parking_lot = "0.10.0" lazy_static = "1.4.0" @@ -39,38 +40,39 @@ hash-db = "0.15.2" serde = "1.0.101" serde_json = "1.0.41" sysinfo = "0.13.3" -sc-keystore = { version = "2.0.0-alpha.8", path = "../keystore" } -sp-io = { version = "2.0.0-alpha.8", path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } -sp-trie = { version = "2.0.0-alpha.8", path = "../../primitives/trie" } -sp-externalities = { version = "0.8.0-alpha.8", path = "../../primitives/externalities" } -sp-utils = { version = "2.0.0-alpha.8", path = "../../primitives/utils" } -sp-version = { version = "2.0.0-alpha.8", path = "../../primitives/version" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../primitives/blockchain" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-session = { version = "2.0.0-alpha.8", path = "../../primitives/session" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../primitives/state-machine" } -sp-application-crypto = { version = "2.0.0-alpha.8", path = "../../primitives/application-crypto" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../primitives/consensus/common" } -sc-network = { version = "0.8.0-alpha.8", path = "../network" } -sc-chain-spec = { version = "2.0.0-alpha.8", path = "../chain-spec" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../api" } -sp-api = { version = "2.0.0-alpha.8", path = "../../primitives/api" } -sc-client-db = { version = "0.8.0-alpha.8", default-features = false, path = "../db" } -codec = { package = "parity-scale-codec", version = "1.3.0" } -sc-executor = { version = "0.8.0-alpha.8", path = "../executor" } -sc-transaction-pool = { version = "2.0.0-alpha.8", path = "../transaction-pool" } -sp-transaction-pool = { version = "2.0.0-alpha.8", path = "../../primitives/transaction-pool" } -sc-rpc-server = { version = "2.0.0-alpha.8", path = "../rpc-servers" } -sc-rpc = { version = "2.0.0-alpha.8", path = "../rpc" } -sc-block-builder = { version = "0.8.0-alpha.8", path = "../block-builder" } -sp-block-builder = { version = "2.0.0-alpha.8", path = "../../primitives/block-builder" } - -sc-telemetry = { version = "2.0.0-alpha.8", path = "../telemetry" } -sc-offchain = { version = "2.0.0-alpha.8", path = "../offchain" } +sc-keystore = { version = "2.0.0-rc4", path = "../keystore" } +sp-io = { version = "2.0.0-rc4", path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } +sp-trie = { version = "2.0.0-rc4", path = "../../primitives/trie" } +sp-externalities = { version = "0.8.0-rc4", path = "../../primitives/externalities" } +sp-utils = { version = "2.0.0-rc4", path = "../../primitives/utils" } +sp-version = { version = "2.0.0-rc4", path = "../../primitives/version" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../primitives/blockchain" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-session = { version = "2.0.0-rc4", path = "../../primitives/session" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../primitives/state-machine" } +sp-application-crypto = { version = "2.0.0-rc4", path = "../../primitives/application-crypto" } +sp-consensus = { version = "0.8.0-rc4", path = "../../primitives/consensus/common" } +sc-network = { version = "0.8.0-rc4", path = "../network" } +sc-chain-spec = { version = "2.0.0-rc4", path = "../chain-spec" } +sc-light = { version = "2.0.0-rc4", path = "../light" } +sc-client-api = { version = "2.0.0-rc4", path = "../api" } +sp-api = { version = "2.0.0-rc4", path = "../../primitives/api" } +sc-client-db = { version = "0.8.0-rc4", default-features = false, path = "../db" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +sc-executor = { version = "0.8.0-rc4", path = "../executor" } +sc-transaction-pool = { version = "2.0.0-rc4", path = "../transaction-pool" } +sp-transaction-pool = { version = "2.0.0-rc4", path = "../../primitives/transaction-pool" } +sc-rpc-server = { version = "2.0.0-rc4", path = "../rpc-servers" } +sc-rpc = { version = "2.0.0-rc4", path = "../rpc" } +sc-block-builder = { version = "0.8.0-rc4", path = "../block-builder" } +sp-block-builder = { version = "2.0.0-rc4", path = "../../primitives/block-builder" } +sc-informant = { version = "0.8.0-rc2", path = "../informant" } +sc-telemetry = { version = "2.0.0-rc4", path = "../telemetry" } +sc-offchain = { version = "2.0.0-rc4", path = "../offchain" } parity-multiaddr = { package = "parity-multiaddr", version = "0.7.3" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" , version = "0.8.0-alpha.8"} -sc-tracing = { version = "2.0.0-alpha.8", path = "../tracing" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" , version = "0.8.0-rc4"} +sc-tracing = { version = "2.0.0-rc4", path = "../tracing" } tracing = "0.1.10" parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } @@ -81,9 +83,12 @@ netstat2 = "0.8.1" [target.'cfg(target_os = "linux")'.dependencies] procfs = '0.7.8' +[target.'cfg(not(target_os = "unknown"))'.dependencies] +tempfile = "3.1.0" +directories = "2.0.2" [dev-dependencies] -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../test-utils/runtime/client" } -sp-consensus-babe = { version = "0.8.0-alpha.8", path = "../../primitives/consensus/babe" } -grandpa = { version = "0.8.0-alpha.8", package = "sc-finality-grandpa", path = "../finality-grandpa" } -grandpa-primitives = { version = "2.0.0-alpha.8", package = "sp-finality-grandpa", path = "../../primitives/finality-grandpa" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../test-utils/runtime/client" } +sp-consensus-babe = { version = "0.8.0-rc4", path = "../../primitives/consensus/babe" } +grandpa = { version = "0.8.0-rc4", package = "sc-finality-grandpa", path = "../finality-grandpa" } +grandpa-primitives = { version = "2.0.0-rc4", package = "sp-finality-grandpa", path = "../../primitives/finality-grandpa" } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 1bbf0658252469fe2b23e519e306410459f7a22b..234356856b313a3674bf6ed146d441f9d211a3fd 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,36 +15,39 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::{Service, NetworkStatus, NetworkState, error::Error, DEFAULT_PROTOCOL_ID, MallocSizeOfWasm}; -use crate::{start_rpc_servers, build_network_future, TransactionPoolAdapter, TaskManager, SpawnTaskHandle}; -use crate::status_sinks; -use crate::config::{Configuration, KeystoreConfig, PrometheusConfig, OffchainWorkerConfig}; -use crate::metrics::MetricsService; + +use crate::{ + NetworkStatus, NetworkState, error::Error, DEFAULT_PROTOCOL_ID, MallocSizeOfWasm, + start_rpc_servers, build_network_future, TransactionPoolAdapter, TaskManager, SpawnTaskHandle, + status_sinks, metrics::MetricsService, + client::{light, Client, ClientConfig}, + config::{Configuration, KeystoreConfig, PrometheusConfig, OffchainWorkerConfig}, +}; use sc_client_api::{ - self, BlockchainEvents, backend::RemoteBackend, light::RemoteBlockchain, execution_extensions::ExtensionsFactory, - ExecutorProvider, CallExecutor, ForkBlocks, BadBlocks, CloneableSpawn, UsageProvider, + self, light::RemoteBlockchain, execution_extensions::ExtensionsFactory, ExecutorProvider, + ForkBlocks, BadBlocks, CloneableSpawn, UsageProvider, backend::RemoteBackend, }; -use crate::client::{Client, ClientConfig}; -use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver}; use sc_chain_spec::get_extension; use sp_consensus::{ - block_validation::{BlockAnnounceValidator, DefaultBlockAnnounceValidator}, + block_validation::{BlockAnnounceValidator, DefaultBlockAnnounceValidator, Chain}, import_queue::ImportQueue, }; use futures::{ Future, FutureExt, StreamExt, future::ready, }; +use jsonrpc_pubsub::manager::SubscriptionManager; use sc_keystore::Store as Keystore; use log::{info, warn, error}; use sc_network::config::{Role, FinalityProofProvider, OnDemand, BoxFinalityProofRequestBuilder}; -use sc_network::{NetworkService, NetworkStateInfo}; +use sc_network::NetworkService; use parking_lot::{Mutex, RwLock}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{ - Block as BlockT, NumberFor, SaturatedConversion, HashFor, + Block as BlockT, NumberFor, SaturatedConversion, HashFor, Zero, BlockIdTo, }; -use sp_api::ProvideRuntimeApi; +use sp_api::{ProvideRuntimeApi, CallApiAt}; use sc_executor::{NativeExecutor, NativeExecutionDispatch, RuntimeInfo}; use std::{ collections::HashMap, @@ -53,14 +56,20 @@ use std::{ }; use wasm_timer::SystemTime; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; -use sp_transaction_pool::{MaintainedTransactionPool, ChainEvent}; -use sp_blockchain; +use sp_transaction_pool::{LocalTransactionPool, MaintainedTransactionPool}; use prometheus_endpoint::Registry; use sc_client_db::{Backend, DatabaseSettings}; use sp_core::traits::CodeExecutor; use sp_runtime::BuildStorage; -use sc_client_api::execution_extensions::ExecutionExtensions; +use sc_client_api::{ + BlockBackend, BlockchainEvents, + backend::StorageProvider, + proof_provider::ProofProvider, + execution_extensions::ExecutionExtensions +}; use sp_core::storage::Storage; +use sp_blockchain::{HeaderMetadata, HeaderBackend}; +use crate::{ServiceComponents, TelemetryOnConnectSinks, RpcHandlers, NetworkStatusSinks}; pub type BackgroundTask = Pin + Send>>; @@ -95,12 +104,61 @@ pub struct ServiceBuilder, finality_proof_provider: Option, transaction_pool: Arc, - rpc_extensions: TRpc, + rpc_extensions_builder: Box + Send>, remote_backend: Option>>, marker: PhantomData<(TBl, TRtApi)>, block_announce_validator_builder: Option) -> Box + Send> + Send>>, } +/// A utility trait for building an RPC extension given a `DenyUnsafe` instance. +/// This is useful since at service definition time we don't know whether the +/// specific interface where the RPC extension will be exposed is safe or not. +/// This trait allows us to lazily build the RPC extension whenever we bind the +/// service to an interface. +pub trait RpcExtensionBuilder { + /// The type of the RPC extension that will be built. + type Output: sc_rpc::RpcExtension; + + /// Returns an instance of the RPC extension for a particular `DenyUnsafe` + /// value, e.g. the RPC extension might not expose some unsafe methods. + fn build(&self, deny: sc_rpc::DenyUnsafe) -> Self::Output; +} + +impl RpcExtensionBuilder for F where + F: Fn(sc_rpc::DenyUnsafe) -> R, + R: sc_rpc::RpcExtension, +{ + type Output = R; + + fn build(&self, deny: sc_rpc::DenyUnsafe) -> Self::Output { + (*self)(deny) + } +} + +/// A utility struct for implementing an `RpcExtensionBuilder` given a cloneable +/// `RpcExtension`, the resulting builder will simply ignore the provided +/// `DenyUnsafe` instance and return a static `RpcExtension` instance. +struct NoopRpcExtensionBuilder(R); + +impl RpcExtensionBuilder for NoopRpcExtensionBuilder where + R: Clone + sc_rpc::RpcExtension, +{ + type Output = R; + + fn build(&self, _deny: sc_rpc::DenyUnsafe) -> Self::Output { + self.0.clone() + } +} + +impl From for NoopRpcExtensionBuilder where + R: sc_rpc::RpcExtension, +{ + fn from(e: R) -> NoopRpcExtensionBuilder { + NoopRpcExtensionBuilder(e) + } +} + + /// Full client type. pub type TFullClient = Client< TFullBackend, @@ -127,19 +185,19 @@ pub type TLightClient = Client< >; /// Light client backend type. -pub type TLightBackend = crate::client::light::backend::Backend< +pub type TLightBackend = sc_light::Backend< sc_client_db::light::LightStorage, HashFor, >; /// Light call executor type. -pub type TLightCallExecutor = crate::client::light::call_executor::GenesisCallExecutor< - crate::client::light::backend::Backend< +pub type TLightCallExecutor = sc_light::GenesisCallExecutor< + sc_light::Backend< sc_client_db::light::LightStorage, HashFor >, crate::client::LocalCallExecutor< - crate::client::light::backend::Backend< + sc_light::Backend< sc_client_db::light::LightStorage, HashFor >, @@ -310,7 +368,7 @@ impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { finality_proof_request_builder: None, finality_proof_provider: None, transaction_pool: Arc::new(()), - rpc_extensions: Default::default(), + rpc_extensions_builder: Box::new(|_| ()), remote_backend: None, block_announce_validator_builder: None, marker: PhantomData, @@ -362,18 +420,18 @@ impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { }; sc_client_db::light::LightStorage::new(db_settings)? }; - let light_blockchain = crate::client::light::new_light_blockchain(db_storage); + let light_blockchain = sc_light::new_light_blockchain(db_storage); let fetch_checker = Arc::new( - crate::client::light::new_fetch_checker::<_, TBl, _>( + sc_light::new_fetch_checker::<_, TBl, _>( light_blockchain.clone(), executor.clone(), Box::new(task_manager.spawn_handle()), ), ); let fetcher = Arc::new(sc_network::config::OnDemand::new(fetch_checker)); - let backend = crate::client::light::new_light_backend(light_blockchain); + let backend = sc_light::new_light_backend(light_blockchain); let remote_blockchain = backend.remote_blockchain(); - let client = Arc::new(crate::client::light::new_light( + let client = Arc::new(light::new_light( backend.clone(), config.chain_spec.as_storage_builder(), executor, @@ -393,7 +451,7 @@ impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { finality_proof_request_builder: None, finality_proof_provider: None, transaction_pool: Arc::new(()), - rpc_extensions: Default::default(), + rpc_extensions_builder: Box::new(|_| ()), remote_backend: Some(remote_blockchain), block_announce_validator_builder: None, marker: PhantomData, @@ -402,8 +460,29 @@ impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { } impl - ServiceBuilder { + ServiceBuilder< + TBl, + TRtApi, + TCl, + TFchr, + TSc, + TImpQu, + TFprb, + TFpp, + TExPool, + TRpc, + Backend + > +{ + /// Returns a reference to the configuration that was stored in this builder. + pub fn config(&self) -> &Configuration { + &self.config + } + + /// Returns a reference to the optional prometheus registry that was stored in this builder. + pub fn prometheus_registry(&self) -> Option<&Registry> { + self.config.prometheus_config.as_ref().map(|config| &config.registry) + } /// Returns a reference to the client that was stored in this builder. pub fn client(&self) -> &Arc { @@ -466,7 +545,7 @@ impl finality_proof_request_builder: self.finality_proof_request_builder, finality_proof_provider: self.finality_proof_provider, transaction_pool: self.transaction_pool, - rpc_extensions: self.rpc_extensions, + rpc_extensions_builder: self.rpc_extensions_builder, remote_backend: self.remote_backend, block_announce_validator_builder: self.block_announce_validator_builder, marker: self.marker, @@ -511,7 +590,7 @@ impl finality_proof_request_builder: self.finality_proof_request_builder, finality_proof_provider: self.finality_proof_provider, transaction_pool: self.transaction_pool, - rpc_extensions: self.rpc_extensions, + rpc_extensions_builder: self.rpc_extensions_builder, remote_backend: self.remote_backend, block_announce_validator_builder: self.block_announce_validator_builder, marker: self.marker, @@ -549,7 +628,7 @@ impl finality_proof_request_builder: self.finality_proof_request_builder, finality_proof_provider, transaction_pool: self.transaction_pool, - rpc_extensions: self.rpc_extensions, + rpc_extensions_builder: self.rpc_extensions_builder, remote_backend: self.remote_backend, block_announce_validator_builder: self.block_announce_validator_builder, marker: self.marker, @@ -615,7 +694,7 @@ impl finality_proof_request_builder: fprb, finality_proof_provider: self.finality_proof_provider, transaction_pool: self.transaction_pool, - rpc_extensions: self.rpc_extensions, + rpc_extensions_builder: self.rpc_extensions_builder, remote_backend: self.remote_backend, block_announce_validator_builder: self.block_announce_validator_builder, marker: self.marker, @@ -648,20 +727,12 @@ impl pub fn with_transaction_pool( self, transaction_pool_builder: impl FnOnce( - sc_transaction_pool::txpool::Options, - Arc, - Option, - Option<&Registry>, - ) -> Result<(UExPool, Option), Error> + &Self, + ) -> Result<(UExPool, Option), Error>, ) -> Result, Error> where TSc: Clone, TFchr: Clone { - let (transaction_pool, background_task) = transaction_pool_builder( - self.config.transaction_pool.clone(), - self.client.clone(), - self.fetcher.clone(), - self.config.prometheus_config.as_ref().map(|config| &config.registry), - )?; + let (transaction_pool, background_task) = transaction_pool_builder(&self)?; if let Some(background_task) = background_task{ self.task_manager.spawn_handle().spawn("txpool-background", background_task); @@ -679,21 +750,30 @@ impl finality_proof_request_builder: self.finality_proof_request_builder, finality_proof_provider: self.finality_proof_provider, transaction_pool: Arc::new(transaction_pool), - rpc_extensions: self.rpc_extensions, + rpc_extensions_builder: self.rpc_extensions_builder, remote_backend: self.remote_backend, block_announce_validator_builder: self.block_announce_validator_builder, marker: self.marker, }) } - /// Defines the RPC extensions to use. - pub fn with_rpc_extensions( + /// Defines the RPC extension builder to use. Unlike `with_rpc_extensions`, + /// this method is useful in situations where the RPC extensions need to + /// access to a `DenyUnsafe` instance to avoid exposing sensitive methods. + pub fn with_rpc_extensions_builder( self, - rpc_ext_builder: impl FnOnce(&Self) -> Result, - ) -> Result, Error> - where TSc: Clone, TFchr: Clone { - let rpc_extensions = rpc_ext_builder(&self)?; + rpc_extensions_builder: impl FnOnce(&Self) -> Result, + ) -> Result< + ServiceBuilder, + Error, + > + where + TSc: Clone, + TFchr: Clone, + URpcBuilder: RpcExtensionBuilder + Send + 'static, + URpc: sc_rpc::RpcExtension, + { + let rpc_extensions_builder = rpc_extensions_builder(&self)?; Ok(ServiceBuilder { config: self.config, @@ -707,13 +787,30 @@ impl finality_proof_request_builder: self.finality_proof_request_builder, finality_proof_provider: self.finality_proof_provider, transaction_pool: self.transaction_pool, - rpc_extensions, + rpc_extensions_builder: Box::new(rpc_extensions_builder), remote_backend: self.remote_backend, block_announce_validator_builder: self.block_announce_validator_builder, marker: self.marker, }) } + /// Defines the RPC extensions to use. + pub fn with_rpc_extensions( + self, + rpc_extensions: impl FnOnce(&Self) -> Result, + ) -> Result< + ServiceBuilder, + Error, + > + where + TSc: Clone, + TFchr: Clone, + URpc: Clone + sc_rpc::RpcExtension + Send + 'static, + { + let rpc_extensions = rpc_extensions(&self)?; + self.with_rpc_extensions_builder(|_| Ok(NoopRpcExtensionBuilder::from(rpc_extensions))) + } + /// Defines the `BlockAnnounceValidator` to use. `DefaultBlockAnnounceValidator` will be used by /// default. pub fn with_block_announce_validator( @@ -735,7 +832,7 @@ impl finality_proof_request_builder: self.finality_proof_request_builder, finality_proof_provider: self.finality_proof_provider, transaction_pool: self.transaction_pool, - rpc_extensions: self.rpc_extensions, + rpc_extensions_builder: self.rpc_extensions_builder, remote_backend: self.remote_backend, block_announce_validator_builder: Some(Box::new(block_announce_validator_builder)), marker: self.marker, @@ -755,6 +852,7 @@ pub trait ServiceBuilderCommand { self, input: impl Read + Seek + Send + 'static, force: bool, + binary: bool, ) -> Pin> + Send>>; /// Performs the blocks export. @@ -786,11 +884,11 @@ pub trait ServiceBuilderCommand { ) -> Result; } -impl +impl ServiceBuilder< TBl, TRtApi, - Client, + TCl, Arc>, TSc, TImpQu, @@ -800,8 +898,12 @@ ServiceBuilder< TRpc, TBackend, > where - Client: ProvideRuntimeApi, - as ProvideRuntimeApi>::Api: + TCl: ProvideRuntimeApi + HeaderMetadata + Chain + + BlockBackend + BlockIdTo + ProofProvider + + HeaderBackend + BlockchainEvents + ExecutorProvider + UsageProvider + + StorageProvider + CallApiAt + + Send + 'static, + >::Api: sp_api::Metadata + sc_offchain::OffchainWorkerApi + sp_transaction_pool::runtime_api::TaggedTransactionQueue + @@ -811,11 +913,10 @@ ServiceBuilder< TBl: BlockT, TRtApi: 'static + Send + Sync, TBackend: 'static + sc_client_api::backend::Backend + Send, - TExec: 'static + CallExecutor + Send + Sync + Clone, TSc: Clone, TImpQu: 'static + ImportQueue, TExPool: MaintainedTransactionPool::Hash> + MallocSizeOfWasm + 'static, - TRpc: sc_rpc::RpcExtension + Clone, + TRpc: sc_rpc::RpcExtension, { /// Set an ExecutionExtensionsFactory @@ -824,27 +925,12 @@ ServiceBuilder< Ok(self) } - /// Builds the service. - pub fn build(self) -> Result, - TSc, - NetworkStatus, - NetworkService::Hash>, - TExPool, - sc_offchain::OffchainWorkers< - Client, - TBackend::OffchainStorage, - TBl - >, - >, Error> - where TExec: CallExecutor, - { + fn build_common(self) -> Result, Error> { let ServiceBuilder { marker: _, mut config, client, - task_manager, + mut task_manager, fetcher: on_demand, backend, keystore, @@ -853,25 +939,19 @@ ServiceBuilder< finality_proof_request_builder, finality_proof_provider, transaction_pool, - rpc_extensions, + rpc_extensions_builder, remote_backend, block_announce_validator_builder, } = self; + let chain_info = client.usage_info().chain; + sp_session::generate_initial_session_keys( client.clone(), - &BlockId::Hash(client.chain_info().best_hash), + &BlockId::Hash(chain_info.best_hash), config.dev_key_seed.clone().map(|s| vec![s]).unwrap_or_default(), )?; - // A side-channel for essential tasks to communicate shutdown. - let (essential_failed_tx, essential_failed_rx) = tracing_unbounded("mpsc_essential_tasks"); - - let import_queue = Box::new(import_queue); - let chain_info = client.chain_info(); - let chain_spec = &config.chain_spec; - - let version = config.impl_version; info!("📦 Highest known block at #{}", chain_info.best_number); telemetry!( SUBSTRATE_INFO; @@ -880,59 +960,27 @@ ServiceBuilder< "best" => ?chain_info.best_hash ); - // make transaction pool available for off-chain runtime calls. - client.execution_extensions() - .register_transaction_pool(Arc::downgrade(&transaction_pool) as _); - - let transaction_pool_adapter = Arc::new(TransactionPoolAdapter { - imports_external_transactions: !matches!(config.role, Role::Light), - pool: transaction_pool.clone(), - client: client.clone(), - }); - - let protocol_id = { - let protocol_id_full = match chain_spec.protocol_id() { - Some(pid) => pid, - None => { - warn!("Using default protocol ID {:?} because none is configured in the \ - chain specs", DEFAULT_PROTOCOL_ID - ); - DEFAULT_PROTOCOL_ID - } - }.as_bytes(); - sc_network::config::ProtocolId::from(protocol_id_full) - }; + let (system_rpc_tx, system_rpc_rx) = tracing_unbounded("mpsc_system_rpc"); - let block_announce_validator = if let Some(f) = block_announce_validator_builder { - f(client.clone()) - } else { - Box::new(DefaultBlockAnnounceValidator::new(client.clone())) - }; + let (network, network_status_sinks, network_future) = build_network( + &config, client.clone(), transaction_pool.clone(), task_manager.spawn_handle(), + on_demand.clone(), block_announce_validator_builder, finality_proof_request_builder, + finality_proof_provider, system_rpc_rx, import_queue + )?; - let network_params = sc_network::config::Params { - role: config.role.clone(), - executor: { - let spawn_handle = task_manager.spawn_handle(); - Some(Box::new(move |fut| { - spawn_handle.spawn("libp2p-node", fut); - })) - }, - network_config: config.network.clone(), - chain: client.clone(), - finality_proof_provider, - finality_proof_request_builder, - on_demand: on_demand.clone(), - transaction_pool: transaction_pool_adapter.clone() as _, - import_queue, - protocol_id, - block_announce_validator, - metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()) - }; + let spawn_handle = task_manager.spawn_handle(); - let has_bootnodes = !network_params.network_config.boot_nodes.is_empty(); - let network_mut = sc_network::NetworkWorker::new(network_params)?; - let network = network_mut.service().clone(); - let network_status_sinks = Arc::new(Mutex::new(status_sinks::StatusSinks::new())); + // The network worker is responsible for gathering all network messages and processing + // them. This is quite a heavy task, and at the time of the writing of this comment it + // frequently happens that this future takes several seconds or in some situations + // even more than a minute until it has processed its entire queue. This is clearly an + // issue, and ideally we would like to fix the network future to take as little time as + // possible, but we also take the extra harm-prevention measure to execute the networking + // future using `spawn_blocking`. + spawn_handle.spawn_blocking( + "network-worker", + network_future + ); let offchain_storage = backend.offchain_storage(); let offchain_workers = match (config.offchain_worker.clone(), offchain_storage.clone()) { @@ -946,108 +994,39 @@ ServiceBuilder< _ => None, }; - let spawn_handle = task_manager.spawn_handle(); - - { - // block notifications - let txpool = Arc::downgrade(&transaction_pool); - let offchain = offchain_workers.as_ref().map(Arc::downgrade); - let notifications_spawn_handle = task_manager.spawn_handle(); - let network_state_info: Arc = network.clone(); - let is_validator = config.role.is_authority(); - - let (import_stream, finality_stream) = ( - client.import_notification_stream().map(|n| ChainEvent::NewBlock { - id: BlockId::Hash(n.hash), - header: n.header, - retracted: n.retracted, - is_new_best: n.is_new_best, - }), - client.finality_notification_stream().map(|n| ChainEvent::Finalized { - hash: n.hash - }) - ); - let events = futures::stream::select(import_stream, finality_stream) - .for_each(move |event| { - // offchain worker is only interested in block import events - if let ChainEvent::NewBlock { ref header, is_new_best, .. } = event { - let offchain = offchain.as_ref().and_then(|o| o.upgrade()); - match offchain { - Some(offchain) if is_new_best => { - notifications_spawn_handle.spawn( - "offchain-on-block", - offchain.on_block_imported( - &header, - network_state_info.clone(), - is_validator, - ), - ); - }, - Some(_) => log::debug!( - target: "sc_offchain", - "Skipping offchain workers for non-canon block: {:?}", - header, - ), - _ => {}, - } - }; - - let txpool = txpool.upgrade(); - if let Some(txpool) = txpool.as_ref() { - notifications_spawn_handle.spawn( - "txpool-maintain", - txpool.maintain(event), - ); - } - - ready(()) - }); + // Inform the tx pool about imported and finalized blocks. + spawn_handle.spawn( + "txpool-notifications", + sc_transaction_pool::notification_future(client.clone(), transaction_pool.clone()), + ); + // Inform the offchain worker about new imported blocks + if let Some(offchain) = offchain_workers.clone() { spawn_handle.spawn( - "txpool-and-offchain-notif", - events, + "offchain-notifications", + sc_offchain::notification_future( + config.role.is_authority(), + client.clone(), + offchain, + task_manager.spawn_handle(), + network.clone() + ) ); } - { - // extrinsic notifications - let network = Arc::downgrade(&network); - let transaction_pool_ = transaction_pool.clone(); - let events = transaction_pool.import_notification_stream() - .for_each(move |hash| { - if let Some(network) = network.upgrade() { - network.propagate_extrinsic(hash); - } - let status = transaction_pool_.status(); - telemetry!(SUBSTRATE_INFO; "txpool.import"; - "ready" => status.ready, - "future" => status.future - ); - ready(()) - }); - - spawn_handle.spawn( - "on-transaction-imported", - events, - ); - } + spawn_handle.spawn( + "on-transaction-imported", + transaction_notifications(transaction_pool.clone(), network.clone()), + ); // Prometheus metrics. - let mut metrics_service = if let Some(PrometheusConfig { port, registry }) = config.prometheus_config.clone() { + let metrics_service = if let Some(PrometheusConfig { port, registry }) = config.prometheus_config.clone() { // Set static metrics. - - - let role_bits = match config.role { - Role::Full => 1u64, - Role::Light => 2u64, - Role::Sentry { .. } => 3u64, - Role::Authority { .. } => 4u64, - }; let metrics = MetricsService::with_prometheus( ®istry, &config.network.node_name, &config.impl_version, - role_bits, + &config.role, )?; spawn_handle.spawn( "prometheus-endpoint", @@ -1060,171 +1039,42 @@ ServiceBuilder< }; // Periodically notify the telemetry. - let transaction_pool_ = transaction_pool.clone(); - let client_ = client.clone(); - let (state_tx, state_rx) = tracing_unbounded::<(NetworkStatus<_>, NetworkState)>("mpsc_netstat1"); - network_status_sinks.lock().push(std::time::Duration::from_millis(5000), state_tx); - let tel_task = state_rx.for_each(move |(net_status, _)| { - let info = client_.usage_info(); - metrics_service.tick( - &info, - &transaction_pool_.status(), - &net_status, - ); - ready(()) - }); - - spawn_handle.spawn( - "telemetry-periodic-send", - tel_task, - ); + spawn_handle.spawn("telemetry-periodic-send", telemetry_periodic_send( + client.clone(), transaction_pool.clone(), metrics_service, network_status_sinks.clone() + )); // Periodically send the network state to the telemetry. - let (netstat_tx, netstat_rx) = tracing_unbounded::<(NetworkStatus<_>, NetworkState)>("mpsc_netstat2"); - network_status_sinks.lock().push(std::time::Duration::from_secs(30), netstat_tx); - let tel_task_2 = netstat_rx.for_each(move |(_, network_state)| { - telemetry!( - SUBSTRATE_INFO; - "system.network_state"; - "state" => network_state, - ); - ready(()) - }); spawn_handle.spawn( "telemetry-periodic-network-state", - tel_task_2, + telemetry_periodic_network_state(network_status_sinks.clone()), ); // RPC - let (system_rpc_tx, system_rpc_rx) = tracing_unbounded("mpsc_system_rpc"); - let gen_handler = |deny_unsafe: sc_rpc::DenyUnsafe| { - use sc_rpc::{chain, state, author, system, offchain}; - - let system_info = sc_rpc::system::SystemInfo { - chain_name: chain_spec.name().into(), - impl_name: config.impl_name.into(), - impl_version: config.impl_version.into(), - properties: chain_spec.properties().clone(), - chain_type: chain_spec.chain_type().clone(), - }; - - let subscriptions = sc_rpc::Subscriptions::new(Arc::new(task_manager.spawn_handle())); - - let (chain, state, child_state) = if let (Some(remote_backend), Some(on_demand)) = - (remote_backend.as_ref(), on_demand.as_ref()) { - // Light clients - let chain = sc_rpc::chain::new_light( - client.clone(), - subscriptions.clone(), - remote_backend.clone(), - on_demand.clone() - ); - let (state, child_state) = sc_rpc::state::new_light( - client.clone(), - subscriptions.clone(), - remote_backend.clone(), - on_demand.clone() - ); - (chain, state, child_state) - - } else { - // Full nodes - let chain = sc_rpc::chain::new_full(client.clone(), subscriptions.clone()); - let (state, child_state) = sc_rpc::state::new_full(client.clone(), subscriptions.clone()); - (chain, state, child_state) - }; - - let author = sc_rpc::author::Author::new( - client.clone(), - transaction_pool.clone(), - subscriptions, - keystore.clone(), - deny_unsafe, - ); - let system = system::System::new(system_info, system_rpc_tx.clone(), deny_unsafe); - - let maybe_offchain_rpc = offchain_storage.clone() - .map(|storage| { - let offchain = sc_rpc::offchain::Offchain::new(storage, deny_unsafe); - // FIXME: Use plain Option (don't collect into HashMap) when we upgrade to jsonrpc 14.1 - // https://github.com/paritytech/jsonrpc/commit/20485387ed06a48f1a70bf4d609a7cde6cf0accf - let delegate = offchain::OffchainApi::to_delegate(offchain); - delegate.into_iter().collect::>() - }).unwrap_or_default(); - - sc_rpc_server::rpc_handler(( - state::StateApi::to_delegate(state), - state::ChildStateApi::to_delegate(child_state), - chain::ChainApi::to_delegate(chain), - maybe_offchain_rpc, - author::AuthorApi::to_delegate(author), - system::SystemApi::to_delegate(system), - rpc_extensions.clone(), - )) - }; + let gen_handler = |deny_unsafe: sc_rpc::DenyUnsafe| gen_handler( + deny_unsafe, &config, &task_manager, client.clone(), transaction_pool.clone(), + keystore.clone(), on_demand.clone(), remote_backend.clone(), &*rpc_extensions_builder, + offchain_storage.clone(), system_rpc_tx.clone() + ); let rpc = start_rpc_servers(&config, gen_handler)?; // This is used internally, so don't restrict access to unsafe RPC - let rpc_handlers = gen_handler(sc_rpc::DenyUnsafe::No); - - // The network worker is responsible for gathering all network messages and processing - // them. This is quite a heavy task, and at the time of the writing of this comment it - // frequently happens that this future takes several seconds or in some situations - // even more than a minute until it has processed its entire queue. This is clearly an - // issue, and ideally we would like to fix the network future to take as little time as - // possible, but we also take the extra harm-prevention measure to execute the networking - // future using `spawn_blocking`. - spawn_handle.spawn_blocking( - "network-worker", - build_network_future( - config.role.clone(), - network_mut, - client.clone(), - network_status_sinks.clone(), - system_rpc_rx, - has_bootnodes, - config.announce_block, - ), - ); + let rpc_handlers = Arc::new(RpcHandlers(gen_handler(sc_rpc::DenyUnsafe::No))); let telemetry_connection_sinks: Arc>>> = Default::default(); // Telemetry let telemetry = config.telemetry_endpoints.clone().map(|endpoints| { - let is_authority = config.role.is_authority(); - let network_id = network.local_peer_id().to_base58(); - let name = config.network.node_name.clone(); - let impl_name = config.impl_name.to_owned(); - let version = version.clone(); - let chain_name = config.chain_spec.name().to_owned(); - let telemetry_connection_sinks_ = telemetry_connection_sinks.clone(); - let telemetry = sc_telemetry::init_telemetry(sc_telemetry::TelemetryConfig { + let genesis_hash = match client.block_hash(Zero::zero()) { + Ok(Some(hash)) => hash, + _ => Default::default(), + }; + + let (telemetry, future) = build_telemetry( + &mut config, endpoints, - wasm_external_transport: config.telemetry_external_transport.take(), - }); - let startup_time = SystemTime::UNIX_EPOCH.elapsed() - .map(|dur| dur.as_millis()) - .unwrap_or(0); - let future = telemetry.clone() - .for_each(move |event| { - // Safe-guard in case we add more events in the future. - let sc_telemetry::TelemetryEvent::Connected = event; - - telemetry!(SUBSTRATE_INFO; "system.connected"; - "name" => name.clone(), - "implementation" => impl_name.clone(), - "version" => version.clone(), - "config" => "", - "chain" => chain_name.clone(), - "authority" => is_authority, - "startup_time" => startup_time, - "network_id" => network_id.clone() - ); - - telemetry_connection_sinks_.lock().retain(|sink| { - sink.unbounded_send(()).is_ok() - }); - ready(()) - }); + telemetry_connection_sinks.clone(), + network.clone(), + genesis_hash, + ); spawn_handle.spawn( "telemetry-worker", @@ -1245,23 +1095,373 @@ ServiceBuilder< } } - Ok(Service { + // Spawn informant task + spawn_handle.spawn("informant", sc_informant::build( + client.clone(), + network_status_sinks.clone(), + transaction_pool.clone(), + config.informant_output_format, + )); + + task_manager.keep_alive((telemetry, config.base_path, rpc, rpc_handlers.clone())); + + Ok(ServiceComponents { client, task_manager, network, - network_status_sinks, select_chain, transaction_pool, - essential_failed_tx, - essential_failed_rx, rpc_handlers, - _rpc: rpc, - _telemetry: telemetry, - _offchain_workers: offchain_workers, - _telemetry_on_connect_sinks: telemetry_connection_sinks.clone(), keystore, - marker: PhantomData::, - prometheus_registry: config.prometheus_config.map(|config| config.registry) + offchain_workers, + telemetry_on_connect_sinks: TelemetryOnConnectSinks(telemetry_connection_sinks), + network_status_sinks: NetworkStatusSinks::new(network_status_sinks), + prometheus_registry: config.prometheus_config.map(|config| config.registry), }) } + + /// Builds the light service. + pub fn build_light(self) -> Result, Error> { + self.build_common() + } +} + +impl +ServiceBuilder< + TBl, + TRtApi, + TCl, + Arc>, + TSc, + TImpQu, + BoxFinalityProofRequestBuilder, + Arc>, + TExPool, + TRpc, + TBackend, +> where + TCl: ProvideRuntimeApi + HeaderMetadata + Chain + + BlockBackend + BlockIdTo + ProofProvider + + HeaderBackend + BlockchainEvents + ExecutorProvider + UsageProvider + + StorageProvider + CallApiAt + + Send + 'static, + >::Api: + sp_api::Metadata + + sc_offchain::OffchainWorkerApi + + sp_transaction_pool::runtime_api::TaggedTransactionQueue + + sp_session::SessionKeys + + sp_api::ApiErrorExt + + sp_api::ApiExt, + TBl: BlockT, + TRtApi: 'static + Send + Sync, + TBackend: 'static + sc_client_api::backend::Backend + Send, + TSc: Clone, + TImpQu: 'static + ImportQueue, + TExPool: MaintainedTransactionPool::Hash> + + LocalTransactionPool::Hash> + + MallocSizeOfWasm + + 'static, + TRpc: sc_rpc::RpcExtension, +{ + + /// Builds the full service. + pub fn build_full(self) -> Result, Error> { + // make transaction pool available for off-chain runtime calls. + self.client.execution_extensions() + .register_transaction_pool(Arc::downgrade(&self.transaction_pool) as _); + + self.build_common() + } +} + +async fn transaction_notifications( + transaction_pool: Arc, + network: Arc::Hash>> +) + where + TBl: BlockT, + TExPool: MaintainedTransactionPool::Hash>, +{ + // transaction notifications + transaction_pool.import_notification_stream() + .for_each(move |hash| { + network.propagate_transaction(hash); + let status = transaction_pool.status(); + telemetry!(SUBSTRATE_INFO; "txpool.import"; + "ready" => status.ready, + "future" => status.future + ); + ready(()) + }) + .await; +} + +// Periodically notify the telemetry. +async fn telemetry_periodic_send( + client: Arc, + transaction_pool: Arc, + mut metrics_service: MetricsService, + network_status_sinks: Arc, NetworkState)>>> +) + where + TBl: BlockT, + TCl: ProvideRuntimeApi + UsageProvider, + TExPool: MaintainedTransactionPool::Hash>, +{ + let (state_tx, state_rx) = tracing_unbounded::<(NetworkStatus<_>, NetworkState)>("mpsc_netstat1"); + network_status_sinks.lock().push(std::time::Duration::from_millis(5000), state_tx); + state_rx.for_each(move |(net_status, _)| { + let info = client.usage_info(); + metrics_service.tick( + &info, + &transaction_pool.status(), + &net_status, + ); + ready(()) + }).await; +} + +async fn telemetry_periodic_network_state( + network_status_sinks: Arc, NetworkState)>>> +) { + // Periodically send the network state to the telemetry. + let (netstat_tx, netstat_rx) = tracing_unbounded::<(NetworkStatus<_>, NetworkState)>("mpsc_netstat2"); + network_status_sinks.lock().push(std::time::Duration::from_secs(30), netstat_tx); + netstat_rx.for_each(move |(_, network_state)| { + telemetry!( + SUBSTRATE_INFO; + "system.network_state"; + "state" => network_state, + ); + ready(()) + }).await; +} + +fn build_telemetry( + config: &mut Configuration, + endpoints: sc_telemetry::TelemetryEndpoints, + telemetry_connection_sinks: Arc>>>, + network: Arc::Hash>>, + genesis_hash: ::Hash, +) -> (sc_telemetry::Telemetry, Pin + Send>>) { + let is_authority = config.role.is_authority(); + let network_id = network.local_peer_id().to_base58(); + let name = config.network.node_name.clone(); + let impl_name = config.impl_name.to_owned(); + let version = config.impl_version; + let chain_name = config.chain_spec.name().to_owned(); + let telemetry = sc_telemetry::init_telemetry(sc_telemetry::TelemetryConfig { + endpoints, + wasm_external_transport: config.telemetry_external_transport.take(), + }); + let startup_time = SystemTime::UNIX_EPOCH.elapsed() + .map(|dur| dur.as_millis()) + .unwrap_or(0); + let future = telemetry.clone() + .for_each(move |event| { + // Safe-guard in case we add more events in the future. + let sc_telemetry::TelemetryEvent::Connected = event; + + telemetry!(SUBSTRATE_INFO; "system.connected"; + "name" => name.clone(), + "implementation" => impl_name.clone(), + "version" => version, + "config" => "", + "chain" => chain_name.clone(), + "genesis_hash" => ?genesis_hash, + "authority" => is_authority, + "startup_time" => startup_time, + "network_id" => network_id.clone() + ); + + telemetry_connection_sinks.lock().retain(|sink| { + sink.unbounded_send(()).is_ok() + }); + ready(()) + }) + .boxed(); + + (telemetry, future) +} + +fn gen_handler( + deny_unsafe: sc_rpc::DenyUnsafe, + config: &Configuration, + task_manager: &TaskManager, + client: Arc, + transaction_pool: Arc, + keystore: Arc>, + on_demand: Option>>, + remote_backend: Option>>, + rpc_extensions_builder: &(dyn RpcExtensionBuilder + Send), + offchain_storage: Option<>::OffchainStorage>, + system_rpc_tx: TracingUnboundedSender> +) -> jsonrpc_pubsub::PubSubHandler + where + TBl: BlockT, + TCl: ProvideRuntimeApi + BlockchainEvents + HeaderBackend + + HeaderMetadata + ExecutorProvider + + CallApiAt + ProofProvider + + StorageProvider + BlockBackend + Send + Sync + 'static, + TExPool: MaintainedTransactionPool::Hash> + 'static, + TBackend: sc_client_api::backend::Backend + 'static, + TRpc: sc_rpc::RpcExtension, + >::Api: + sp_session::SessionKeys + + sp_api::Metadata, +{ + use sc_rpc::{chain, state, author, system, offchain}; + + let system_info = sc_rpc::system::SystemInfo { + chain_name: config.chain_spec.name().into(), + impl_name: config.impl_name.into(), + impl_version: config.impl_version.into(), + properties: config.chain_spec.properties(), + chain_type: config.chain_spec.chain_type(), + }; + + let subscriptions = SubscriptionManager::new(Arc::new(task_manager.spawn_handle())); + + let (chain, state, child_state) = if let (Some(remote_backend), Some(on_demand)) = + (remote_backend, on_demand) { + // Light clients + let chain = sc_rpc::chain::new_light( + client.clone(), + subscriptions.clone(), + remote_backend.clone(), + on_demand.clone() + ); + let (state, child_state) = sc_rpc::state::new_light( + client.clone(), + subscriptions.clone(), + remote_backend.clone(), + on_demand.clone() + ); + (chain, state, child_state) + + } else { + // Full nodes + let chain = sc_rpc::chain::new_full(client.clone(), subscriptions.clone()); + let (state, child_state) = sc_rpc::state::new_full(client.clone(), subscriptions.clone()); + (chain, state, child_state) + }; + + let author = sc_rpc::author::Author::new( + client.clone(), + transaction_pool.clone(), + subscriptions, + keystore.clone(), + deny_unsafe, + ); + let system = system::System::new(system_info, system_rpc_tx.clone(), deny_unsafe); + + let maybe_offchain_rpc = offchain_storage.clone() + .map(|storage| { + let offchain = sc_rpc::offchain::Offchain::new(storage, deny_unsafe); + // FIXME: Use plain Option (don't collect into HashMap) when we upgrade to jsonrpc 14.1 + // https://github.com/paritytech/jsonrpc/commit/20485387ed06a48f1a70bf4d609a7cde6cf0accf + let delegate = offchain::OffchainApi::to_delegate(offchain); + delegate.into_iter().collect::>() + }).unwrap_or_default(); + + sc_rpc_server::rpc_handler(( + state::StateApi::to_delegate(state), + state::ChildStateApi::to_delegate(child_state), + chain::ChainApi::to_delegate(chain), + maybe_offchain_rpc, + author::AuthorApi::to_delegate(author), + system::SystemApi::to_delegate(system), + rpc_extensions_builder.build(deny_unsafe), + )) +} + +fn build_network( + config: &Configuration, + client: Arc, + transaction_pool: Arc, + spawn_handle: SpawnTaskHandle, + on_demand: Option>>, + block_announce_validator_builder: Option) -> Box + Send> + Send + >>, + finality_proof_request_builder: Option>, + finality_proof_provider: Option>>, + system_rpc_rx: TracingUnboundedReceiver>, + import_queue: TImpQu +) -> Result< + ( + Arc::Hash>>, + Arc, NetworkState)>>>, + Pin + Send>> + ), + Error +> + where + TBl: BlockT, + TCl: ProvideRuntimeApi + HeaderMetadata + Chain + + BlockBackend + BlockIdTo + ProofProvider + + HeaderBackend + BlockchainEvents + 'static, + TExPool: MaintainedTransactionPool::Hash> + 'static, + TImpQu: ImportQueue + 'static, +{ + let transaction_pool_adapter = Arc::new(TransactionPoolAdapter { + imports_external_transactions: !matches!(config.role, Role::Light), + pool: transaction_pool.clone(), + client: client.clone(), + }); + + let protocol_id = { + let protocol_id_full = match config.chain_spec.protocol_id() { + Some(pid) => pid, + None => { + warn!("Using default protocol ID {:?} because none is configured in the \ + chain specs", DEFAULT_PROTOCOL_ID + ); + DEFAULT_PROTOCOL_ID + } + }.as_bytes(); + sc_network::config::ProtocolId::from(protocol_id_full) + }; + + let block_announce_validator = if let Some(f) = block_announce_validator_builder { + f(client.clone()) + } else { + Box::new(DefaultBlockAnnounceValidator) + }; + + let network_params = sc_network::config::Params { + role: config.role.clone(), + executor: { + Some(Box::new(move |fut| { + spawn_handle.spawn("libp2p-node", fut); + })) + }, + network_config: config.network.clone(), + chain: client.clone(), + finality_proof_provider, + finality_proof_request_builder, + on_demand: on_demand.clone(), + transaction_pool: transaction_pool_adapter.clone() as _, + import_queue: Box::new(import_queue), + protocol_id, + block_announce_validator, + metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()) + }; + + let has_bootnodes = !network_params.network_config.boot_nodes.is_empty(); + let network_mut = sc_network::NetworkWorker::new(network_params)?; + let network = network_mut.service().clone(); + let network_status_sinks = Arc::new(Mutex::new(status_sinks::StatusSinks::new())); + + let future = build_network_future( + config.role.clone(), + network_mut, + client.clone(), + network_status_sinks.clone(), + system_rpc_rx, + has_bootnodes, + config.announce_block, + ).boxed(); + + Ok((network, network_status_sinks, future)) } diff --git a/client/service/src/chain_ops.rs b/client/service/src/chain_ops.rs index 7206ab6b3a529dc26428afc51af79b1032915f69..cb4ed24b60b624c78a6640ab25ba4cd1ceff7a6b 100644 --- a/client/service/src/chain_ops.rs +++ b/client/service/src/chain_ops.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Chain utilities. use crate::error; @@ -24,10 +25,10 @@ use sc_chain_spec::ChainSpec; use log::{warn, info}; use futures::{future, prelude::*}; use sp_runtime::traits::{ - Block as BlockT, NumberFor, One, Zero, Header, SaturatedConversion + Block as BlockT, NumberFor, One, Zero, Header, SaturatedConversion, MaybeSerializeDeserialize, }; use sp_runtime::generic::{BlockId, SignedBlock}; -use codec::{Decode, Encode, IoReader}; +use codec::{Decode, Encode, IoReader as CodecIoReader}; use crate::client::{Client, LocalCallExecutor}; use sp_consensus::{ BlockOrigin, @@ -38,12 +39,252 @@ use sp_core::storage::{StorageKey, well_known_keys, ChildInfo, Storage, StorageC use sc_client_api::{StorageProvider, BlockBackend, UsageProvider}; use std::{io::{Read, Write, Seek}, pin::Pin, collections::HashMap}; +use std::time::{Duration, Instant}; +use futures_timer::Delay; +use std::task::Poll; +use serde_json::{de::IoRead as JsonIoRead, Deserializer, StreamDeserializer}; +use std::convert::{TryFrom, TryInto}; +use sp_runtime::traits::{CheckedDiv, Saturating}; + +/// Number of blocks we will add to the queue before waiting for the queue to catch up. +const MAX_PENDING_BLOCKS: u64 = 1_024; + +/// Number of milliseconds to wait until next poll. +const DELAY_TIME: u64 = 2_000; + +/// Number of milliseconds that must have passed between two updates. +const TIME_BETWEEN_UPDATES: u64 = 3_000; /// Build a chain spec json pub fn build_spec(spec: &dyn ChainSpec, raw: bool) -> error::Result { spec.as_json(raw).map_err(Into::into) } + +/// Helper enum that wraps either a binary decoder (from parity-scale-codec), or a JSON decoder (from serde_json). +/// Implements the Iterator Trait, calling `next()` will decode the next SignedBlock and return it. +enum BlockIter where + R: std::io::Read + std::io::Seek, +{ + Binary { + // Total number of blocks we are expecting to decode. + num_expected_blocks: u64, + // Number of blocks we have decoded thus far. + read_block_count: u64, + // Reader to the data, used for decoding new blocks. + reader: CodecIoReader, + }, + Json { + // Nubmer of blocks we have decoded thus far. + read_block_count: u64, + // Stream to the data, used for decoding new blocks. + reader: StreamDeserializer<'static, JsonIoRead, SignedBlock>, + }, +} + +impl BlockIter where + R: Read + Seek + 'static, + B: BlockT + MaybeSerializeDeserialize, +{ + fn new(input: R, binary: bool) -> Result { + if binary { + let mut reader = CodecIoReader(input); + // If the file is encoded in binary format, it is expected to first specify the number + // of blocks that are going to be decoded. We read it and add it to our enum struct. + let num_expected_blocks: u64 = Decode::decode(&mut reader) + .map_err(|e| format!("Failed to decode the number of blocks: {:?}", e))?; + Ok(BlockIter::Binary { + num_expected_blocks, + read_block_count: 0, + reader, + }) + } else { + let stream_deser = Deserializer::from_reader(input) + .into_iter::>(); + Ok(BlockIter::Json { + reader: stream_deser, + read_block_count: 0, + }) + } + } + + /// Returns the number of blocks read thus far. + fn read_block_count(&self) -> u64 { + match self { + BlockIter::Binary { read_block_count, .. } + | BlockIter::Json { read_block_count, .. } + => *read_block_count, + } + } + + /// Returns the total number of blocks to be imported, if possible. + fn num_expected_blocks(&self) -> Option { + match self { + BlockIter::Binary { num_expected_blocks, ..} => Some(*num_expected_blocks), + BlockIter::Json {..} => None + } + } +} + +impl Iterator for BlockIter where + R: Read + Seek + 'static, + B: BlockT + MaybeSerializeDeserialize, +{ + type Item = Result, String>; + + fn next(&mut self) -> Option { + match self { + BlockIter::Binary { num_expected_blocks, read_block_count, reader } => { + if read_block_count < num_expected_blocks { + let block_result: Result, _> = SignedBlock::::decode(reader) + .map_err(|e| e.to_string()); + *read_block_count += 1; + Some(block_result) + } else { + // `read_block_count` == `num_expected_blocks` so we've read enough blocks. + None + } + } + BlockIter::Json { reader, read_block_count } => { + let res = Some(reader.next()?.map_err(|e| e.to_string())); + *read_block_count += 1; + res + } + } + } +} + +/// Imports the SignedBlock to the queue. +fn import_block_to_queue( + signed_block: SignedBlock, + queue: &mut TImpQu, + force: bool +) where + TBl: BlockT + MaybeSerializeDeserialize, + TImpQu: 'static + ImportQueue, +{ + let (header, extrinsics) = signed_block.block.deconstruct(); + let hash = header.hash(); + // import queue handles verification and importing it into the client. + queue.import_blocks(BlockOrigin::File, vec![ + IncomingBlock:: { + hash, + header: Some(header), + body: Some(extrinsics), + justification: signed_block.justification, + origin: None, + allow_missing_state: false, + import_existing: force, + } + ]); +} + +/// Returns true if we have imported every block we were supposed to import, else returns false. +fn importing_is_done( + num_expected_blocks: Option, + read_block_count: u64, + imported_blocks: u64 +) -> bool { + if let Some(num_expected_blocks) = num_expected_blocks { + imported_blocks >= num_expected_blocks + } else { + imported_blocks >= read_block_count + } +} + +/// Structure used to log the block importing speed. +struct Speedometer { + best_number: NumberFor, + last_number: Option>, + last_update: Instant, +} + +impl Speedometer { + /// Creates a fresh Speedometer. + fn new() -> Self { + Self { + best_number: NumberFor::::from(0), + last_number: None, + last_update: Instant::now(), + } + } + + /// Calculates `(best_number - last_number) / (now - last_update)` and + /// logs the speed of import. + fn display_speed(&self) { + // Number of milliseconds elapsed since last time. + let elapsed_ms = { + let elapsed = self.last_update.elapsed(); + let since_last_millis = elapsed.as_secs() * 1000; + let since_last_subsec_millis = elapsed.subsec_millis() as u64; + since_last_millis + since_last_subsec_millis + }; + + // Number of blocks that have been imported since last time. + let diff = match self.last_number { + None => return, + Some(n) => self.best_number.saturating_sub(n) + }; + + if let Ok(diff) = TryInto::::try_into(diff) { + // If the number of blocks can be converted to a regular integer, then it's easy: just + // do the math and turn it into a `f64`. + let speed = diff.saturating_mul(10_000).checked_div(u128::from(elapsed_ms)) + .map_or(0.0, |s| s as f64) / 10.0; + info!("📦 Current best block: {} ({:4.1} bps)", self.best_number, speed); + } else { + // If the number of blocks can't be converted to a regular integer, then we need a more + // algebraic approach and we stay within the realm of integers. + let one_thousand = NumberFor::::from(1_000); + let elapsed = NumberFor::::from( + >::try_from(elapsed_ms).unwrap_or(u32::max_value()) + ); + + let speed = diff.saturating_mul(one_thousand).checked_div(&elapsed) + .unwrap_or_else(Zero::zero); + info!("📦 Current best block: {} ({} bps)", self.best_number, speed) + } + } + + /// Updates the Speedometer. + fn update(&mut self, best_number: NumberFor) { + self.last_number = Some(self.best_number); + self.best_number = best_number; + self.last_update = Instant::now(); + } + + // If more than TIME_BETWEEN_UPDATES has elapsed since last update, + // then print and update the speedometer. + fn notify_user(&mut self, best_number: NumberFor) { + let delta = Duration::from_millis(TIME_BETWEEN_UPDATES); + if Instant::now().duration_since(self.last_update) >= delta { + self.display_speed(); + self.update(best_number); + } + } +} + +/// Different State that the `import_blocks` future could be in. +enum ImportState where + R: Read + Seek + 'static, + B: BlockT + MaybeSerializeDeserialize, +{ + /// We are reading from the BlockIter structure, adding those blocks to the queue if possible. + Reading{block_iter: BlockIter}, + /// The queue is full (contains at least MAX_PENDING_BLOCKS blocks) and we are waiting for it to catch up. + WaitingForImportQueueToCatchUp{ + block_iter: BlockIter, + delay: Delay, + block: SignedBlock + }, + // We have added all the blocks to the queue but they are still being processed. + WaitingForImportQueueToFinish{ + num_expected_blocks: Option, + read_block_count: u64, + delay: Delay, + }, +} + impl< TBl, TRtApi, TBackend, TExecDisp, TFchr, TSc, TImpQu, TFprb, TFpp, @@ -53,7 +294,7 @@ impl< Client>, TBl, TRtApi>, TFchr, TSc, TImpQu, TFprb, TFpp, TExPool, TRpc, Backend > where - TBl: BlockT, + TBl: BlockT + MaybeSerializeDeserialize, TBackend: 'static + sc_client_api::backend::Backend + Send, TExecDisp: 'static + NativeExecutionDispatch, TImpQu: 'static + ImportQueue, @@ -67,6 +308,7 @@ impl< mut self, input: impl Read + Seek + Send + 'static, force: bool, + binary: bool, ) -> Pin> + Send>> { struct WaitLink { imported_blocks: u64, @@ -86,7 +328,7 @@ impl< fn blocks_processed( &mut self, imported: usize, - _count: usize, + _num_expected_blocks: usize, results: Vec<(Result>, BlockImportError>, B::Hash)> ) { self.imported_blocks += imported as u64; @@ -101,10 +343,20 @@ impl< } } - let mut io_reader_input = IoReader(input); - let mut count = None::; - let mut read_block_count = 0; let mut link = WaitLink::new(); + let block_iter_res: Result, String> = BlockIter::new(input, binary); + + let block_iter = match block_iter_res { + Ok(block_iter) => block_iter, + Err(e) => { + // We've encountered an error while creating the block iterator + // so we can just return a future that returns an error. + return future::ready(Err(Error::Other(e))).boxed() + } + }; + + let mut state = Some(ImportState::Reading{block_iter}); + let mut speedometer = Speedometer::::new(); // Importing blocks is implemented as a future, because we want the operation to be // interruptible. @@ -116,85 +368,103 @@ impl< let import = future::poll_fn(move |cx| { let client = &self.client; let queue = &mut self.import_queue; - - // Start by reading the number of blocks if not done so already. - let count = match count { - Some(c) => c, - None => { - let c: u64 = match Decode::decode(&mut io_reader_input) { - Ok(c) => c, - Err(err) => { - let err = format!("Error reading file: {}", err); - return std::task::Poll::Ready(Err(From::from(err))); + match state.take().expect("state should never be None; qed") { + ImportState::Reading{mut block_iter} => { + match block_iter.next() { + None => { + // The iterator is over: we now need to wait for the import queue to finish. + let num_expected_blocks = block_iter.num_expected_blocks(); + let read_block_count = block_iter.read_block_count(); + let delay = Delay::new(Duration::from_millis(DELAY_TIME)); + state = Some(ImportState::WaitingForImportQueueToFinish{num_expected_blocks, read_block_count, delay}); }, - }; - info!("📦 Importing {} blocks", c); - count = Some(c); - c - } - }; - - // Read blocks from the input. - if read_block_count < count { - match SignedBlock::::decode(&mut io_reader_input) { - Ok(signed) => { - let (header, extrinsics) = signed.block.deconstruct(); - let hash = header.hash(); - // import queue handles verification and importing it into the client - queue.import_blocks(BlockOrigin::File, vec![ - IncomingBlock:: { - hash, - header: Some(header), - body: Some(extrinsics), - justification: signed.justification, - origin: None, - allow_missing_state: false, - import_existing: force, + Some(block_result) => { + let read_block_count = block_iter.read_block_count(); + match block_result { + Ok(block) => { + if read_block_count - link.imported_blocks >= MAX_PENDING_BLOCKS { + // The queue is full, so do not add this block and simply wait until + // the queue has made some progress. + let delay = Delay::new(Duration::from_millis(DELAY_TIME)); + state = Some(ImportState::WaitingForImportQueueToCatchUp{block_iter, delay, block}); + } else { + // Queue is not full, we can keep on adding blocks to the queue. + import_block_to_queue(block, queue, force); + state = Some(ImportState::Reading{block_iter}); + } + } + Err(e) => { + return Poll::Ready( + Err(Error::Other(format!("Error reading block #{}: {}", read_block_count, e)))) + } } - ]); + } } - Err(e) => { - warn!("Error reading block data at {}: {}", read_block_count, e); - return std::task::Poll::Ready(Ok(())); + }, + ImportState::WaitingForImportQueueToCatchUp{block_iter, mut delay, block} => { + let read_block_count = block_iter.read_block_count(); + if read_block_count - link.imported_blocks >= MAX_PENDING_BLOCKS { + // Queue is still full, so wait until there is room to insert our block. + match Pin::new(&mut delay).poll(cx) { + Poll::Pending => { + state = Some(ImportState::WaitingForImportQueueToCatchUp{block_iter, delay, block}); + return Poll::Pending + }, + Poll::Ready(_) => { + delay.reset(Duration::from_millis(DELAY_TIME)); + }, + } + state = Some(ImportState::WaitingForImportQueueToCatchUp{block_iter, delay, block}); + } else { + // Queue is no longer full, so we can add our block to the queue. + import_block_to_queue(block, queue, force); + // Switch back to Reading state. + state = Some(ImportState::Reading{block_iter}); + } + }, + ImportState::WaitingForImportQueueToFinish{num_expected_blocks, read_block_count, mut delay} => { + // All the blocks have been added to the queue, which doesn't mean they + // have all been properly imported. + if importing_is_done(num_expected_blocks, read_block_count, link.imported_blocks) { + // Importing is done, we can log the result and return. + info!( + "🎉 Imported {} blocks. Best: #{}", + read_block_count, client.chain_info().best_number + ); + return Poll::Ready(Ok(())) + } else { + // Importing is not done, we still have to wait for the queue to finish. + // Wait for the delay, because we know the queue is lagging behind. + match Pin::new(&mut delay).poll(cx) { + Poll::Pending => { + state = Some(ImportState::WaitingForImportQueueToFinish{num_expected_blocks, read_block_count, delay}); + return Poll::Pending + }, + Poll::Ready(_) => { + delay.reset(Duration::from_millis(DELAY_TIME)); + }, + } + + state = Some(ImportState::WaitingForImportQueueToFinish{num_expected_blocks, read_block_count, delay}); } } - - read_block_count += 1; - if read_block_count % 1000 == 0 { - info!("#{} blocks were added to the queue", read_block_count); - } - - cx.waker().wake_by_ref(); - return std::task::Poll::Pending; } - let blocks_before = link.imported_blocks; queue.poll_actions(cx, &mut link); - if link.has_error { - info!( - "Stopping after #{} blocks because of an error", - link.imported_blocks, - ); - return std::task::Poll::Ready(Ok(())); - } + let best_number = client.chain_info().best_number; + speedometer.notify_user(best_number); - if link.imported_blocks / 1000 != blocks_before / 1000 { - info!( - "#{} blocks were imported (#{} left)", - link.imported_blocks, - count - link.imported_blocks - ); + if link.has_error { + return Poll::Ready(Err( + Error::Other( + format!("Stopping after #{} blocks because of an error", link.imported_blocks) + ) + )) } - if link.imported_blocks >= count { - info!("🎉 Imported {} blocks. Best: #{}", read_block_count, client.chain_info().best_number); - return std::task::Poll::Ready(Ok(())); - - } else { - // Polling the import queue will re-schedule the task when ready. - return std::task::Poll::Pending; - } + cx.waker().wake_by_ref(); + Poll::Pending }); Box::pin(import) } @@ -227,7 +497,7 @@ impl< let client = &self.client; if last < block { - return std::task::Poll::Ready(Err("Invalid block range specified".into())); + return Poll::Ready(Err("Invalid block range specified".into())); } if !wrote_header { @@ -251,19 +521,19 @@ impl< } }, // Reached end of the chain. - None => return std::task::Poll::Ready(Ok(())), + None => return Poll::Ready(Ok(())), } if (block % 10000.into()).is_zero() { info!("#{}", block); } if block == last { - return std::task::Poll::Ready(Ok(())); + return Poll::Ready(Ok(())); } block += One::one(); // Re-schedule the task in order to continue the operation. cx.waker().wake_by_ref(); - std::task::Poll::Pending + Poll::Pending }); Box::pin(export) @@ -294,7 +564,7 @@ impl< 1u64.encode_to(&mut buf); block.encode_to(&mut buf); let reader = std::io::Cursor::new(buf); - self.import_blocks(reader, true) + self.import_blocks(reader, true, true) } Ok(None) => Box::pin(future::err("Unknown block".into())), Err(e) => Box::pin(future::err(format!("Error reading block: {:?}", e).into())), diff --git a/client/service/src/client/block_rules.rs b/client/service/src/client/block_rules.rs index 8ea9c42483ace90383a4a8c06d61e215b3d6cd51..247d09197b604c7335c5141f6409b9b727c67b97 100644 --- a/client/service/src/client/block_rules.rs +++ b/client/service/src/client/block_rules.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Client fixed chain specification rules use std::collections::{HashMap, HashSet}; diff --git a/client/service/src/client/call_executor.rs b/client/service/src/client/call_executor.rs index 093d74e3b3cba9782aa250e212e6d8cee1507b98..049bd888b13c78ca4f67eaadfa7dfa551b01c313 100644 --- a/client/service/src/client/call_executor.rs +++ b/client/service/src/client/call_executor.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use std::{sync::Arc, panic::UnwindSafe, result, cell::RefCell}; use codec::{Encode, Decode}; use sp_runtime::{ diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 09b9929a325a33572faea77cd67b5dbcee3db822..2f101465d516f935701a8ff2f622f3902515ef2b 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate Client use std::{ @@ -27,8 +28,9 @@ use parking_lot::{Mutex, RwLock}; use codec::{Encode, Decode}; use hash_db::Prefix; use sp_core::{ - ChangesTrieConfiguration, convert_hash, NativeOrEncoded, - storage::{StorageKey, PrefixedStorageKey, StorageData, well_known_keys, ChildInfo}, + convert_hash, + storage::{well_known_keys, ChildInfo, PrefixedStorageKey, StorageData, StorageKey}, + ChangesTrieConfiguration, ExecutionContext, NativeOrEncoded, }; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; use sp_runtime::{ @@ -79,15 +81,13 @@ use sc_client_api::{ KeyIterator, CallExecutor, ExecutorProvider, ProofProvider, cht, UsageProvider }; -use sp_utils::mpsc::tracing_unbounded; +use sp_utils::mpsc::{TracingUnboundedSender, tracing_unbounded}; use sp_blockchain::Error; use prometheus_endpoint::Registry; use super::{ - genesis, - light::{call_executor::prove_execution, fetcher::ChangesProof}, - block_rules::{BlockRules, LookupResult as BlockLookupResult}, + genesis, block_rules::{BlockRules, LookupResult as BlockLookupResult}, }; -use futures::channel::mpsc; +use sc_light::{call_executor::prove_execution, fetcher::ChangesProof}; use rand::Rng; #[cfg(feature="test-helpers")] @@ -98,7 +98,7 @@ use { super::call_executor::LocalCallExecutor, }; -type NotificationSinks = Mutex>>; +type NotificationSinks = Mutex>>; /// Substrate Client pub struct Client where Block: BlockT { @@ -353,13 +353,6 @@ impl Client where self.executor.runtime_version(id) } - /// Get block hash by number. - pub fn block_hash(&self, - block_number: <::Header as HeaderT>::Number - ) -> sp_blockchain::Result> { - self.backend.blockchain().hash(block_number) - } - /// Reads given header and generates CHT-based header proof for CHT of given size. pub fn header_proof_with_cht_size( &self, @@ -753,11 +746,6 @@ impl Client where ) = storage_changes.into_inner(); if self.config.offchain_indexing_api { - // if let Some(mut offchain_storage) = self.backend.offchain_storage() { - // offchain_sc.iter().for_each(|(k,v)| { - // offchain_storage.set(b"block-import-info", k,v) - // }); - // } operation.op.update_offchain_storage(offchain_sc)?; } @@ -786,15 +774,15 @@ impl Client where NewBlockState::Normal }; - let retracted = if is_new_best { + let tree_route = if is_new_best && info.best_hash != parent_hash { let route_from_best = sp_blockchain::tree_route( self.backend.blockchain(), info.best_hash, parent_hash, )?; - route_from_best.retracted().iter().rev().map(|e| e.hash.clone()).collect() + Some(route_from_best) } else { - Vec::default() + None }; trace!( @@ -825,7 +813,7 @@ impl Client where header: import_headers.into_post(), is_new_best, storage_changes, - retracted, + tree_route, }) } @@ -864,9 +852,15 @@ impl Client where // block. (true, ref mut storage_changes @ None, Some(ref body)) => { let runtime_api = self.runtime_api(); + let execution_context = if import_block.origin == BlockOrigin::NetworkInitialSync { + ExecutionContext::Syncing + } else { + ExecutionContext::Importing + }; - runtime_api.execute_block( + runtime_api.execute_block_with_context( &at, + execution_context, Block::new(import_block.header.clone(), body.clone()), )?; @@ -1047,7 +1041,7 @@ impl Client where origin: notify_import.origin, header: notify_import.header, is_new_best: notify_import.is_new_best, - retracted: notify_import.retracted, + tree_route: notify_import.tree_route.map(Arc::new), }; self.import_notification_sinks.lock() @@ -1924,6 +1918,10 @@ impl BlockBackend for Client fn justification(&self, id: &BlockId) -> sp_blockchain::Result> { self.backend.blockchain().justification(*id) } + + fn block_hash(&self, number: NumberFor) -> sp_blockchain::Result> { + self.backend.blockchain().hash(number) + } } impl backend::AuxStore for Client diff --git a/client/service/src/client/genesis.rs b/client/service/src/client/genesis.rs index 66436ce81f0720469ac1d26fa899695c7a079a69..4df08025e38264ab0a5f5b09631c95e4681d1b10 100644 --- a/client/service/src/client/genesis.rs +++ b/client/service/src/client/genesis.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Tool for creating the genesis block. use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Hash as HashT, Zero}; diff --git a/client/service/src/client/light/mod.rs b/client/service/src/client/light.rs similarity index 63% rename from client/service/src/client/light/mod.rs rename to client/service/src/client/light.rs index 7cc13cfb6b01b6feb56df988b03958c512a91d84..8b9b65fc2faddade57ead7c676fbfc8fd453dc2f 100644 --- a/client/service/src/client/light/mod.rs +++ b/client/service/src/client/light.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,12 +15,8 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Light client components. -pub mod backend; -pub mod blockchain; -pub mod call_executor; -pub mod fetcher; +//! Light client utilities. use std::sync::Arc; @@ -36,24 +32,8 @@ use super::client::{Client,ClientConfig}; use sc_client_api::{ light::Storage as BlockchainStorage, CloneableSpawn, }; -use self::backend::Backend; -use self::blockchain::Blockchain; -use self::call_executor::GenesisCallExecutor; -use self::fetcher::LightDataChecker; +use sc_light::{Backend, GenesisCallExecutor}; -/// Create an instance of light client blockchain backend. -pub fn new_light_blockchain>(storage: S) -> Arc> { - Arc::new(Blockchain::new(storage)) -} - -/// Create an instance of light client backend. -pub fn new_light_backend(blockchain: Arc>) -> Arc>> - where - B: BlockT, - S: BlockchainStorage, -{ - Arc::new(Backend::new(blockchain)) -} /// Create an instance of light client. pub fn new_light( @@ -78,7 +58,12 @@ pub fn new_light( S: BlockchainStorage + 'static, E: CodeExecutor + RuntimeInfo + Clone + 'static, { - let local_executor = LocalCallExecutor::new(backend.clone(), code_executor, spawn_handle.clone(), ClientConfig::default()); + let local_executor = LocalCallExecutor::new( + backend.clone(), + code_executor, + spawn_handle.clone(), + ClientConfig::default() + ); let executor = GenesisCallExecutor::new(backend.clone(), local_executor); Client::new( backend, @@ -91,15 +76,3 @@ pub fn new_light( ClientConfig::default(), ) } - -/// Create an instance of fetch data checker. -pub fn new_fetch_checker>( - blockchain: Arc>, - executor: E, - spawn_handle: Box, -) -> LightDataChecker, B, S> - where - E: CodeExecutor, -{ - LightDataChecker::new(blockchain, executor, spawn_handle) -} diff --git a/client/service/src/client/mod.rs b/client/service/src/client/mod.rs index 996b885fe524a8193302905fb4c7de4c069dbd15..7c96f61a7867a6f68aab035e69494db8e53d748a 100644 --- a/client/service/src/client/mod.rs +++ b/client/service/src/client/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate Client and associated logic. //! //! The [`Client`] is one of the most important components of Substrate. It mainly comprises two diff --git a/client/service/src/config.rs b/client/service/src/config.rs index 0cc43dac48145a8b512e5c2266909ba88d8b146c..5015ce7facc6f1984d8c9d23af33fc4dfcd338d4 100644 --- a/client/service/src/config.rs +++ b/client/service/src/config.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Service configuration. pub use sc_client_db::{Database, PruningMode, DatabaseSettingsSrc as DatabaseConfig}; @@ -23,14 +24,17 @@ pub use sc_network::config::{ExtTransport, MultiaddrWithPeerId, NetworkConfigura pub use sc_executor::WasmExecutionMethod; use sc_client_api::execution_extensions::ExecutionStrategies; -use std::{future::Future, path::{PathBuf, Path}, pin::Pin, net::SocketAddr, sync::Arc}; +use std::{io, future::Future, path::{PathBuf, Path}, pin::Pin, net::SocketAddr, sync::Arc}; pub use sc_transaction_pool::txpool::Options as TransactionPoolOptions; use sc_chain_spec::ChainSpec; -use sp_core::crypto::Protected; +use sp_core::crypto::SecretString; pub use sc_telemetry::TelemetryEndpoints; use prometheus_endpoint::Registry; +#[cfg(not(target_os = "unknown"))] +use tempfile::TempDir; /// Service configuration. +#[derive(Debug)] pub struct Configuration { /// Implementation name pub impl_name: &'static str, @@ -39,7 +43,7 @@ pub struct Configuration { /// Node role. pub role: Role, /// How to spawn background tasks. Mandatory, otherwise creating a `Service` will error. - pub task_executor: Arc + Send>>, TaskType) + Send + Sync>, + pub task_executor: TaskExecutor, /// Extrinsic pool configuration. pub transaction_pool: TransactionPoolOptions, /// Network configuration. @@ -64,6 +68,8 @@ pub struct Configuration { pub rpc_http: Option, /// RPC over Websockets binding address. `None` if disabled. pub rpc_ws: Option, + /// RPC over IPC binding path. `None` if disabled. + pub rpc_ipc: Option, /// Maximum number of connections for WebSockets RPC server. `None` if default. pub rpc_ws_max_connections: Option, /// CORS settings for HTTP & WS servers. `None` if all origins are allowed. @@ -101,6 +107,10 @@ pub struct Configuration { pub max_runtime_instances: usize, /// Announce block automatically after they have been imported pub announce_block: bool, + /// Base path of the configuration + pub base_path: Option, + /// Configuration of the output format that the informant uses. + pub informant_output_format: sc_informant::OutputFormat, } /// Type for tasks spawned by the executor. @@ -113,14 +123,14 @@ pub enum TaskType { } /// Configuration of the client keystore. -#[derive(Clone)] +#[derive(Debug, Clone)] pub enum KeystoreConfig { /// Keystore at a path on-disk. Recommended for native nodes. Path { /// The path of the keystore. path: PathBuf, /// Node keystore's password. - password: Option> + password: Option }, /// In-memory keystore. Recommended for in-browser nodes. InMemory, @@ -136,7 +146,7 @@ impl KeystoreConfig { } } /// Configuration of the database of the client. -#[derive(Clone, Default)] +#[derive(Debug, Clone, Default)] pub struct OffchainWorkerConfig { /// If this is allowed. pub enabled: bool, @@ -145,7 +155,7 @@ pub struct OffchainWorkerConfig { } /// Configuration of the Prometheus endpoint. -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct PrometheusConfig { /// Port to use. pub port: SocketAddr, @@ -190,3 +200,126 @@ impl Default for RpcMethods { RpcMethods::Auto } } + +/// The base path that is used for everything that needs to be write on disk to run a node. +#[derive(Debug)] +pub enum BasePath { + /// A temporary directory is used as base path and will be deleted when dropped. + #[cfg(not(target_os = "unknown"))] + Temporary(TempDir), + /// A path on the disk. + Permanenent(PathBuf), +} + +impl BasePath { + /// Create a `BasePath` instance using a temporary directory prefixed with "substrate" and use + /// it as base path. + /// + /// Note: the temporary directory will be created automatically and deleted when the `BasePath` + /// instance is dropped. + #[cfg(not(target_os = "unknown"))] + pub fn new_temp_dir() -> io::Result { + Ok(BasePath::Temporary( + tempfile::Builder::new().prefix("substrate").tempdir()?, + )) + } + + /// Create a `BasePath` instance based on an existing path on disk. + /// + /// Note: this function will not ensure that the directory exist nor create the directory. It + /// will also not delete the directory when the instance is dropped. + pub fn new>(path: P) -> BasePath { + BasePath::Permanenent(path.as_ref().to_path_buf()) + } + + /// Create a base path from values describing the project. + #[cfg(not(target_os = "unknown"))] + pub fn from_project(qualifier: &str, organization: &str, application: &str) -> BasePath { + BasePath::new( + directories::ProjectDirs::from(qualifier, organization, application) + .expect("app directories exist on all supported platforms; qed") + .data_local_dir(), + ) + } + + /// Retrieve the base path. + pub fn path(&self) -> &Path { + match self { + #[cfg(not(target_os = "unknown"))] + BasePath::Temporary(temp_dir) => temp_dir.path(), + BasePath::Permanenent(path) => path.as_path(), + } + } +} + +impl std::convert::From for BasePath { + fn from(path: PathBuf) -> Self { + BasePath::new(path) + } +} + +type TaskExecutorInner = Arc + Send>>, TaskType) + Send + Sync>; + +/// Callable object that execute tasks. +/// +/// This struct can be created easily using `Into`. +/// +/// # Examples +/// +/// ## Using tokio +/// +/// ``` +/// # use sc_service::TaskExecutor; +/// # mod tokio { pub mod runtime { +/// # #[derive(Clone)] +/// # pub struct Runtime; +/// # impl Runtime { +/// # pub fn new() -> Result { Ok(Runtime) } +/// # pub fn handle(&self) -> &Self { &self } +/// # pub fn spawn(&self, _: std::pin::Pin + Send>>) {} +/// # } +/// # } } +/// use tokio::runtime::Runtime; +/// +/// let runtime = Runtime::new().unwrap(); +/// let handle = runtime.handle().clone(); +/// let task_executor: TaskExecutor = (move |future, _task_type| { +/// handle.spawn(future); +/// }).into(); +/// ``` +/// +/// ## Using async-std +/// +/// ``` +/// # use sc_service::TaskExecutor; +/// # mod async_std { pub mod task { +/// # pub fn spawn(_: std::pin::Pin + Send>>) {} +/// # } } +/// let task_executor: TaskExecutor = (|future, _task_type| { +/// async_std::task::spawn(future); +/// }).into(); +/// ``` +#[derive(Clone)] +pub struct TaskExecutor(TaskExecutorInner); + +impl std::fmt::Debug for TaskExecutor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "TaskExecutor") + } +} + +impl std::convert::From for TaskExecutor +where + F: Fn(Pin + Send>>, TaskType) + Send + Sync + 'static, +{ + fn from(x: F) -> Self { + Self(Arc::new(x)) + } +} + +impl TaskExecutor { + /// Spawns a new asynchronous task. + pub fn spawn(&self, future: Pin + Send>>, task_type: TaskType) { + self.0(future, task_type) + } +} diff --git a/client/service/src/error.rs b/client/service/src/error.rs index 0b2bbd8d56acdedceb800c7cb6be4d551649291e..ffe1b39405501d224b7c56f40facb11776775f10 100644 --- a/client/service/src/error.rs +++ b/client/service/src/error.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Errors that can occur during the service operation. use sc_network; diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index ebd2b99ef1d31956d849eec21fc5eb977dda41c8..c3c8f60e689ad3cf043bac2c82cf87b1dd687ac7 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate service. Starts a thread that spins up the network, client, and extrinsic pool. //! Manages communication between them. @@ -32,40 +33,34 @@ mod builder; pub mod client; #[cfg(not(feature = "test-helpers"))] mod client; -mod status_sinks; mod task_manager; use std::{io, pin::Pin}; -use std::marker::PhantomData; use std::net::SocketAddr; use std::collections::HashMap; use std::time::Duration; use wasm_timer::Instant; -use std::task::{Poll, Context}; +use std::task::Poll; use parking_lot::Mutex; -use client::Client; -use futures::{ - Future, FutureExt, Stream, StreamExt, - compat::*, - sink::SinkExt, - task::{Spawn, FutureObj, SpawnError}, -}; -use sc_network::{NetworkService, network_state::NetworkState, PeerId}; +use futures::{Future, FutureExt, Stream, StreamExt, compat::*}; +use sc_network::{NetworkStatus, network_state::NetworkState, PeerId}; use log::{log, warn, debug, error, Level}; use codec::{Encode, Decode}; use sp_runtime::generic::BlockId; -use sp_runtime::traits::{NumberFor, Block as BlockT}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use parity_util_mem::MallocSizeOf; -use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sp_utils::{status_sinks, mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}}; pub use self::error::Error; pub use self::builder::{ new_full_client, new_client, ServiceBuilder, ServiceBuilderCommand, TFullClient, TLightClient, TFullBackend, TLightBackend, - TFullCallExecutor, TLightCallExecutor, + TFullCallExecutor, TLightCallExecutor, RpcExtensionBuilder, +}; +pub use config::{ + BasePath, Configuration, DatabaseConfig, PruningMode, Role, RpcMethods, TaskExecutor, TaskType, }; -pub use config::{Configuration, DatabaseConfig, PruningMode, Role, RpcMethods, TaskType}; pub use sc_chain_spec::{ ChainSpec, GenericChainSpec, Properties, RuntimeGenesis, Extension as ChainSpecExtension, NoExtension, ChainType, @@ -83,13 +78,8 @@ pub use sc_network::config::{ }; pub use sc_tracing::TracingReceiver; pub use task_manager::SpawnTaskHandle; -use task_manager::TaskManager; -use sp_blockchain::{HeaderBackend, HeaderMetadata}; -use sp_api::{ApiExt, ConstructRuntimeApi, ApiErrorExt}; -use sc_client_api::{ - Backend as BackendT, BlockchainEvents, CallExecutor, UsageProvider, -}; -use sp_block_builder::BlockBuilder; +pub use task_manager::TaskManager; +use sc_client_api::{Backend, BlockchainEvents}; const DEFAULT_PROTOCOL_ID: &str = "sup"; @@ -103,79 +93,10 @@ impl MallocSizeOfWasm for T {} #[cfg(target_os = "unknown")] impl MallocSizeOfWasm for T {} -/// Substrate service. -pub struct Service { - client: Arc, - task_manager: TaskManager, - select_chain: Option, - network: Arc, - /// Sinks to propagate network status updates. - /// For each element, every time the `Interval` fires we push an element on the sender. - network_status_sinks: Arc>>, - transaction_pool: Arc, - /// Send a signal when a spawned essential task has concluded. The next time - /// the service future is polled it should complete with an error. - essential_failed_tx: TracingUnboundedSender<()>, - /// A receiver for spawned essential-tasks concluding. - essential_failed_rx: TracingUnboundedReceiver<()>, - rpc_handlers: sc_rpc_server::RpcHandler, - _rpc: Box, - _telemetry: Option, - _telemetry_on_connect_sinks: Arc>>>, - _offchain_workers: Option>, - keystore: sc_keystore::KeyStorePtr, - marker: PhantomData, - prometheus_registry: Option, -} - -impl Unpin for Service {} - -/// Abstraction over a Substrate service. -pub trait AbstractService: Future> + Send + Unpin + Spawn + 'static { - /// Type of block of this chain. - type Block: BlockT; - /// Backend storage for the client. - type Backend: 'static + BackendT; - /// How to execute calls towards the runtime. - type CallExecutor: 'static + CallExecutor + Send + Sync + Clone; - /// API that the runtime provides. - type RuntimeApi: Send + Sync; - /// Chain selection algorithm. - type SelectChain: sp_consensus::SelectChain; - /// Transaction pool. - type TransactionPool: TransactionPool + MallocSizeOfWasm; - /// The generic Client type, the bounds here are the ones specifically required by - /// internal crates like sc_informant. - type Client: - HeaderMetadata + UsageProvider - + BlockchainEvents + HeaderBackend + Send + Sync; - - /// Get event stream for telemetry connection established events. - fn telemetry_on_connect_stream(&self) -> TracingUnboundedReceiver<()>; - - /// return a shared instance of Telemetry (if enabled) - fn telemetry(&self) -> Option; - - /// Spawns a task in the background that runs the future passed as parameter. - /// - /// Information about this task will be reported to Prometheus. - /// - /// The task name is a `&'static str` as opposed to a `String`. The reason for that is that - /// in order to avoid memory consumption issues with the Prometheus metrics, the set of - /// possible task names has to be bounded. - fn spawn_task(&self, name: &'static str, task: impl Future + Send + 'static); - - /// Spawns a task in the background that runs the future passed as - /// parameter. The given task is considered essential, i.e. if it errors we - /// trigger a service exit. - fn spawn_essential_task(&self, name: &'static str, task: impl Future + Send + 'static); - - /// Returns a handle for spawning tasks. - fn spawn_task_handle(&self) -> SpawnTaskHandle; - - /// Returns the keystore that stores keys. - fn keystore(&self) -> sc_keystore::KeyStorePtr; +/// RPC handlers that can perform RPC queries. +pub struct RpcHandlers(sc_rpc_server::RpcHandler); +impl RpcHandlers { /// Starts an RPC query. /// /// The query is passed as a string and must be a JSON text similar to what an HTTP client @@ -185,164 +106,76 @@ pub trait AbstractService: Future> + Send + Unpin + S /// /// If the request subscribes you to events, the `Sender` in the `RpcSession` object is used to /// send back spontaneous events. - fn rpc_query(&self, mem: &RpcSession, request: &str) -> Pin> + Send>>; - - /// Get shared client instance. - fn client(&self) -> Arc; - - /// Get clone of select chain. - fn select_chain(&self) -> Option; - - /// Get shared network instance. - fn network(&self) - -> Arc::Hash>>; - - /// Returns a receiver that periodically receives a status of the network. - fn network_status(&self, interval: Duration) -> TracingUnboundedReceiver<(NetworkStatus, NetworkState)>; - - /// Get shared transaction pool instance. - fn transaction_pool(&self) -> Arc; - - /// Get a handle to a future that will resolve on exit. - #[deprecated(note = "Use `spawn_task`/`spawn_essential_task` instead, those functions will attach on_exit signal.")] - fn on_exit(&self) -> ::exit_future::Exit; - - /// Get the prometheus metrics registry, if available. - fn prometheus_registry(&self) -> Option; -} - -impl AbstractService for - Service, TSc, NetworkStatus, - NetworkService, TExPool, TOc> -where - TBl: BlockT, - TBackend: 'static + BackendT, - TExec: 'static + CallExecutor + Send + Sync + Clone, - TRtApi: 'static + Send + Sync + ConstructRuntimeApi>, - >>::RuntimeApi: - sp_api::Core - + ApiExt - + ApiErrorExt - + BlockBuilder, - TSc: sp_consensus::SelectChain + 'static + Clone + Send + Unpin, - TExPool: 'static + TransactionPool + MallocSizeOfWasm, - TOc: 'static + Send + Sync, -{ - type Block = TBl; - type Backend = TBackend; - type CallExecutor = TExec; - type RuntimeApi = TRtApi; - type SelectChain = TSc; - type TransactionPool = TExPool; - type Client = Client; - - fn telemetry_on_connect_stream(&self) -> TracingUnboundedReceiver<()> { - let (sink, stream) = tracing_unbounded("mpsc_telemetry_on_connect"); - self._telemetry_on_connect_sinks.lock().push(sink); - stream - } - - fn telemetry(&self) -> Option { - self._telemetry.as_ref().map(|t| t.clone()) - } - - fn keystore(&self) -> sc_keystore::KeyStorePtr { - self.keystore.clone() - } - - fn spawn_task(&self, name: &'static str, task: impl Future + Send + 'static) { - self.task_manager.spawn(name, task) - } - - fn spawn_essential_task(&self, name: &'static str, task: impl Future + Send + 'static) { - let mut essential_failed = self.essential_failed_tx.clone(); - let essential_task = std::panic::AssertUnwindSafe(task) - .catch_unwind() - .map(move |_| { - error!("Essential task `{}` failed. Shutting down service.", name); - let _ = essential_failed.send(()); - }); - - let _ = self.spawn_task(name, essential_task); - } - - fn spawn_task_handle(&self) -> SpawnTaskHandle { - self.task_manager.spawn_handle() - } - - fn rpc_query(&self, mem: &RpcSession, request: &str) -> Pin> + Send>> { - Box::pin( - self.rpc_handlers.handle_request(request, mem.metadata.clone()) - .compat() - .map(|res| res.expect("this should never fail")) - ) - } - - fn client(&self) -> Arc { - self.client.clone() - } - - fn select_chain(&self) -> Option { - self.select_chain.clone() + pub fn rpc_query(&self, mem: &RpcSession, request: &str) + -> Pin> + Send>> { + self.0.handle_request(request, mem.metadata.clone()) + .compat() + .map(|res| res.expect("this should never fail")) + .boxed() } +} - fn network(&self) - -> Arc::Hash>> - { - self.network.clone() +/// Sinks to propagate network status updates. +/// For each element, every time the `Interval` fires we push an element on the sender. +pub struct NetworkStatusSinks( + Arc, NetworkState)>>>, +); + +impl NetworkStatusSinks { + fn new( + sinks: Arc, NetworkState)>>> + ) -> Self { + Self(sinks) } - fn network_status(&self, interval: Duration) -> TracingUnboundedReceiver<(NetworkStatus, NetworkState)> { + /// Returns a receiver that periodically receives a status of the network. + pub fn network_status(&self, interval: Duration) + -> TracingUnboundedReceiver<(NetworkStatus, NetworkState)> { let (sink, stream) = tracing_unbounded("mpsc_network_status"); - self.network_status_sinks.lock().push(interval, sink); + self.0.lock().push(interval, sink); stream } - - fn transaction_pool(&self) -> Arc { - self.transaction_pool.clone() - } - - fn on_exit(&self) -> exit_future::Exit { - self.task_manager.on_exit() - } - - fn prometheus_registry(&self) -> Option { - self.prometheus_registry.clone() - } } -impl Future for - Service -{ - type Output = Result<(), Error>; - - fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { - let this = Pin::into_inner(self); +/// Sinks to propagate telemetry connection established events. +pub struct TelemetryOnConnectSinks(pub Arc>>>); - match Pin::new(&mut this.essential_failed_rx).poll_next(cx) { - Poll::Pending => {}, - Poll::Ready(_) => { - // Ready(None) should not be possible since we hold a live - // sender. - return Poll::Ready(Err(Error::Other("Essential task failed.".into()))); - } - } - - // The service future never ends. - Poll::Pending +impl TelemetryOnConnectSinks { + /// Get event stream for telemetry connection established events. + pub fn on_connect_stream(&self) -> TracingUnboundedReceiver<()> { + let (sink, stream) =tracing_unbounded("mpsc_telemetry_on_connect"); + self.0.lock().push(sink); + stream } } -impl Spawn for - Service -{ - fn spawn_obj( - &self, - future: FutureObj<'static, ()> - ) -> Result<(), SpawnError> { - self.task_manager.spawn_handle().spawn("unnamed", future); - Ok(()) - } +/// The individual components of the chain, built by the service builder. You are encouraged to +/// deconstruct this into its fields. +pub struct ServiceComponents, TSc, TExPool, TCl> { + /// A blockchain client. + pub client: Arc, + /// A shared transaction pool instance. + pub transaction_pool: Arc, + /// The chain task manager. + pub task_manager: TaskManager, + /// A keystore that stores keys. + pub keystore: sc_keystore::KeyStorePtr, + /// A shared network instance. + pub network: Arc::Hash>>, + /// RPC handlers that can perform RPC queries. + pub rpc_handlers: Arc, + /// A shared instance of the chain selection algorithm. + pub select_chain: Option, + /// Sinks to propagate network status updates. + pub network_status_sinks: NetworkStatusSinks, + /// A prometheus metrics registry, (if enabled). + pub prometheus_registry: Option, + /// Shared Telemetry connection sinks, + pub telemetry_on_connect_sinks: TelemetryOnConnectSinks, + /// A shared offchain workers instance. + pub offchain_workers: Option>>, } /// Builds a never-ending future that continuously polls the network. @@ -369,11 +202,16 @@ fn build_network_future< // We poll `imported_blocks_stream`. while let Poll::Ready(Some(notification)) = Pin::new(&mut imported_blocks_stream).poll_next(cx) { - network.on_block_imported(notification.header, notification.is_new_best); - if announce_imported_blocks { network.service().announce_block(notification.hash, Vec::new()); } + + if let sp_consensus::BlockOrigin::Own = notification.origin { + network.service().own_block_imported( + notification.hash, + notification.header.number().clone(), + ); + } } // We poll `finality_notification_stream`, but we only take the last event. @@ -488,25 +326,6 @@ fn build_network_future< }) } -/// Overview status of the network. -#[derive(Clone)] -pub struct NetworkStatus { - /// Current global sync state. - pub sync_state: sc_network::SyncState, - /// Target sync block number. - pub best_seen_block: Option>, - /// Number of peers participating in syncing. - pub num_sync_peers: u32, - /// Total number of connected peers - pub num_connected_peers: usize, - /// Total number of active peers. - pub num_active_peers: usize, - /// Downloaded bytes per second averaged over the past few seconds. - pub average_download_per_sec: u64, - /// Uploaded bytes per second averaged over the past few seconds. - pub average_upload_per_sec: u64, -} - #[cfg(not(target_os = "unknown"))] // Wrapper for HTTP and WS servers that makes sure they are properly shut down. mod waiting { @@ -520,6 +339,16 @@ mod waiting { } } + pub struct IpcServer(pub Option); + impl Drop for IpcServer { + fn drop(&mut self) { + if let Some(server) = self.0.take() { + server.close_handle().close(); + let _ = server.wait(); + } + } + } + pub struct WsServer(pub Option); impl Drop for WsServer { fn drop(&mut self) { @@ -555,8 +384,8 @@ fn start_rpc_servers sc_rpc_server::RpcHandler, methods: &RpcMethods) -> sc_rpc::DenyUnsafe { - let is_exposed_addr = addr.map(|x| x.ip().is_loopback()).unwrap_or(false); + fn deny_unsafe(addr: &SocketAddr, methods: &RpcMethods) -> sc_rpc::DenyUnsafe { + let is_exposed_addr = !addr.ip().is_loopback(); match (is_exposed_addr, methods) { | (_, RpcMethods::Unsafe) | (false, RpcMethods::Auto) => sc_rpc::DenyUnsafe::No, @@ -565,12 +394,13 @@ fn start_rpc_servers sc_rpc_server::RpcHandler sc_rpc_server::RpcHandler. -use std::convert::TryFrom; + +use std::{convert::TryFrom, time::SystemTime}; use crate::NetworkStatus; use prometheus_endpoint::{register, Gauge, U64, F64, Registry, PrometheusError, Opts, GaugeVec}; @@ -24,6 +25,7 @@ use sp_runtime::traits::{NumberFor, Block, SaturatedConversion, UniqueSaturatedI use sp_transaction_pool::PoolStatus; use sp_utils::metrics::register_globals; use sc_client_api::ClientInfo; +use sc_network::config::Role; use sysinfo::{self, ProcessExt, SystemExt}; @@ -78,6 +80,13 @@ impl PrometheusMetrics { register_globals(registry)?; + let start_time_since_epoch = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default(); + register(Gauge::::new( + "process_start_time_seconds", + "Number of seconds between the UNIX epoch and the moment the process started", + )?, registry)?.set(start_time_since_epoch.as_secs()); + Ok(Self { // system #[cfg(all(any(unix, windows), not(target_os = "android")))] @@ -252,10 +261,17 @@ impl MetricsService { impl MetricsService { - pub fn with_prometheus(registry: &Registry, name: &str, version: &str, roles: u64) + pub fn with_prometheus(registry: &Registry, name: &str, version: &str, role: &Role) -> Result { - PrometheusMetrics::setup(registry, name, version, roles).map(|p| { + let role_bits = match role { + Role::Full => 1u64, + Role::Light => 2u64, + Role::Sentry { .. } => 3u64, + Role::Authority { .. } => 4u64, + }; + + PrometheusMetrics::setup(registry, name, version, role_bits).map(|p| { Self::inner_new(Some(p)) }) } diff --git a/client/service/src/task_manager.rs b/client/service/src/task_manager.rs index 553ca9c326d8ba344c9add9d87dfa1cc83b9ed3f..b6cc26005570a0a33bdf7e8af4c5e327459ceda8 100644 --- a/client/service/src/task_manager.rs +++ b/client/service/src/task_manager.rs @@ -13,14 +13,15 @@ //! Substrate service tasks management module. -use std::{panic, pin::Pin, result::Result, sync::Arc}; +use std::{panic, result::Result, pin::Pin}; use exit_future::Signal; use log::debug; use futures::{ - Future, FutureExt, + Future, FutureExt, StreamExt, future::{select, Either, BoxFuture}, compat::*, task::{Spawn, FutureObj, SpawnError}, + sink::SinkExt, }; use prometheus_endpoint::{ exponential_buckets, register, @@ -28,18 +29,16 @@ use prometheus_endpoint::{ CounterVec, HistogramOpts, HistogramVec, Opts, Registry, U64 }; use sc_client_api::CloneableSpawn; -use crate::config::TaskType; +use sp_utils::mpsc::{TracingUnboundedSender, TracingUnboundedReceiver, tracing_unbounded}; +use crate::{config::{TaskExecutor, TaskType}, Error}; mod prometheus_future; -/// Type alias for service task executor (usually runtime). -pub type ServiceTaskExecutor = Arc + Send>>, TaskType) + Send + Sync>; - /// An handle for spawning tasks in the service. #[derive(Clone)] pub struct SpawnTaskHandle { on_exit: exit_future::Exit, - executor: ServiceTaskExecutor, + executor: TaskExecutor, metrics: Option, } @@ -112,7 +111,7 @@ impl SpawnTaskHandle { } }; - (self.executor)(Box::pin(future), task_type); + self.executor.spawn(Box::pin(future), task_type); } } @@ -124,10 +123,14 @@ impl Spawn for SpawnTaskHandle { } } -impl sp_core::traits::SpawnBlocking for SpawnTaskHandle { +impl sp_core::traits::SpawnNamed for SpawnTaskHandle { fn spawn_blocking(&self, name: &'static str, future: BoxFuture<'static, ()>) { self.spawn_blocking(name, future); } + + fn spawn(&self, name: &'static str, future: BoxFuture<'static, ()>) { + self.spawn(name, future); + } } impl sc_client_api::CloneableSpawn for SpawnTaskHandle { @@ -145,6 +148,63 @@ impl futures01::future::Executor for SpawnTaskHandle { } } +/// A wrapper over `SpawnTaskHandle` that will notify a receiver whenever any +/// task spawned through it fails. The service should be on the receiver side +/// and will shut itself down whenever it receives any message, i.e. an +/// essential task has failed. +pub struct SpawnEssentialTaskHandle { + essential_failed_tx: TracingUnboundedSender<()>, + inner: SpawnTaskHandle, +} + +impl SpawnEssentialTaskHandle { + /// Creates a new `SpawnEssentialTaskHandle`. + pub fn new( + essential_failed_tx: TracingUnboundedSender<()>, + spawn_task_handle: SpawnTaskHandle, + ) -> SpawnEssentialTaskHandle { + SpawnEssentialTaskHandle { + essential_failed_tx, + inner: spawn_task_handle, + } + } + + /// Spawns the given task with the given name. + /// + /// See also [`SpawnTaskHandle::spawn`]. + pub fn spawn(&self, name: &'static str, task: impl Future + Send + 'static) { + self.spawn_inner(name, task, TaskType::Async) + } + + /// Spawns the blocking task with the given name. + /// + /// See also [`SpawnTaskHandle::spawn_blocking`]. + pub fn spawn_blocking( + &self, + name: &'static str, + task: impl Future + Send + 'static, + ) { + self.spawn_inner(name, task, TaskType::Blocking) + } + + fn spawn_inner( + &self, + name: &'static str, + task: impl Future + Send + 'static, + task_type: TaskType, + ) { + let mut essential_failed = self.essential_failed_tx.clone(); + let essential_task = std::panic::AssertUnwindSafe(task) + .catch_unwind() + .map(move |_| { + log::error!("Essential task `{}` failed. Shutting down service.", name); + let _ = essential_failed.send(()); + }); + + let _ = self.inner.spawn_inner(name, essential_task, task_type); + } +} + /// Helper struct to manage background/async tasks in Service. pub struct TaskManager { /// A future that resolves when the service has exited, this is useful to @@ -153,19 +213,28 @@ pub struct TaskManager { /// A signal that makes the exit future above resolve, fired on service drop. signal: Option, /// How to spawn background tasks. - executor: ServiceTaskExecutor, + executor: TaskExecutor, /// Prometheus metric where to report the polling times. metrics: Option, + /// Send a signal when a spawned essential task has concluded. The next time + /// the service future is polled it should complete with an error. + essential_failed_tx: TracingUnboundedSender<()>, + /// A receiver for spawned essential-tasks concluding. + essential_failed_rx: TracingUnboundedReceiver<()>, + /// Things to keep alive until the task manager is dropped. + keep_alive: Box, } impl TaskManager { /// If a Prometheus registry is passed, it will be used to report statistics about the /// service tasks. pub(super) fn new( - executor: ServiceTaskExecutor, + executor: TaskExecutor, prometheus_registry: Option<&Registry> ) -> Result { let (signal, on_exit) = exit_future::signal(); + // A side-channel for essential tasks to communicate shutdown. + let (essential_failed_tx, essential_failed_rx) = tracing_unbounded("mpsc_essential_tasks"); let metrics = prometheus_registry.map(Metrics::register).transpose()?; @@ -174,17 +243,15 @@ impl TaskManager { signal: Some(signal), executor, metrics, + essential_failed_tx, + essential_failed_rx, + keep_alive: Box::new(()), }) } - /// Spawn background/async task, which will be aware on exit signal. - /// - /// See also the documentation of [`SpawnTaskHandler::spawn`]. - pub(super) fn spawn(&self, name: &'static str, task: impl Future + Send + 'static) { - self.spawn_handle().spawn(name, task) - } - pub(super) fn spawn_handle(&self) -> SpawnTaskHandle { + /// Get a handle for spawning tasks. + pub fn spawn_handle(&self) -> SpawnTaskHandle { SpawnTaskHandle { on_exit: self.on_exit.clone(), executor: self.executor.clone(), @@ -192,18 +259,37 @@ impl TaskManager { } } - /// Clone on exit signal. - pub(super) fn on_exit(&self) -> exit_future::Exit { - self.on_exit.clone() + /// Get a handle for spawning essential tasks. + pub fn spawn_essential_handle(&self) -> SpawnEssentialTaskHandle { + SpawnEssentialTaskHandle::new(self.essential_failed_tx.clone(), self.spawn_handle()) + } + + /// Return a future that will end if an essential task fails. + pub fn future<'a>(&'a mut self) -> Pin> + Send + 'a>> { + Box::pin(async move { + self.essential_failed_rx.next().await; + + Err(Error::Other("Essential task failed.".into())) + }) + } + + /// Signal to terminate all the running tasks. + pub fn terminate(&mut self) { + if let Some(signal) = self.signal.take() { + let _ = signal.fire(); + } + } + + /// Set what the task manager should keep alivei + pub(super) fn keep_alive(&mut self, to_keep_alive: T) { + self.keep_alive = Box::new(to_keep_alive); } } impl Drop for TaskManager { fn drop(&mut self) { debug!(target: "service", "Tasks manager shutdown"); - if let Some(signal) = self.signal.take() { - let _ = signal.fire(); - } + self.terminate(); } } diff --git a/client/service/test/Cargo.toml b/client/service/test/Cargo.toml index 76420f913906339f0029867ed6747ac3ad906ce5..6a886ebcbaccb02139337ee121d609e181f199d3 100644 --- a/client/service/test/Cargo.toml +++ b/client/service/test/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-service-test" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -20,24 +20,25 @@ log = "0.4.8" env_logger = "0.7.0" fdlimit = "0.1.4" parking_lot = "0.10.0" -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } -sp-api = { version = "2.0.0-alpha.8", path = "../../../primitives/api" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../../primitives/state-machine" } -sp-externalities = { version = "0.8.0-alpha.8", path = "../../../primitives/externalities" } -sp-trie = { version = "2.0.0-alpha.8", path = "../../../primitives/trie" } -sp-storage = { version = "2.0.0-alpha.8", path = "../../../primitives/storage" } -sc-client-db = { version = "0.8.0-alpha.8", default-features = false, path = "../../db" } +sc-light = { version = "2.0.0-rc4", path = "../../light" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } +sp-api = { version = "2.0.0-rc4", path = "../../../primitives/api" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../../primitives/state-machine" } +sp-externalities = { version = "0.8.0-rc4", path = "../../../primitives/externalities" } +sp-trie = { version = "2.0.0-rc4", path = "../../../primitives/trie" } +sp-storage = { version = "2.0.0-rc4", path = "../../../primitives/storage" } +sc-client-db = { version = "0.8.0-rc4", default-features = false, path = "../../db" } futures = { version = "0.3.1", features = ["compat"] } -sc-service = { version = "0.8.0-alpha.8", default-features = false, features = ["test-helpers"], path = "../../service" } -sc-network = { version = "0.8.0-alpha.8", path = "../../network" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/common" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-transaction-pool = { version = "2.0.0-alpha.8", path = "../../../primitives/transaction-pool" } -substrate-test-runtime = { version = "2.0.0-alpha.8", path = "../../../test-utils/runtime" } -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../../test-utils/runtime/client" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../../api" } -sc-block-builder = { version = "0.8.0-alpha.8", path = "../../block-builder" } -sc-executor = { version = "0.8.0-alpha.8", path = "../../executor" } -sp-panic-handler = { version = "2.0.0-alpha.8", path = "../../../primitives/panic-handler" } -parity-scale-codec = "1.3.0" +sc-service = { version = "0.8.0-rc4", default-features = false, features = ["test-helpers"], path = "../../service" } +sc-network = { version = "0.8.0-rc4", path = "../../network" } +sp-consensus = { version = "0.8.0-rc4", path = "../../../primitives/consensus/common" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-transaction-pool = { version = "2.0.0-rc4", path = "../../../primitives/transaction-pool" } +substrate-test-runtime = { version = "2.0.0-rc4", path = "../../../test-utils/runtime" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../../test-utils/runtime/client" } +sc-client-api = { version = "2.0.0-rc4", path = "../../api" } +sc-block-builder = { version = "0.8.0-rc4", path = "../../block-builder" } +sc-executor = { version = "0.8.0-rc4", path = "../../executor" } +sp-panic-handler = { version = "2.0.0-rc4", path = "../../../primitives/panic-handler" } +parity-scale-codec = "1.3.1" diff --git a/client/service/test/src/client/db.rs b/client/service/test/src/client/db.rs index 134075a11df519267053e5f5eaa3fec4e036c8cd..36d49732246e579d610a707d827c0b5d49c00ea6 100644 --- a/client/service/test/src/client/db.rs +++ b/client/service/test/src/client/db.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use sp_core::offchain::{OffchainStorage, storage::InMemOffchainStorage}; use std::sync::Arc; @@ -53,4 +54,4 @@ fn in_memory_offchain_storage() { assert!(!storage.compare_and_set(b"B", b"A", Some(b""), b"Y")); assert!(storage.compare_and_set(b"B", b"A", None, b"X")); assert_eq!(storage.get(b"B", b"A"), Some(b"X".to_vec())); -} \ No newline at end of file +} diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index 0ad5ba78fb06def53aae406ea24b2acc4176cbdc..e72c290d43bbeaf363a241692663b440e8570d43 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -16,7 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use sc_service::client::light::{ +use sc_light::{ call_executor::{ GenesisCallExecutor, check_execution_proof, @@ -40,7 +40,13 @@ use sp_api::{InitializeBlock, StorageTransactionCache, ProofRecorder, OffchainOv use sp_consensus::{BlockOrigin}; use sc_executor::{NativeExecutor, WasmExecutionMethod, RuntimeVersion, NativeVersion}; use sp_core::{H256, tasks::executor as tasks_executor, NativeOrEncoded}; -use sc_client_api::{blockchain::Info, backend::NewBlockState, Backend as ClientBackend, ProofProvider, in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, AuxStore, Storage, CallExecutor, cht, ExecutionStrategy, StorageProof, BlockImportOperation, RemoteCallRequest, StorageProvider, ChangesProof, RemoteBodyRequest, RemoteReadRequest, RemoteChangesRequest, FetchChecker, RemoteReadChildRequest, RemoteHeaderRequest}; +use sc_client_api::{ + blockchain::Info, backend::NewBlockState, Backend as ClientBackend, ProofProvider, + in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, + AuxStore, Storage, CallExecutor, cht, ExecutionStrategy, StorageProof, BlockImportOperation, + RemoteCallRequest, StorageProvider, ChangesProof, RemoteBodyRequest, RemoteReadRequest, + RemoteChangesRequest, FetchChecker, RemoteReadChildRequest, RemoteHeaderRequest, BlockBackend, +}; use sp_externalities::Extensions; use sc_block_builder::BlockBuilderProvider; use sp_blockchain::{ diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 0791c421bddcd6f8f666d6cba7f8c72f1bb89b5f..2124f0ced412294b134a153642dd8e2bec581133 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use parity_scale_codec::{Encode, Decode, Joiner}; use sc_executor::native_executor_instance; use sp_state_machine::{StateMachine, OverlayedChanges, ExecutionStrategy, InMemoryBackend}; diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index faffce3e0a69447aebbce0f53e45f65f43c250ee..5a676e5263c8a749421bce84a3c9ef5b5fbcf982 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,35 +15,39 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Service integration test utils. use std::iter; -use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::Arc; use std::net::Ipv4Addr; use std::pin::Pin; use std::time::Duration; -use log::info; +use log::{info, debug}; use futures01::{Future, Stream, Poll}; use futures::{FutureExt as _, TryFutureExt as _}; use tempfile::TempDir; use tokio::{runtime::Runtime, prelude::FutureExt}; use tokio::timer::Interval; use sc_service::{ - AbstractService, + TaskManager, GenericChainSpec, ChainSpecExtension, Configuration, - config::{DatabaseConfig, KeystoreConfig}, + config::{BasePath, DatabaseConfig, KeystoreConfig}, RuntimeGenesis, Role, Error, - TaskType, + TaskExecutor, + client::Client, }; use sp_blockchain::HeaderBackend; use sc_network::{multiaddr, Multiaddr}; use sc_network::config::{NetworkConfiguration, TransportConfig}; use sp_runtime::{generic::BlockId, traits::Block as BlockT}; use sp_transaction_pool::TransactionPool; +use sc_client_api::{Backend, CallExecutor}; +use parking_lot::Mutex; #[cfg(test)] mod client; @@ -53,47 +57,100 @@ const MAX_WAIT_TIME: Duration = Duration::from_secs(60 * 3); struct TestNet { runtime: Runtime, - authority_nodes: Vec<(usize, SyncService, U, Multiaddr)>, - full_nodes: Vec<(usize, SyncService, U, Multiaddr)>, - light_nodes: Vec<(usize, SyncService, Multiaddr)>, + authority_nodes: Vec<(usize, F, U, Multiaddr)>, + full_nodes: Vec<(usize, F, U, Multiaddr)>, + light_nodes: Vec<(usize, L, Multiaddr)>, chain_spec: GenericChainSpec, base_port: u16, nodes: usize, } -/// Wraps around an `Arc` and implements `Future`. -pub struct SyncService(Arc>); +pub trait TestNetNode: Clone + Future + Send + 'static { + type Block: BlockT; + type Backend: Backend; + type Executor: CallExecutor + Send + Sync; + type RuntimeApi: Send + Sync; + type TransactionPool: TransactionPool; -impl SyncService { - pub fn get(&self) -> MutexGuard { - self.0.lock().unwrap() - } + fn client(&self) -> Arc>; + fn transaction_pool(&self) -> Arc; + fn network(&self) -> Arc::Hash>>; } -impl Clone for SyncService { - fn clone(&self) -> Self { - Self(self.0.clone()) +pub struct TestNetComponents { + task_manager: Arc>, + client: Arc>, + transaction_pool: Arc, + network: Arc::Hash>>, +} + +impl +TestNetComponents { + pub fn new( + task_manager: TaskManager, + client: Arc>, + network: Arc::Hash>>, + transaction_pool: Arc, + ) -> Self { + Self { + client, transaction_pool, network, + task_manager: Arc::new(Mutex::new(task_manager)), + } } } -impl From for SyncService { - fn from(service: T) -> Self { - SyncService(Arc::new(Mutex::new(service))) + +impl Clone for +TestNetComponents { + fn clone(&self) -> Self { + Self { + task_manager: self.task_manager.clone(), + client: self.client.clone(), + transaction_pool: self.transaction_pool.clone(), + network: self.network.clone(), + } } } -impl> + Unpin> Future for SyncService { +impl Future for + TestNetComponents +{ type Item = (); type Error = sc_service::Error; fn poll(&mut self) -> Poll { - let mut f = self.0.lock().unwrap(); - futures::compat::Compat::new(&mut *f).poll() + futures::compat::Compat::new(&mut self.task_manager.lock().future()).poll() + } +} + +impl TestNetNode for +TestNetComponents + where + TBl: BlockT, + TBackend: sc_client_api::Backend + Send + Sync + 'static, + TExec: CallExecutor + Send + Sync + 'static, + TRtApi: Send + Sync + 'static, + TExPool: TransactionPool + Send + Sync + 'static, +{ + type Block = TBl; + type Backend = TBackend; + type Executor = TExec; + type RuntimeApi = TRtApi; + type TransactionPool = TExPool; + + fn client(&self) -> Arc> { + self.client.clone() + } + fn transaction_pool(&self) -> Arc { + self.transaction_pool.clone() + } + fn network(&self) -> Arc::Hash>> { + self.network.clone() } } impl TestNet -where F: Send + 'static, L: Send +'static, U: Clone + Send + 'static +where F: Clone + Send + 'static, L: Clone + Send +'static, U: Clone + Send + 'static { pub fn run_until_all_full( &mut self, @@ -101,8 +158,8 @@ where F: Send + 'static, L: Send +'static, U: Clone + Send + 'static light_predicate: LP, ) where - FP: Send + Fn(usize, &SyncService) -> bool + 'static, - LP: Send + Fn(usize, &SyncService) -> bool + 'static, + FP: Send + Fn(usize, &F) -> bool + 'static, + LP: Send + Fn(usize, &L) -> bool + 'static, { let full_nodes = self.full_nodes.clone(); let light_nodes = self.light_nodes.clone(); @@ -141,7 +198,7 @@ fn node_config, role: Role, - task_executor: Arc + Send>>, TaskType) + Send + Sync>, + task_executor: TaskExecutor, key_seed: Option, base_port: u16, root: &TempDir, @@ -193,6 +250,7 @@ fn node_config TestNet where - F: AbstractService, - L: AbstractService, + F: TestNetNode, + L: TestNetNode, E: ChainSpecExtension + Clone + 'static + Send, G: RuntimeGenesis + 'static, { @@ -253,58 +313,66 @@ impl TestNet where authorities: impl Iterator Result<(F, U), Error>)> ) { let executor = self.runtime.executor(); + let task_executor: TaskExecutor = { + let executor = executor.clone(); + (move |fut: Pin + Send>>, _| { + executor.spawn(fut.unit_error().compat()); + }).into() + }; for (key, authority) in authorities { - let task_executor = { - let executor = executor.clone(); - Arc::new(move |fut: Pin + Send>>, _| executor.spawn(fut.unit_error().compat())) - }; let node_config = node_config( self.nodes, &self.chain_spec, Role::Authority { sentry_nodes: Vec::new() }, - task_executor, + task_executor.clone(), Some(key), self.base_port, &temp, ); let addr = node_config.network.listen_addresses.iter().next().unwrap().clone(); let (service, user_data) = authority(node_config).expect("Error creating test node service"); - let service = SyncService::from(service); executor.spawn(service.clone().map_err(|_| ())); - let addr = addr.with(multiaddr::Protocol::P2p(service.get().network().local_peer_id().clone().into())); + let addr = addr.with(multiaddr::Protocol::P2p(service.network().local_peer_id().clone().into())); self.authority_nodes.push((self.nodes, service, user_data, addr)); self.nodes += 1; } for full in full { - let task_executor = { - let executor = executor.clone(); - Arc::new(move |fut: Pin + Send>>, _| executor.spawn(fut.unit_error().compat())) - }; - let node_config = node_config(self.nodes, &self.chain_spec, Role::Full, task_executor, None, self.base_port, &temp); + let node_config = node_config( + self.nodes, + &self.chain_spec, + Role::Full, + task_executor.clone(), + None, + self.base_port, + &temp, + ); let addr = node_config.network.listen_addresses.iter().next().unwrap().clone(); let (service, user_data) = full(node_config).expect("Error creating test node service"); - let service = SyncService::from(service); executor.spawn(service.clone().map_err(|_| ())); - let addr = addr.with(multiaddr::Protocol::P2p(service.get().network().local_peer_id().clone().into())); + let addr = addr.with(multiaddr::Protocol::P2p(service.network().local_peer_id().clone().into())); self.full_nodes.push((self.nodes, service, user_data, addr)); self.nodes += 1; } for light in light { - let task_executor = { - let executor = executor.clone(); - Arc::new(move |fut: Pin + Send>>, _| executor.spawn(fut.unit_error().compat())) - }; - let node_config = node_config(self.nodes, &self.chain_spec, Role::Light, task_executor, None, self.base_port, &temp); + let node_config = node_config( + self.nodes, + &self.chain_spec, + Role::Light, + task_executor.clone(), + None, + self.base_port, + &temp, + ); let addr = node_config.network.listen_addresses.iter().next().unwrap().clone(); - let service = SyncService::from(light(node_config).expect("Error creating test node service")); + let service = light(node_config).expect("Error creating test node service"); executor.spawn(service.clone().map_err(|_| ())); - let addr = addr.with(multiaddr::Protocol::P2p(service.get().network().local_peer_id().clone().into())); + let addr = addr.with(multiaddr::Protocol::P2p(service.network().local_peer_id().clone().into())); self.light_nodes.push((self.nodes, service, addr)); self.nodes += 1; } @@ -323,9 +391,9 @@ pub fn connectivity( E: ChainSpecExtension + Clone + 'static + Send, G: RuntimeGenesis + 'static, Fb: Fn(Configuration) -> Result, - F: AbstractService, + F: TestNetNode, Lb: Fn(Configuration) -> Result, - L: AbstractService, + L: TestNetNode, { const NUM_FULL_NODES: usize = 5; const NUM_LIGHT_NODES: usize = 5; @@ -349,19 +417,25 @@ pub fn connectivity( info!("Checking star topology"); let first_address = network.full_nodes[0].3.clone(); for (_, service, _, _) in network.full_nodes.iter().skip(1) { - service.get().network().add_reserved_peer(first_address.to_string()) + service.network().add_reserved_peer(first_address.to_string()) .expect("Error adding reserved peer"); } for (_, service, _) in network.light_nodes.iter() { - service.get().network().add_reserved_peer(first_address.to_string()) + service.network().add_reserved_peer(first_address.to_string()) .expect("Error adding reserved peer"); } network.run_until_all_full( - move |_index, service| service.get().network().num_connected() - == expected_full_connections, - move |_index, service| service.get().network().num_connected() - == expected_light_connections, + move |_index, service| { + let connected = service.network().num_connected(); + debug!("Got {}/{} full connections...", connected, expected_full_connections); + connected == expected_full_connections + }, + move |_index, service| { + let connected = service.network().num_connected(); + debug!("Got {}/{} light connections...", connected, expected_light_connections); + connected == expected_light_connections + }, ); network.runtime @@ -390,24 +464,30 @@ pub fn connectivity( for i in 0..max_nodes { if i != 0 { if let Some((_, service, _, node_id)) = network.full_nodes.get(i) { - service.get().network().add_reserved_peer(address.to_string()) + service.network().add_reserved_peer(address.to_string()) .expect("Error adding reserved peer"); address = node_id.clone(); } } if let Some((_, service, node_id)) = network.light_nodes.get(i) { - service.get().network().add_reserved_peer(address.to_string()) + service.network().add_reserved_peer(address.to_string()) .expect("Error adding reserved peer"); address = node_id.clone(); } } network.run_until_all_full( - move |_index, service| service.get().network().num_connected() - == expected_full_connections, - move |_index, service| service.get().network().num_connected() - == expected_light_connections, + move |_index, service| { + let connected = service.network().num_connected(); + debug!("Got {}/{} full connections...", connected, expected_full_connections); + connected == expected_full_connections + }, + move |_index, service| { + let connected = service.network().num_connected(); + debug!("Got {}/{} light connections...", connected, expected_light_connections); + connected == expected_light_connections + }, ); } temp.close().expect("Error removing temp dir"); @@ -422,9 +502,9 @@ pub fn sync( mut extrinsic_factory: ExF ) where Fb: Fn(Configuration) -> Result<(F, U), Error>, - F: AbstractService, + F: TestNetNode, Lb: Fn(Configuration) -> Result, - L: AbstractService, + L: TestNetNode, B: FnMut(&F, &mut U), ExF: FnMut(&F, &U) -> ::Extrinsic, U: Clone + Send + 'static, @@ -454,38 +534,41 @@ pub fn sync( info!("Generating #{}", i + 1); } - make_block_and_import(&first_service.get(), first_user_data); + make_block_and_import(&first_service, first_user_data); } + network.full_nodes[0].1.network().update_chain(); network.full_nodes[0].3.clone() }; info!("Running sync"); for (_, service, _, _) in network.full_nodes.iter().skip(1) { - service.get().network().add_reserved_peer(first_address.to_string()).expect("Error adding reserved peer"); + service.network().add_reserved_peer(first_address.to_string()) + .expect("Error adding reserved peer"); } for (_, service, _) in network.light_nodes.iter() { - service.get().network().add_reserved_peer(first_address.to_string()).expect("Error adding reserved peer"); + service.network().add_reserved_peer(first_address.to_string()) + .expect("Error adding reserved peer"); } network.run_until_all_full( |_index, service| - service.get().client().info().best_number == (NUM_BLOCKS as u32).into(), + service.client().info().best_number == (NUM_BLOCKS as u32).into(), |_index, service| - service.get().client().info().best_number == (NUM_BLOCKS as u32).into(), + service.client().info().best_number == (NUM_BLOCKS as u32).into(), ); info!("Checking extrinsic propagation"); let first_service = network.full_nodes[0].1.clone(); let first_user_data = &network.full_nodes[0].2; - let best_block = BlockId::number(first_service.get().client().info().best_number); - let extrinsic = extrinsic_factory(&first_service.get(), first_user_data); + let best_block = BlockId::number(first_service.client().info().best_number); + let extrinsic = extrinsic_factory(&first_service, first_user_data); let source = sp_transaction_pool::TransactionSource::External; futures::executor::block_on( - first_service.get().transaction_pool().submit_one(&best_block, source, extrinsic) + first_service.transaction_pool().submit_one(&best_block, source, extrinsic) ).expect("failed to submit extrinsic"); network.run_until_all_full( - |_index, service| service.get().transaction_pool().ready().count() == 1, + |_index, service| service.transaction_pool().ready().count() == 1, |_index, _service| true, ); } @@ -497,9 +580,9 @@ pub fn consensus( authorities: impl IntoIterator ) where Fb: Fn(Configuration) -> Result, - F: AbstractService, + F: TestNetNode, Lb: Fn(Configuration) -> Result, - L: AbstractService, + L: TestNetNode, E: ChainSpecExtension + Clone + 'static + Send, G: RuntimeGenesis + 'static, { @@ -519,19 +602,22 @@ pub fn consensus( info!("Checking consensus"); let first_address = network.authority_nodes[0].3.clone(); for (_, service, _, _) in network.full_nodes.iter() { - service.get().network().add_reserved_peer(first_address.to_string()).expect("Error adding reserved peer"); + service.network().add_reserved_peer(first_address.to_string()) + .expect("Error adding reserved peer"); } for (_, service, _) in network.light_nodes.iter() { - service.get().network().add_reserved_peer(first_address.to_string()).expect("Error adding reserved peer"); + service.network().add_reserved_peer(first_address.to_string()) + .expect("Error adding reserved peer"); } for (_, service, _, _) in network.authority_nodes.iter().skip(1) { - service.get().network().add_reserved_peer(first_address.to_string()).expect("Error adding reserved peer"); + service.network().add_reserved_peer(first_address.to_string()) + .expect("Error adding reserved peer"); } network.run_until_all_full( |_index, service| - service.get().client().info().finalized_number >= (NUM_BLOCKS as u32 / 2).into(), + service.client().info().finalized_number >= (NUM_BLOCKS as u32 / 2).into(), |_index, service| - service.get().client().info().best_number >= (NUM_BLOCKS as u32 / 2).into(), + service.client().info().best_number >= (NUM_BLOCKS as u32 / 2).into(), ); info!("Adding more peers"); @@ -544,15 +630,17 @@ pub fn consensus( (0..0).map(|_| (String::new(), { |cfg| full_builder(cfg).map(|s| (s, ())) })), ); for (_, service, _, _) in network.full_nodes.iter() { - service.get().network().add_reserved_peer(first_address.to_string()).expect("Error adding reserved peer"); + service.network().add_reserved_peer(first_address.to_string()) + .expect("Error adding reserved peer"); } for (_, service, _) in network.light_nodes.iter() { - service.get().network().add_reserved_peer(first_address.to_string()).expect("Error adding reserved peer"); + service.network().add_reserved_peer(first_address.to_string()) + .expect("Error adding reserved peer"); } network.run_until_all_full( |_index, service| - service.get().client().info().finalized_number >= (NUM_BLOCKS as u32).into(), + service.client().info().finalized_number >= (NUM_BLOCKS as u32).into(), |_index, service| - service.get().client().info().best_number >= (NUM_BLOCKS as u32).into(), + service.client().info().best_number >= (NUM_BLOCKS as u32).into(), ); } diff --git a/client/state-db/Cargo.toml b/client/state-db/Cargo.toml index 17b9340afdabbfd17c79969f9f8cc2a394845750..7cc8d41e767ab6d6e659e200bd9b0972407e4ab8 100644 --- a/client/state-db/Cargo.toml +++ b/client/state-db/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-state-db" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -14,9 +14,9 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] parking_lot = "0.10.0" log = "0.4.8" -sc-client-api = { version = "2.0.0-alpha.8", path = "../api" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } +sc-client-api = { version = "2.0.0-rc4", path = "../api" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +codec = { package = "parity-scale-codec", version = "1.3.1", features = ["derive"] } parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } parity-util-mem-derive = "0.1.0" diff --git a/client/state-db/src/lib.rs b/client/state-db/src/lib.rs index 60255011023967187ab7e2afbc8c7c7703d47560..61470894e487e3884bc873301ff17e39345a0877 100644 --- a/client/state-db/src/lib.rs +++ b/client/state-db/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! State database maintenance. Handles canonicalization and pruning in the database. The input to //! this module is a `ChangeSet` which is basically a list of key-value pairs (trie nodes) that //! were added or deleted during block execution. diff --git a/client/state-db/src/noncanonical.rs b/client/state-db/src/noncanonical.rs index e9b4c829e146f69a682be229107e12533ea3cc75..d77f20c50d05f30916f024666607dc7f3f16d440 100644 --- a/client/state-db/src/noncanonical.rs +++ b/client/state-db/src/noncanonical.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Canonicalization window. //! Maintains trees of block overlays and allows discarding trees/roots //! The overlays are added in `insert` and removed in `canonicalize`. diff --git a/client/state-db/src/pruning.rs b/client/state-db/src/pruning.rs index 3ab4a61da4c15808db646066fdbaa15c959fbe5c..69b07c285fad8318152611d9473c508367a9f73c 100644 --- a/client/state-db/src/pruning.rs +++ b/client/state-db/src/pruning.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Pruning window. //! //! For each block we maintain a list of nodes pending deletion. diff --git a/client/state-db/src/test.rs b/client/state-db/src/test.rs index d445ce6b6d70c13093a767932463dfe0ebea6532..11ce4ad8226202d0f08a9e77bdddfb8e7772deaa 100644 --- a/client/state-db/src/test.rs +++ b/client/state-db/src/test.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Test utils use std::collections::HashMap; diff --git a/client/telemetry/Cargo.toml b/client/telemetry/Cargo.toml index 4a3f302fb6e75bc41e51572eb0e98353aa8683ee..8d4aecc46816644242b0bcb18e06699c4065df6f 100644 --- a/client/telemetry/Cargo.toml +++ b/client/telemetry/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-telemetry" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] description = "Telemetry utils" edition = "2018" @@ -14,12 +14,11 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -bytes = "0.5" parking_lot = "0.10.0" futures = "0.3.4" futures-timer = "3.0.1" wasm-timer = "0.2.0" -libp2p = { version = "0.18.1", default-features = false, features = ["websocket", "wasm-ext", "tcp", "dns"] } +libp2p = { version = "0.20.1", default-features = false, features = ["dns", "tcp-async-std", "wasm-ext", "websocket"] } log = "0.4.8" pin-project = "0.4.6" rand = "0.7.2" diff --git a/client/telemetry/src/lib.rs b/client/telemetry/src/lib.rs index b959697f390366d99309569feaf9254ed8c75920..315bedbe5b6d92d4a96fa5085658d95c1c99a29b 100644 --- a/client/telemetry/src/lib.rs +++ b/client/telemetry/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Telemetry utilities. //! //! Calling `init_telemetry` registers a global `slog` logger using `slog_scope::set_global_logger`. diff --git a/client/telemetry/src/worker.rs b/client/telemetry/src/worker.rs index 04689f76869e92acefeddb4832d135c1e8419f7c..e01ac62d12dc36fdab9e4feebec33eed662ba93c 100644 --- a/client/telemetry/src/worker.rs +++ b/client/telemetry/src/worker.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Contains the object that makes the telemetry work. //! //! # Usage @@ -27,7 +28,6 @@ //! events indicating what happened since the latest polling. //! -use bytes::BytesMut; use futures::{prelude::*, ready}; use libp2p::{core::transport::OptionalTransport, Multiaddr, Transport, wasm_ext}; use log::{trace, warn, error}; @@ -60,8 +60,8 @@ impl, I> StreamAndSink for T {} type WsTrans = libp2p::core::transport::boxed::Boxed< Pin, + Vec, + Item = Result, io::Error>, Error = io::Error > + Send>>, io::Error @@ -91,12 +91,12 @@ impl TelemetryWorker { libp2p::websocket::framed::WsConfig::new(inner) .and_then(|connec, _| { let connec = connec - .with(|item: BytesMut| { + .with(|item| { let item = libp2p::websocket::framed::OutgoingData::Binary(item); future::ready(Ok::<_, io::Error>(item)) }) .try_filter(|item| future::ready(item.is_data())) - .map_ok(|data| BytesMut::from(data.as_ref())); + .map_ok(|data| data.into_bytes()); future::ready(Ok::<_, io::Error>(connec)) }) }); @@ -188,7 +188,7 @@ impl TelemetryWorker { /// For some context, we put this object around the `wasm_ext::ExtTransport` in order to make sure /// that each telemetry message maps to one single call to `write` in the WASM FFI. #[pin_project::pin_project] -struct StreamSink(#[pin] T, Option); +struct StreamSink(#[pin] T, Option>); impl From for StreamSink { fn from(inner: T) -> StreamSink { @@ -197,15 +197,15 @@ impl From for StreamSink { } impl Stream for StreamSink { - type Item = Result; + type Item = Result, io::Error>; fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { let this = self.project(); - let mut buf = [0; 128]; + let mut buf = vec![0; 128]; match ready!(AsyncRead::poll_read(this.0, cx, &mut buf)) { Ok(0) => Poll::Ready(None), Ok(n) => { - let buf: BytesMut = buf[..n].into(); + buf.truncate(n); Poll::Ready(Some(Ok(buf))) }, Err(err) => Poll::Ready(Some(Err(err))), @@ -231,7 +231,7 @@ impl StreamSink { } } -impl Sink for StreamSink { +impl Sink> for StreamSink { type Error = io::Error; fn poll_ready(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { @@ -239,7 +239,7 @@ impl Sink for StreamSink { Poll::Ready(Ok(())) } - fn start_send(self: Pin<&mut Self>, item: BytesMut) -> Result<(), Self::Error> { + fn start_send(self: Pin<&mut Self>, item: Vec) -> Result<(), Self::Error> { let this = self.project(); debug_assert!(this.1.is_none()); *this.1 = Some(item); diff --git a/client/telemetry/src/worker/node.rs b/client/telemetry/src/worker/node.rs index 0662ecab543c3cb6a9f6ffea9dbc1ef8ca6f4b7e..eef7ca7e815536aaaa26f8e2d20f17dc67e2f1e5 100644 --- a/client/telemetry/src/worker/node.rs +++ b/client/telemetry/src/worker/node.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,9 +15,9 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Contains the `Node` struct, which handles communications with a single telemetry endpoint. -use bytes::BytesMut; use futures::prelude::*; use futures_timer::Delay; use libp2p::Multiaddr; @@ -56,7 +56,7 @@ struct NodeSocketConnected { /// Where to send data. sink: TTrans::Output, /// Queue of packets to send. - pending: VecDeque, + pending: VecDeque>, /// If true, we need to flush the sink. need_flush: bool, /// A timeout for the socket to write data. @@ -102,15 +102,15 @@ impl Node { impl Node where TTrans: Clone + Unpin, TTrans::Dial: Unpin, - TTrans::Output: Sink - + Stream> + TTrans::Output: Sink, Error = TSinkErr> + + Stream, TSinkErr>> + Unpin, TSinkErr: fmt::Debug { /// Sends a WebSocket frame to the node. Returns an error if we are not connected to the node. /// /// After calling this method, you should call `poll` in order for it to be properly processed. - pub fn send_message(&mut self, payload: impl Into) -> Result<(), ()> { + pub fn send_message(&mut self, payload: impl Into>) -> Result<(), ()> { if let NodeSocket::Connected(NodeSocketConnected { pending, .. }) = &mut self.socket { if pending.len() <= MAX_PENDING { trace!(target: "telemetry", "Adding log entry to queue for {:?}", self.addr); @@ -202,8 +202,8 @@ fn gen_rand_reconnect_delay() -> Delay { } impl NodeSocketConnected -where TTrans::Output: Sink - + Stream> +where TTrans::Output: Sink, Error = TSinkErr> + + Stream, TSinkErr>> + Unpin { /// Processes the queue of messages for the connected socket. diff --git a/client/tracing/Cargo.toml b/client/tracing/Cargo.toml index 4e1924180b724b744ff6cf5b0ab7cc9e15b12d94..c4564e5fe53f59ff53f00997af0c85f9ce15ce96 100644 --- a/client/tracing/Cargo.toml +++ b/client/tracing/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-tracing" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" authors = ["Parity Technologies "] edition = "2018" @@ -15,12 +15,14 @@ targets = ["x86_64-unknown-linux-gnu"] erased-serde = "0.3.9" log = { version = "0.4.8" } parking_lot = "0.10.0" +rustc-hash = "1.1.0" serde = "1.0.101" serde_json = "1.0.41" slog = { version = "2.5.2", features = ["nested-values"] } tracing-core = "0.1.7" +sp-tracing = { version = "2.0.0-rc2", path = "../../primitives/tracing" } -sc-telemetry = { version = "2.0.0-alpha.8", path = "../telemetry" } +sc-telemetry = { version = "2.0.0-rc4", path = "../telemetry" } [dev-dependencies] tracing = "0.1.10" diff --git a/client/tracing/src/lib.rs b/client/tracing/src/lib.rs index d450700ed3c49f9f41d27999aad6753fbe5f84bf..c62b8d5b1e9c95f8a02bde84fad19c101ac075e3 100644 --- a/client/tracing/src/lib.rs +++ b/client/tracing/src/lib.rs @@ -24,7 +24,7 @@ //! //! Currently we provide `Log` (default), `Telemetry` variants for `Receiver` -use std::collections::HashMap; +use rustc_hash::FxHashMap; use std::fmt; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; @@ -38,10 +38,14 @@ use tracing_core::{ Level, metadata::Metadata, span::{Attributes, Id, Record}, - subscriber::Subscriber + subscriber::Subscriber, }; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; +use sp_tracing::proxy::{WASM_NAME_KEY, WASM_TARGET_KEY, WASM_TRACE_IDENTIFIER}; + +const ZERO_DURATION: Duration = Duration::from_nanos(0); +const PROXY_TARGET: &'static str = "sp_tracing::proxy"; /// Used to configure how to receive the metrics #[derive(Debug, Clone)] @@ -58,36 +62,55 @@ impl Default for TracingReceiver { } } +/// A handler for tracing `SpanDatum` +pub trait TraceHandler: Send + Sync { + /// Process a `SpanDatum` + fn process_span(&self, span: SpanDatum); +} + +/// Represents a single instance of a tracing span #[derive(Debug)] -struct SpanDatum { - id: u64, - name: &'static str, - target: &'static str, - level: Level, - line: u32, - start_time: Instant, - overall_time: Duration, - values: Visitor, +pub struct SpanDatum { + pub id: u64, + pub name: String, + pub target: String, + pub level: Level, + pub line: u32, + pub start_time: Instant, + pub overall_time: Duration, + pub values: Visitor, } +/// Holds associated values for a tracing span #[derive(Clone, Debug)] -struct Visitor(Vec<(String, String)>); +pub struct Visitor(FxHashMap); + +impl Visitor { + /// Consume the Visitor, returning the inner FxHashMap + pub fn into_inner(self) -> FxHashMap { + self.0 + } +} impl Visit for Visitor { fn record_i64(&mut self, field: &Field, value: i64) { - self.record_debug(field, &value) + self.0.insert(field.name().to_string(), value.to_string()); } fn record_u64(&mut self, field: &Field, value: u64) { - self.record_debug(field, &value) + self.0.insert(field.name().to_string(), value.to_string()); } fn record_bool(&mut self, field: &Field, value: bool) { - self.record_debug(field, &value) + self.0.insert(field.name().to_string(), value.to_string()); + } + + fn record_str(&mut self, field: &Field, value: &str) { + self.0.insert(field.name().to_string(), value.to_owned()); } fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { - self.0.push((field.name().to_string(), format!("{:?}",value))); + self.0.insert(field.name().to_string(), format!("{:?}", value)); } } @@ -105,7 +128,7 @@ impl Serialize for Visitor { impl fmt::Display for Visitor { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let values = self.0.iter().map(|(k,v)| format!("{}={}",k,v)).collect::>().join(", "); + let values = self.0.iter().map(|(k, v)| format!("{}={}", k, v)).collect::>().join(", "); write!(f, "{}", values) } } @@ -135,23 +158,50 @@ impl Value for Visitor { pub struct ProfilingSubscriber { next_id: AtomicU64, targets: Vec<(String, Level)>, - receiver: TracingReceiver, - span_data: Mutex>, + trace_handler: Box, + span_data: Mutex>, } impl ProfilingSubscriber { - /// Takes a `Receiver` and a comma separated list of targets, - /// either with a level: "pallet=trace" - /// or without: "pallet". - pub fn new(receiver: TracingReceiver, targets: &str) -> Self { + /// Takes a `TracingReceiver` and a comma separated list of targets, + /// either with a level: "pallet=trace,frame=debug" + /// or without: "pallet,frame" in which case the level defaults to `trace`. + /// wasm_tracing indicates whether to enable wasm traces + pub fn new(receiver: TracingReceiver, targets: &str) -> ProfilingSubscriber { + match receiver { + TracingReceiver::Log => Self::new_with_handler(Box::new(LogTraceHandler), targets), + TracingReceiver::Telemetry => Self::new_with_handler( + Box::new(TelemetryTraceHandler), + targets, + ), + } + } + + /// Allows use of a custom TraceHandler to create a new instance of ProfilingSubscriber. + /// Takes a comma separated list of targets, + /// either with a level, eg: "pallet=trace" + /// or without: "pallet" in which case the level defaults to `trace`. + /// wasm_tracing indicates whether to enable wasm traces + pub fn new_with_handler(trace_handler: Box, targets: &str) + -> ProfilingSubscriber + { let targets: Vec<_> = targets.split(',').map(|s| parse_target(s)).collect(); ProfilingSubscriber { next_id: AtomicU64::new(1), targets, - receiver, - span_data: Mutex::new(HashMap::new()), + trace_handler, + span_data: Mutex::new(FxHashMap::default()), } } + + fn check_target(&self, target: &str, level: &Level) -> bool { + for t in &self.targets { + if target.starts_with(t.0.as_str()) && level <= &t.1 { + return true; + } + } + false + } } // Default to TRACE if no level given or unable to parse Level @@ -173,36 +223,45 @@ fn parse_target(s: &str) -> (String, Level) { impl Subscriber for ProfilingSubscriber { fn enabled(&self, metadata: &Metadata<'_>) -> bool { - for t in &self.targets { - if metadata.target().starts_with(t.0.as_str()) && metadata.level() <= &t.1 { - log::debug!("Enabled target: {}, level: {}", metadata.target(), metadata.level()); - return true; - } else { - log::debug!("Disabled target: {}, level: {}", metadata.target(), metadata.level()); - } + if metadata.target() == PROXY_TARGET || self.check_target(metadata.target(), metadata.level()) { + log::debug!(target: "tracing", "Enabled target: {}, level: {}", metadata.target(), metadata.level()); + true + } else { + log::debug!(target: "tracing", "Disabled target: {}, level: {}", metadata.target(), metadata.level()); + false } - false } fn new_span(&self, attrs: &Attributes<'_>) -> Id { let id = self.next_id.fetch_add(1, Ordering::Relaxed); - let mut values = Visitor(Vec::new()); + let mut values = Visitor(FxHashMap::default()); attrs.record(&mut values); + // If this is a wasm trace, check if target/level is enabled + if let Some(wasm_target) = values.0.get(WASM_TARGET_KEY) { + if !self.check_target(wasm_target, attrs.metadata().level()) { + return Id::from_u64(id); + } + } let span_datum = SpanDatum { id, - name: attrs.metadata().name(), - target: attrs.metadata().target(), + name: attrs.metadata().name().to_owned(), + target: attrs.metadata().target().to_owned(), level: attrs.metadata().level().clone(), line: attrs.metadata().line().unwrap_or(0), start_time: Instant::now(), - overall_time: Duration::from_nanos(0), + overall_time: ZERO_DURATION, values, }; self.span_data.lock().insert(id, span_datum); Id::from_u64(id) } - fn record(&self, _span: &Id, _values: &Record<'_>) {} + fn record(&self, span: &Id, values: &Record<'_>) { + let mut span_data = self.span_data.lock(); + if let Some(s) = span_data.get_mut(&span.into_u64()) { + values.record(&mut s.values); + } + } fn record_follows_from(&self, _span: &Id, _follows: &Id) {} @@ -213,65 +272,89 @@ impl Subscriber for ProfilingSubscriber { let start_time = Instant::now(); if let Some(mut s) = span_data.get_mut(&span.into_u64()) { s.start_time = start_time; - } else { - log::warn!("Tried to enter span {:?} that has already been closed!", span); } } fn exit(&self, span: &Id) { - let mut span_data = self.span_data.lock(); let end_time = Instant::now(); + let mut span_data = self.span_data.lock(); if let Some(mut s) = span_data.get_mut(&span.into_u64()) { s.overall_time = end_time - s.start_time + s.overall_time; } } fn try_close(&self, span: Id) -> bool { - let mut span_data = self.span_data.lock(); - if let Some(data) = span_data.remove(&span.into_u64()) { - self.send_span(data); + let span_datum = { + let mut span_data = self.span_data.lock(); + span_data.remove(&span.into_u64()) + }; + if let Some(mut span_datum) = span_datum { + if span_datum.name == WASM_TRACE_IDENTIFIER { + span_datum.values.0.insert("wasm".to_owned(), "true".to_owned()); + if let Some(n) = span_datum.values.0.remove(WASM_NAME_KEY) { + span_datum.name = n; + } + if let Some(t) = span_datum.values.0.remove(WASM_TARGET_KEY) { + span_datum.target = t; + } + } + if self.check_target(&span_datum.target, &span_datum.level) { + self.trace_handler.process_span(span_datum); + } }; true } } -impl ProfilingSubscriber { - fn send_span(&self, span_datum: SpanDatum) { - match self.receiver { - TracingReceiver::Log => print_log(span_datum), - TracingReceiver::Telemetry => send_telemetry(span_datum), - } +/// TraceHandler for sending span data to the logger +pub struct LogTraceHandler; + +fn log_level(level: Level) -> log::Level { + match level { + Level::TRACE => log::Level::Trace, + Level::DEBUG => log::Level::Debug, + Level::INFO => log::Level::Info, + Level::WARN => log::Level::Warn, + Level::ERROR => log::Level::Error, } } -fn print_log(span_datum: SpanDatum) { - if span_datum.values.0.is_empty() { - log::info!("TRACING: {} {}: {}, line: {}, time: {}", - span_datum.level, - span_datum.target, - span_datum.name, - span_datum.line, - span_datum.overall_time.as_nanos(), - ); - } else { - log::info!("TRACING: {} {}: {}, line: {}, time: {}, {}", - span_datum.level, - span_datum.target, - span_datum.name, - span_datum.line, - span_datum.overall_time.as_nanos(), - span_datum.values - ); +impl TraceHandler for LogTraceHandler { + fn process_span(&self, span_datum: SpanDatum) { + if span_datum.values.0.is_empty() { + log::log!( + log_level(span_datum.level), + "{}: {}, time: {}", + span_datum.target, + span_datum.name, + span_datum.overall_time.as_nanos(), + ); + } else { + log::log!( + log_level(span_datum.level), + "{}: {}, time: {}, {}", + span_datum.target, + span_datum.name, + span_datum.overall_time.as_nanos(), + span_datum.values, + ); + } } } -fn send_telemetry(span_datum: SpanDatum) { - telemetry!(SUBSTRATE_INFO; "tracing.profiling"; - "name" => span_datum.name, - "target" => span_datum.target, - "line" => span_datum.line, - "time" => span_datum.overall_time.as_nanos(), - "values" => span_datum.values - ); +/// TraceHandler for sending span data to telemetry, +/// Please see telemetry documentation for details on how to specify endpoints and +/// set the required telemetry level to activate tracing messages +pub struct TelemetryTraceHandler; + +impl TraceHandler for TelemetryTraceHandler { + fn process_span(&self, span_datum: SpanDatum) { + telemetry!(SUBSTRATE_INFO; "tracing.profiling"; + "name" => span_datum.name, + "target" => span_datum.target, + "line" => span_datum.line, + "time" => span_datum.overall_time.as_nanos(), + "values" => span_datum.values + ); + } } - diff --git a/client/transaction-pool/Cargo.toml b/client/transaction-pool/Cargo.toml index a7b50c847c4d449e95dfa69ab61da3693cc4ec88..bd271d8ba13104be5e253bc3f81d50594445679d 100644 --- a/client/transaction-pool/Cargo.toml +++ b/client/transaction-pool/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-transaction-pool" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -12,7 +12,7 @@ description = "Substrate transaction pool implementation." targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0" } +codec = { package = "parity-scale-codec", version = "1.3.1" } derive_more = "0.99.2" futures = { version = "0.3.1", features = ["compat"] } futures-diagnose = "1.0" @@ -20,21 +20,23 @@ intervalier = "0.4.0" log = "0.4.8" parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } parking_lot = "0.10.0" -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-alpha.8"} -sc-client-api = { version = "2.0.0-alpha.8", path = "../api" } -sc-transaction-graph = { version = "2.0.0-alpha.8", path = "./graph" } -sp-api = { version = "2.0.0-alpha.8", path = "../../primitives/api" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } -sp-tracing = { version = "2.0.0-alpha.8", path = "../../primitives/tracing" } -sp-transaction-pool = { version = "2.0.0-alpha.8", path = "../../primitives/transaction-pool" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../primitives/blockchain" } -sp-utils = { version = "2.0.0-alpha.8", path = "../../primitives/utils" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-rc4"} +sc-client-api = { version = "2.0.0-rc4", path = "../api" } +sc-transaction-graph = { version = "2.0.0-rc4", path = "./graph" } +sp-api = { version = "2.0.0-rc4", path = "../../primitives/api" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } +sp-tracing = { version = "2.0.0-rc4", path = "../../primitives/tracing" } +sp-transaction-pool = { version = "2.0.0-rc4", path = "../../primitives/transaction-pool" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../primitives/blockchain" } +sp-utils = { version = "2.0.0-rc4", path = "../../primitives/utils" } wasm-timer = "0.2" [dev-dependencies] assert_matches = "1.3.0" hex = "0.4" -sp-keyring = { version = "2.0.0-alpha.8", path = "../../primitives/keyring" } -substrate-test-runtime-transaction-pool = { version = "2.0.0-alpha.8", path = "../../test-utils/runtime/transaction-pool" } -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../test-utils/runtime/client" } +sp-keyring = { version = "2.0.0-rc4", path = "../../primitives/keyring" } +sp-consensus = { version = "0.8.0-rc4", path = "../../primitives/consensus/common" } +substrate-test-runtime-transaction-pool = { version = "2.0.0-rc4", path = "../../test-utils/runtime/transaction-pool" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../test-utils/runtime/client" } +sc-block-builder = { version = "0.8.0-rc4", path = "../block-builder" } diff --git a/client/transaction-pool/graph/Cargo.toml b/client/transaction-pool/graph/Cargo.toml index eaa70743e25eb7112548853b1990b188b02a68fc..0a30b3a4c92d43af0cb05d22263f0d6e1fcfd01d 100644 --- a/client/transaction-pool/graph/Cargo.toml +++ b/client/transaction-pool/graph/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-transaction-graph" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" @@ -18,18 +18,18 @@ log = "0.4.8" parking_lot = "0.10.0" serde = { version = "1.0.101", features = ["derive"] } wasm-timer = "0.2" -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } -sp-utils = { version = "2.0.0-alpha.8", path = "../../../primitives/utils" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-transaction-pool = { version = "2.0.0-alpha.8", path = "../../../primitives/transaction-pool" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } +sp-utils = { version = "2.0.0-rc4", path = "../../../primitives/utils" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-transaction-pool = { version = "2.0.0-rc4", path = "../../../primitives/transaction-pool" } parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } linked-hash-map = "0.5.2" [dev-dependencies] assert_matches = "1.3.0" -codec = { package = "parity-scale-codec", version = "1.3.0" } -substrate-test-runtime = { version = "2.0.0-alpha.8", path = "../../../test-utils/runtime" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +substrate-test-runtime = { version = "2.0.0-rc4", path = "../../../test-utils/runtime" } criterion = "0.3" [[bench]] diff --git a/client/transaction-pool/graph/benches/basics.rs b/client/transaction-pool/graph/benches/basics.rs index 544d31e176e5249ca729acbb365ddcfd3546da40..bb10086bd4a558330d8d36e4bf3b92d791fdcae4 100644 --- a/client/transaction-pool/graph/benches/basics.rs +++ b/client/transaction-pool/graph/benches/basics.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use criterion::{criterion_group, criterion_main, Criterion}; use futures::{future::{ready, Ready}, executor::block_on}; @@ -50,7 +51,6 @@ fn to_tag(nonce: u64, from: AccountId) -> Tag { impl ChainApi for TestApi { type Block = Block; - type Hash = H256; type Error = sp_transaction_pool::error::Error; type ValidationFuture = Ready>; type BodyFuture = Ready>>>; @@ -106,7 +106,7 @@ impl ChainApi for TestApi { }) } - fn hash_and_length(&self, uxt: &ExtrinsicFor) -> (Self::Hash, usize) { + fn hash_and_length(&self, uxt: &ExtrinsicFor) -> (H256, usize) { let encoded = uxt.encode(); (blake2_256(&encoded).into(), encoded.len()) } diff --git a/client/transaction-pool/graph/src/base_pool.rs b/client/transaction-pool/graph/src/base_pool.rs index 452f9c9feb1b97493b0a3de1a11dedc72801a368..0128e94675e23ff8329988f768e9d87defc2bfdf 100644 --- a/client/transaction-pool/graph/src/base_pool.rs +++ b/client/transaction-pool/graph/src/base_pool.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! A basic version of the dependency graph. //! //! For a more full-featured pool, have a look at the `pool` module. diff --git a/client/transaction-pool/graph/src/error.rs b/client/transaction-pool/graph/src/error.rs index c04f90dac0367dacdc12aafc7cabf76d3cccc507..392ddaa39be6f2d809792985b283cf5630d09792 100644 --- a/client/transaction-pool/graph/src/error.rs +++ b/client/transaction-pool/graph/src/error.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Transaction pool errors. use sp_runtime::transaction_validity::{ diff --git a/client/transaction-pool/graph/src/future.rs b/client/transaction-pool/graph/src/future.rs index beff5bb2ccff5f83f104edf9cdf2e72e06f94e57..80e6825d4ff9d3097f485d8fe86ca388ef0a3197 100644 --- a/client/transaction-pool/graph/src/future.rs +++ b/client/transaction-pool/graph/src/future.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use std::{ collections::{HashMap, HashSet}, fmt, diff --git a/client/transaction-pool/graph/src/lib.rs b/client/transaction-pool/graph/src/lib.rs index 632f7d3feb0e33086ef7d4176f5e3e01ea5142cd..bf220ce22973a97190ad7a0d5da2ba50e9e0acd6 100644 --- a/client/transaction-pool/graph/src/lib.rs +++ b/client/transaction-pool/graph/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Generic Transaction Pool //! //! The pool is based on dependency graph between transactions @@ -31,14 +32,13 @@ mod pool; mod ready; mod rotator; mod validated_pool; +mod tracked_map; pub mod base_pool; pub mod watcher; pub use self::base_pool::Transaction; pub use self::pool::{ - Pool, - Options, ChainApi, EventStream, ExtrinsicFor, - BlockHash, ExHash, NumberFor, TransactionFor, - ValidatedTransaction, + Pool, Options, ChainApi, EventStream, ExtrinsicFor, ExtrinsicHash, + BlockHash, NumberFor, TransactionFor, ValidatedTransaction, }; diff --git a/client/transaction-pool/graph/src/listener.rs b/client/transaction-pool/graph/src/listener.rs index c8bf2c9d39f3bfe648fa3d2ef9f1eb0cf9eb0d98..1bc3720fa6b85e8eea21b75d1a15bffd9f210758 100644 --- a/client/transaction-pool/graph/src/listener.rs +++ b/client/transaction-pool/graph/src/listener.rs @@ -6,7 +6,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -16,25 +16,26 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use std::{ - collections::HashMap, hash, + collections::HashMap, hash, fmt::Debug, }; use linked_hash_map::LinkedHashMap; use serde::Serialize; -use crate::{watcher, ChainApi, BlockHash}; +use crate::{watcher, ChainApi, ExtrinsicHash, BlockHash}; use log::{debug, trace, warn}; use sp_runtime::traits; /// Extrinsic pool default listener. pub struct Listener { - watchers: HashMap>>, - finality_watchers: LinkedHashMap, Vec>, + watchers: HashMap>>, + finality_watchers: LinkedHashMap, Vec>, } /// Maximum number of blocks awaiting finality at any time. const MAX_FINALITY_WATCHERS: usize = 512; -impl Default for Listener { +impl Default for Listener { fn default() -> Self { Listener { watchers: Default::default(), @@ -44,7 +45,7 @@ impl Default for Listener { } impl Listener { - fn fire(&mut self, hash: &H, fun: F) where F: FnOnce(&mut watcher::Sender>) { + fn fire(&mut self, hash: &H, fun: F) where F: FnOnce(&mut watcher::Sender>) { let clean = if let Some(h) = self.watchers.get_mut(hash) { fun(h); h.is_done() @@ -60,7 +61,7 @@ impl Listener { /// Creates a new watcher for given verified extrinsic. /// /// The watcher can be used to subscribe to life-cycle events of that extrinsic. - pub fn create_watcher(&mut self, hash: H) -> watcher::Watcher> { + pub fn create_watcher(&mut self, hash: H) -> watcher::Watcher> { let sender = self.watchers.entry(hash.clone()).or_insert_with(watcher::Sender::default); sender.new_watcher(hash) } @@ -73,7 +74,7 @@ impl Listener { /// New transaction was added to the ready pool or promoted from the future pool. pub fn ready(&mut self, tx: &H, old: Option<&H>) { - trace!(target: "txpool", "[{:?}] Ready (replaced: {:?})", tx, old); + trace!(target: "txpool", "[{:?}] Ready (replaced with {:?})", tx, old); self.fire(tx, |watcher| watcher.ready()); if let Some(old) = old { self.fire(old, |watcher| watcher.usurped(tx.clone())); @@ -88,7 +89,7 @@ impl Listener { /// Transaction was dropped from the pool because of the limit. pub fn dropped(&mut self, tx: &H, by: Option<&H>) { - trace!(target: "txpool", "[{:?}] Dropped (replaced by {:?})", tx, by); + trace!(target: "txpool", "[{:?}] Dropped (replaced with {:?})", tx, by); self.fire(tx, |watcher| match by { Some(t) => watcher.usurped(t.clone()), None => watcher.dropped(), @@ -98,9 +99,9 @@ impl Listener { /// Transaction was removed as invalid. pub fn invalid(&mut self, tx: &H, warn: bool) { if warn { - warn!(target: "txpool", "Extrinsic invalid: {:?}", tx); + warn!(target: "txpool", "[{:?}] Extrinsic invalid", tx); } else { - debug!(target: "txpool", "Extrinsic invalid: {:?}", tx); + debug!(target: "txpool", "[{:?}] Extrinsic invalid", tx); } self.fire(tx, |watcher| watcher.invalid()); } @@ -133,6 +134,7 @@ impl Listener { pub fn finalized(&mut self, block_hash: BlockHash) { if let Some(hashes) = self.finality_watchers.remove(&block_hash) { for hash in hashes { + log::debug!(target: "txpool", "[{:?}] Sent finalization event (block {:?})", hash, block_hash); self.fire(&hash, |s| s.finalized(block_hash)) } } diff --git a/client/transaction-pool/graph/src/pool.rs b/client/transaction-pool/graph/src/pool.rs index cef2f0b62e698465c0209e1b68bf1d8dfb84feb6..e4d81c38ae38eb43d2cc653b27060da0cdf597f4 100644 --- a/client/transaction-pool/graph/src/pool.rs +++ b/client/transaction-pool/graph/src/pool.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,20 +15,18 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use std::{ - hash, collections::HashMap, sync::Arc, }; -use crate::base_pool as base; -use crate::watcher::Watcher; -use serde::Serialize; +use crate::{base_pool as base, watcher::Watcher}; use futures::{Future, FutureExt}; use sp_runtime::{ generic::BlockId, - traits::{self, SaturatedConversion}, + traits::{self, SaturatedConversion, Block as BlockT}, transaction_validity::{ TransactionValidity, TransactionTag as Tag, TransactionValidityError, TransactionSource, }, @@ -43,19 +41,19 @@ pub use crate::validated_pool::ValidatedTransaction; /// Modification notification event stream type; pub type EventStream = TracingUnboundedReceiver; -/// Extrinsic hash type for a pool. -pub type ExHash = ::Hash; /// Block hash type for a pool. pub type BlockHash = <::Block as traits::Block>::Hash; +/// Extrinsic hash type for a pool. +pub type ExtrinsicHash = <::Block as traits::Block>::Hash; /// Extrinsic type for a pool. pub type ExtrinsicFor = <::Block as traits::Block>::Extrinsic; /// Block number type for the ChainApi pub type NumberFor = traits::NumberFor<::Block>; /// A type of transaction stored in the pool -pub type TransactionFor = Arc, ExtrinsicFor>>; +pub type TransactionFor = Arc, ExtrinsicFor>>; /// A type of validated transaction stored in the pool. pub type ValidatedTransactionFor = ValidatedTransaction< - ExHash, + ExtrinsicHash, ExtrinsicFor, ::Error, >; @@ -63,15 +61,15 @@ pub type ValidatedTransactionFor = ValidatedTransaction< /// Concrete extrinsic validation and query logic. pub trait ChainApi: Send + Sync { /// Block type. - type Block: traits::Block; - /// Transaction Hash type - type Hash: hash::Hash + Eq + traits::Member + Serialize; + type Block: BlockT; /// Error type. type Error: From + error::IntoPoolError; /// Validate transaction future. type ValidationFuture: Future> + Send + Unpin; /// Body future (since block body might be remote) - type BodyFuture: Future::Extrinsic>>, Self::Error>> + Unpin + Send + 'static; + type BodyFuture: Future< + Output = Result::Extrinsic>>, Self::Error> + > + Unpin + Send + 'static; /// Verify extrinsic at given block. fn validate_transaction( @@ -82,13 +80,19 @@ pub trait ChainApi: Send + Sync { ) -> Self::ValidationFuture; /// Returns a block number given the block id. - fn block_id_to_number(&self, at: &BlockId) -> Result>, Self::Error>; + fn block_id_to_number( + &self, + at: &BlockId, + ) -> Result>, Self::Error>; /// Returns a block hash given the block id. - fn block_id_to_hash(&self, at: &BlockId) -> Result>, Self::Error>; + fn block_id_to_hash( + &self, + at: &BlockId, + ) -> Result::Hash>, Self::Error>; /// Returns hash and encoding length of the extrinsic. - fn hash_and_length(&self, uxt: &ExtrinsicFor) -> (Self::Hash, usize); + fn hash_and_length(&self, uxt: &ExtrinsicFor) -> (ExtrinsicHash, usize); /// Returns a block body given the block id. fn block_body(&self, at: &BlockId) -> Self::BodyFuture; @@ -129,7 +133,6 @@ pub struct Pool { #[cfg(not(target_os = "unknown"))] impl parity_util_mem::MallocSizeOf for Pool where - B::Hash: parity_util_mem::MallocSizeOf, ExtrinsicFor: parity_util_mem::MallocSizeOf, { fn size_of(&self, ops: &mut parity_util_mem::MallocSizeOfOps) -> usize { @@ -152,7 +155,7 @@ impl Pool { source: TransactionSource, xts: T, force: bool, - ) -> Result, B::Error>>, B::Error> where + ) -> Result, B::Error>>, B::Error> where T: IntoIterator>, { let validated_pool = self.validated_pool.clone(); @@ -171,7 +174,7 @@ impl Pool { at: &BlockId, source: TransactionSource, xt: ExtrinsicFor, - ) -> Result, B::Error> { + ) -> Result, B::Error> { self.submit_at(at, source, std::iter::once(xt), false) .map(|import_result| import_result.and_then(|mut import_result| import_result .pop() @@ -186,7 +189,7 @@ impl Pool { at: &BlockId, source: TransactionSource, xt: ExtrinsicFor, - ) -> Result, BlockHash>, B::Error> { + ) -> Result, ExtrinsicHash>, B::Error> { let block_number = self.resolve_block_number(at)?; let (_, tx) = self.verify_one( at, block_number, source, xt, false @@ -197,7 +200,7 @@ impl Pool { /// Resubmit some transaction that were validated elsewhere. pub fn resubmit( &self, - revalidated_transactions: HashMap, ValidatedTransactionFor>, + revalidated_transactions: HashMap, ValidatedTransactionFor>, ) { let now = Instant::now(); @@ -214,7 +217,11 @@ impl Pool { /// Used to clear the pool from transactions that were part of recently imported block. /// The main difference from the `prune` is that we do not revalidate any transactions /// and ignore unknown passed hashes. - pub fn prune_known(&self, at: &BlockId, hashes: &[ExHash]) -> Result<(), B::Error> { + pub fn prune_known( + &self, + at: &BlockId, + hashes: &[ExtrinsicHash], + ) -> Result<(), B::Error> { // Get details of all extrinsics that are already in the pool let in_pool_tags = self.validated_pool.extrinsics_tags(hashes) .into_iter().filter_map(|x| x).flat_map(|x| x); @@ -298,7 +305,7 @@ impl Pool { &self, at: &BlockId, tags: impl IntoIterator, - known_imported_hashes: impl IntoIterator> + Clone, + known_imported_hashes: impl IntoIterator> + Clone, ) -> Result<(), B::Error> { log::debug!(target: "txpool", "Pruning at {:?}", at); // Prune all transactions that provide given tags @@ -335,7 +342,7 @@ impl Pool { } /// Returns transaction hash - pub fn hash_of(&self, xt: &ExtrinsicFor) -> ExHash { + pub fn hash_of(&self, xt: &ExtrinsicFor) -> ExtrinsicHash { self.validated_pool.api().hash_and_length(xt).0 } @@ -352,7 +359,7 @@ impl Pool { at: &BlockId, xts: impl IntoIterator)>, force: bool, - ) -> Result, ValidatedTransactionFor>, B::Error> { + ) -> Result, ValidatedTransactionFor>, B::Error> { // we need a block number to compute tx validity let block_number = self.resolve_block_number(at)?; let mut result = HashMap::new(); @@ -378,7 +385,7 @@ impl Pool { source: TransactionSource, xt: ExtrinsicFor, force: bool, - ) -> (ExHash, ValidatedTransactionFor) { + ) -> (ExtrinsicHash, ValidatedTransactionFor) { let (hash, bytes) = self.validated_pool.api().hash_and_length(&xt); if !force && self.validated_pool.is_banned(&hash) { return ( @@ -443,9 +450,12 @@ mod tests { use futures::executor::block_on; use super::*; use sp_transaction_pool::TransactionStatus; - use sp_runtime::transaction_validity::{ValidTransaction, InvalidTransaction, TransactionSource}; + use sp_runtime::{ + traits::Hash, + transaction_validity::{ValidTransaction, InvalidTransaction, TransactionSource}, + }; use codec::Encode; - use substrate_test_runtime::{Block, Extrinsic, Transfer, H256, AccountId}; + use substrate_test_runtime::{Block, Extrinsic, Transfer, H256, AccountId, Hashing}; use assert_matches::assert_matches; use wasm_timer::Instant; use crate::base_pool::Limit; @@ -456,14 +466,13 @@ mod tests { #[derive(Clone, Debug, Default)] struct TestApi { delay: Arc>>>, - invalidate: Arc>>, - clear_requirements: Arc>>, - add_requirements: Arc>>, + invalidate: Arc>>, + clear_requirements: Arc>>, + add_requirements: Arc>>, } impl ChainApi for TestApi { type Block = Block; - type Hash = u64; type Error = error::Error; type ValidationFuture = futures::future::Ready>; type BodyFuture = futures::future::Ready>>>; @@ -517,7 +526,10 @@ mod tests { } /// Returns a block number given the block id. - fn block_id_to_number(&self, at: &BlockId) -> Result>, Self::Error> { + fn block_id_to_number( + &self, + at: &BlockId, + ) -> Result>, Self::Error> { Ok(match at { BlockId::Number(num) => Some(*num), BlockId::Hash(_) => None, @@ -525,7 +537,10 @@ mod tests { } /// Returns a block hash given the block id. - fn block_id_to_hash(&self, at: &BlockId) -> Result>, Self::Error> { + fn block_id_to_hash( + &self, + at: &BlockId, + ) -> Result::Hash>, Self::Error> { Ok(match at { BlockId::Number(num) => Some(H256::from_low_u64_be(*num)).into(), BlockId::Hash(_) => None, @@ -533,12 +548,10 @@ mod tests { } /// Hash the extrinsic. - fn hash_and_length(&self, uxt: &ExtrinsicFor) -> (Self::Hash, usize) { - let len = uxt.encode().len(); - ( - (H256::from(uxt.transfer().from.clone()).to_low_u64_be() << 5) + uxt.transfer().nonce, - len - ) + fn hash_and_length(&self, uxt: &ExtrinsicFor) -> (BlockHash, usize) { + let encoded = uxt.encode(); + let len = encoded.len(); + (Hashing::hash(&encoded), len) } fn block_body(&self, _id: &BlockId) -> Self::BodyFuture { @@ -598,19 +611,19 @@ mod tests { #[test] fn should_notify_about_pool_events() { - let stream = { + let (stream, hash0, hash1) = { // given let pool = pool(); let stream = pool.validated_pool().import_notification_stream(); // when - let _hash = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { + let hash0 = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, nonce: 0, }))).unwrap(); - let _hash = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { + let hash1 = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt(Transfer { from: AccountId::from_h256(H256::from_low_u64_be(1)), to: AccountId::from_h256(H256::from_low_u64_be(2)), amount: 5, @@ -626,13 +639,14 @@ mod tests { assert_eq!(pool.validated_pool().status().ready, 2); assert_eq!(pool.validated_pool().status().future, 1); - stream + + (stream, hash0, hash1) }; // then let mut it = futures::executor::block_on_stream(stream); - assert_eq!(it.next(), Some(32)); - assert_eq!(it.next(), Some(33)); + assert_eq!(it.next(), Some(hash0)); + assert_eq!(it.next(), Some(hash1)); assert_eq!(it.next(), None); } @@ -794,7 +808,10 @@ mod tests { // then let mut stream = futures::executor::block_on_stream(watcher.into_stream()); assert_eq!(stream.next(), Some(TransactionStatus::Ready)); - assert_eq!(stream.next(), Some(TransactionStatus::InBlock(H256::from_low_u64_be(2).into()))); + assert_eq!( + stream.next(), + Some(TransactionStatus::InBlock(H256::from_low_u64_be(2).into())), + ); } #[test] @@ -811,14 +828,19 @@ mod tests { assert_eq!(pool.validated_pool().status().future, 0); // when - block_on(pool.prune_tags(&BlockId::Number(2), vec![vec![0u8]], vec![2u64])).unwrap(); + block_on( + pool.prune_tags(&BlockId::Number(2), vec![vec![0u8]], vec![watcher.hash().clone()]), + ).unwrap(); assert_eq!(pool.validated_pool().status().ready, 0); assert_eq!(pool.validated_pool().status().future, 0); // then let mut stream = futures::executor::block_on_stream(watcher.into_stream()); assert_eq!(stream.next(), Some(TransactionStatus::Ready)); - assert_eq!(stream.next(), Some(TransactionStatus::InBlock(H256::from_low_u64_be(2).into()))); + assert_eq!( + stream.next(), + Some(TransactionStatus::InBlock(H256::from_low_u64_be(2).into())), + ); } #[test] diff --git a/client/transaction-pool/graph/src/ready.rs b/client/transaction-pool/graph/src/ready.rs index e759e318a1821ee7bd66126c05de9f4a4fe30e14..b98512b05d5cfc4418f3b11a9e0cba74d9453b66 100644 --- a/client/transaction-pool/graph/src/ready.rs +++ b/client/transaction-pool/graph/src/ready.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use std::{ collections::{HashMap, HashSet, BTreeSet}, cmp, @@ -24,15 +25,17 @@ use std::{ use serde::Serialize; use log::trace; -use parking_lot::RwLock; use sp_runtime::traits::Member; use sp_runtime::transaction_validity::{ TransactionTag as Tag, }; use sp_transaction_pool::error; -use crate::future::WaitingTransaction; -use crate::base_pool::Transaction; +use crate::{ + base_pool::Transaction, + future::WaitingTransaction, + tracked_map::{self, ReadOnlyTrackedMap, TrackedMap}, +}; /// An in-pool transaction reference. /// @@ -112,11 +115,17 @@ pub struct ReadyTransactions { /// tags that are provided by Ready transactions provided_tags: HashMap, /// Transactions that are ready (i.e. don't have any requirements external to the pool) - ready: Arc>>>, + ready: TrackedMap>, /// Best transactions that are ready to be included to the block without any other previous transaction. best: BTreeSet>, } +impl tracked_map::Size for ReadyTx { + fn size(&self) -> usize { + self.transaction.transaction.bytes + } +} + impl Default for ReadyTransactions { fn default() -> Self { ReadyTransactions { @@ -266,12 +275,7 @@ impl ReadyTransactions { ) -> Vec>> { let mut removed = vec![]; let mut ready = self.ready.write(); - loop { - let hash = match to_remove.pop() { - Some(hash) => hash, - None => return removed, - }; - + while let Some(hash) = to_remove.pop() { if let Some(mut tx) = ready.remove(&hash) { let invalidated = tx.transaction.transaction.provides .iter() @@ -310,6 +314,8 @@ impl ReadyTransactions { removed.push(tx.transaction.transaction); } } + + removed } /// Removes transactions that provide given tag. @@ -321,17 +327,16 @@ impl ReadyTransactions { let mut removed = vec![]; let mut to_remove = vec![tag]; - loop { - let tag = match to_remove.pop() { - Some(tag) => tag, - None => return removed, - }; - + while let Some(tag) = to_remove.pop() { let res = self.provided_tags.remove(&tag) - .and_then(|hash| self.ready.write().remove(&hash)); + .and_then(|hash| self.ready.write().remove(&hash)); if let Some(tx) = res { let unlocks = tx.unlocks; + + // Make sure we remove it from best txs + self.best.remove(&tx.transaction); + let tx = tx.transaction.transaction; // prune previous transactions as well @@ -394,6 +399,8 @@ impl ReadyTransactions { removed.push(tx); } } + + removed } /// Checks if the transaction is providing the same tags as other transactions. @@ -467,18 +474,18 @@ impl ReadyTransactions { /// Returns number of transactions in this queue. pub fn len(&self) -> usize { - self.ready.read().len() + self.ready.len() } /// Returns sum of encoding lengths of all transactions in this queue. pub fn bytes(&self) -> usize { - self.ready.read().values().fold(0, |acc, tx| acc + tx.transaction.transaction.bytes) + self.ready.bytes() } } /// Iterator of ready transactions ordered by priority. pub struct BestIterator { - all: Arc>>>, + all: ReadOnlyTrackedMap>, awaiting: HashMap)>, best: BTreeSet>, } diff --git a/client/transaction-pool/graph/src/rotator.rs b/client/transaction-pool/graph/src/rotator.rs index 9ce6a43b1eefa39a5c4f8f2765ab4b9f90bca4e4..65e21d0d4b5068262a6caf00add896ab51f4a710 100644 --- a/client/transaction-pool/graph/src/rotator.rs +++ b/client/transaction-pool/graph/src/rotator.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Rotate extrinsic inside the pool. //! //! Keeps only recent extrinsic and discard the ones kept for a significant amount of time. diff --git a/client/transaction-pool/graph/src/tracked_map.rs b/client/transaction-pool/graph/src/tracked_map.rs new file mode 100644 index 0000000000000000000000000000000000000000..c799eb0b96ea146b16951d3822502a55940b763b --- /dev/null +++ b/client/transaction-pool/graph/src/tracked_map.rs @@ -0,0 +1,189 @@ +// This file is part of Substrate. + +// Copyright (C) 2018-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use std::{ + collections::HashMap, + sync::{Arc, atomic::{AtomicIsize, Ordering as AtomicOrdering}}, +}; +use parking_lot::{RwLock, RwLockWriteGuard, RwLockReadGuard}; + +/// Something that can report it's size. +pub trait Size { + fn size(&self) -> usize; +} + +/// Map with size tracking. +/// +/// Size reported might be slightly off and only approximately true. +#[derive(Debug, parity_util_mem::MallocSizeOf)] +pub struct TrackedMap { + index: Arc>>, + bytes: AtomicIsize, + length: AtomicIsize, +} + +impl Default for TrackedMap { + fn default() -> Self { + Self { + index: Arc::new(HashMap::default().into()), + bytes: 0.into(), + length: 0.into(), + } + } +} + +impl TrackedMap { + /// Current tracked length of the content. + pub fn len(&self) -> usize { + std::cmp::max(self.length.load(AtomicOrdering::Relaxed), 0) as usize + } + + /// Current sum of content length. + pub fn bytes(&self) -> usize { + std::cmp::max(self.bytes.load(AtomicOrdering::Relaxed), 0) as usize + } + + /// Read-only clone of the interior. + pub fn clone(&self) -> ReadOnlyTrackedMap { + ReadOnlyTrackedMap(self.index.clone()) + } + + /// Lock map for read. + pub fn read<'a>(&'a self) -> TrackedMapReadAccess<'a, K, V> { + TrackedMapReadAccess { + inner_guard: self.index.read(), + } + } + + /// Lock map for write. + pub fn write<'a>(&'a self) -> TrackedMapWriteAccess<'a, K, V> { + TrackedMapWriteAccess { + inner_guard: self.index.write(), + bytes: &self.bytes, + length: &self.length, + } + } +} + +/// Read-only access to map. +/// +/// The only thing can be done is .read(). +pub struct ReadOnlyTrackedMap(Arc>>); + +impl ReadOnlyTrackedMap +where + K: Eq + std::hash::Hash +{ + /// Lock map for read. + pub fn read<'a>(&'a self) -> TrackedMapReadAccess<'a, K, V> { + TrackedMapReadAccess { + inner_guard: self.0.read(), + } + } +} + +pub struct TrackedMapReadAccess<'a, K, V> { + inner_guard: RwLockReadGuard<'a, HashMap>, +} + +impl<'a, K, V> TrackedMapReadAccess<'a, K, V> +where + K: Eq + std::hash::Hash +{ + /// Returns true if map contains key. + pub fn contains_key(&self, key: &K) -> bool { + self.inner_guard.contains_key(key) + } + + /// Returns reference to the contained value by key, if exists. + pub fn get(&self, key: &K) -> Option<&V> { + self.inner_guard.get(key) + } + + /// Returns iterator over all values. + pub fn values(&self) -> std::collections::hash_map::Values { + self.inner_guard.values() + } +} + +pub struct TrackedMapWriteAccess<'a, K, V> { + bytes: &'a AtomicIsize, + length: &'a AtomicIsize, + inner_guard: RwLockWriteGuard<'a, HashMap>, +} + +impl<'a, K, V> TrackedMapWriteAccess<'a, K, V> +where + K: Eq + std::hash::Hash, V: Size +{ + /// Insert value and return previous (if any). + pub fn insert(&mut self, key: K, val: V) -> Option { + let new_bytes = val.size(); + self.bytes.fetch_add(new_bytes as isize, AtomicOrdering::Relaxed); + self.length.fetch_add(1, AtomicOrdering::Relaxed); + self.inner_guard.insert(key, val).and_then(|old_val| { + self.bytes.fetch_sub(old_val.size() as isize, AtomicOrdering::Relaxed); + self.length.fetch_sub(1, AtomicOrdering::Relaxed); + Some(old_val) + }) + } + + /// Remove value by key. + pub fn remove(&mut self, key: &K) -> Option { + let val = self.inner_guard.remove(key); + if let Some(size) = val.as_ref().map(Size::size) { + self.bytes.fetch_sub(size as isize, AtomicOrdering::Relaxed); + self.length.fetch_sub(1, AtomicOrdering::Relaxed); + } + val + } + + /// Returns mutable reference to the contained value by key, if exists. + pub fn get_mut(&mut self, key: &K) -> Option<&mut V> { + self.inner_guard.get_mut(key) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + impl Size for i32 { + fn size(&self) -> usize { *self as usize / 10 } + } + + #[test] + fn basic() { + let map = TrackedMap::default(); + map.write().insert(5, 10); + map.write().insert(6, 20); + + assert_eq!(map.bytes(), 3); + assert_eq!(map.len(), 2); + + map.write().insert(6, 30); + + assert_eq!(map.bytes(), 4); + assert_eq!(map.len(), 2); + + map.write().remove(&6); + assert_eq!(map.bytes(), 1); + assert_eq!(map.len(), 1); + } +} \ No newline at end of file diff --git a/client/transaction-pool/graph/src/validated_pool.rs b/client/transaction-pool/graph/src/validated_pool.rs index 4a32c4c58c753e3eaee8ba1e56a5ed34ac8ee13b..d730b892e350245097f74853aea9fd54dd8b9de3 100644 --- a/client/transaction-pool/graph/src/validated_pool.rs +++ b/client/transaction-pool/graph/src/validated_pool.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,18 +15,18 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use std::{ collections::{HashSet, HashMap}, hash, sync::Arc, }; -use crate::{base_pool as base, BlockHash}; +use crate::base_pool as base; use crate::listener::Listener; use crate::rotator::PoolRotator; use crate::watcher::Watcher; use serde::Serialize; -use log::{debug, warn}; use parking_lot::{Mutex, RwLock}; use sp_runtime::{ @@ -39,7 +39,9 @@ use wasm_timer::Instant; use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use crate::base_pool::PruneStatus; -use crate::pool::{EventStream, Options, ChainApi, ExHash, ExtrinsicFor, TransactionFor}; +use crate::pool::{ + EventStream, Options, ChainApi, BlockHash, ExtrinsicHash, ExtrinsicFor, TransactionFor, +}; /// Pre-validated transaction. Validated pool only accepts transactions wrapped in this enum. #[derive(Debug)] @@ -82,7 +84,7 @@ impl ValidatedTransaction { /// A type of validated transaction stored in the pool. pub type ValidatedTransactionFor = ValidatedTransaction< - ExHash, + ExtrinsicHash, ExtrinsicFor, ::Error, >; @@ -91,19 +93,18 @@ pub type ValidatedTransactionFor = ValidatedTransaction< pub struct ValidatedPool { api: Arc, options: Options, - listener: RwLock, B>>, + listener: RwLock, B>>, pool: RwLock, + ExtrinsicHash, ExtrinsicFor, >>, - import_notification_sinks: Mutex>>>, - rotator: PoolRotator>, + import_notification_sinks: Mutex>>>, + rotator: PoolRotator>, } #[cfg(not(target_os = "unknown"))] impl parity_util_mem::MallocSizeOf for ValidatedPool where - B::Hash: parity_util_mem::MallocSizeOf, ExtrinsicFor: parity_util_mem::MallocSizeOf, { fn size_of(&self, ops: &mut parity_util_mem::MallocSizeOfOps) -> usize { @@ -127,17 +128,17 @@ impl ValidatedPool { } /// Bans given set of hashes. - pub fn ban(&self, now: &Instant, hashes: impl IntoIterator>) { + pub fn ban(&self, now: &Instant, hashes: impl IntoIterator>) { self.rotator.ban(now, hashes) } /// Returns true if transaction with given hash is currently banned from the pool. - pub fn is_banned(&self, hash: &ExHash) -> bool { + pub fn is_banned(&self, hash: &ExtrinsicHash) -> bool { self.rotator.is_banned(hash) } /// Imports a bunch of pre-validated transactions to the pool. - pub fn submit(&self, txs: T) -> Vec, B::Error>> where + pub fn submit(&self, txs: T) -> Vec, B::Error>> where T: IntoIterator> { let results = txs.into_iter() @@ -158,7 +159,7 @@ impl ValidatedPool { } /// Submit single pre-validated transaction to the pool. - fn submit_one(&self, tx: ValidatedTransactionFor) -> Result, B::Error> { + fn submit_one(&self, tx: ValidatedTransactionFor) -> Result, B::Error> { match tx { ValidatedTransaction::Valid(tx) => { let imported = self.pool.write().import(tx)?; @@ -183,16 +184,16 @@ impl ValidatedPool { } } - fn enforce_limits(&self) -> HashSet> { + fn enforce_limits(&self) -> HashSet> { let status = self.pool.read().status(); let ready_limit = &self.options.ready; let future_limit = &self.options.future; - debug!(target: "txpool", "Pool Status: {:?}", status); + log::debug!(target: "txpool", "Pool Status: {:?}", status); if ready_limit.is_exceeded(status.ready, status.ready_bytes) || future_limit.is_exceeded(status.future, status.future_bytes) { - debug!( + log::debug!( target: "txpool", "Enforcing limits ({}/{}kB ready, {}/{}kB future", ready_limit.count, ready_limit.total_bytes / 1024, @@ -208,8 +209,11 @@ impl ValidatedPool { self.rotator.ban(&Instant::now(), removed.iter().map(|x| x.clone())); removed }; + if !removed.is_empty() { + log::debug!(target: "txpool", "Enforcing limits: {} dropped", removed.len()); + } + // run notifications - debug!(target: "txpool", "Enforcing limits: {} dropped", removed.len()); let mut listener = self.listener.write(); for h in &removed { listener.dropped(h, None); @@ -225,7 +229,7 @@ impl ValidatedPool { pub fn submit_and_watch( &self, tx: ValidatedTransactionFor, - ) -> Result, BlockHash>, B::Error> { + ) -> Result, ExtrinsicHash>, B::Error> { match tx { ValidatedTransaction::Valid(tx) => { let hash = self.api.hash_and_length(&tx.data).0; @@ -247,7 +251,7 @@ impl ValidatedPool { /// /// Removes and then submits passed transactions and all dependent transactions. /// Transactions that are missing from the pool are not submitted. - pub fn resubmit(&self, mut updated_transactions: HashMap, ValidatedTransactionFor>) { + pub fn resubmit(&self, mut updated_transactions: HashMap, ValidatedTransactionFor>) { #[derive(Debug, Clone, Copy, PartialEq)] enum Status { Future, Ready, Failed, Dropped }; @@ -323,7 +327,7 @@ impl ValidatedPool { // we do not want to fail if single transaction import has failed // nor we do want to propagate this error, because it could tx unknown to caller // => let's just notify listeners (and issue debug message) - warn!( + log::warn!( target: "txpool", "[{:?}] Removing invalid transaction from update: {}", hash, @@ -366,7 +370,7 @@ impl ValidatedPool { } /// For each extrinsic, returns tags that it provides (if known), or None (if it is unknown). - pub fn extrinsics_tags(&self, hashes: &[ExHash]) -> Vec>> { + pub fn extrinsics_tags(&self, hashes: &[ExtrinsicHash]) -> Vec>> { self.pool.read().by_hashes(&hashes) .into_iter() .map(|existing_in_pool| existing_in_pool @@ -375,7 +379,7 @@ impl ValidatedPool { } /// Get ready transaction by hash - pub fn ready_by_hash(&self, hash: &ExHash) -> Option> { + pub fn ready_by_hash(&self, hash: &ExtrinsicHash) -> Option> { self.pool.read().ready_by_hash(hash) } @@ -383,7 +387,7 @@ impl ValidatedPool { pub fn prune_tags( &self, tags: impl IntoIterator, - ) -> Result, ExtrinsicFor>, B::Error> { + ) -> Result, ExtrinsicFor>, B::Error> { // Perform tag-based pruning in the base pool let status = self.pool.write().prune_tags(tags); // Notify event listeners of all transactions @@ -405,8 +409,8 @@ impl ValidatedPool { pub fn resubmit_pruned( &self, at: &BlockId, - known_imported_hashes: impl IntoIterator> + Clone, - pruned_hashes: Vec>, + known_imported_hashes: impl IntoIterator> + Clone, + pruned_hashes: Vec>, pruned_xts: Vec>, ) -> Result<(), B::Error> { debug_assert_eq!(pruned_hashes.len(), pruned_xts.len()); @@ -437,7 +441,7 @@ impl ValidatedPool { pub fn fire_pruned( &self, at: &BlockId, - hashes: impl Iterator>, + hashes: impl Iterator>, ) -> Result<(), B::Error> { let header_hash = self.api.block_id_to_hash(at)? .ok_or_else(|| error::Error::InvalidBlockId(format!("{:?}", at)).into())?; @@ -470,7 +474,7 @@ impl ValidatedPool { .map(|tx| tx.hash.clone()) .collect::>() }; - let futures_to_remove: Vec> = { + let futures_to_remove: Vec> = { let p = self.pool.read(); let mut hashes = Vec::new(); for tx in p.futures() { @@ -491,7 +495,7 @@ impl ValidatedPool { /// Get rotator reference. #[cfg(test)] - pub fn rotator(&self) -> &PoolRotator> { + pub fn rotator(&self) -> &PoolRotator> { &self.rotator } @@ -504,14 +508,14 @@ impl ValidatedPool { /// /// Consumers of this stream should use the `ready` method to actually get the /// pending transactions in the right order. - pub fn import_notification_stream(&self) -> EventStream> { + pub fn import_notification_stream(&self) -> EventStream> { let (sink, stream) = tracing_unbounded("mpsc_import_notifications"); self.import_notification_sinks.lock().push(sink); stream } /// Invoked when extrinsics are broadcasted. - pub fn on_broadcasted(&self, propagated: HashMap, Vec>) { + pub fn on_broadcasted(&self, propagated: HashMap, Vec>) { let mut listener = self.listener.write(); for (hash, peers) in propagated.into_iter() { listener.broadcasted(&hash, peers); @@ -524,20 +528,20 @@ impl ValidatedPool { /// to prevent them from entering the pool right away. /// Note this is not the case for the dependent transactions - those may /// still be valid so we want to be able to re-import them. - pub fn remove_invalid(&self, hashes: &[ExHash]) -> Vec> { + pub fn remove_invalid(&self, hashes: &[ExtrinsicHash]) -> Vec> { // early exit in case there is no invalid transactions. if hashes.is_empty() { return vec![]; } - debug!(target: "txpool", "Removing invalid transactions: {:?}", hashes); + log::debug!(target: "txpool", "Removing invalid transactions: {:?}", hashes); // temporarily ban invalid transactions self.rotator.ban(&Instant::now(), hashes.iter().cloned()); let invalid = self.pool.write().remove_subtree(hashes); - debug!(target: "txpool", "Removed invalid transactions: {:?}", invalid); + log::debug!(target: "txpool", "Removed invalid transactions: {:?}", invalid); let mut listener = self.listener.write(); for tx in &invalid { @@ -559,7 +563,7 @@ impl ValidatedPool { /// Notify all watchers that transactions in the block with hash have been finalized pub async fn on_block_finalized(&self, block_hash: BlockHash) -> Result<(), B::Error> { - debug!(target: "txpool", "Attempting to notify watchers of finalization for {}", block_hash); + log::trace!(target: "txpool", "Attempting to notify watchers of finalization for {}", block_hash); self.listener.write().finalized(block_hash); Ok(()) } diff --git a/client/transaction-pool/graph/src/watcher.rs b/client/transaction-pool/graph/src/watcher.rs index 098e468d227fe66a5fc91da032634e36cca9e424..9d9a91bb23f69acb0ad42a1eba4dcadecc856c66 100644 --- a/client/transaction-pool/graph/src/watcher.rs +++ b/client/transaction-pool/graph/src/watcher.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Extrinsics status updates. use futures::Stream; diff --git a/client/transaction-pool/src/api.rs b/client/transaction-pool/src/api.rs index 90d667d613b2b93984d39087aa0faac5e56b5fe2..10ac4aa46960d04fa51cd67feb70de4a07b3364a 100644 --- a/client/transaction-pool/src/api.rs +++ b/client/transaction-pool/src/api.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Chain api required for the transaction pool. use std::{marker::PhantomData, pin::Pin, sync::Arc}; @@ -24,9 +25,7 @@ use futures::{ }; use sc_client_api::{ - blockchain::HeaderBackend, - light::{Fetcher, RemoteCallRequest, RemoteBodyRequest}, - BlockBackend, + blockchain::HeaderBackend, light::{Fetcher, RemoteCallRequest, RemoteBodyRequest}, BlockBackend, }; use sp_runtime::{ generic::BlockId, traits::{self, Block as BlockT, BlockIdTo, Header as HeaderT, Hash as HashT}, @@ -44,10 +43,7 @@ pub struct FullChainApi { _marker: PhantomData, } -impl FullChainApi where - Block: BlockT, - Client: ProvideRuntimeApi + BlockIdTo, -{ +impl FullChainApi { /// Create new transaction pool logic. pub fn new(client: Arc) -> Self { FullChainApi { @@ -57,22 +53,24 @@ impl FullChainApi where .name_prefix("txpool-verifier") .create() .expect("Failed to spawn verifier threads, that are critical for node operation."), - _marker: Default::default() + _marker: Default::default(), } } } -impl sc_transaction_graph::ChainApi for FullChainApi where +impl sc_transaction_graph::ChainApi for FullChainApi +where Block: BlockT, Client: ProvideRuntimeApi + BlockBackend + BlockIdTo, Client: Send + Sync + 'static, Client::Api: TaggedTransactionQueue, - sp_api::ApiErrorFor: Send, + sp_api::ApiErrorFor: Send + std::fmt::Display, { type Block = Block; - type Hash = Block::Hash; type Error = error::Error; - type ValidationFuture = Pin> + Send>>; + type ValidationFuture = Pin< + Box> + Send> + >; type BodyFuture = Ready::Extrinsic>>>>; fn block_body(&self, id: &BlockId) -> Self::BodyFuture { @@ -89,29 +87,15 @@ impl sc_transaction_graph::ChainApi for FullChainApi, _>( - &at, |v| v >= 2, - ) - .unwrap_or_default() - }; - - sp_tracing::enter_span!("runtime::validate_transaction"); - let res = if has_v2 { - runtime_api.validate_transaction(&at, source, uxt) - } else { - #[allow(deprecated)] // old validate_transaction - runtime_api.validate_transaction_before_version_2(&at, uxt) - }; - let res = res.map_err(|e| Error::RuntimeApi(format!("{:?}", e))); - if let Err(e) = tx.send(res) { - log::warn!("Unable to send a validate transaction result: {:?}", e); - } - })); + self.pool.spawn_ok(futures_diagnose::diagnose( + "validate-transaction", + async move { + let res = validate_transaction_blocking(&*client, &at, source, uxt); + if let Err(e) = tx.send(res) { + log::warn!("Unable to send a validate transaction result: {:?}", e); + } + }, + )); Box::pin(async move { match rx.await { @@ -135,13 +119,72 @@ impl sc_transaction_graph::ChainApi for FullChainApi) -> (Self::Hash, usize) { + fn hash_and_length( + &self, + ex: &sc_transaction_graph::ExtrinsicFor, + ) -> (sc_transaction_graph::ExtrinsicHash, usize) { ex.using_encoded(|x| { ( as traits::Hash>::hash(x), x.len()) }) } } +/// Helper function to validate a transaction using a full chain API. +/// This method will call into the runtime to perform the validation. +fn validate_transaction_blocking( + client: &Client, + at: &BlockId, + source: TransactionSource, + uxt: sc_transaction_graph::ExtrinsicFor>, +) -> error::Result +where + Block: BlockT, + Client: ProvideRuntimeApi + BlockBackend + BlockIdTo, + Client: Send + Sync + 'static, + Client::Api: TaggedTransactionQueue, + sp_api::ApiErrorFor: Send + std::fmt::Display, +{ + sp_tracing::enter_span!("validate_transaction"); + let runtime_api = client.runtime_api(); + let has_v2 = sp_tracing::tracing_span! { "check_version"; + runtime_api + .has_api_with::, _>(&at, |v| v >= 2) + .unwrap_or_default() + }; + + sp_tracing::enter_span!("runtime::validate_transaction"); + let res = if has_v2 { + runtime_api.validate_transaction(&at, source, uxt) + } else { + #[allow(deprecated)] // old validate_transaction + runtime_api.validate_transaction_before_version_2(&at, uxt) + }; + + res.map_err(|e| Error::RuntimeApi(e.to_string())) +} + +impl FullChainApi +where + Block: BlockT, + Client: ProvideRuntimeApi + BlockBackend + BlockIdTo, + Client: Send + Sync + 'static, + Client::Api: TaggedTransactionQueue, + sp_api::ApiErrorFor: Send + std::fmt::Display, +{ + /// Validates a transaction by calling into the runtime, same as + /// `validate_transaction` but blocks the current thread when performing + /// validation. Only implemented for `FullChainApi` since we can call into + /// the runtime locally. + pub fn validate_transaction_blocking( + &self, + at: &BlockId, + source: TransactionSource, + uxt: sc_transaction_graph::ExtrinsicFor, + ) -> error::Result { + validate_transaction_blocking(&*self.client, at, source, uxt) + } +} + /// The transaction pool logic for light client. pub struct LightChainApi { client: Arc, @@ -149,11 +192,7 @@ pub struct LightChainApi { _phantom: PhantomData, } -impl LightChainApi where - Block: BlockT, - Client: HeaderBackend, - F: Fetcher, -{ +impl LightChainApi { /// Create new transaction pool logic. pub fn new(client: Arc, fetcher: Arc) -> Self { LightChainApi { @@ -164,16 +203,23 @@ impl LightChainApi where } } -impl sc_transaction_graph::ChainApi for LightChainApi where - Block: BlockT, - Client: HeaderBackend + 'static, - F: Fetcher + 'static, +impl sc_transaction_graph::ChainApi for + LightChainApi where + Block: BlockT, + Client: HeaderBackend + 'static, + F: Fetcher + 'static, { type Block = Block; - type Hash = Block::Hash; type Error = error::Error; - type ValidationFuture = Box> + Send + Unpin>; - type BodyFuture = Pin::Extrinsic>>>> + Send>>; + type ValidationFuture = Box< + dyn Future> + Send + Unpin + >; + type BodyFuture = Pin< + Box< + dyn Future::Extrinsic>>>> + + Send + > + >; fn validate_transaction( &self, @@ -210,15 +256,24 @@ impl sc_transaction_graph::ChainApi for LightChainApi) -> error::Result>> { + fn block_id_to_number( + &self, + at: &BlockId, + ) -> error::Result>> { Ok(self.client.block_number_from_id(at)?) } - fn block_id_to_hash(&self, at: &BlockId) -> error::Result>> { + fn block_id_to_hash( + &self, + at: &BlockId, + ) -> error::Result>> { Ok(self.client.block_hash_from_id(at)?) } - fn hash_and_length(&self, ex: &sc_transaction_graph::ExtrinsicFor) -> (Self::Hash, usize) { + fn hash_and_length( + &self, + ex: &sc_transaction_graph::ExtrinsicFor, + ) -> (sc_transaction_graph::ExtrinsicHash, usize) { ex.using_encoded(|x| { (<::Hashing as HashT>::hash(x), x.len()) }) diff --git a/client/transaction-pool/src/error.rs b/client/transaction-pool/src/error.rs index feccf4a7b0aa8337347b91b6ec73c21299781fed..c0f795df1801a0c59313aac28346bb6797ff04b7 100644 --- a/client/transaction-pool/src/error.rs +++ b/client/transaction-pool/src/error.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Transaction pool error. use sp_transaction_pool::error::Error as TxPoolError; diff --git a/client/transaction-pool/src/lib.rs b/client/transaction-pool/src/lib.rs index 62b99ef50549fa74d913a47381e54a69fae5dc44..ea8b4bf9dec81cf9903fc7e922f7013a3b499f74 100644 --- a/client/transaction-pool/src/lib.rs +++ b/client/transaction-pool/src/lib.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Substrate transaction pool implementation. #![recursion_limit="256"] @@ -27,14 +28,14 @@ mod metrics; pub mod error; -#[cfg(any(feature = "test-helpers", test))] +#[cfg(test)] pub mod testing; pub use sc_transaction_graph as txpool; pub use crate::api::{FullChainApi, LightChainApi}; -use std::{collections::HashMap, sync::Arc, pin::Pin}; -use futures::{prelude::*, future::ready, channel::oneshot}; +use std::{collections::{HashMap, HashSet}, sync::Arc, pin::Pin}; +use futures::{prelude::*, future::{self, ready}, channel::oneshot}; use parking_lot::Mutex; use sp_runtime::{ @@ -46,14 +47,19 @@ use sp_transaction_pool::{ TransactionStatusStreamFor, MaintainedTransactionPool, PoolFuture, ChainEvent, TransactionSource, }; +use sc_transaction_graph::{ChainApi, ExtrinsicHash}; use wasm_timer::Instant; use prometheus_endpoint::Registry as PrometheusRegistry; use crate::metrics::MetricsLink as PrometheusMetrics; -type BoxedReadyIterator = Box>> + Send>; +type BoxedReadyIterator = Box< + dyn Iterator>> + Send +>; -type ReadyIteratorFor = BoxedReadyIterator, sc_transaction_graph::ExtrinsicFor>; +type ReadyIteratorFor = BoxedReadyIterator< + sc_transaction_graph::ExtrinsicHash, sc_transaction_graph::ExtrinsicFor +>; type PolledIterator = Pin> + Send>>; @@ -61,7 +67,7 @@ type PolledIterator = Pin where Block: BlockT, - PoolApi: sc_transaction_graph::ChainApi, + PoolApi: ChainApi, { pool: Arc>, api: Arc, @@ -115,8 +121,7 @@ impl ReadyPoll { #[cfg(not(target_os = "unknown"))] impl parity_util_mem::MallocSizeOf for BasicPool where - PoolApi: sc_transaction_graph::ChainApi, - PoolApi::Hash: parity_util_mem::MallocSizeOf, + PoolApi: ChainApi, Block: BlockT, { fn size_of(&self, ops: &mut parity_util_mem::MallocSizeOfOps) -> usize { @@ -145,7 +150,7 @@ pub enum RevalidationType { impl BasicPool where Block: BlockT, - PoolApi: sc_transaction_graph::ChainApi + 'static, + PoolApi: ChainApi + 'static, { /// Create new basic transaction pool with provided api. /// @@ -225,11 +230,13 @@ impl BasicPool impl TransactionPool for BasicPool where Block: BlockT, - PoolApi: 'static + sc_transaction_graph::ChainApi, + PoolApi: 'static + ChainApi, { type Block = PoolApi::Block; - type Hash = sc_transaction_graph::ExHash; - type InPoolTransaction = sc_transaction_graph::base_pool::Transaction, TransactionFor>; + type Hash = sc_transaction_graph::ExtrinsicHash; + type InPoolTransaction = sc_transaction_graph::base_pool::Transaction< + TxHash, TransactionFor + >; type Error = PoolApi::Error; fn submit_at( @@ -324,6 +331,7 @@ impl TransactionPool for BasicPool fn ready_at(&self, at: NumberFor) -> PolledIterator { if self.ready_poll.lock().updated_at() >= at { + log::trace!(target: "txpool", "Transaction pool already processed block #{}", at); let iterator: ReadyIteratorFor = Box::new(self.pool.validated_pool().ready()); return Box::pin(futures::future::ready(iterator)); } @@ -344,6 +352,59 @@ impl TransactionPool for BasicPool } } +impl sp_transaction_pool::LocalTransactionPool + for BasicPool, Block> +where + Block: BlockT, + Client: sp_api::ProvideRuntimeApi + + sc_client_api::BlockBackend + + sp_runtime::traits::BlockIdTo, + Client: Send + Sync + 'static, + Client::Api: sp_transaction_pool::runtime_api::TaggedTransactionQueue, + sp_api::ApiErrorFor: Send + std::fmt::Display, +{ + type Block = Block; + type Hash = sc_transaction_graph::ExtrinsicHash>; + type Error = as ChainApi>::Error; + + fn submit_local( + &self, + at: &BlockId, + xt: sp_transaction_pool::LocalTransactionFor, + ) -> Result { + use sc_transaction_graph::ValidatedTransaction; + use sp_runtime::traits::SaturatedConversion; + use sp_runtime::transaction_validity::TransactionValidityError; + + let validity = self + .api + .validate_transaction_blocking(at, TransactionSource::Local, xt.clone())? + .map_err(|e| { + Self::Error::Pool(match e { + TransactionValidityError::Invalid(i) => i.into(), + TransactionValidityError::Unknown(u) => u.into(), + }) + })?; + + let (hash, bytes) = self.pool.validated_pool().api().hash_and_length(&xt); + let block_number = self + .api + .block_id_to_number(at)? + .ok_or_else(|| error::Error::BlockIdConversion(format!("{:?}", at)))?; + + let validated = ValidatedTransaction::valid_at( + block_number.saturated_into::(), + hash.clone(), + TransactionSource::Local, + xt, + bytes, + validity, + ); + + self.pool.validated_pool().submit(vec![validated]).remove(0) + } +} + #[cfg_attr(test, derive(Debug))] enum RevalidationStatus { /// The revalidation has never been completed. @@ -428,22 +489,51 @@ impl RevalidationStatus { } } +/// Prune the known txs for the given block. +async fn prune_known_txs_for_block>( + block_id: BlockId, + api: &Api, + pool: &sc_transaction_graph::Pool, +) -> Vec> { + let hashes = api.block_body(&block_id).await + .unwrap_or_else(|e| { + log::warn!("Prune known transactions: error request {:?}!", e); + None + }) + .unwrap_or_default() + .into_iter() + .map(|tx| pool.hash_of(&tx)) + .collect::>(); + + log::trace!(target: "txpool", "Pruning transactions: {:?}", hashes); + + if let Err(e) = pool.prune_known(&block_id, &hashes) { + log::error!("Cannot prune known in the pool {:?}!", e); + } + + hashes +} + impl MaintainedTransactionPool for BasicPool where Block: BlockT, - PoolApi: 'static + sc_transaction_graph::ChainApi, + PoolApi: 'static + ChainApi, { fn maintain(&self, event: ChainEvent) -> Pin + Send>> { match event { - ChainEvent::NewBlock { id, retracted, .. } => { - let id = id.clone(); + ChainEvent::NewBlock { hash, tree_route, is_new_best, .. } => { let pool = self.pool.clone(); let api = self.api.clone(); + let id = BlockId::hash(hash); let block_number = match api.block_id_to_number(&id) { Ok(Some(number)) => number, _ => { - log::trace!(target: "txpool", "Skipping chain event - no number for that block {:?}", id); + log::trace!( + target: "txpool", + "Skipping chain event - no number for that block {:?}", + id, + ); return Box::pin(ready(())); } }; @@ -454,40 +544,59 @@ impl MaintainedTransactionPool for BasicPool Some(20.into()), ); let revalidation_strategy = self.revalidation_strategy.clone(); - let retracted = retracted.clone(); let revalidation_queue = self.revalidation_queue.clone(); let ready_poll = self.ready_poll.clone(); + let metrics = self.metrics.clone(); async move { - // We don't query block if we won't prune anything - if !pool.validated_pool().status().is_empty() { - let hashes = api.block_body(&id).await - .unwrap_or_else(|e| { - log::warn!("Prune known transactions: error request {:?}!", e); - None - }) - .unwrap_or_default() - .into_iter() - .map(|tx| pool.hash_of(&tx)) - .collect::>(); - - if let Err(e) = pool.prune_known(&id, &hashes) { - log::error!("Cannot prune known in the pool {:?}!", e); + // We keep track of everything we prune so that later we won't add + // tranactions with those hashes from the retracted blocks. + let mut pruned_log = HashSet::>::new(); + + // If there is a tree route, we use this to prune known tx based on the enacted + // blocks. Before pruning enacted transactions, we inform the listeners about + // retracted blocks and their transactions. This order is important, because + // if we enact and retract the same transaction at the same time, we want to + // send first the retract and than the prune event. + if let Some(ref tree_route) = tree_route { + for retracted in tree_route.retracted() { + // notify txs awaiting finality that it has been retracted + pool.validated_pool().on_block_retracted(retracted.hash.clone()); } + + future::join_all( + tree_route + .enacted() + .iter() + .map(|h| + prune_known_txs_for_block( + BlockId::Hash(h.hash.clone()), + &*api, + &*pool, + ), + ), + ).await.into_iter().for_each(|enacted_log|{ + pruned_log.extend(enacted_log); + }) } - let extra_pool = pool.clone(); - // After #5200 lands, this arguably might be moved to the handler of "all blocks notification". - ready_poll.lock().trigger(block_number, move || Box::new(extra_pool.validated_pool().ready())); + // If this is a new best block, we need to prune its transactions from the pool. + if is_new_best { + pruned_log.extend(prune_known_txs_for_block(id.clone(), &*api, &*pool).await); + } + + metrics.report( + |metrics| metrics.block_transactions_pruned.inc_by(pruned_log.len() as u64) + ); - if next_action.resubmit { + if let (true, Some(tree_route)) = (next_action.resubmit, tree_route) { let mut resubmit_transactions = Vec::new(); - for retracted_hash in retracted { - // notify txs awaiting finality that it has been retracted - pool.validated_pool().on_block_retracted(retracted_hash.clone()); + for retracted in tree_route.retracted() { + let hash = retracted.hash.clone(); - let block_transactions = api.block_body(&BlockId::hash(retracted_hash.clone())).await + let block_transactions = api.block_body(&BlockId::hash(hash)) + .await .unwrap_or_else(|e| { log::warn!("Failed to fetch block body {:?}!", e); None @@ -496,25 +605,63 @@ impl MaintainedTransactionPool for BasicPool .into_iter() .filter(|tx| tx.is_signed().unwrap_or(true)); - resubmit_transactions.extend(block_transactions); + let mut resubmitted_to_report = 0; + + resubmit_transactions.extend( + block_transactions.into_iter().filter(|tx| { + let tx_hash = pool.hash_of(&tx); + let contains = pruned_log.contains(&tx_hash); + + // need to count all transactions, not just filtered, here + resubmitted_to_report += 1; + + if !contains { + log::debug!( + target: "txpool", + "[{:?}]: Resubmitting from retracted block {:?}", + tx_hash, + hash, + ); + } + !contains + }) + ); + + metrics.report( + |metrics| metrics.block_transactions_resubmitted.inc_by(resubmitted_to_report) + ); } + if let Err(e) = pool.submit_at( &id, // These transactions are coming from retracted blocks, we should // simply consider them external. TransactionSource::External, resubmit_transactions, - true + true, ).await { log::debug!( target: "txpool", - "[{:?}] Error re-submitting transactions: {:?}", id, e + "[{:?}] Error re-submitting transactions: {:?}", + id, + e, ) } } + let extra_pool = pool.clone(); + // After #5200 lands, this arguably might be moved to the + // handler of "all blocks notification". + ready_poll.lock().trigger( + block_number, + move || Box::new(extra_pool.validated_pool().ready()), + ); + if next_action.revalidate { - let hashes = pool.validated_pool().ready().map(|tx| tx.hash.clone()).collect(); + let hashes = pool.validated_pool() + .ready() + .map(|tx| tx.hash.clone()) + .collect(); revalidation_queue.revalidate_later(block_number, hashes).await; } @@ -536,3 +683,23 @@ impl MaintainedTransactionPool for BasicPool } } } + +/// Inform the transaction pool about imported and finalized blocks. +pub async fn notification_future( + client: Arc, + txpool: Arc +) + where + Block: BlockT, + Client: sc_client_api::BlockchainEvents, + Pool: MaintainedTransactionPool, +{ + let import_stream = client.import_notification_stream().map(Into::into).fuse(); + let finality_stream = client.finality_notification_stream() + .map(Into::into) + .fuse(); + + futures::stream::select(import_stream, finality_stream) + .for_each(|evt| txpool.maintain(evt)) + .await +} diff --git a/client/transaction-pool/src/metrics.rs b/client/transaction-pool/src/metrics.rs index 3b0e48cbfc8561a0b1c8a8ba4f45b43f91a51de7..d5a10dfd6f4bdf7ed66551a243a7e3b91070ca47 100644 --- a/client/transaction-pool/src/metrics.rs +++ b/client/transaction-pool/src/metrics.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Transaction pool Prometheus metrics. use std::sync::Arc; @@ -47,6 +48,8 @@ pub struct Metrics { pub validations_scheduled: Counter, pub validations_finished: Counter, pub validations_invalid: Counter, + pub block_transactions_pruned: Counter, + pub block_transactions_resubmitted: Counter, } impl Metrics { @@ -73,6 +76,20 @@ impl Metrics { )?, registry, )?, + block_transactions_pruned: register( + Counter::new( + "sub_txpool_block_transactions_pruned", + "Total number of transactions that was requested to be pruned by block events", + )?, + registry, + )?, + block_transactions_resubmitted: register( + Counter::new( + "sub_txpool_block_transactions_resubmitted", + "Total number of transactions that was requested to be resubmitted by block events", + )?, + registry, + )?, }) } } diff --git a/client/transaction-pool/src/revalidation.rs b/client/transaction-pool/src/revalidation.rs index f8f7280417e546a0c52506f49a9972f992801149..af9a76c055b6bc0fb643c4f54377ab30e5d35789 100644 --- a/client/transaction-pool/src/revalidation.rs +++ b/client/transaction-pool/src/revalidation.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,11 +15,12 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Pool periodic revalidation. use std::{sync::Arc, pin::Pin, collections::{HashMap, HashSet, BTreeMap}}; -use sc_transaction_graph::{ChainApi, Pool, ExHash, NumberFor, ValidatedTransaction}; +use sc_transaction_graph::{ChainApi, Pool, ExtrinsicHash, NumberFor, ValidatedTransaction}; use sp_runtime::traits::{Zero, SaturatedConversion}; use sp_runtime::generic::BlockId; use sp_runtime::transaction_validity::TransactionValidityError; @@ -33,12 +34,12 @@ const BACKGROUND_REVALIDATION_INTERVAL: Duration = Duration::from_millis(200); #[cfg(test)] pub const BACKGROUND_REVALIDATION_INTERVAL: Duration = Duration::from_millis(1); -const BACKGROUND_REVALIDATION_BATCH_SIZE: usize = 20; +const MIN_BACKGROUND_REVALIDATION_BATCH_SIZE: usize = 20; /// Payload from queue to worker. struct WorkerPayload { at: NumberFor, - transactions: Vec>, + transactions: Vec>, } /// Async revalidation worker. @@ -48,8 +49,8 @@ struct RevalidationWorker { api: Arc, pool: Arc>, best_block: NumberFor, - block_ordered: BTreeMap, HashSet>>, - members: HashMap, NumberFor>, + block_ordered: BTreeMap, HashSet>>, + members: HashMap, NumberFor>, } impl Unpin for RevalidationWorker {} @@ -62,18 +63,22 @@ async fn batch_revalidate( pool: Arc>, api: Arc, at: NumberFor, - batch: impl IntoIterator>, + batch: impl IntoIterator>, ) { let mut invalid_hashes = Vec::new(); let mut revalidated = HashMap::new(); - for ext_hash in batch { - let ext = match pool.validated_pool().ready_by_hash(&ext_hash) { - Some(ext) => ext, - None => continue, - }; - - match api.validate_transaction(&BlockId::Number(at), ext.source, ext.data.clone()).await { + let validation_results = futures::future::join_all( + batch.into_iter().filter_map(|ext_hash| { + pool.validated_pool().ready_by_hash(&ext_hash).map(|ext| { + api.validate_transaction(&BlockId::Number(at), ext.source, ext.data.clone()) + .map(move |validation_result| (validation_result, ext_hash, ext)) + }) + }) + ).await; + + for (validation_result, ext_hash, ext) in validation_results { + match validation_result { Ok(Err(TransactionValidityError::Invalid(err))) => { log::debug!(target: "txpool", "[{:?}]: Revalidation: invalid {:?}", ext_hash, err); invalid_hashes.push(ext_hash); @@ -128,22 +133,22 @@ impl RevalidationWorker { } } - fn prepare_batch(&mut self) -> Vec> { + fn prepare_batch(&mut self) -> Vec> { let mut queued_exts = Vec::new(); - let mut left = BACKGROUND_REVALIDATION_BATCH_SIZE; + let mut left = std::cmp::max(MIN_BACKGROUND_REVALIDATION_BATCH_SIZE, self.members.len() / 4); // Take maximum of count transaction by order // which they got into the pool while left > 0 { let first_block = match self.block_ordered.keys().next().cloned() { - Some(bn) => bn, - None => break, + Some(bn) => bn, + None => break, }; let mut block_drained = false; if let Some(extrinsics) = self.block_ordered.get_mut(&first_block) { let to_queue = extrinsics.iter().take(left).cloned().collect::>(); if to_queue.len() == extrinsics.len() { - block_drained = true; + block_drained = true; } else { for xt in &to_queue { extrinsics.remove(xt); @@ -154,7 +159,7 @@ impl RevalidationWorker { } if block_drained { - self.block_ordered.remove(&first_block); + self.block_ordered.remove(&first_block); } } @@ -177,7 +182,7 @@ impl RevalidationWorker { for ext_hash in transactions { // we don't add something that already scheduled for revalidation if self.members.contains_key(&ext_hash) { - log::debug!( + log::trace!( target: "txpool", "[{:?}] Skipped adding for revalidation: Already there.", ext_hash, @@ -244,6 +249,16 @@ impl RevalidationWorker { Some(worker_payload) => { this.best_block = worker_payload.at; this.push(worker_payload); + + if this.members.len() > 0 { + log::debug!( + target: "txpool", + "Updated revalidation queue at {}. Transactions: {:?}", + this.best_block, + this.members, + ); + } + continue; }, // R.I.P. worker! @@ -323,9 +338,9 @@ where /// If queue configured with background worker, this will return immediately. /// If queue configured without background worker, this will resolve after /// revalidation is actually done. - pub async fn revalidate_later(&self, at: NumberFor, transactions: Vec>) { + pub async fn revalidate_later(&self, at: NumberFor, transactions: Vec>) { if transactions.len() > 0 { - log::debug!(target: "txpool", "Added {} transactions to revalidation queue", transactions.len()); + log::debug!(target: "txpool", "Sent {} transactions to revalidation queue", transactions.len()); } if let Some(ref to_worker) = self.background { @@ -348,9 +363,7 @@ mod tests { use sp_transaction_pool::TransactionSource; use substrate_test_runtime_transaction_pool::{TestApi, uxt}; use futures::executor::block_on; - use substrate_test_runtime_client::{ - AccountKeyring::*, - }; + use substrate_test_runtime_client::AccountKeyring::*; fn setup() -> (Arc, Pool) { let test_api = Arc::new(TestApi::empty()); diff --git a/client/transaction-pool/src/testing/mod.rs b/client/transaction-pool/src/testing/mod.rs index c4bf184a2b59cfa96692623e626817c2960452cf..350c4137c37b23a5d1fdfde94f0b19acde7cbdb4 100644 --- a/client/transaction-pool/src/testing/mod.rs +++ b/client/transaction-pool/src/testing/mod.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,6 +15,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + //! Tests for top-level transaction pool api mod pool; diff --git a/client/transaction-pool/src/testing/pool.rs b/client/transaction-pool/src/testing/pool.rs index 32701d3c2d6a282d4f13e8c683054b69cad16fa8..5ad79a6f75d87905403847d1605c225157034fa9 100644 --- a/client/transaction-pool/src/testing/pool.rs +++ b/client/transaction-pool/src/testing/pool.rs @@ -5,7 +5,7 @@ // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or +// the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, @@ -15,21 +15,26 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . + use crate::*; use sp_transaction_pool::TransactionStatus; -use futures::executor::block_on; +use futures::executor::{block_on, block_on_stream}; use txpool::{self, Pool}; use sp_runtime::{ generic::BlockId, transaction_validity::{ValidTransaction, TransactionSource, InvalidTransaction}, }; use substrate_test_runtime_client::{ - runtime::{Block, Hash, Index, Header, Extrinsic, Transfer}, - AccountKeyring::*, + runtime::{Block, Hash, Index, Header, Extrinsic, Transfer}, AccountKeyring::*, + ClientBlockImportExt, }; use substrate_test_runtime_transaction_pool::{TestApi, uxt}; use futures::{prelude::*, task::Poll}; use codec::Encode; +use std::collections::BTreeSet; +use sc_client_api::client::BlockchainEvents; +use sc_block_builder::BlockBuilderProvider; +use sp_consensus::BlockOrigin; fn pool() -> Pool { Pool::new(Default::default(), TestApi::with_alice_nonce(209).into()) @@ -41,7 +46,7 @@ fn maintained_pool() -> ( intervalier::BackSignalControl, ) { let (pool, background_task, notifier) = BasicPool::new_test( - std::sync::Arc::new(TestApi::with_alice_nonce(209)) + Arc::new(TestApi::with_alice_nonce(209)), ); let thread_pool = futures::executor::ThreadPool::new().unwrap(); @@ -49,16 +54,6 @@ fn maintained_pool() -> ( (pool, thread_pool, notifier) } -fn header(number: u64) -> Header { - Header { - number, - digest: Default::default(), - extrinsics_root: Default::default(), - parent_hash: Default::default(), - state_root: Default::default(), - } -} - const SOURCE: TransactionSource = TransactionSource::External; #[test] @@ -111,6 +106,7 @@ fn prune_tags_should_work() { let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![209, 210]); + pool.validated_pool().api().push_block(1, Vec::new()); block_on( pool.prune_tags( &BlockId::number(1), @@ -139,6 +135,37 @@ fn should_ban_invalid_transactions() { block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt.clone())).unwrap_err(); } +#[test] +fn only_prune_on_new_best() { + let pool = maintained_pool().0; + let uxt = uxt(Alice, 209); + + let _ = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, uxt.clone()) + ).expect("1. Imported"); + let header = pool.api.push_block(1, vec![uxt.clone()]); + assert_eq!(pool.status().ready, 1); + + let event = ChainEvent::NewBlock { + hash: header.hash(), + is_new_best: false, + header: header.clone(), + tree_route: None, + }; + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 1); + + let header = pool.api.push_block(2, vec![uxt]); + let event = ChainEvent::NewBlock { + hash: header.hash(), + is_new_best: true, + header: header.clone(), + tree_route: None, + }; + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); +} + #[test] fn should_correctly_prune_transactions_providing_more_than_one_tag() { let api = Arc::new(TestApi::with_alice_nonce(209)); @@ -152,6 +179,7 @@ fn should_correctly_prune_transactions_providing_more_than_one_tag() { // remove the transaction that just got imported. api.increment_nonce(Alice.into()); + api.push_block(1, Vec::new()); block_on(pool.prune_tags(&BlockId::number(1), vec![vec![209]], vec![])).expect("1. Pruned"); assert_eq!(pool.validated_pool().status().ready, 0); // it's re-imported to future @@ -159,6 +187,7 @@ fn should_correctly_prune_transactions_providing_more_than_one_tag() { // so now let's insert another transaction that also provides the 155 api.increment_nonce(Alice.into()); + api.push_block(2, Vec::new()); let xt = uxt(Alice, 211); block_on(pool.submit_one(&BlockId::number(2), SOURCE, xt.clone())).expect("2. Imported"); assert_eq!(pool.validated_pool().status().ready, 1); @@ -168,30 +197,36 @@ fn should_correctly_prune_transactions_providing_more_than_one_tag() { // prune it and make sure the pool is empty api.increment_nonce(Alice.into()); + api.push_block(3, Vec::new()); block_on(pool.prune_tags(&BlockId::number(3), vec![vec![155]], vec![])).expect("2. Pruned"); assert_eq!(pool.validated_pool().status().ready, 0); assert_eq!(pool.validated_pool().status().future, 2); } -fn block_event(id: u64) -> ChainEvent { +fn block_event(header: Header) -> ChainEvent { ChainEvent::NewBlock { - id: BlockId::number(id), + hash: header.hash(), is_new_best: true, - retracted: vec![], - header: header(id), + tree_route: None, + header, } } -fn block_event_with_retracted(id: u64, retracted: Vec) -> ChainEvent { +fn block_event_with_retracted( + header: Header, + retracted_start: Hash, + api: &TestApi, +) -> ChainEvent { + let tree_route = api.tree_route(retracted_start, header.parent_hash).expect("Tree route exists"); + ChainEvent::NewBlock { - id: BlockId::number(id), + hash: header.hash(), is_new_best: true, - retracted: retracted, - header: header(id), + tree_route: Some(Arc::new(tree_route)), + header, } } - #[test] fn should_prune_old_during_maintenance() { let xt = uxt(Alice, 209); @@ -201,9 +236,9 @@ fn should_prune_old_during_maintenance() { block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - pool.api.push_block(1, vec![xt.clone()]); + let header = pool.api.push_block(1, vec![xt.clone()]); - block_on(pool.maintain(block_event(1))); + block_on(pool.maintain(block_event(header))); assert_eq!(pool.status().ready, 0); } @@ -218,9 +253,9 @@ fn should_revalidate_during_maintenance() { assert_eq!(pool.status().ready, 2); assert_eq!(pool.api.validation_requests().len(), 2); - pool.api.push_block(1, vec![xt1.clone()]); + let header = pool.api.push_block(1, vec![xt1.clone()]); - block_on(pool.maintain(block_event(1))); + block_on(pool.maintain(block_event(header))); assert_eq!(pool.status().ready, 1); block_on(notifier.next()); @@ -231,37 +266,54 @@ fn should_revalidate_during_maintenance() { #[test] fn should_resubmit_from_retracted_during_maintenance() { let xt = uxt(Alice, 209); - let retracted_hash = Hash::random(); let (pool, _guard, _notifier) = maintained_pool(); block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - pool.api.push_block(1, vec![]); - pool.api.push_fork_block(retracted_hash, vec![xt.clone()]); + let header = pool.api.push_block(1, vec![]); + let fork_header = pool.api.push_block(1, vec![]); - let event = block_event_with_retracted(1, vec![retracted_hash]); + let event = block_event_with_retracted(header, fork_header.hash(), &*pool.api); block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 1); } + +#[test] +fn should_not_resubmit_from_retracted_during_maintenance_if_tx_is_also_in_enacted() { + let xt = uxt(Alice, 209); + + let (pool, _guard, _notifier) = maintained_pool(); + + block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); + assert_eq!(pool.status().ready, 1); + + let header = pool.api.push_block(1, vec![xt.clone()]); + let fork_header = pool.api.push_block(1, vec![xt]); + + let event = block_event_with_retracted(header, fork_header.hash(), &*pool.api); + + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); +} + #[test] fn should_not_retain_invalid_hashes_from_retracted() { let xt = uxt(Alice, 209); - let retracted_hash = Hash::random(); let (pool, _guard, mut notifier) = maintained_pool(); block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - pool.api.push_block(1, vec![]); - pool.api.push_fork_block(retracted_hash, vec![xt.clone()]); + let header = pool.api.push_block(1, vec![]); + let fork_header = pool.api.push_block(1, vec![xt.clone()]); pool.api.add_invalid(&xt); - let event = block_event_with_retracted(1, vec![retracted_hash]); + let event = block_event_with_retracted(header, fork_header.hash(), &*pool.api); block_on(pool.maintain(event)); block_on(notifier.next()); @@ -278,17 +330,17 @@ fn should_revalidate_transaction_multiple_times() { block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - pool.api.push_block(1, vec![xt.clone()]); + let header = pool.api.push_block(1, vec![xt.clone()]); - block_on(pool.maintain(block_event(1))); + block_on(pool.maintain(block_event(header))); block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - pool.api.push_block(2, vec![]); + let header = pool.api.push_block(2, vec![]); pool.api.add_invalid(&xt); - block_on(pool.maintain(block_event(2))); + block_on(pool.maintain(block_event(header))); block_on(notifier.next()); assert_eq!(pool.status().ready, 0); @@ -306,15 +358,15 @@ fn should_revalidate_across_many_blocks() { block_on(pool.submit_one(&BlockId::number(1), SOURCE, xt2.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 2); - pool.api.push_block(1, vec![]); - block_on(pool.maintain(block_event(1))); + let header = pool.api.push_block(1, vec![]); + block_on(pool.maintain(block_event(header))); block_on(notifier.next()); block_on(pool.submit_one(&BlockId::number(2), SOURCE, xt3.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 3); - pool.api.push_block(2, vec![xt1.clone()]); - block_on(pool.maintain(block_event(2))); + let header = pool.api.push_block(2, vec![xt1.clone()]); + block_on(pool.maintain(block_event(header))); block_on(notifier.next()); assert_eq!(pool.status().ready, 2); @@ -359,7 +411,8 @@ fn should_push_watchers_during_maintaince() { pool.api.add_invalid(&tx4); // clear timer events if any - block_on(pool.maintain(block_event(0))); + let header = pool.api.push_block(1, vec![]); + block_on(pool.maintain(block_event(header))); block_on(notifier.next()); // then @@ -376,8 +429,9 @@ fn should_push_watchers_during_maintaince() { ); // when - let header_hash = pool.api.push_block(1, vec![tx0, tx1, tx2]).hash(); - block_on(pool.maintain(block_event(1))); + let header = pool.api.push_block(2, vec![tx0, tx1, tx2]); + let header_hash = header.hash(); + block_on(pool.maintain(block_event(header))); let event = ChainEvent::Finalized { hash: header_hash.clone() }; block_on(pool.maintain(event)); @@ -388,15 +442,24 @@ fn should_push_watchers_during_maintaince() { // events for hash2 are: Ready, InBlock assert_eq!( futures::executor::block_on_stream(watcher0).collect::>(), - vec![TransactionStatus::Ready, TransactionStatus::InBlock(header_hash.clone()), TransactionStatus::Finalized(header_hash.clone())], + vec![ + TransactionStatus::Ready, + TransactionStatus::InBlock(header_hash.clone()), + TransactionStatus::Finalized(header_hash.clone())], ); assert_eq!( futures::executor::block_on_stream(watcher1).collect::>(), - vec![TransactionStatus::Ready, TransactionStatus::InBlock(header_hash.clone()), TransactionStatus::Finalized(header_hash.clone())], + vec![ + TransactionStatus::Ready, + TransactionStatus::InBlock(header_hash.clone()), + TransactionStatus::Finalized(header_hash.clone())], ); assert_eq!( futures::executor::block_on_stream(watcher2).collect::>(), - vec![TransactionStatus::Ready, TransactionStatus::InBlock(header_hash.clone()), TransactionStatus::Finalized(header_hash.clone())], + vec![ + TransactionStatus::Ready, + TransactionStatus::InBlock(header_hash.clone()), + TransactionStatus::Finalized(header_hash.clone())], ); } @@ -422,12 +485,12 @@ fn finalization() { ).expect("1. Imported"); pool.api.push_block(2, vec![xt.clone()]); - let header = pool.api.chain().read().header_by_number.get(&2).cloned().unwrap(); + let header = pool.api.chain().read().block_by_number.get(&2).unwrap()[0].header().clone(); let event = ChainEvent::NewBlock { - id: BlockId::Hash(header.hash()), + hash: header.hash(), is_new_best: true, header: header.clone(), - retracted: vec![] + tree_route: None, }; block_on(pool.maintain(event)); @@ -466,7 +529,6 @@ fn fork_aware_finalization() { let c2; let d2; - // block B1 { let watcher = block_on( @@ -477,10 +539,10 @@ fn fork_aware_finalization() { assert_eq!(pool.status().ready, 1); let event = ChainEvent::NewBlock { - id: BlockId::Number(2), + hash: header.hash(), is_new_best: true, header: header.clone(), - retracted: vec![], + tree_route: None, }; b1 = header.hash(); block_on(pool.maintain(event)); @@ -491,16 +553,16 @@ fn fork_aware_finalization() { // block C2 { - let header = pool.api.push_fork_block_with_parent(b1, vec![from_dave.clone()]); + let header = pool.api.push_block_with_parent(b1, vec![from_dave.clone()]); from_dave_watcher = block_on( pool.submit_and_watch(&BlockId::number(1), SOURCE, from_dave.clone()) ).expect("1. Imported"); assert_eq!(pool.status().ready, 1); let event = ChainEvent::NewBlock { - id: BlockId::Hash(header.hash()), + hash: header.hash(), is_new_best: true, header: header.clone(), - retracted: vec![] + tree_route: None, }; c2 = header.hash(); block_on(pool.maintain(event)); @@ -513,13 +575,13 @@ fn fork_aware_finalization() { pool.submit_and_watch(&BlockId::number(1), SOURCE, from_bob.clone()) ).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - let header = pool.api.push_fork_block_with_parent(c2, vec![from_bob.clone()]); + let header = pool.api.push_block_with_parent(c2, vec![from_bob.clone()]); let event = ChainEvent::NewBlock { - id: BlockId::Hash(header.hash()), + hash: header.hash(), is_new_best: true, header: header.clone(), - retracted: vec![] + tree_route: None, }; d2 = header.hash(); block_on(pool.maintain(event)); @@ -535,14 +597,10 @@ fn fork_aware_finalization() { let header = pool.api.push_block(3, vec![from_charlie.clone()]); canon_watchers.push((watcher, header.hash())); - let event = ChainEvent::NewBlock { - id: BlockId::Number(3), - is_new_best: true, - header: header.clone(), - retracted: vec![c2, d2], - }; + let event = block_event_with_retracted(header.clone(), d2, &*pool.api); block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 2); + let event = ChainEvent::Finalized { hash: header.hash() }; block_on(pool.maintain(event)); } @@ -558,10 +616,10 @@ fn fork_aware_finalization() { canon_watchers.push((w, header.hash())); let event = ChainEvent::NewBlock { - id: BlockId::Hash(header.hash()), + hash: header.hash(), is_new_best: true, header: header.clone(), - retracted: vec![] + tree_route: None, }; d1 = header.hash(); block_on(pool.maintain(event)); @@ -577,10 +635,10 @@ fn fork_aware_finalization() { let header = pool.api.push_block(5, vec![from_dave, from_bob]); e1 = header.hash(); let event = ChainEvent::NewBlock { - id: BlockId::Hash(header.hash()), + hash: header.hash(), is_new_best: true, header: header.clone(), - retracted: vec![] + tree_route: None, }; block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 0); @@ -598,7 +656,7 @@ fn fork_aware_finalization() { { - let mut stream= futures::executor::block_on_stream(from_dave_watcher); + let mut stream = futures::executor::block_on_stream(from_dave_watcher); assert_eq!(stream.next(), Some(TransactionStatus::Ready)); assert_eq!(stream.next(), Some(TransactionStatus::InBlock(c2.clone()))); assert_eq!(stream.next(), Some(TransactionStatus::Retracted(c2))); @@ -609,7 +667,7 @@ fn fork_aware_finalization() { } { - let mut stream= futures::executor::block_on_stream(from_bob_watcher); + let mut stream = futures::executor::block_on_stream(from_bob_watcher); assert_eq!(stream.next(), Some(TransactionStatus::Ready)); assert_eq!(stream.next(), Some(TransactionStatus::InBlock(d2.clone()))); assert_eq!(stream.next(), Some(TransactionStatus::Retracted(d2))); @@ -620,6 +678,277 @@ fn fork_aware_finalization() { } } +/// Tests that when pruning and retracing a tx by the same event, we generate +/// the correct events in the correct order. +#[test] +fn prune_and_retract_tx_at_same_time() { + let api = TestApi::empty(); + // starting block A1 (last finalized.) + api.push_block(1, vec![]); + + let (pool, _background, _) = BasicPool::new_test(api.into()); + + let from_alice = uxt(Alice, 1); + pool.api.increment_nonce(Alice.into()); + + let watcher = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, from_alice.clone()) + ).expect("1. Imported"); + + // Block B1 + let b1 = { + let header = pool.api.push_block(2, vec![from_alice.clone()]); + assert_eq!(pool.status().ready, 1); + + let event = ChainEvent::NewBlock { + hash: header.hash(), + is_new_best: true, + header: header.clone(), + tree_route: None, + }; + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); + header.hash() + }; + + // Block B2 + let b2 = { + let header = pool.api.push_block(2, vec![from_alice.clone()]); + assert_eq!(pool.status().ready, 0); + + let event = block_event_with_retracted(header.clone(), b1, &*pool.api); + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); + + let event = ChainEvent::Finalized { hash: header.hash() }; + block_on(pool.maintain(event)); + + header.hash() + }; + + { + let mut stream = futures::executor::block_on_stream(watcher); + assert_eq!(stream.next(), Some(TransactionStatus::Ready)); + assert_eq!(stream.next(), Some(TransactionStatus::InBlock(b1.clone()))); + assert_eq!(stream.next(), Some(TransactionStatus::Retracted(b1))); + assert_eq!(stream.next(), Some(TransactionStatus::InBlock(b2.clone()))); + assert_eq!(stream.next(), Some(TransactionStatus::Finalized(b2))); + assert_eq!(stream.next(), None); + } +} + + +/// This test ensures that transactions from a fork are re-submitted if +/// the forked block is not part of the retracted blocks. This happens as the +/// retracted block list only contains the route from the old best to the new +/// best, without any further forks. +/// +/// Given the following: +/// +/// -> D0 (old best, tx0) +/// / +/// C - -> D1 (tx1) +/// \ +/// -> D2 (new best) +/// +/// Retracted will contain `D0`, but we need to re-submit `tx0` and `tx1` as both +/// blocks are not part of the canonical chain. +#[test] +fn resubmit_tx_of_fork_that_is_not_part_of_retracted() { + let api = TestApi::empty(); + // starting block A1 (last finalized.) + api.push_block(1, vec![]); + + let (pool, _background, _) = BasicPool::new_test(api.into()); + + let tx0 = uxt(Alice, 1); + let tx1 = uxt(Dave, 2); + pool.api.increment_nonce(Alice.into()); + pool.api.increment_nonce(Dave.into()); + + let d0; + + // Block D0 + { + let _ = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, tx0.clone()) + ).expect("1. Imported"); + let header = pool.api.push_block(2, vec![tx0.clone()]); + assert_eq!(pool.status().ready, 1); + + let event = ChainEvent::NewBlock { + hash: header.hash(), + is_new_best: true, + header: header.clone(), + tree_route: None, + }; + d0 = header.hash(); + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); + } + + // Block D1 + { + let _ = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, tx1.clone()) + ).expect("1. Imported"); + let header = pool.api.push_block(2, vec![tx1.clone()]); + assert_eq!(pool.status().ready, 1); + let event = ChainEvent::NewBlock { + hash: header.hash(), + is_new_best: false, + header: header.clone(), + tree_route: None, + }; + block_on(pool.maintain(event)); + + // Only transactions from new best should be pruned + assert_eq!(pool.status().ready, 1); + } + + // Block D2 + { + let header = pool.api.push_block(2, vec![]); + let event = block_event_with_retracted(header, d0, &*pool.api); + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 2); + } +} + +#[test] +fn resubmit_from_retracted_fork() { + let api = TestApi::empty(); + // starting block A1 (last finalized.) + api.push_block(1, vec![]); + + let (pool, _background, _) = BasicPool::new_test(api.into()); + + let tx0 = uxt(Alice, 1); + let tx1 = uxt(Dave, 2); + let tx2 = uxt(Bob, 3); + + // Transactions of the fork that will be enacted later + let tx3 = uxt(Eve, 1); + let tx4 = uxt(Ferdie, 2); + let tx5 = uxt(One, 3); + + pool.api.increment_nonce(Alice.into()); + pool.api.increment_nonce(Dave.into()); + pool.api.increment_nonce(Bob.into()); + pool.api.increment_nonce(Eve.into()); + pool.api.increment_nonce(Ferdie.into()); + pool.api.increment_nonce(One.into()); + + // Block D0 + { + let _ = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, tx0.clone()) + ).expect("1. Imported"); + let header = pool.api.push_block(2, vec![tx0.clone()]); + assert_eq!(pool.status().ready, 1); + + let event = ChainEvent::NewBlock { + hash: header.hash(), + is_new_best: true, + header: header.clone(), + tree_route: None, + }; + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); + } + + // Block E0 + { + let _ = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, tx1.clone()) + ).expect("1. Imported"); + let header = pool.api.push_block(3, vec![tx1.clone()]); + let event = ChainEvent::NewBlock { + hash: header.hash(), + is_new_best: true, + header: header.clone(), + tree_route: None, + }; + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); + } + + // Block F0 + let f0 = { + let _ = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, tx2.clone()) + ).expect("1. Imported"); + let header = pool.api.push_block(4, vec![tx2.clone()]); + let event = ChainEvent::NewBlock { + hash: header.hash(), + is_new_best: true, + header: header.clone(), + tree_route: None, + }; + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 0); + header.hash() + }; + + // Block D1 + let d1 = { + let _ = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, tx3.clone()) + ).expect("1. Imported"); + let header = pool.api.push_block(2, vec![tx3.clone()]); + let event = ChainEvent::NewBlock { + hash: header.hash(), + is_new_best: false, + header: header.clone(), + tree_route: None, + }; + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 1); + header.hash() + }; + + // Block E1 + let e1 = { + let _ = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, tx4.clone()) + ).expect("1. Imported"); + let header = pool.api.push_block_with_parent(d1.clone(), vec![tx4.clone()]); + let event = ChainEvent::NewBlock { + hash: header.hash(), + is_new_best: false, + header: header.clone(), + tree_route: None, + }; + block_on(pool.maintain(event)); + assert_eq!(pool.status().ready, 2); + header.hash() + }; + + // Block F1 + let f1_header = { + let _ = block_on( + pool.submit_and_watch(&BlockId::number(1), SOURCE, tx5.clone()) + ).expect("1. Imported"); + let header = pool.api.push_block_with_parent(e1.clone(), vec![tx5.clone()]); + // Don't announce the block event to the pool directly, because we will + // re-org to this block. + assert_eq!(pool.status().ready, 3); + header + }; + + let ready = pool.ready().map(|t| t.data.encode()).collect::>(); + let expected_ready = vec![tx3, tx4, tx5].iter().map(Encode::encode).collect::>(); + assert_eq!(expected_ready, ready); + + let event = block_event_with_retracted(f1_header, f0, &*pool.api); + block_on(pool.maintain(event)); + + assert_eq!(pool.status().ready, 3); + let ready = pool.ready().map(|t| t.data.encode()).collect::>(); + let expected_ready = vec![tx0, tx1, tx2].iter().map(Encode::encode).collect::>(); + assert_eq!(expected_ready, ready); +} + #[test] fn ready_set_should_not_resolve_before_block_update() { let (pool, _guard, _notifier) = maintained_pool(); @@ -632,12 +961,12 @@ fn ready_set_should_not_resolve_before_block_update() { #[test] fn ready_set_should_resolve_after_block_update() { let (pool, _guard, _notifier) = maintained_pool(); - pool.api.push_block(1, vec![]); + let header = pool.api.push_block(1, vec![]); let xt1 = uxt(Alice, 209); block_on(pool.submit_one(&BlockId::number(1), SOURCE, xt1.clone())).expect("1. Imported"); - block_on(pool.maintain(block_event(1))); + block_on(pool.maintain(block_event(header))); assert!(pool.ready_at(1).now_or_never().is_some()); } @@ -645,7 +974,7 @@ fn ready_set_should_resolve_after_block_update() { #[test] fn ready_set_should_eventually_resolve_when_block_update_arrives() { let (pool, _guard, _notifier) = maintained_pool(); - pool.api.push_block(1, vec![]); + let header = pool.api.push_block(1, vec![]); let xt1 = uxt(Alice, 209); @@ -659,7 +988,7 @@ fn ready_set_should_eventually_resolve_when_block_update_arrives() { panic!("Ready set should not be ready before block update!"); } - block_on(pool.maintain(block_event(1))); + block_on(pool.maintain(block_event(header))); match ready_set_future.poll_unpin(&mut context) { Poll::Pending => { @@ -677,6 +1006,7 @@ fn should_not_accept_old_signatures() { use std::convert::TryFrom; let client = Arc::new(substrate_test_runtime_client::new()); + let pool = Arc::new( BasicPool::new_test(Arc::new(FullChainApi::new(client))).0 ); @@ -694,7 +1024,11 @@ fn should_not_accept_old_signatures() { "c427eb672e8c441c86d31f1a81b22b43102058e9ce237cabe9897ea5099ffd426cd1c6a1f4f2869c3df57901d36bedcb295657adb3a4355add86ed234eb83108" ).expect("hex invalid")[..]).expect("signature construction failed"); - let xt = Extrinsic::Transfer { transfer, signature: old_singature, exhaust_resources_when_not_first: false }; + let xt = Extrinsic::Transfer { + transfer, + signature: old_singature, + exhaust_resources_when_not_first: false, + }; assert_matches::assert_matches!( block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())), @@ -704,3 +1038,56 @@ fn should_not_accept_old_signatures() { "Should be invalid transaction with bad proof", ); } + +#[test] +fn import_notification_to_pool_maintain_works() { + let mut client = Arc::new(substrate_test_runtime_client::new()); + + let pool = Arc::new( + BasicPool::new_test(Arc::new(FullChainApi::new(client.clone()))).0 + ); + + // Prepare the extrisic, push it to the pool and check that it was added. + let xt = uxt(Alice, 0); + block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); + assert_eq!(pool.status().ready, 1); + + let mut import_stream = block_on_stream(client.import_notification_stream()); + + // Build the block with the transaction included + let mut block_builder = client.new_block(Default::default()).unwrap(); + block_builder.push(xt).unwrap(); + let block = block_builder.build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + + // Get the notification of the block import and maintain the pool with it, + // Now, the pool should not contain any transactions. + let evt = import_stream.next().expect("Importing a block leads to an event"); + block_on(pool.maintain(evt.into())); + assert_eq!(pool.status().ready, 0); +} + +// When we prune transactions, we need to make sure that we remove +#[test] +fn pruning_a_transaction_should_remove_it_from_best_transaction() { + let (pool, _guard, _notifier) = maintained_pool(); + + let xt1 = Extrinsic::IncludeData(Vec::new()); + + block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt1.clone())).expect("1. Imported"); + let header = pool.api.push_block(1, vec![xt1.clone()]); + + // This will prune `xt1`. + block_on(pool.maintain(block_event(header))); + + // Submit the tx again. + block_on(pool.submit_one(&BlockId::number(1), SOURCE, xt1.clone())).expect("2. Imported"); + + let mut iterator = block_on(pool.ready_at(1)); + + assert_eq!(iterator.next().unwrap().data, xt1.clone()); + + // If the tx was not removed from the best txs, the tx would be + // returned a second time by the iterator. + assert!(iterator.next().is_none()); +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 84c469386acef223c4dc3f03b3bbbb7d5fee8edc..78fc85acc623afe058b5bd05a4a9a4bcb899660c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,6 +6,100 @@ The format is based on [Keep a Changelog]. ## Unreleased +## 2.0.0-rc3 -> 2.0.0-rc4 (Rhinoceros) + +Runtime +------- + +* Staking Payout Creates Controller (#6496) +* `pallet-scheduler`: Check that `when` is not in the past (#6480) +* Fix `sp-api` handling of multiple arguments (#6484) +* Fix issues with `Operational` transactions validity and prioritization. (#6435) +* pallet-atomic-swap: generialized swap action (#6421) +* Avoid multisig reentrancy (#6445) +* Root origin use no filter by default. Scheduler and Democracy dispatch without asserting BaseCallFilter (#6408) +* Scale and increase validator count (#6417) +* Pallet: Atomic Swap (#6349) +* Restrict remove_proxies (#6383) +* Stored call in multisig (#6319) +* Allow Sudo to do anything (#6375) +* vesting: Force Vested Transfer (#6368) +* Add events for balance reserve and unreserve functions (#6330) +* Introduce frozen indices. (#6307) + +Client +------ + +* client/network/service: Add primary dimension to connection metrics (#6472) +* Fix Babe secondary plain slots claiming (#6451) +* add network propagated metrics (#6438) +* client/authority-discovery: Compare PeerIds and not Multihashes (#6414) +* Update sync chain info on own block import (#6424) +* Remove --legacy-network-protocol CLI flag (#6411) +* Runtime interface to add support for tracing from wasm (#6381) +* Remove penalty on duplicate Status message (#6377) +* Fix the broken weight multiplier update function (#6334) +* client/authority-discovery: Don't add own address to priority group (#6370) +* Split the service initialisation up into seperate functions (#6332) +* Fix transaction pool event sending (#6341) +* Add a [prefix]_process_start_time_seconds metric (#6315) +* new crate sc-light (#6235) +* Allow adding a prefix to the informant (#6174) + +API +--- + +* seal: Remove ext_dispatch_call and ext_get_runtime_storage (#6464) +* seal: Refactor ext_gas_price (#6478) +* Implement nested storage transactions (#6269) +* Allow empty values in the storage (#6364) +* add system_dryRun (#6300) +* Introduce in-origin filtering (#6318) +* add extend_lock for StorageLock (#6323) +* Deprecate FunctionOf and remove its users (#6340) +* transaction-pool: expose blocking api for tx submission (#6325) + + +## 2.0.0-rc2 -> 2.0.0-rc3 + +Runtime +------- + +* Introduce stacked filtering (#6273) +* Allow "anonymous" proxied accounts (#6236) +* Allow over-weight collective proposals to be closed (#6163) +* Fix Election when ForceNone V1 (#6166) + +Client +------ + +* Make transaction pool prune transactions only of canonical blocks (#6123) +* Rename all the election operations (#6245) +* Sentry nodes and validator nodes also imply reserved (#6251) +* Fix peerset not filtering incoming connections in reserved-only (#6249) +* Use Subscription Manager from `jsonrpc-pubsub` (#6208) +* Add a Substrate networking Grafana dashboard template (#6171) +* Add subkey inspect-node-key (#6153) + +## 2.0.0-rc1 -> 2.0.0-rc2 + +(nothing of note) + +## 2.0.0-alpha.8 -> 2.0.0-rc1 + +Runtime +------- + +* Allow operational recovery path if on_initialize use fullblock. (#6089) +* Maximum extrinsic weight limit (#6067) + +Client +------ + +* Add JSON format to import blocks and set it as default (#5816) +* Upgrade to libp2p v0.19 - Changes the default PeerId representation (#6064) + + ## 2.0.0-alpha.7 -> 2.0.0-alpha.8 **License Changed** diff --git a/docs/CODEOWNERS b/docs/CODEOWNERS index 7559a9ee2b27f547fae30ea98ace8f8ad8e694f4..d9342de399503bd157ebb9779ee38db1ecda736f 100644 --- a/docs/CODEOWNERS +++ b/docs/CODEOWNERS @@ -18,15 +18,7 @@ # are more recognizable on GitHub, you can use them for mentioning unlike an email. # - The latest matching rule, if multiple, takes precedence. -# Wasm execution and the wasm side of Substrate Runtime Interface -/client/executor/ @pepyakin -/primitives/io/ @pepyakin @NikVolf - -# Crypto, execution extensions, etc. -/primitives/core/ @NikVolf - # Block production -/primitives/authorship/ @NikVolf /client/basic-authorship/ @NikVolf # Sandboxing capability of Substrate Runtime @@ -41,45 +33,32 @@ /client/offchain/ @tomusdrw /primitives/offchain/ @tomusdrw -# Everything that has RPC in it -/bin/node/rpc/ @tomusdrw -/bin/node/rpc-client/ @tomusdrw -/client/rpc/ @tomusdrw -/primitives/rpc/ @tomusdrw - # GRANDPA, BABE, consensus stuff -/frame/babe/ @andresilva @DemiMarie-parity +/frame/babe/ @andresilva /frame/grandpa/ @andresilva /client/finality-grandpa/ @andresilva -/client/consensus/babe/ @andresilva @DemiMarie-parity -/client/consensus/slots/ @andresilva @DemiMarie-parity +/client/consensus/babe/ @andresilva +/client/consensus/slots/ @andresilva /client/consensus/pow/ @sorpaas /primitives/consensus/pow/ @sorpaas # Contracts /frame/contracts/ @pepyakin -/frame/contracts/src/wasm/runtime.rs @Robbepop # EVM /frame/evm/ @sorpaas -# NPoS and Governance and Phragmén +# NPoS and election /frame/staking/ @kianenigma /frame/elections/ @kianenigma /frame/elections-phragmen/ @kianenigma -/primitives/phragmen/ @kianenigma +/primitives/npos-elections/ @kianenigma # Fixed point arithmetic /primitives/sp-arithmetic/ @kianenigma -# End to end testing of substrate node -/bin/node/executor/ @kianenigma - # Transaction weight stuff -/frame/support/src/weights.rs @kianenigma - -# Support crates -/frame/support/ @kianenigma +/frame/support/src/weights.rs @shawntabrizi # Authority discovery /client/authority-discovery/ @mxinden @@ -87,7 +66,3 @@ # Prometheus endpoint /utils/prometheus/ @mxinden - -# CLI API -/client/cli @cecton -/client/cli-derive @cecton diff --git a/docs/CONTRIBUTING.adoc b/docs/CONTRIBUTING.adoc index b573aef50d874b304f3b135b4cffd08811fcfdea..ec747d6693bc5f2910f398628bd92782b93f5bd0 100644 --- a/docs/CONTRIBUTING.adoc +++ b/docs/CONTRIBUTING.adoc @@ -19,17 +19,30 @@ There are a few basic ground-rules for contributors (including the maintainer(s) == Merge Process -Merging pull requests once CI is successful: +*In General* -. A PR needs to be reviewed and approved by project maintainers unless: - - it does not alter any logic (e.g. comments, dependencies, docs), then it may be tagged https://github.com/paritytech/substrate/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aopen+label%3AA2-insubstantial[`insubstantial`] and merged by its author once CI is complete. - - it is an urgent fix with no large change to logic, then it may be merged after a non-author contributor has approved the review once CI is complete. +A PR needs to be reviewed and approved by project maintainers unless: +- it does not alter any logic (e.g. comments, dependencies, docs), then it may be tagged https://github.com/paritytech/substrate/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aopen+label%3AA2-insubstantial[`insubstantial`] and merged by its author once CI is complete. +- it is an urgent fix with no large change to logic, then it may be merged after a non-author contributor has approved the review once CI is complete. + +*Labels TLDR:* + +- `A-*` Pull request status. ONE REQUIRED. +- `B-*` Changelog and/or Runtime-upgrade post composition markers. ONE REQUIRED. (used by automation) +- `C-*` Release notes release-priority markers. EXACTLY ONE REQUIRED. (used by automation) +- `D-*` More general tags on the PR denoting various implications and requirements. + +*Process:* + +. Please tag each PR with exactly one `A`, `B` and `C` label at the minimum. . Once a PR is ready for review please add the https://github.com/paritytech/substrate/pulls?q=is%3Apr+is%3Aopen+label%3AA0-pleasereview[`A0-pleasereview`] label. Generally PRs should sit with this label for 48 hours in order to garner feedback. It may be merged before if all relevant parties had a look at it. -. If the first review is not an approval, swap `A0-pleasereview` to any label `[A3, A7]` to indicate that the PR has received some feedback, but needs further work. For example. https://github.com/paritytech/substrate/labels/A3-inprogress[`A3-inprogress`] is a general indicator that the PR is work in progress and https://github.com/paritytech/substrate/labels/A4-gotissues[`A4-gotissues`] means that it has significant problems that need fixing. Once the work is done, change the label back to `A0-pleasereview`. You might end up swapping a few times back and forth to climb up the A label group. Once a PR is https://github.com/paritytech/substrate/labels/A8-mergeoncegreen[`A8-mergeoncegreen`], it is ready to merge. +. If the first review is not an approval, swap `A0-pleasereview` to any label `[A3, A7]` to indicate that the PR has received some feedback, but needs further work. For example. https://github.com/paritytech/substrate/labels/A3-inprogress[`A3-inprogress`] is a general indicator that the PR is work in progress and https://github.com/paritytech/substrate/labels/A4-gotissues[`A4-gotissues`] means that it has significant problems that need fixing. Once the work is done, change the label back to `A0-pleasereview`. You might end up swapping a few times back and forth to climb up the A label group. Once a PR is https://github.com/paritytech/substrate/labels/A8-mergeoncegreen[`A8-mergeoncegreen`], it is ready to merge. . PRs must be tagged with respect to _release notes_ with https://github.com/paritytech/substrate/labels/B0-silent[`B0-silent`] and `B1-..`. The former indicates that no changes should be mentioned in any release notes. The latter indicates that the changes should be reported in the corresponding release note -. PRs that break the external API must be tagged with https://github.com/paritytech/substrate/labels/B2-breaksapi[`B2-breaksapi`], when it changes the FRAME or consensus of running system with https://github.com/paritytech/substrate/labels/B3-breaksconsensus[`B3-breaksconsensus`] -. No PR should be merged until all reviews' comments are addressed. +. PRs that break the external API must be tagged with https://github.com/paritytech/substrate/labels/D2-breaksapi[`D2-breaksapi`], when it changes the FRAME or consensus of running system with https://github.com/paritytech/substrate/labels/B3-breaksconsensus[`B3-breaksconsensus`]. +. PRs should be labeled with their release importance via the `C1-C9`. +. PRs should be categorized into projects. +. No PR should be merged until all reviews' comments are addressed and CI is successful. *Reviewing pull requests*: @@ -49,19 +62,19 @@ When reviewing a pull request, the end-goal is to suggest useful changes to the === Updating Polkadot as well -**All pull requests will be checked agains either Polkadot master, or your provided Polkadot companion PR**. That is, If your PR changes the external APIs or interfaces used by Polkadot. If you tagged the PR with `breaksapi` or `breaksconsensus` this is most certainly the case, in all other cases check for it by running step 1 below. +**All pull requests will be checked against either Polkadot master, or your provided Polkadot companion PR**. That is, If your PR changes the external APIs or interfaces used by Polkadot. If you tagged the PR with `breaksapi` or `breaksconsensus` this is most certainly the case, in all other cases check for it by running step 1 below. To create a Polkadot companion PR: . Pull latest Polkadot master (or clone it, if you haven't yet). . Override your local cargo config to point to your local substrate (pointing to your WIP branch): place `paths = ["path/to/substrate"]` in `~/.cargo/config`. . Make the changes required and build polkadot locally. -. Submit all this as a PR against the Polkadot Repo. Link to your Polkadot PR in the _description_ of your Substrate PR as "polkadot companion: [URL]" OR use the same name for your Polkdadot branch as the Substrate branch. +. Submit all this as a PR against the Polkadot Repo. Link to your Polkadot PR in the _description_ of your Substrate PR as "polkadot companion: [URL]" OR use the same name for your Polkdadot branch as the Substrate branch. . Now you should see that the `check_polkadot` CI job will build your Substrate PR agains the mentioned Polkadot branch in your PR description. . Wait for reviews on both -. Once both PRs have been green lit, they can both be merged 🍻. +. Once both PRs have been green lit, they can both be merged 🍻. -If your PR is reviewed well, but a Polkadot PR is missing, signal it with https://github.com/paritytech/substrate/labels/A7-needspolkadotpr[`A7-needspolkadotpr`] to prevent it from getting automatically merged. +If your PR is reviewed well, but a Polkadot PR is missing, signal it with https://github.com/paritytech/substrate/labels/A7-needspolkadotpr[`A7-needspolkadotpr`] to prevent it from getting automatically merged. As there might be multiple pending PRs that might conflict with one another, a) you should not merge the substrate PR until the Polkadot PR has also been reviewed and b) both should be merged pretty quickly after another to not block others. @@ -69,6 +82,14 @@ As there might be multiple pending PRs that might conflict with one another, a) We use https://github.com/paritytech/substrate/labels[labels] to manage PRs and issues and communicate state of a PR. Please familiarize yourself with them. Furthermore we are organizing issues in https://github.com/paritytech/substrate/milestones[milestones]. Best way to get started is to a pick a ticket from the current milestone tagged https://github.com/paritytech/substrate/issues?q=is%3Aissue+is%3Aopen+label%3AQ2-easy[`easy`] or https://github.com/paritytech/substrate/issues?q=is%3Aissue+is%3Aopen+label%3AQ3-medium[`medium`] and get going or https://github.com/paritytech/substrate/issues?q=is%3Aissue+is%3Aopen+label%3AX1-mentor[`mentor`] and get in contact with the mentor offering their support on that larger task. +== Issues +Please label issues with the following labels: + +. `I-*` Issue severity and type. EXACTLY ONE REQUIRED. +. `P-*` Issue priority. AT MOST ONE ALLOWED. +. `Q-*` Issue difficulty. AT MOST ONE ALLOWED. +. `Z-*` More general tags on the issue, denoting context and resolution. + == Releases Declaring formal releases remains the prerogative of the project maintainer(s). diff --git a/docs/PULL_REQUEST_TEMPLATE.md b/docs/PULL_REQUEST_TEMPLATE.md index fa2c15e6c02fea0f8f8bba9e27059f0e0617b2d6..8ca6ba9b01fe21795d8ce57a99c736ac67220586 100644 --- a/docs/PULL_REQUEST_TEMPLATE.md +++ b/docs/PULL_REQUEST_TEMPLATE.md @@ -6,17 +6,24 @@ Before you submitting, please check that: - What does it do? - What important points reviewers should know? - Is there something left for follow-up PRs? -- [ ] You labeled the PR with appropriate labels if you have permissions to do so. +- [ ] You labeled the PR appropriately if you have permissions to do so: + - [ ] `A*` for PR status (**one required**) + - [ ] `B*` for changelog (**one required**) + - [ ] `C*` for release notes (**exactly one required**) + - [ ] `D*` for various implications/requirements + - [ ] Github's project assignment - [ ] You mentioned a related issue if this PR related to it, e.g. `Fixes #228` or `Related #1337`. - [ ] You asked any particular reviewers to review. If you aren't sure, start with GH suggestions. -- [ ] Your PR adheres [the style guide](https://wiki.parity.io/Substrate-Style-Guide) - - In particular, mind the maximal line length. +- [ ] Your PR adheres to [the style guide](https://wiki.parity.io/Substrate-Style-Guide) + - In particular, mind the maximal line length of 100 (120 in exceptional circumstances). - There is no commented code checked in unless necessary. - Any panickers have a proof or removed. - [ ] You bumped the runtime version if there are breaking changes in the **runtime**. - [ ] You updated any rustdocs which may have changed - [ ] Has the PR altered the external API or interfaces used by Polkadot? Do you have the corresponding Polkadot PR ready? +Refer to [the contributing guide](https://github.com/paritytech/substrate/blob/master/docs/CONTRIBUTING.adoc) for details. + After you've read this notice feel free to remove it. Thank you! diff --git a/frame/assets/Cargo.toml b/frame/assets/Cargo.toml index 6dc6e3e513dbf1cc6f6aff482ddb5e51592de70a..4deb7b8a9bb1bde7fc4f83ecce1d509e1c843944 100644 --- a/frame/assets/Cargo.toml +++ b/frame/assets/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-assets" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,18 +13,18 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } # Needed for various traits. In our case, `OnFinalize`. -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } # Needed for type-safe access to storage DB. -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } # `system` module provides us with all sorts of useful stuff and macros depend on it being around. -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.8", path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", path = "../../primitives/io" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-std = { version = "2.0.0-rc4", path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", path = "../../primitives/io" } [features] default = ["std"] diff --git a/frame/assets/src/lib.rs b/frame/assets/src/lib.rs index 6211694c945978f7bfd0e664e7bd63f1f0432b45..159546ccb3a4f72d22609637302dd4ed8f3d2d5c 100644 --- a/frame/assets/src/lib.rs +++ b/frame/assets/src/lib.rs @@ -134,7 +134,7 @@ #![cfg_attr(not(feature = "std"), no_std)] use frame_support::{Parameter, decl_module, decl_event, decl_storage, decl_error, ensure}; -use sp_runtime::traits::{Member, AtLeast32Bit, Zero, StaticLookup}; +use sp_runtime::traits::{Member, AtLeast32Bit, AtLeast32BitUnsigned, Zero, StaticLookup}; use frame_system::{self as system, ensure_signed}; use sp_runtime::traits::One; @@ -144,7 +144,7 @@ pub trait Trait: frame_system::Trait { type Event: From> + Into<::Event>; /// The units in which we record balances. - type Balance: Member + Parameter + AtLeast32Bit + Default + Copy; + type Balance: Member + Parameter + AtLeast32BitUnsigned + Default + Copy; /// The arithmetic type of asset identifier. type AssetId: Parameter + AtLeast32Bit + Default + Copy; @@ -257,6 +257,8 @@ decl_storage! { /// The next asset identifier up for grabs. NextAssetId get(fn next_asset_id): T::AssetId; /// The total unit supply of an asset. + /// + /// TWOX-NOTE: `AssetId` is trusted, so this is safe. TotalSupply: map hasher(twox_64_concat) T::AssetId => T::Balance; } } @@ -302,6 +304,7 @@ mod tests { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type Call = (); @@ -317,6 +320,7 @@ mod tests { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); diff --git a/frame/atomic-swap/Cargo.toml b/frame/atomic-swap/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..829c40b675c35945eb58ed8a73aaa08050ed405e --- /dev/null +++ b/frame/atomic-swap/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "pallet-atomic-swap" +version = "2.0.0-rc4" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME atomic swap pallet" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +serde = { version = "1.0.101", optional = true } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } + +[dev-dependencies] +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } + +[features] +default = ["std"] +std = [ + "serde", + "codec/std", + "frame-support/std", + "frame-system/std", + "sp-runtime/std", + "sp-std/std", + "sp-io/std", + "sp-core/std", +] diff --git a/frame/atomic-swap/src/lib.rs b/frame/atomic-swap/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..56aa67310fb48f985e7344e034134c1ced659f1f --- /dev/null +++ b/frame/atomic-swap/src/lib.rs @@ -0,0 +1,327 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Atomic Swap +//! +//! A module for atomically sending funds. +//! +//! - [`atomic_swap::Trait`](./trait.Trait.html) +//! - [`Call`](./enum.Call.html) +//! - [`Module`](./struct.Module.html) +//! +//! ## Overview +//! +//! A module for atomically sending funds from an origin to a target. A proof +//! is used to allow the target to approve (claim) the swap. If the swap is not +//! claimed within a specified duration of time, the sender may cancel it. +//! +//! ## Interface +//! +//! ### Dispatchable Functions +//! +//! * `create_swap` - called by a sender to register a new atomic swap +//! * `claim_swap` - called by the target to approve a swap +//! * `cancel_swap` - may be called by a sender after a specified duration + +// Ensure we're `no_std` when compiling for Wasm. +#![cfg_attr(not(feature = "std"), no_std)] + +mod tests; + +use sp_std::{prelude::*, marker::PhantomData, ops::{Deref, DerefMut}}; +use sp_io::hashing::blake2_256; +use frame_support::{ + Parameter, decl_module, decl_storage, decl_event, decl_error, ensure, + traits::{Get, Currency, ReservableCurrency, BalanceStatus}, + weights::Weight, + dispatch::DispatchResult, +}; +use frame_system::{self as system, ensure_signed}; +use codec::{Encode, Decode}; +use sp_runtime::RuntimeDebug; + +/// Pending atomic swap operation. +#[derive(Clone, Eq, PartialEq, RuntimeDebug, Encode, Decode)] +pub struct PendingSwap { + /// Source of the swap. + pub source: T::AccountId, + /// Action of this swap. + pub action: T::SwapAction, + /// End block of the lock. + pub end_block: T::BlockNumber, +} + +/// Hashed proof type. +pub type HashedProof = [u8; 32]; + +/// Definition of a pending atomic swap action. It contains the following three phrases: +/// +/// - **Reserve**: reserve the resources needed for a swap. This is to make sure that **Claim** +/// succeeds with best efforts. +/// - **Claim**: claim any resources reserved in the first phrase. +/// - **Cancel**: cancel any resources reserved in the first phrase. +pub trait SwapAction { + /// Reserve the resources needed for the swap, from the given `source`. The reservation is + /// allowed to fail. If that is the case, the the full swap creation operation is cancelled. + fn reserve(&self, source: &T::AccountId) -> DispatchResult; + /// Claim the reserved resources, with `source` and `target`. Returns whether the claim + /// succeeds. + fn claim(&self, source: &T::AccountId, target: &T::AccountId) -> bool; + /// Weight for executing the operation. + fn weight(&self) -> Weight; + /// Cancel the resources reserved in `source`. + fn cancel(&self, source: &T::AccountId); +} + +/// A swap action that only allows transferring balances. +#[derive(Clone, RuntimeDebug, Eq, PartialEq, Encode, Decode)] +pub struct BalanceSwapAction> { + value: ::AccountId>>::Balance, + _marker: PhantomData, +} + +impl BalanceSwapAction where + C: ReservableCurrency, +{ + /// Create a new swap action value of balance. + pub fn new(value: ::AccountId>>::Balance) -> Self { + Self { value, _marker: PhantomData } + } +} + +impl Deref for BalanceSwapAction where + C: ReservableCurrency, +{ + type Target = ::AccountId>>::Balance; + + fn deref(&self) -> &Self::Target { + &self.value + } +} + +impl DerefMut for BalanceSwapAction where + C: ReservableCurrency, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.value + } +} + +impl SwapAction for BalanceSwapAction where + C: ReservableCurrency, +{ + fn reserve(&self, source: &T::AccountId) -> DispatchResult { + C::reserve(&source, self.value) + } + + fn claim(&self, source: &T::AccountId, target: &T::AccountId) -> bool { + C::repatriate_reserved(source, target, self.value, BalanceStatus::Free).is_ok() + } + + fn weight(&self) -> Weight { + T::DbWeight::get().reads_writes(1, 1) + } + + fn cancel(&self, source: &T::AccountId) { + C::unreserve(source, self.value); + } +} + +/// Atomic swap's pallet configuration trait. +pub trait Trait: frame_system::Trait { + /// The overarching event type. + type Event: From> + Into<::Event>; + /// Swap action. + type SwapAction: SwapAction + Parameter; + /// Limit of proof size. + /// + /// Atomic swap is only atomic if once the proof is revealed, both parties can submit the proofs + /// on-chain. If A is the one that generates the proof, then it requires that either: + /// - A's blockchain has the same proof length limit as B's blockchain. + /// - Or A's blockchain has shorter proof length limit as B's blockchain. + /// + /// If B sees A is on a blockchain with larger proof length limit, then it should kindly refuse + /// to accept the atomic swap request if A generates the proof, and asks that B generates the + /// proof instead. + type ProofLimit: Get; +} + +decl_storage! { + trait Store for Module as AtomicSwap { + pub PendingSwaps: double_map + hasher(twox_64_concat) T::AccountId, hasher(blake2_128_concat) HashedProof + => Option>; + } +} + +decl_error! { + pub enum Error for Module { + /// Swap already exists. + AlreadyExist, + /// Swap proof is invalid. + InvalidProof, + /// Proof is too large. + ProofTooLarge, + /// Source does not match. + SourceMismatch, + /// Swap has already been claimed. + AlreadyClaimed, + /// Swap does not exist. + NotExist, + /// Claim action mismatch. + ClaimActionMismatch, + /// Duration has not yet passed for the swap to be cancelled. + DurationNotPassed, + } +} + +decl_event!( + /// Event of atomic swap pallet. + pub enum Event where + AccountId = ::AccountId, + PendingSwap = PendingSwap, + { + /// Swap created. + NewSwap(AccountId, HashedProof, PendingSwap), + /// Swap claimed. The last parameter indicates whether the execution succeeds. + SwapClaimed(AccountId, HashedProof, bool), + /// Swap cancelled. + SwapCancelled(AccountId, HashedProof), + } +); + +decl_module! { + /// Module definition of atomic swap pallet. + pub struct Module for enum Call where origin: T::Origin { + type Error = Error; + + fn deposit_event() = default; + + /// Register a new atomic swap, declaring an intention to send funds from origin to target + /// on the current blockchain. The target can claim the fund using the revealed proof. If + /// the fund is not claimed after `duration` blocks, then the sender can cancel the swap. + /// + /// The dispatch origin for this call must be _Signed_. + /// + /// - `target`: Receiver of the atomic swap. + /// - `hashed_proof`: The blake2_256 hash of the secret proof. + /// - `balance`: Funds to be sent from origin. + /// - `duration`: Locked duration of the atomic swap. For safety reasons, it is recommended + /// that the revealer uses a shorter duration than the counterparty, to prevent the + /// situation where the revealer reveals the proof too late around the end block. + #[weight = T::DbWeight::get().reads_writes(1, 1).saturating_add(40_000_000)] + fn create_swap( + origin, + target: T::AccountId, + hashed_proof: HashedProof, + action: T::SwapAction, + duration: T::BlockNumber, + ) { + let source = ensure_signed(origin)?; + ensure!( + !PendingSwaps::::contains_key(&target, hashed_proof), + Error::::AlreadyExist + ); + + action.reserve(&source)?; + + let swap = PendingSwap { + source, + action, + end_block: frame_system::Module::::block_number() + duration, + }; + PendingSwaps::::insert(target.clone(), hashed_proof.clone(), swap.clone()); + + Self::deposit_event( + RawEvent::NewSwap(target, hashed_proof, swap) + ); + } + + /// Claim an atomic swap. + /// + /// The dispatch origin for this call must be _Signed_. + /// + /// - `proof`: Revealed proof of the claim. + /// - `action`: Action defined in the swap, it must match the entry in blockchain. Otherwise + /// the operation fails. This is used for weight calculation. + #[weight = T::DbWeight::get().reads_writes(1, 1) + .saturating_add(40_000_000) + .saturating_add((proof.len() as Weight).saturating_mul(100)) + .saturating_add(action.weight()) + ] + fn claim_swap( + origin, + proof: Vec, + action: T::SwapAction, + ) -> DispatchResult { + ensure!( + proof.len() <= T::ProofLimit::get() as usize, + Error::::ProofTooLarge, + ); + + let target = ensure_signed(origin)?; + let hashed_proof = blake2_256(&proof); + + let swap = PendingSwaps::::get(&target, hashed_proof) + .ok_or(Error::::InvalidProof)?; + ensure!(swap.action == action, Error::::ClaimActionMismatch); + + let succeeded = swap.action.claim(&swap.source, &target); + + PendingSwaps::::remove(target.clone(), hashed_proof.clone()); + + Self::deposit_event( + RawEvent::SwapClaimed(target, hashed_proof, succeeded) + ); + + Ok(()) + } + + /// Cancel an atomic swap. Only possible after the originally set duration has passed. + /// + /// The dispatch origin for this call must be _Signed_. + /// + /// - `target`: Target of the original atomic swap. + /// - `hashed_proof`: Hashed proof of the original atomic swap. + #[weight = T::DbWeight::get().reads_writes(1, 1).saturating_add(40_000_000)] + fn cancel_swap( + origin, + target: T::AccountId, + hashed_proof: HashedProof, + ) { + let source = ensure_signed(origin)?; + + let swap = PendingSwaps::::get(&target, hashed_proof) + .ok_or(Error::::NotExist)?; + ensure!( + swap.source == source, + Error::::SourceMismatch, + ); + ensure!( + frame_system::Module::::block_number() >= swap.end_block, + Error::::DurationNotPassed, + ); + + swap.action.cancel(&swap.source); + PendingSwaps::::remove(&target, hashed_proof.clone()); + + Self::deposit_event( + RawEvent::SwapCancelled(target, hashed_proof) + ); + } + } +} diff --git a/frame/atomic-swap/src/tests.rs b/frame/atomic-swap/src/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..d04ffab2052835024e4a12e818e4384804a0625b --- /dev/null +++ b/frame/atomic-swap/src/tests.rs @@ -0,0 +1,157 @@ +#![cfg(test)] + +use super::*; + +use frame_support::{ + impl_outer_origin, parameter_types, weights::Weight, +}; +use sp_core::H256; +// The testing primitives are very useful for avoiding having to work with signatures +// or public keys. `u64` is used as the `AccountId` and no `Signature`s are required. +use sp_runtime::{ + Perbill, + testing::Header, + traits::{BlakeTwo256, IdentityLookup}, +}; + +impl_outer_origin! { + pub enum Origin for Test where system = frame_system {} +} + +// For testing the pallet, we construct most of a mock runtime. This means +// first constructing a configuration type (`Test`) which `impl`s each of the +// configuration traits of pallets we want to use. +#[derive(Clone, Eq, Debug, PartialEq)] +pub struct Test; +parameter_types! { + pub const BlockHashCount: u64 = 250; + pub const MaximumBlockWeight: Weight = 1024; + pub const MaximumBlockLength: u32 = 2 * 1024; + pub const AvailableBlockRatio: Perbill = Perbill::one(); +} +impl frame_system::Trait for Test { + type BaseCallFilter = (); + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Call = (); + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = (); + type BlockHashCount = BlockHashCount; + type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; + type MaximumBlockLength = MaximumBlockLength; + type AvailableBlockRatio = AvailableBlockRatio; + type Version = (); + type ModuleToIndex = (); + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); +} +parameter_types! { + pub const ExistentialDeposit: u64 = 1; +} +impl pallet_balances::Trait for Test { + type Balance = u64; + type DustRemoval = (); + type Event = (); + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; +} +parameter_types! { + pub const ProofLimit: u32 = 1024; + pub const ExpireDuration: u64 = 100; +} +impl Trait for Test { + type Event = (); + type SwapAction = BalanceSwapAction; + type ProofLimit = ProofLimit; +} +type System = frame_system::Module; +type Balances = pallet_balances::Module; +type AtomicSwap = Module; + +const A: u64 = 1; +const B: u64 = 2; + +pub fn new_test_ext() -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let genesis = pallet_balances::GenesisConfig:: { + balances: vec![ + (A, 100), + (B, 200), + ], + }; + genesis.assimilate_storage(&mut t).unwrap(); + t.into() +} + +#[test] +fn two_party_successful_swap() { + let mut chain1 = new_test_ext(); + let mut chain2 = new_test_ext(); + + // A generates a random proof. Keep it secret. + let proof: [u8; 2] = [4, 2]; + // The hashed proof is the blake2_256 hash of the proof. This is public. + let hashed_proof = blake2_256(&proof); + + // A creates the swap on chain1. + chain1.execute_with(|| { + AtomicSwap::create_swap( + Origin::signed(A), + B, + hashed_proof.clone(), + BalanceSwapAction::new(50), + 1000, + ).unwrap(); + + assert_eq!(Balances::free_balance(A), 100 - 50); + assert_eq!(Balances::free_balance(B), 200); + }); + + // B creates the swap on chain2. + chain2.execute_with(|| { + AtomicSwap::create_swap( + Origin::signed(B), + A, + hashed_proof.clone(), + BalanceSwapAction::new(75), + 1000, + ).unwrap(); + + assert_eq!(Balances::free_balance(A), 100); + assert_eq!(Balances::free_balance(B), 200 - 75); + }); + + // A reveals the proof and claims the swap on chain2. + chain2.execute_with(|| { + AtomicSwap::claim_swap( + Origin::signed(A), + proof.to_vec(), + BalanceSwapAction::new(75), + ).unwrap(); + + assert_eq!(Balances::free_balance(A), 100 + 75); + assert_eq!(Balances::free_balance(B), 200 - 75); + }); + + // B use the revealed proof to claim the swap on chain1. + chain1.execute_with(|| { + AtomicSwap::claim_swap( + Origin::signed(B), + proof.to_vec(), + BalanceSwapAction::new(50), + ).unwrap(); + + assert_eq!(Balances::free_balance(A), 100 - 50); + assert_eq!(Balances::free_balance(B), 200 + 50); + }); +} diff --git a/frame/aura/Cargo.toml b/frame/aura/Cargo.toml index f052f8c7e4fbd0fb6a7854019c011ce98d0d73b9..a648be5f10b17048e7fa1408b704ffe6609a7bb8 100644 --- a/frame/aura/Cargo.toml +++ b/frame/aura/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-aura" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,23 +12,23 @@ description = "FRAME AURA consensus pallet" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/application-crypto" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/inherents" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } +sp-application-crypto = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/application-crypto" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/inherents" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } serde = { version = "1.0.101", optional = true } -pallet-session = { version = "2.0.0-alpha.8", default-features = false, path = "../session" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.8"} -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -sp-consensus-aura = { path = "../../primitives/consensus/aura", default-features = false, version = "0.8.0-alpha.8"} -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -sp-timestamp = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/timestamp" } -pallet-timestamp = { version = "2.0.0-alpha.8", default-features = false, path = "../timestamp" } +pallet-session = { version = "2.0.0-rc4", default-features = false, path = "../session" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +sp-consensus-aura = { version = "0.8.0-rc4", path = "../../primitives/consensus/aura", default-features = false } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +sp-timestamp = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/timestamp" } +pallet-timestamp = { version = "2.0.0-rc4", default-features = false, path = "../timestamp" } [dev-dependencies] +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-io ={ version = "2.0.0-rc4", path = "../../primitives/io" } lazy_static = "1.4.0" parking_lot = "0.10.0" @@ -38,8 +38,6 @@ std = [ "sp-application-crypto/std", "codec/std", "sp-inherents/std", - "sp-io/std", - "sp-core/std", "sp-std/std", "serde", "sp-runtime/std", diff --git a/frame/aura/src/lib.rs b/frame/aura/src/lib.rs index d124ef0017e5ff292abb927e3c6983a1f10c48b6..ca3d1f15f421bfae638a69a45f012a76e407e54a 100644 --- a/frame/aura/src/lib.rs +++ b/frame/aura/src/lib.rs @@ -162,6 +162,27 @@ impl FindAuthor for Module { } } +/// We can not implement `FindAuthor` twice, because the compiler does not know if +/// `u32 == T::AuthorityId` and thus, prevents us to implement the trait twice. +#[doc(hidden)] +pub struct FindAccountFromAuthorIndex(sp_std::marker::PhantomData<(T, Inner)>); + +impl> FindAuthor + for FindAccountFromAuthorIndex +{ + fn find_author<'a, I>(digests: I) -> Option + where I: 'a + IntoIterator + { + let i = Inner::find_author(digests)?; + + let validators = >::authorities(); + validators.get(i as usize).map(|k| k.clone()) + } +} + +/// Find the authority ID of the Aura authority who authored the current block. +pub type AuraAuthorId = FindAccountFromAuthorIndex>; + impl IsMember for Module { fn is_member(authority_id: &T::AuthorityId) -> bool { Self::authorities() diff --git a/frame/aura/src/mock.rs b/frame/aura/src/mock.rs index db4448bdeff105ff8a7de82bbf56fb73a7e44be9..db2c86492f5eb8e8e51a96cbd640bea11e293c7b 100644 --- a/frame/aura/src/mock.rs +++ b/frame/aura/src/mock.rs @@ -46,6 +46,7 @@ parameter_types! { } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -61,6 +62,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); diff --git a/frame/authority-discovery/Cargo.toml b/frame/authority-discovery/Cargo.toml index a9f352a8bcfbd914531fd78017f83215d3dd75ea..3270437ce8bd2786346795b4949c8bbdaa0c9041 100644 --- a/frame/authority-discovery/Cargo.toml +++ b/frame/authority-discovery/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-authority-discovery" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,20 +12,20 @@ description = "FRAME pallet for authority discovery" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-authority-discovery = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/authority-discovery" } -sp-application-crypto = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/application-crypto" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } +sp-authority-discovery = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/authority-discovery" } +sp-application-crypto = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/application-crypto" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } serde = { version = "1.0.101", optional = true } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -pallet-session = { version = "2.0.0-alpha.8", features = ["historical" ], path = "../session", default-features = false } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +pallet-session = { version = "2.0.0-rc4", features = ["historical" ], path = "../session", default-features = false } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } [dev-dependencies] -sp-staking = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/staking" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-io = { version = "2.0.0-rc4", path = "../../primitives/io" } +sp-staking = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/staking" } [features] default = ["std"] @@ -33,8 +33,6 @@ std = [ "sp-application-crypto/std", "sp-authority-discovery/std", "codec/std", - "sp-core/std", - "sp-io/std", "sp-std/std", "serde", "pallet-session/std", diff --git a/frame/authority-discovery/src/lib.rs b/frame/authority-discovery/src/lib.rs index 022ce362811da02b3325bc5e3a42992a4e1aedb6..f6008c9719d748905a8436ba021cddc84ce78b28 100644 --- a/frame/authority-discovery/src/lib.rs +++ b/frame/authority-discovery/src/lib.rs @@ -143,6 +143,7 @@ mod tests { } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = BlockNumber; @@ -158,6 +159,7 @@ mod tests { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); diff --git a/frame/authorship/Cargo.toml b/frame/authorship/Cargo.toml index 46ce41e1483435ed35341fa898593a078763cb91..08114eb4016f14af32c22100a9cc5ddfe05df6c2 100644 --- a/frame/authorship/Cargo.toml +++ b/frame/authorship/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-authorship" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" description = "Block and Uncle Author tracking for the FRAME" authors = ["Parity Technologies "] edition = "2018" @@ -12,27 +12,27 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/inherents" } -sp-authorship = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/authorship" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.8"} +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/inherents" } +sp-authorship = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/authorship" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } impl-trait-for-tuples = "0.1.3" +[dev-dependencies] +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-io ={ version = "2.0.0-rc4", path = "../../primitives/io" } + [features] default = ["std"] std = [ "codec/std", - "sp-core/std", "sp-inherents/std", "sp-runtime/std", "sp-std/std", "frame-support/std", "frame-system/std", - "sp-io/std", "sp-authorship/std", ] diff --git a/frame/authorship/src/lib.rs b/frame/authorship/src/lib.rs index 7605e780978f408b007d25a7fd3ba6c7c155290f..3023f8a2d399b51eaad70cc72a7578f3c7a2563f 100644 --- a/frame/authorship/src/lib.rs +++ b/frame/authorship/src/lib.rs @@ -418,6 +418,7 @@ mod tests { } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -433,6 +434,7 @@ mod tests { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); diff --git a/frame/babe/Cargo.toml b/frame/babe/Cargo.toml index 3340f7a71e33bfd453fd0b3d72833b1dac7a185f..845acce5f24277576982004e805a508c31c0b1bc 100644 --- a/frame/babe/Cargo.toml +++ b/frame/babe/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-babe" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,30 +12,30 @@ description = "Consensus extension module for BABE consensus. Collects on-chain targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true } -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/inherents" } -sp-application-crypto = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/application-crypto" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-staking = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/staking" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -pallet-timestamp = { version = "2.0.0-alpha.8", default-features = false, path = "../timestamp" } -sp-timestamp = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/timestamp" } -pallet-session = { version = "2.0.0-alpha.8", default-features = false, path = "../session" } -sp-consensus-babe = { version = "0.8.0-alpha.8", default-features = false, path = "../../primitives/consensus/babe" } -sp-consensus-vrf = { version = "0.8.0-alpha.8", default-features = false, path = "../../primitives/consensus/vrf" } -sp-io = { path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.8"} +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/inherents" } +sp-application-crypto = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/application-crypto" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/staking" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +pallet-timestamp = { version = "2.0.0-rc4", default-features = false, path = "../timestamp" } +sp-timestamp = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/timestamp" } +pallet-session = { version = "2.0.0-rc4", default-features = false, path = "../session" } +sp-consensus-babe = { version = "0.8.0-rc4", default-features = false, path = "../../primitives/consensus/babe" } +sp-consensus-vrf = { version = "0.8.0-rc4", default-features = false, path = "../../primitives/consensus/vrf" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } [features] default = ["std"] std = [ - "serde", "codec/std", + "serde", "sp-std/std", "sp-application-crypto/std", "frame-support/std", diff --git a/frame/babe/src/lib.rs b/frame/babe/src/lib.rs index 6d32c2224935cb918cac8dab12f895684d6c2c8a..91421739327ab0e885da3af3d86491ff1210132f 100644 --- a/frame/babe/src/lib.rs +++ b/frame/babe/src/lib.rs @@ -42,7 +42,7 @@ use sp_inherents::{InherentIdentifier, InherentData, ProvideInherent, MakeFatalE use sp_consensus_babe::{ BABE_ENGINE_ID, ConsensusLog, BabeAuthorityWeight, SlotNumber, inherents::{INHERENT_IDENTIFIER, BabeInherentData}, - digests::{NextEpochDescriptor, PreDigest}, + digests::{NextEpochDescriptor, NextConfigDescriptor, PreDigest}, }; use sp_consensus_vrf::schnorrkel; pub use sp_consensus_babe::{AuthorityId, VRF_OUTPUT_LENGTH, RANDOMNESS_LENGTH, PUBLIC_KEY_LENGTH}; @@ -136,6 +136,9 @@ decl_storage! { // variable to its underlying value. pub Randomness get(fn randomness): schnorrkel::Randomness; + /// Next epoch configuration, if changed. + NextEpochConfig: Option; + /// Next epoch randomness. NextRandomness: schnorrkel::Randomness; @@ -149,6 +152,8 @@ decl_storage! { /// We reset all segments and return to `0` at the beginning of every /// epoch. SegmentIndex build(|_| 0): u32; + + /// TWOX-NOTE: `SegmentIndex` is an increasing integer, so this is okay. UnderConstruction: map hasher(twox_64_concat) u32 => Vec; /// Temporary value (cleared at block finalization) which is `Some` @@ -364,6 +369,15 @@ impl Module { }) } + /// Plan an epoch config change. The epoch config change is recorded and will be enacted on the + /// next call to `enact_epoch_change`. The config will be activated one epoch after. Multiple calls to this + /// method will replace any existing planned config change that had not been enacted yet. + pub fn plan_config_change( + config: NextConfigDescriptor, + ) { + NextEpochConfig::put(config); + } + /// DANGEROUS: Enact an epoch change. Should be done on every block where `should_epoch_change` has returned `true`, /// and the caller is the only caller of this function. /// @@ -399,12 +413,15 @@ impl Module { // so that nodes can track changes. let next_randomness = NextRandomness::get(); - let next = NextEpochDescriptor { + let next_epoch = NextEpochDescriptor { authorities: next_authorities, randomness: next_randomness, }; + Self::deposit_consensus(ConsensusLog::NextEpochData(next_epoch)); - Self::deposit_consensus(ConsensusLog::NextEpochData(next)) + if let Some(next_config) = NextEpochConfig::take() { + Self::deposit_consensus(ConsensusLog::NextConfigData(next_config)); + } } // finds the start slot of the current epoch. only guaranteed to diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index bdeb284887d5a0138bb8de758ddeb701f0a49522..b977ea90448f32f0d5afe46a70b92d5670e34932 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -57,6 +57,7 @@ parameter_types! { } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -73,6 +74,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type ModuleToIndex = (); diff --git a/frame/babe/src/tests.rs b/frame/babe/src/tests.rs index be2d3ed036e08c10e172b8a27a63919969c49dae..ecb3639fc573bd66230ada5ce052e966d4edc8b3 100644 --- a/frame/babe/src/tests.rs +++ b/frame/babe/src/tests.rs @@ -22,6 +22,7 @@ use mock::*; use frame_support::traits::OnFinalize; use pallet_session::ShouldEndSession; use sp_core::crypto::IsWrappedBy; +use sp_consensus_babe::AllowedSlots; use sp_consensus_vrf::schnorrkel::{VRFOutput, VRFProof}; const EMPTY_RANDOMNESS: [u8; 32] = [ @@ -150,3 +151,35 @@ fn can_predict_next_epoch_change() { assert_eq!(Babe::next_expected_epoch_change(System::block_number()), Some(5 + 2)); }) } + +#[test] +fn can_enact_next_config() { + new_test_ext(0).1.execute_with(|| { + assert_eq!(::EpochDuration::get(), 3); + // this sets the genesis slot to 6; + go_to_block(1, 6); + assert_eq!(Babe::genesis_slot(), 6); + assert_eq!(Babe::current_slot(), 6); + assert_eq!(Babe::epoch_index(), 0); + go_to_block(2, 7); + + Babe::plan_config_change(NextConfigDescriptor::V1 { + c: (1, 4), + allowed_slots: AllowedSlots::PrimarySlots, + }); + + progress_to_block(4); + Babe::on_finalize(9); + let header = System::finalize(); + + let consensus_log = sp_consensus_babe::ConsensusLog::NextConfigData( + sp_consensus_babe::digests::NextConfigDescriptor::V1 { + c: (1, 4), + allowed_slots: AllowedSlots::PrimarySlots, + } + ); + let consensus_digest = DigestItem::Consensus(BABE_ENGINE_ID, consensus_log.encode()); + + assert_eq!(header.digest.logs[2], consensus_digest.clone()) + }); +} diff --git a/frame/balances/Cargo.toml b/frame/balances/Cargo.toml index 29802a68d2908c62907f24c4ae7b769ced208bcf..88c8657d4744b28f739408cfab74dfed8c63e0d3 100644 --- a/frame/balances/Cargo.toml +++ b/frame/balances/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-balances" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,17 +13,17 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../benchmarking", optional = true } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -pallet-transaction-payment = { version = "2.0.0-alpha.8", path = "../transaction-payment" } +sp-io = { version = "2.0.0-rc4", path = "../../primitives/io" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +pallet-transaction-payment = { version = "2.0.0-rc4", path = "../transaction-payment" } [features] default = ["std"] @@ -31,7 +31,6 @@ std = [ "serde", "codec/std", "sp-std/std", - "sp-io/std", "sp-runtime/std", "frame-benchmarking/std", "frame-support/std", diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index d10909b73954fc68bd166db7167d69d50b31edce..62402c78630c7d3bad0a9d58abe2b129da74f178 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -170,7 +170,7 @@ use frame_support::{ use sp_runtime::{ RuntimeDebug, DispatchResult, DispatchError, traits::{ - Zero, AtLeast32Bit, StaticLookup, Member, CheckedAdd, CheckedSub, + Zero, AtLeast32BitUnsigned, StaticLookup, Member, CheckedAdd, CheckedSub, MaybeSerializeDeserialize, Saturating, Bounded, }, }; @@ -180,7 +180,7 @@ pub use self::imbalances::{PositiveImbalance, NegativeImbalance}; pub trait Subtrait: frame_system::Trait { /// The balance of an account. - type Balance: Parameter + Member + AtLeast32Bit + Codec + Default + Copy + + type Balance: Parameter + Member + AtLeast32BitUnsigned + Codec + Default + Copy + MaybeSerializeDeserialize + Debug; /// The minimum amount required to keep an account open. @@ -192,7 +192,7 @@ pub trait Subtrait: frame_system::Trait { pub trait Trait: frame_system::Trait { /// The balance of an account. - type Balance: Parameter + Member + AtLeast32Bit + Codec + Default + Copy + + type Balance: Parameter + Member + AtLeast32BitUnsigned + Codec + Default + Copy + MaybeSerializeDeserialize + Debug; /// Handler for the unbalanced reduction when removing a dust account. @@ -230,6 +230,13 @@ decl_event!( BalanceSet(AccountId, Balance, Balance), /// Some amount was deposited (e.g. for transaction fees). Deposit(AccountId, Balance), + /// Some balance was reserved (moved from free to reserved). + Reserved(AccountId, Balance), + /// Some balance was unreserved (moved from reserved to free). + Unreserved(AccountId, Balance), + /// Some balance was moved from the reserve of the first account to the second account. + /// Final argument indicates the destination balance type. + ReserveRepatriated(AccountId, AccountId, Balance, Status), } ); @@ -365,9 +372,6 @@ decl_storage! { /// The balance of an account. /// - /// NOTE: THIS MAY NEVER BE IN EXISTENCE AND YET HAVE A `total().is_zero()`. If the total - /// is ever zero, then the entry *MUST* be removed. - /// /// NOTE: This is only used in the case that this module is used to store balances. pub Account: map hasher(blake2_128_concat) T::AccountId => AccountData; @@ -384,10 +388,6 @@ decl_storage! { config(balances): Vec<(T::AccountId, T::Balance)>; // ^^ begin, length, amount liquid at genesis build(|config: &GenesisConfig| { - assert!( - >::ExistentialDeposit::get() > Zero::zero(), - "The existential deposit should be greater than zero." - ); for (_, balance) in &config.balances { assert!( *balance >= >::ExistentialDeposit::get(), @@ -605,11 +605,11 @@ impl, I: Instance> Module { /// /// NOTE: LOW-LEVEL: This will not attempt to maintain total issuance. It is expected that /// the caller will do this. - fn mutate_account( + pub fn mutate_account( who: &T::AccountId, f: impl FnOnce(&mut AccountData) -> R ) -> R { - Self::try_mutate_account(who, |a| -> Result { Ok(f(a)) }) + Self::try_mutate_account(who, |a, _| -> Result { Ok(f(a)) }) .expect("Error is infallible; qed") } @@ -624,13 +624,13 @@ impl, I: Instance> Module { /// the caller will do this. fn try_mutate_account( who: &T::AccountId, - f: impl FnOnce(&mut AccountData) -> Result + f: impl FnOnce(&mut AccountData, bool) -> Result ) -> Result { T::AccountStore::try_mutate_exists(who, |maybe_account| { + let is_new = maybe_account.is_none(); let mut account = maybe_account.take().unwrap_or_default(); - let was_zero = account.total().is_zero(); - f(&mut account).map(move |result| { - let maybe_endowed = if was_zero { Some(account.free) } else { None }; + f(&mut account, is_new).map(move |result| { + let maybe_endowed = if is_new { Some(account.free) } else { None }; *maybe_account = Self::post_mutation(who, account); (maybe_endowed, result) }) @@ -848,6 +848,7 @@ impl, I: Instance> PartialEq for ElevatedTrait { } impl, I: Instance> Eq for ElevatedTrait {} impl, I: Instance> frame_system::Trait for ElevatedTrait { + type BaseCallFilter = T::BaseCallFilter; type Origin = T::Origin; type Call = T::Call; type Index = T::Index; @@ -861,8 +862,9 @@ impl, I: Instance> frame_system::Trait for ElevatedTrait { type BlockHashCount = T::BlockHashCount; type MaximumBlockWeight = T::MaximumBlockWeight; type DbWeight = T::DbWeight; - type BlockExecutionWeight = (); - type ExtrinsicBaseWeight = (); + type BlockExecutionWeight = T::BlockExecutionWeight; + type ExtrinsicBaseWeight = T::ExtrinsicBaseWeight; + type MaximumExtrinsicWeight = T::MaximumBlockWeight; type MaximumBlockLength = T::MaximumBlockLength; type AvailableBlockRatio = T::AvailableBlockRatio; type Version = T::Version; @@ -965,8 +967,8 @@ impl, I: Instance> Currency for Module where ) -> DispatchResult { if value.is_zero() || transactor == dest { return Ok(()) } - Self::try_mutate_account(dest, |to_account| -> DispatchResult { - Self::try_mutate_account(transactor, |from_account| -> DispatchResult { + Self::try_mutate_account(dest, |to_account, _| -> DispatchResult { + Self::try_mutate_account(transactor, |from_account, _| -> DispatchResult { from_account.free = from_account.free.checked_sub(&value) .ok_or(Error::::InsufficientBalance)?; @@ -1037,8 +1039,8 @@ impl, I: Instance> Currency for Module where ) -> Result { if value.is_zero() { return Ok(PositiveImbalance::zero()) } - Self::try_mutate_account(who, |account| -> Result { - ensure!(!account.total().is_zero(), Error::::DeadAccount); + Self::try_mutate_account(who, |account, is_new| -> Result { + ensure!(!is_new, Error::::DeadAccount); account.free = account.free.checked_add(&value).ok_or(Error::::Overflow)?; Ok(PositiveImbalance::new(value)) }) @@ -1056,10 +1058,10 @@ impl, I: Instance> Currency for Module where ) -> Self::PositiveImbalance { if value.is_zero() { return Self::PositiveImbalance::zero() } - Self::try_mutate_account(who, |account| -> Result { + Self::try_mutate_account(who, |account, is_new| -> Result { // bail if not yet created and this operation wouldn't be enough to create it. let ed = T::ExistentialDeposit::get(); - ensure!(value >= ed || !account.total().is_zero(), Self::PositiveImbalance::zero()); + ensure!(value >= ed || !is_new, Self::PositiveImbalance::zero()); // defensive only: overflow should never happen, however in case it does, then this // operation is a no-op. @@ -1080,7 +1082,7 @@ impl, I: Instance> Currency for Module where ) -> result::Result { if value.is_zero() { return Ok(NegativeImbalance::zero()); } - Self::try_mutate_account(who, |account| + Self::try_mutate_account(who, |account, _| -> Result { let new_free_account = account.free.checked_sub(&value) @@ -1104,7 +1106,7 @@ impl, I: Instance> Currency for Module where fn make_free_balance_be(who: &T::AccountId, value: Self::Balance) -> SignedImbalance { - Self::try_mutate_account(who, |account| + Self::try_mutate_account(who, |account, is_new| -> Result, ()> { let ed = T::ExistentialDeposit::get(); @@ -1115,7 +1117,7 @@ impl, I: Instance> Currency for Module where // equal and opposite cause (returned as an Imbalance), then in the // instance that there's no other accounts on the system at all, we might // underflow the issuance and our arithmetic will be off. - ensure!(value.saturating_add(account.reserved) >= ed || !account.total().is_zero(), ()); + ensure!(value.saturating_add(account.reserved) >= ed || !is_new, ()); let imbalance = if account.free <= value { SignedImbalance::Positive(PositiveImbalance::new(value - account.free)) @@ -1153,11 +1155,14 @@ impl, I: Instance> ReservableCurrency for Module fn reserve(who: &T::AccountId, value: Self::Balance) -> DispatchResult { if value.is_zero() { return Ok(()) } - Self::try_mutate_account(who, |account| -> DispatchResult { + Self::try_mutate_account(who, |account, _| -> DispatchResult { account.free = account.free.checked_sub(&value).ok_or(Error::::InsufficientBalance)?; account.reserved = account.reserved.checked_add(&value).ok_or(Error::::Overflow)?; - Self::ensure_can_withdraw(who, value, WithdrawReason::Reserve.into(), account.free) - }) + Self::ensure_can_withdraw(&who, value.clone(), WithdrawReason::Reserve.into(), account.free) + })?; + + Self::deposit_event(RawEvent::Reserved(who.clone(), value)); + Ok(()) } /// Unreserve some funds, returning any amount that was unable to be unreserved. @@ -1166,14 +1171,17 @@ impl, I: Instance> ReservableCurrency for Module fn unreserve(who: &T::AccountId, value: Self::Balance) -> Self::Balance { if value.is_zero() { return Zero::zero() } - Self::mutate_account(who, |account| { + let actual = Self::mutate_account(who, |account| { let actual = cmp::min(account.reserved, value); account.reserved -= actual; // defensive only: this can never fail since total issuance which is at least free+reserved // fits into the same data type. account.free = account.free.saturating_add(actual); - value - actual - }) + actual + }); + + Self::deposit_event(RawEvent::Unreserved(who.clone(), actual.clone())); + value - actual } /// Slash from reserved balance, returning the negative imbalance created, @@ -1214,18 +1222,21 @@ impl, I: Instance> ReservableCurrency for Module }; } - Self::try_mutate_account(beneficiary, |to_account| -> Result { - ensure!(!to_account.total().is_zero(), Error::::DeadAccount); - Self::try_mutate_account(slashed, |from_account| -> Result { + let actual = Self::try_mutate_account(beneficiary, |to_account, is_new|-> Result { + ensure!(!is_new, Error::::DeadAccount); + Self::try_mutate_account(slashed, |from_account, _| -> Result { let actual = cmp::min(from_account.reserved, value); match status { Status::Free => to_account.free = to_account.free.checked_add(&actual).ok_or(Error::::Overflow)?, Status::Reserved => to_account.reserved = to_account.reserved.checked_add(&actual).ok_or(Error::::Overflow)?, } from_account.reserved -= actual; - Ok(value - actual) + Ok(actual) }) - }) + })?; + + Self::deposit_event(RawEvent::ReserveRepatriated(slashed.clone(), beneficiary.clone(), actual, status)); + Ok(value - actual) } } @@ -1236,7 +1247,14 @@ impl, I: Instance> ReservableCurrency for Module /// storage (which is the default in most examples and tests) then there's no need.** impl, I: Instance> OnKilledAccount for Module { fn on_killed_account(who: &T::AccountId) { - Account::::remove(who); + Account::::mutate_exists(who, |account| { + let total = account.as_ref().map(|acc| acc.total()).unwrap_or_default(); + if !total.is_zero() { + T::DustRemoval::on_unbalanced(NegativeImbalance::new(total)); + Self::deposit_event(RawEvent::DustLost(who.clone(), total)); + } + *account = None; + }); } } @@ -1256,12 +1274,9 @@ where ) { if amount.is_zero() || reasons.is_none() { return } let mut new_lock = Some(BalanceLock { id, amount, reasons: reasons.into() }); - let mut locks = Self::locks(who).into_iter().filter_map(|l| - if l.id == id { - new_lock.take() - } else { - Some(l) - }).collect::>(); + let mut locks = Self::locks(who).into_iter() + .filter_map(|l| if l.id == id { new_lock.take() } else { Some(l) }) + .collect::>(); if let Some(lock) = new_lock { locks.push(lock) } @@ -1310,7 +1325,7 @@ impl, I: Instance> IsDeadAccount for Module wher T::Balance: MaybeSerializeDeserialize + Debug { fn is_dead_account(who: &T::AccountId) -> bool { - // this should always be exactly equivalent to `Self::account(who).total().is_zero()` + // this should always be exactly equivalent to `Self::account(who).total().is_zero()` if ExistentialDeposit > 0 !T::AccountStore::is_explicit(who) } } diff --git a/frame/balances/src/tests.rs b/frame/balances/src/tests.rs index 9cfdc147b4a70502fff7b8f433b03f2f5e43e839..210c75631da6387b8f7386248059ff473f135d06 100644 --- a/frame/balances/src/tests.rs +++ b/frame/balances/src/tests.rs @@ -26,6 +26,7 @@ impl sp_runtime::traits::Dispatchable for CallWithDispatchInfo { type Trait = (); type Info = frame_support::weights::DispatchInfo; type PostInfo = frame_support::weights::PostDispatchInfo; + fn dispatch(self, _origin: Self::Origin) -> sp_runtime::DispatchResultWithInfo { panic!("Do not use dummy implementation for dispatch."); @@ -37,7 +38,7 @@ macro_rules! decl_tests { ($test:ty, $ext_builder:ty, $existential_deposit:expr) => { use crate::*; - use sp_runtime::{Fixed128, traits::{SignedExtension, BadOrigin}}; + use sp_runtime::{FixedPointNumber, traits::{SignedExtension, BadOrigin}}; use frame_support::{ assert_noop, assert_ok, assert_err, traits::{ @@ -45,7 +46,7 @@ macro_rules! decl_tests { Currency, ReservableCurrency, ExistenceRequirement::AllowDeath, StoredMap } }; - use pallet_transaction_payment::ChargeTransactionPayment; + use pallet_transaction_payment::{ChargeTransactionPayment, Multiplier}; use frame_system::RawOrigin; const ID_1: LockIdentifier = *b"1 "; @@ -61,6 +62,18 @@ macro_rules! decl_tests { DispatchInfo { weight: w, ..Default::default() } } + fn events() -> Vec { + let evt = System::events().into_iter().map(|evt| evt.event).collect::>(); + + System::reset_events(); + + evt + } + + fn last_event() -> Event { + system::Module::::events().pop().expect("Event expected").event + } + #[test] fn basic_locking_should_work() { <$ext_builder>::default().existential_deposit(1).monied(true).build().execute_with(|| { @@ -154,7 +167,7 @@ macro_rules! decl_tests { .monied(true) .build() .execute_with(|| { - pallet_transaction_payment::NextFeeMultiplier::put(Fixed128::from_natural(1)); + pallet_transaction_payment::NextFeeMultiplier::put(Multiplier::saturating_from_integer(1)); Balances::set_lock(ID_1, &1, 10, WithdrawReason::Reserve.into()); assert_noop!( >::transfer(&1, &2, 1, AllowDeath), @@ -162,7 +175,7 @@ macro_rules! decl_tests { ); assert_noop!( >::reserve(&1, 1), - Error::<$test, _>::LiquidityRestrictions + Error::<$test, _>::LiquidityRestrictions, ); assert!( as SignedExtension>::pre_dispatch( ChargeTransactionPayment::from(1), @@ -477,6 +490,10 @@ macro_rules! decl_tests { let _ = Balances::deposit_creating(&2, 1); assert_ok!(Balances::reserve(&1, 110)); assert_ok!(Balances::repatriate_reserved(&1, &2, 41, Status::Free), 0); + assert_eq!( + last_event(), + Event::balances(RawEvent::ReserveRepatriated(1, 2, 41, Status::Free)), + ); assert_eq!(Balances::reserved_balance(1), 69); assert_eq!(Balances::free_balance(1), 0); assert_eq!(Balances::reserved_balance(2), 0); @@ -674,5 +691,101 @@ macro_rules! decl_tests { assert_eq!(Balances::reserved_balance(1), 0); }); } + + #[test] + fn emit_events_with_reserve_and_unreserve() { + <$ext_builder>::default() + .build() + .execute_with(|| { + let _ = Balances::deposit_creating(&1, 100); + + System::set_block_number(2); + let _ = Balances::reserve(&1, 10); + + assert_eq!( + last_event(), + Event::balances(RawEvent::Reserved(1, 10)), + ); + + System::set_block_number(3); + let _ = Balances::unreserve(&1, 5); + + assert_eq!( + last_event(), + Event::balances(RawEvent::Unreserved(1, 5)), + ); + + System::set_block_number(4); + let _ = Balances::unreserve(&1, 6); + + // should only unreserve 5 + assert_eq!( + last_event(), + Event::balances(RawEvent::Unreserved(1, 5)), + ); + }); + } + + #[test] + fn emit_events_with_existential_deposit() { + <$ext_builder>::default() + .existential_deposit(100) + .build() + .execute_with(|| { + assert_ok!(Balances::set_balance(RawOrigin::Root.into(), 1, 100, 0)); + + assert_eq!( + events(), + [ + Event::system(system::RawEvent::NewAccount(1)), + Event::balances(RawEvent::Endowed(1, 100)), + Event::balances(RawEvent::BalanceSet(1, 100, 0)), + ] + ); + + let _ = Balances::slash(&1, 1); + + assert_eq!( + events(), + [ + Event::balances(RawEvent::DustLost(1, 99)), + Event::system(system::RawEvent::KilledAccount(1)) + ] + ); + }); + } + + #[test] + fn emit_events_with_no_existential_deposit_suicide() { + <$ext_builder>::default() + .existential_deposit(0) + .build() + .execute_with(|| { + assert_ok!(Balances::set_balance(RawOrigin::Root.into(), 1, 100, 0)); + + assert_eq!( + events(), + [ + Event::system(system::RawEvent::NewAccount(1)), + Event::balances(RawEvent::Endowed(1, 100)), + Event::balances(RawEvent::BalanceSet(1, 100, 0)), + ] + ); + + let _ = Balances::slash(&1, 100); + + // no events + assert_eq!(events(), []); + + assert_ok!(System::suicide(Origin::signed(1))); + + assert_eq!( + events(), + [ + Event::system(system::RawEvent::KilledAccount(1)) + ] + ); + }); + } } } diff --git a/frame/balances/src/tests_composite.rs b/frame/balances/src/tests_composite.rs index e78171376c16b1106f9873f0677abf32947f1b4a..81cb3449a823751b2f3dcbc27cab73e049fba997 100644 --- a/frame/balances/src/tests_composite.rs +++ b/frame/balances/src/tests_composite.rs @@ -21,14 +21,14 @@ use sp_runtime::{ Perbill, - traits::{ConvertInto, IdentityLookup}, + traits::IdentityLookup, testing::Header, }; use sp_core::H256; use sp_io; -use frame_support::{impl_outer_origin, parameter_types}; +use frame_support::{impl_outer_origin, impl_outer_event, parameter_types}; use frame_support::traits::Get; -use frame_support::weights::{Weight, DispatchInfo}; +use frame_support::weights::{Weight, DispatchInfo, IdentityFee}; use std::cell::RefCell; use crate::{GenesisConfig, Module, Trait, decl_tests, tests::CallWithDispatchInfo}; @@ -37,6 +37,17 @@ impl_outer_origin!{ pub enum Origin for Test {} } +mod balances { + pub use crate::Event; +} + +impl_outer_event! { + pub enum Event for Test { + system, + balances, + } +} + thread_local! { static EXISTENTIAL_DEPOSIT: RefCell = RefCell::new(0); } @@ -56,6 +67,7 @@ parameter_types! { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -65,12 +77,13 @@ impl frame_system::Trait for Test { type AccountId = u64; type Lookup = IdentityLookup; type Header = Header; - type Event = (); + type Event = Event; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -86,13 +99,13 @@ impl pallet_transaction_payment::Trait for Test { type Currency = Module; type OnTransactionPayment = (); type TransactionByteFee = TransactionByteFee; - type WeightToFee = ConvertInto; + type WeightToFee = IdentityFee; type FeeMultiplierUpdate = (); } impl Trait for Test { type Balance = u64; type DustRemoval = (); - type Event = (); + type Event = Event; type ExistentialDeposit = ExistentialDeposit; type AccountStore = system::Module; } @@ -137,7 +150,10 @@ impl ExtBuilder { vec![] }, }.assimilate_storage(&mut t).unwrap(); - t.into() + + let mut ext = sp_io::TestExternalities::new(t); + ext.execute_with(|| System::set_block_number(1)); + ext } } diff --git a/frame/balances/src/tests_local.rs b/frame/balances/src/tests_local.rs index 736afe9cbada6dbd17234fd90f9775a089a79a49..54ab22af33c6cf1648abfdd7258026ae42554d3c 100644 --- a/frame/balances/src/tests_local.rs +++ b/frame/balances/src/tests_local.rs @@ -21,14 +21,14 @@ use sp_runtime::{ Perbill, - traits::{ConvertInto, IdentityLookup}, + traits::IdentityLookup, testing::Header, }; use sp_core::H256; use sp_io; -use frame_support::{impl_outer_origin, parameter_types}; +use frame_support::{impl_outer_origin, impl_outer_event, parameter_types}; use frame_support::traits::{Get, StorageMapShim}; -use frame_support::weights::{Weight, DispatchInfo}; +use frame_support::weights::{Weight, DispatchInfo, IdentityFee}; use std::cell::RefCell; use crate::{GenesisConfig, Module, Trait, decl_tests, tests::CallWithDispatchInfo}; @@ -37,6 +37,17 @@ impl_outer_origin!{ pub enum Origin for Test {} } +mod balances { + pub use crate::Event; +} + +impl_outer_event! { + pub enum Event for Test { + system, + balances, + } +} + thread_local! { static EXISTENTIAL_DEPOSIT: RefCell = RefCell::new(0); } @@ -56,6 +67,7 @@ parameter_types! { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -65,12 +77,13 @@ impl frame_system::Trait for Test { type AccountId = u64; type Lookup = IdentityLookup; type Header = Header; - type Event = (); + type Event = Event; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -86,13 +99,13 @@ impl pallet_transaction_payment::Trait for Test { type Currency = Module; type OnTransactionPayment = (); type TransactionByteFee = TransactionByteFee; - type WeightToFee = ConvertInto; + type WeightToFee = IdentityFee; type FeeMultiplierUpdate = (); } impl Trait for Test { type Balance = u64; type DustRemoval = (); - type Event = (); + type Event = Event; type ExistentialDeposit = ExistentialDeposit; type AccountStore = StorageMapShim< super::Account, @@ -145,8 +158,45 @@ impl ExtBuilder { vec![] }, }.assimilate_storage(&mut t).unwrap(); - t.into() + + let mut ext = sp_io::TestExternalities::new(t); + ext.execute_with(|| System::set_block_number(1)); + ext } } decl_tests!{ Test, ExtBuilder, EXISTENTIAL_DEPOSIT } + +#[test] +fn emit_events_with_no_existential_deposit_suicide_with_dust() { + ::default() + .existential_deposit(0) + .build() + .execute_with(|| { + assert_ok!(Balances::set_balance(RawOrigin::Root.into(), 1, 100, 0)); + + assert_eq!( + events(), + [ + Event::system(system::RawEvent::NewAccount(1)), + Event::balances(RawEvent::Endowed(1, 100)), + Event::balances(RawEvent::BalanceSet(1, 100, 0)), + ] + ); + + let _ = Balances::slash(&1, 99); + + // no events + assert_eq!(events(), []); + + assert_ok!(System::suicide(Origin::signed(1))); + + assert_eq!( + events(), + [ + Event::balances(RawEvent::DustLost(1, 1)), + Event::system(system::RawEvent::KilledAccount(1)) + ] + ); + }); +} diff --git a/frame/benchmark/Cargo.toml b/frame/benchmark/Cargo.toml index 1c564241514daa3d22a02d9d5810ddda55cc3077..79d9a8d7711e0c06a9c7ca1994909416e8cee51d 100644 --- a/frame/benchmark/Cargo.toml +++ b/frame/benchmark/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-benchmark" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -14,12 +14,12 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../benchmarking", optional = true } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } [features] default = ["std"] diff --git a/frame/benchmarking/Cargo.toml b/frame/benchmarking/Cargo.toml index b963b73483a9f54d102a804de7acdf425af026f3..0823ec626cb4f3b2b6b22895bc55c277cf75ab1c 100644 --- a/frame/benchmarking/Cargo.toml +++ b/frame/benchmarking/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-benchmarking" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -14,14 +14,14 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] linregress = "0.1" paste = "0.1" -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-api = { version = "2.0.0-alpha.8", path = "../../primitives/api", default-features = false } -sp-runtime-interface = { version = "2.0.0-alpha.8", path = "../../primitives/runtime-interface", default-features = false } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime", default-features = false } -sp-std = { version = "2.0.0-alpha.8", path = "../../primitives/std", default-features = false } -sp-io = { path = "../../primitives/io", default-features = false, version = "2.0.0-alpha.8"} -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +sp-api = { version = "2.0.0-rc4", path = "../../primitives/api", default-features = false } +sp-runtime-interface = { version = "2.0.0-rc4", path = "../../primitives/runtime-interface", default-features = false } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime", default-features = false } +sp-std = { version = "2.0.0-rc4", path = "../../primitives/std", default-features = false } +sp-io = { version = "2.0.0-rc4", path = "../../primitives/io", default-features = false } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } [features] default = [ "std" ] diff --git a/frame/benchmarking/src/analysis.rs b/frame/benchmarking/src/analysis.rs index 04464309755505a29702bc3fb2a32cbc8563a54b..621f3a2941ff5a1c59525b2003c167e7e8b81059 100644 --- a/frame/benchmarking/src/analysis.rs +++ b/frame/benchmarking/src/analysis.rs @@ -29,24 +29,40 @@ pub struct Analysis { model: Option, } +pub enum BenchmarkSelector { + ExtrinsicTime, + StorageRootTime, + Reads, + Writes, +} + impl Analysis { - pub fn median_slopes(r: &Vec) -> Option { - let results = r[0].0.iter().enumerate().map(|(i, &(param, _))| { + pub fn median_slopes(r: &Vec, selector: BenchmarkSelector) -> Option { + let results = r[0].components.iter().enumerate().map(|(i, &(param, _))| { let mut counted = BTreeMap::, usize>::new(); - for (params, _, _) in r.iter() { - let mut p = params.iter().map(|x| x.1).collect::>(); + for result in r.iter() { + let mut p = result.components.iter().map(|x| x.1).collect::>(); p[i] = 0; *counted.entry(p).or_default() += 1; } let others: Vec = counted.iter().max_by_key(|i| i.1).expect("r is not empty; qed").0.clone(); let values = r.iter() .filter(|v| - v.0.iter() + v.components.iter() .map(|x| x.1) .zip(others.iter()) .enumerate() .all(|(j, (v1, v2))| j == i || v1 == *v2) - ).map(|(ps, v, _)| (ps[i].1, *v)) + ).map(|result| { + // Extract the data we are interested in analyzing + let data = match selector { + BenchmarkSelector::ExtrinsicTime => result.extrinsic_time, + BenchmarkSelector::StorageRootTime => result.storage_root_time, + BenchmarkSelector::Reads => result.reads.into(), + BenchmarkSelector::Writes => result.writes.into(), + }; + (result.components[i].1, data) + }) .collect::>(); (format!("{:?}", param), i, others, values) }).collect::>(); @@ -97,12 +113,18 @@ impl Analysis { }) } - pub fn min_squares_iqr(r: &Vec) -> Option { + pub fn min_squares_iqr(r: &Vec, selector: BenchmarkSelector) -> Option { let mut results = BTreeMap::, Vec>::new(); - for &(ref params, t, _) in r.iter() { - let p = params.iter().map(|x| x.1).collect::>(); - results.entry(p).or_default().push(t); + for result in r.iter() { + let p = result.components.iter().map(|x| x.1).collect::>(); + results.entry(p).or_default().push(match selector { + BenchmarkSelector::ExtrinsicTime => result.extrinsic_time, + BenchmarkSelector::StorageRootTime => result.storage_root_time, + BenchmarkSelector::Reads => result.reads.into(), + BenchmarkSelector::Writes => result.writes.into(), + }) } + for (_, rs) in results.iter_mut() { rs.sort(); let ql = rs.len() / 4; @@ -111,7 +133,7 @@ impl Analysis { let mut data = vec![("Y", results.iter().flat_map(|x| x.1.iter().map(|v| *v as f64)).collect())]; - let names = r[0].0.iter().map(|x| format!("{:?}", x.0)).collect::>(); + let names = r[0].components.iter().map(|x| format!("{:?}", x.0)).collect::>(); data.extend(names.iter() .enumerate() .map(|(i, p)| ( @@ -217,40 +239,88 @@ impl std::fmt::Display for Analysis { } } +impl std::fmt::Debug for Analysis { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", self.base)?; + for (&m, n) in self.slopes.iter().zip(self.names.iter()) { + write!(f, " + ({} * {})", m, n)?; + } + write!(f,"") + } +} + #[cfg(test)] mod tests { use super::*; use crate::BenchmarkParameter; + fn benchmark_result( + components: Vec<(BenchmarkParameter, u32)>, + extrinsic_time: u128, + storage_root_time: u128, + reads: u32, + writes: u32, + ) -> BenchmarkResults { + BenchmarkResults { + components, + extrinsic_time, + storage_root_time, + reads, + repeat_reads: 0, + writes, + repeat_writes: 0, + } + } + #[test] fn analysis_median_slopes_should_work() { - let a = Analysis::median_slopes(&vec![ - (vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0), - (vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0), - (vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0), - ]).unwrap(); - assert_eq!(a.base, 10_000_000); - assert_eq!(a.slopes, vec![1_000_000, 100_000]); + let data = vec![ + benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0, 3, 10), + benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0, 4, 10), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0, 5, 10), + benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0, 6, 10), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0, 5, 2), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0, 5, 6), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0, 5, 14), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0, 5, 20), + ]; + + let extrinsic_time = Analysis::median_slopes(&data, BenchmarkSelector::ExtrinsicTime).unwrap(); + assert_eq!(extrinsic_time.base, 10_000_000); + assert_eq!(extrinsic_time.slopes, vec![1_000_000, 100_000]); + + let reads = Analysis::median_slopes(&data, BenchmarkSelector::Reads).unwrap(); + assert_eq!(reads.base, 2); + assert_eq!(reads.slopes, vec![1, 0]); + + let writes = Analysis::median_slopes(&data, BenchmarkSelector::Writes).unwrap(); + assert_eq!(writes.base, 0); + assert_eq!(writes.slopes, vec![0, 2]); } #[test] fn analysis_median_min_squares_should_work() { - let a = Analysis::min_squares_iqr(&vec![ - (vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0), - (vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0), - (vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0), - ]).unwrap(); - assert_eq!(a.base, 10_000_000); - assert_eq!(a.slopes, vec![1_000_000, 100_000]); + let data = vec![ + benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0, 3, 10), + benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0, 4, 10), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0, 5, 10), + benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0, 6, 10), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0, 5, 2), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0, 5, 6), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0, 5, 14), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0, 5, 20), + ]; + + let extrinsic_time = Analysis::min_squares_iqr(&data, BenchmarkSelector::ExtrinsicTime).unwrap(); + assert_eq!(extrinsic_time.base, 10_000_000); + assert_eq!(extrinsic_time.slopes, vec![1_000_000, 100_000]); + + let reads = Analysis::min_squares_iqr(&data, BenchmarkSelector::Reads).unwrap(); + assert_eq!(reads.base, 2); + assert_eq!(reads.slopes, vec![1, 0]); + + let writes = Analysis::min_squares_iqr(&data, BenchmarkSelector::Writes).unwrap(); + assert_eq!(writes.base, 0); + assert_eq!(writes.slopes, vec![0, 2]); } } diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index ae9ef90764cd2387fb03f89533f74afa4cbcc1e6..7a7848305a01c509c808b1939cc42feda1847bcd 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -26,10 +26,11 @@ mod analysis; pub use utils::*; #[cfg(feature = "std")] -pub use analysis::Analysis; +pub use analysis::{Analysis, BenchmarkSelector}; #[doc(hidden)] pub use sp_io::storage::root as storage_root; -pub use sp_runtime::traits::{Dispatchable, Zero}; +pub use sp_runtime::traits::Zero; +pub use frame_support; pub use paste; /// Construct pallet benchmarks for weighing dispatchables. @@ -84,6 +85,8 @@ pub use paste; /// Example: /// ```ignore /// benchmarks! { +/// where_clause { where T::A: From } // Optional line to give additional bound on `T`. +/// /// // common parameter; just one for this example. /// // will be `1`, `MAX_LENGTH` or any value inbetween /// _ { @@ -172,6 +175,7 @@ pub use paste; #[macro_export] macro_rules! benchmarks { ( + $( where_clause { where $( $where_ty:ty: $where_bound:path ),* $(,)? } )? _ { $( let $common:ident in $common_from:tt .. $common_to:expr => $common_instancer:expr; @@ -181,6 +185,7 @@ macro_rules! benchmarks { ) => { $crate::benchmarks_iter!( NO_INSTANCE + { $( $( $where_ty: $where_bound ),* )? } { $( { $common , $common_from , $common_to , $common_instancer } )* } ( ) $( $rest )* @@ -188,9 +193,11 @@ macro_rules! benchmarks { } } +/// Same as [`benchmarks`] but for instantiable module. #[macro_export] macro_rules! benchmarks_instance { ( + $( where_clause { where $( $where_ty:ty: $where_bound:path ),* $(,)? } )? _ { $( let $common:ident in $common_from:tt .. $common_to:expr => $common_instancer:expr; @@ -200,6 +207,7 @@ macro_rules! benchmarks_instance { ) => { $crate::benchmarks_iter!( INSTANCE + { $( $( $where_ty: $where_bound ),* )? } { $( { $common , $common_from , $common_to , $common_instancer } )* } ( ) $( $rest )* @@ -208,11 +216,12 @@ macro_rules! benchmarks_instance { } #[macro_export] -#[allow(missing_docs)] +#[doc(hidden)] macro_rules! benchmarks_iter { // mutation arm: ( $instance:ident + { $( $where_clause:tt )* } { $( $common:tt )* } ( $( $names:ident )* ) $name:ident { $( $code:tt )* }: _ ( $origin:expr $( , $arg:expr )* ) @@ -221,6 +230,7 @@ macro_rules! benchmarks_iter { ) => { $crate::benchmarks_iter! { $instance + { $( $where_clause )* } { $( $common )* } ( $( $names )* ) $name { $( $code )* }: $name ( $origin $( , $arg )* ) @@ -231,6 +241,7 @@ macro_rules! benchmarks_iter { // no instance mutation arm: ( NO_INSTANCE + { $( $where_clause:tt )* } { $( $common:tt )* } ( $( $names:ident )* ) $name:ident { $( $code:tt )* }: $dispatch:ident ( $origin:expr $( , $arg:expr )* ) @@ -239,10 +250,13 @@ macro_rules! benchmarks_iter { ) => { $crate::benchmarks_iter! { NO_INSTANCE + { $( $where_clause )* } { $( $common )* } ( $( $names )* ) $name { $( $code )* }: { - as $crate::Dispatchable>::dispatch(Call::::$dispatch($($arg),*), $origin.into())?; + < + Call as $crate::frame_support::traits::UnfilteredDispatchable + >::dispatch_bypass_filter(Call::::$dispatch($($arg),*), $origin.into())?; } verify $postcode $( $rest )* @@ -251,6 +265,7 @@ macro_rules! benchmarks_iter { // instance mutation arm: ( INSTANCE + { $( $where_clause:tt )* } { $( $common:tt )* } ( $( $names:ident )* ) $name:ident { $( $code:tt )* }: $dispatch:ident ( $origin:expr $( , $arg:expr )* ) @@ -259,10 +274,13 @@ macro_rules! benchmarks_iter { ) => { $crate::benchmarks_iter! { INSTANCE + { $( $where_clause )* } { $( $common )* } ( $( $names )* ) $name { $( $code )* }: { - as $crate::Dispatchable>::dispatch(Call::::$dispatch($($arg),*), $origin.into())?; + < + Call as $crate::frame_support::traits::UnfilteredDispatchable + >::dispatch_bypass_filter(Call::::$dispatch($($arg),*), $origin.into())?; } verify $postcode $( $rest )* @@ -271,6 +289,7 @@ macro_rules! benchmarks_iter { // iteration arm: ( $instance:ident + { $( $where_clause:tt )* } { $( $common:tt )* } ( $( $names:ident )* ) $name:ident { $( $code:tt )* }: $eval:block @@ -280,29 +299,34 @@ macro_rules! benchmarks_iter { $crate::benchmark_backend! { $instance $name + { $( $where_clause )* } { $( $common )* } { } { $eval } { $( $code )* } $postcode } + + #[cfg(test)] + $crate::impl_benchmark_test!( { $( $where_clause )* } $instance $name ); + $crate::benchmarks_iter!( $instance + { $( $where_clause )* } { $( $common )* } ( $( $names )* $name ) $( $rest )* ); }; // iteration-exit arm - ( $instance:ident { $( $common:tt )* } ( $( $names:ident )* ) ) => { - $crate::selected_benchmark!( $instance $( $names ),* ); - $crate::impl_benchmark!( $instance $( $names ),* ); - #[cfg(test)] - $crate::impl_benchmark_tests!( $instance $( $names ),* ); + ( $instance:ident { $( $where_clause:tt )* } { $( $common:tt )* } ( $( $names:ident )* ) ) => { + $crate::selected_benchmark!( { $( $where_clause)* } $instance $( $names ),* ); + $crate::impl_benchmark!( { $( $where_clause )* } $instance $( $names ),* ); }; // add verify block to _() format ( $instance:ident + { $( $where_clause:tt )* } { $( $common:tt )* } ( $( $names:ident )* ) $name:ident { $( $code:tt )* }: _ ( $origin:expr $( , $arg:expr )* ) @@ -310,6 +334,7 @@ macro_rules! benchmarks_iter { ) => { $crate::benchmarks_iter! { $instance + { $( $where_clause )* } { $( $common )* } ( $( $names )* ) $name { $( $code )* }: _ ( $origin $( , $arg )* ) @@ -320,6 +345,7 @@ macro_rules! benchmarks_iter { // add verify block to name() format ( $instance:ident + { $( $where_clause:tt )* } { $( $common:tt )* } ( $( $names:ident )* ) $name:ident { $( $code:tt )* }: $dispatch:ident ( $origin:expr $( , $arg:expr )* ) @@ -327,6 +353,7 @@ macro_rules! benchmarks_iter { ) => { $crate::benchmarks_iter! { $instance + { $( $where_clause )* } { $( $common )* } ( $( $names )* ) $name { $( $code )* }: $dispatch ( $origin $( , $arg )* ) @@ -337,6 +364,7 @@ macro_rules! benchmarks_iter { // add verify block to {} format ( $instance:ident + { $( $where_clause:tt )* } { $( $common:tt )* } ( $( $names:ident )* ) $name:ident { $( $code:tt )* }: $eval:block @@ -344,6 +372,7 @@ macro_rules! benchmarks_iter { ) => { $crate::benchmarks_iter!( $instance + { $( $where_clause )* } { $( $common )* } ( $( $names )* ) $name { $( $code )* }: $eval @@ -354,10 +383,12 @@ macro_rules! benchmarks_iter { } #[macro_export] -#[allow(missing_docs)] +#[doc(hidden)] macro_rules! benchmark_backend { // parsing arms ($instance:ident $name:ident { + $( $where_clause:tt )* + } { $( $common:tt )* } { $( PRE { $( $pre_parsed:tt )* } )* @@ -366,13 +397,15 @@ macro_rules! benchmark_backend { $( $rest:tt )* } $postcode:block) => { $crate::benchmark_backend! { - $instance $name { $( $common )* } { + $instance $name { $( $where_clause )* } { $( $common )* } { $( PRE { $( $pre_parsed )* } )* PRE { $pre_id , $pre_ty , $pre_ex } } { $eval } { $( $rest )* } $postcode } }; ($instance:ident $name:ident { + $( $where_clause:tt )* + } { $( $common:tt )* } { $( $parsed:tt )* @@ -381,7 +414,7 @@ macro_rules! benchmark_backend { $( $rest:tt )* } $postcode:block) => { $crate::benchmark_backend! { - $instance $name { $( $common )* } { + $instance $name { $( $where_clause )* } { $( $common )* } { $( $parsed )* PARAM { $param , $param_from , $param_to , $param_instancer } } { $eval } { $( $rest )* } $postcode @@ -389,6 +422,8 @@ macro_rules! benchmark_backend { }; // mutation arm to look after defaulting to a common param ($instance:ident $name:ident { + $( $where_clause:tt )* + } { $( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )* } { $( $parsed:tt )* @@ -397,7 +432,7 @@ macro_rules! benchmark_backend { $( $rest:tt )* } $postcode:block) => { $crate::benchmark_backend! { - $instance $name { + $instance $name { $( $where_clause )* } { $( { $common , $common_from , $common_to , $common_instancer } )* } { $( $parsed )* @@ -412,6 +447,8 @@ macro_rules! benchmark_backend { }; // mutation arm to look after defaulting only the range to common param ($instance:ident $name:ident { + $( $where_clause:tt )* + } { $( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )* } { $( $parsed:tt )* @@ -420,7 +457,7 @@ macro_rules! benchmark_backend { $( $rest:tt )* } $postcode:block) => { $crate::benchmark_backend! { - $instance $name { + $instance $name { $( $where_clause )* } { $( { $common , $common_from , $common_to , $common_instancer } )* } { $( $parsed )* @@ -435,6 +472,8 @@ macro_rules! benchmark_backend { }; // mutation arm to look after a single tt for param_from. ($instance:ident $name:ident { + $( $where_clause:tt )* + } { $( $common:tt )* } { $( $parsed:tt )* @@ -443,7 +482,7 @@ macro_rules! benchmark_backend { $( $rest:tt )* } $postcode:block) => { $crate::benchmark_backend! { - $instance $name { $( $common )* } { $( $parsed )* } { $eval } { + $instance $name { $( $where_clause )* } { $( $common )* } { $( $parsed )* } { $eval } { let $param in ( $param_from ) .. $param_to => $param_instancer; $( $rest )* } $postcode @@ -451,6 +490,8 @@ macro_rules! benchmark_backend { }; // mutation arm to look after the default tail of `=> ()` ($instance:ident $name:ident { + $( $where_clause:tt )* + } { $( $common:tt )* } { $( $parsed:tt )* @@ -459,7 +500,7 @@ macro_rules! benchmark_backend { $( $rest:tt )* } $postcode:block) => { $crate::benchmark_backend! { - $instance $name { $( $common )* } { $( $parsed )* } { $eval } { + $instance $name { $( $where_clause )* } { $( $common )* } { $( $parsed )* } { $eval } { let $param in $param_from .. $param_to => (); $( $rest )* } $postcode @@ -467,6 +508,8 @@ macro_rules! benchmark_backend { }; // mutation arm to look after `let _ =` ($instance:ident $name:ident { + $( $where_clause:tt )* + } { $( $common:tt )* } { $( $parsed:tt )* @@ -475,7 +518,7 @@ macro_rules! benchmark_backend { $( $rest:tt )* } $postcode:block) => { $crate::benchmark_backend! { - $instance $name { $( $common )* } { $( $parsed )* } { $eval } { + $instance $name { $( $where_clause )* } { $( $common )* } { $( $parsed )* } { $eval } { let $pre_id : _ = $pre_ex; $( $rest )* } $postcode @@ -483,6 +526,8 @@ macro_rules! benchmark_backend { }; // no instance actioning arm (NO_INSTANCE $name:ident { + $( $where_clause:tt )* + } { $( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )* } { $( PRE { $pre_id:tt , $pre_ty:ty , $pre_ex:expr } )* @@ -491,7 +536,9 @@ macro_rules! benchmark_backend { #[allow(non_camel_case_types)] struct $name; #[allow(unused_variables)] - impl $crate::BenchmarkingSetup for $name { + impl $crate::BenchmarkingSetup for $name + where $( $where_clause )* + { fn components(&self) -> Vec<($crate::BenchmarkParameter, u32, u32)> { vec! [ $( @@ -508,7 +555,9 @@ macro_rules! benchmark_backend { )* $( // Prepare instance - let $param = components.iter().find(|&c| c.0 == $crate::BenchmarkParameter::$param).unwrap().1; + let $param = components.iter() + .find(|&c| c.0 == $crate::BenchmarkParameter::$param) + .unwrap().1; )* $( let $pre_id : $pre_ty = $pre_ex; @@ -527,7 +576,9 @@ macro_rules! benchmark_backend { )* $( // Prepare instance - let $param = components.iter().find(|&c| c.0 == $crate::BenchmarkParameter::$param).unwrap().1; + let $param = components.iter() + .find(|&c| c.0 == $crate::BenchmarkParameter::$param) + .unwrap().1; )* $( let $pre_id : $pre_ty = $pre_ex; @@ -541,6 +592,8 @@ macro_rules! benchmark_backend { }; // instance actioning arm (INSTANCE $name:ident { + $( $where_clause:tt )* + } { $( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )* } { $( PRE { $pre_id:tt , $pre_ty:ty , $pre_ex:expr } )* @@ -549,7 +602,9 @@ macro_rules! benchmark_backend { #[allow(non_camel_case_types)] struct $name; #[allow(unused_variables)] - impl, I: Instance> $crate::BenchmarkingSetupInstance for $name { + impl, I: Instance> $crate::BenchmarkingSetupInstance for $name + where $( $where_clause )* + { fn components(&self) -> Vec<($crate::BenchmarkParameter, u32, u32)> { vec! [ $( @@ -566,7 +621,9 @@ macro_rules! benchmark_backend { )* $( // Prepare instance - let $param = components.iter().find(|&c| c.0 == $crate::BenchmarkParameter::$param).unwrap().1; + let $param = components.iter() + .find(|&c| c.0 == $crate::BenchmarkParameter::$param) + .unwrap().1; )* $( let $pre_id : $pre_ty = $pre_ex; @@ -585,7 +642,9 @@ macro_rules! benchmark_backend { )* $( // Prepare instance - let $param = components.iter().find(|&c| c.0 == $crate::BenchmarkParameter::$param).unwrap().1; + let $param = components.iter() + .find(|&c| c.0 == $crate::BenchmarkParameter::$param) + .unwrap().1; )* $( let $pre_id : $pre_ty = $pre_ex; @@ -599,23 +658,25 @@ macro_rules! benchmark_backend { } } -/// Creates a `SelectedBenchmark` enum implementing `BenchmarkingSetup`. -/// -/// Every variant must implement [`BenchmarkingSetup`]. -/// -/// ```nocompile -/// -/// struct Transfer; -/// impl BenchmarkingSetup for Transfer { ... } -/// -/// struct SetBalance; -/// impl BenchmarkingSetup for SetBalance { ... } -/// -/// selected_benchmark!(Transfer, SetBalance); -/// ``` +// Creates a `SelectedBenchmark` enum implementing `BenchmarkingSetup`. +// +// Every variant must implement [`BenchmarkingSetup`]. +// +// ```nocompile +// +// struct Transfer; +// impl BenchmarkingSetup for Transfer { ... } +// +// struct SetBalance; +// impl BenchmarkingSetup for SetBalance { ... } +// +// selected_benchmark!(Transfer, SetBalance); +// ``` #[macro_export] +#[doc(hidden)] macro_rules! selected_benchmark { ( + { $( $where_clause:tt )* } NO_INSTANCE $( $bench:ident ),* ) => { // The list of available benchmarks for this pallet. @@ -625,7 +686,9 @@ macro_rules! selected_benchmark { } // Allow us to select a benchmark from the list of available benchmarks. - impl $crate::BenchmarkingSetup for SelectedBenchmark { + impl $crate::BenchmarkingSetup for SelectedBenchmark + where $( $where_clause )* + { fn components(&self) -> Vec<($crate::BenchmarkParameter, u32, u32)> { match self { $( Self::$bench => <$bench as $crate::BenchmarkingSetup>::components(&$bench), )* @@ -650,6 +713,7 @@ macro_rules! selected_benchmark { } }; ( + { $( $where_clause:tt )* } INSTANCE $( $bench:ident ),* ) => { // The list of available benchmarks for this pallet. @@ -659,7 +723,9 @@ macro_rules! selected_benchmark { } // Allow us to select a benchmark from the list of available benchmarks. - impl, I: Instance> $crate::BenchmarkingSetupInstance for SelectedBenchmark { + impl, I: Instance> $crate::BenchmarkingSetupInstance for SelectedBenchmark + where $( $where_clause )* + { fn components(&self) -> Vec<($crate::BenchmarkParameter, u32, u32)> { match self { $( Self::$bench => <$bench as $crate::BenchmarkingSetupInstance>::components(&$bench), )* @@ -686,12 +752,14 @@ macro_rules! selected_benchmark { } #[macro_export] +#[doc(hidden)] macro_rules! impl_benchmark { ( + { $( $where_clause:tt )* } NO_INSTANCE $( $name:ident ),* ) => { impl $crate::Benchmarking<$crate::BenchmarkResults> for Module - where T: frame_system::Trait + where T: frame_system::Trait, $( $where_clause )* { fn benchmarks() -> Vec<&'static [u8]> { vec![ $( stringify!($name).as_ref() ),* ] @@ -703,6 +771,7 @@ macro_rules! impl_benchmark { highest_range_values: &[u32], steps: &[u32], repeat: u32, + whitelist: &[Vec] ) -> Result, &'static str> { // Map the input to the selected benchmark. let extrinsic = sp_std::str::from_utf8(extrinsic) @@ -712,6 +781,9 @@ macro_rules! impl_benchmark { _ => return Err("Could not find extrinsic."), }; + // Add whitelist to DB + $crate::benchmarking::set_whitelist(whitelist.to_vec()); + // Warm up the DB $crate::benchmarking::commit_db(); $crate::benchmarking::wipe_db(); @@ -758,8 +830,11 @@ macro_rules! impl_benchmark { // Run the benchmark `repeat` times. for _ in 0..repeat { - // Set up the externalities environment for the setup we want to benchmark. - let closure_to_benchmark = >::instance(&selected_benchmark, &c)?; + // Set up the externalities environment for the setup we want to + // benchmark. + let closure_to_benchmark = < + SelectedBenchmark as $crate::BenchmarkingSetup + >::instance(&selected_benchmark, &c)?; // Set the block number to at least 1 so events are deposited. if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { @@ -770,13 +845,30 @@ macro_rules! impl_benchmark { // This will enable worst case scenario for reading from the database. $crate::benchmarking::commit_db(); + // Reset the read/write counter so we don't count operations in the setup process. + $crate::benchmarking::reset_read_write_count(); + // Time the extrinsic logic. - frame_support::debug::trace!(target: "benchmark", "Start Benchmark: {:?} {:?}", name, component_value); + frame_support::debug::trace!( + target: "benchmark", + "Start Benchmark: {:?} {:?}", name, component_value + ); + let start_extrinsic = $crate::benchmarking::current_time(); closure_to_benchmark()?; let finish_extrinsic = $crate::benchmarking::current_time(); let elapsed_extrinsic = finish_extrinsic - start_extrinsic; - frame_support::debug::trace!(target: "benchmark", "End Benchmark: {} ns", elapsed_extrinsic); + // Commit the changes to get proper write count + $crate::benchmarking::commit_db(); + frame_support::debug::trace!( + target: "benchmark", + "End Benchmark: {} ns", elapsed_extrinsic + ); + let read_write_count = $crate::benchmarking::read_write_count(); + frame_support::debug::trace!( + target: "benchmark", + "Read/Write Count {:?}", read_write_count + ); // Time the storage root recalculation. let start_storage_root = $crate::benchmarking::current_time(); @@ -784,7 +876,15 @@ macro_rules! impl_benchmark { let finish_storage_root = $crate::benchmarking::current_time(); let elapsed_storage_root = finish_storage_root - start_storage_root; - results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root)); + results.push($crate::BenchmarkResults { + components: c.clone(), + extrinsic_time: elapsed_extrinsic, + storage_root_time: elapsed_storage_root, + reads: read_write_count.0, + repeat_reads: read_write_count.1, + writes: read_write_count.2, + repeat_writes: read_write_count.3, + }); // Wipe the DB back to the genesis state. $crate::benchmarking::wipe_db(); @@ -796,10 +896,12 @@ macro_rules! impl_benchmark { } }; ( + { $( $where_clause:tt )* } INSTANCE $( $name:ident ),* ) => { - impl, I: Instance> $crate::Benchmarking<$crate::BenchmarkResults> for Module - where T: frame_system::Trait + impl, I: Instance> $crate::Benchmarking<$crate::BenchmarkResults> + for Module + where T: frame_system::Trait, $( $where_clause )* { fn benchmarks() -> Vec<&'static [u8]> { vec![ $( stringify!($name).as_ref() ),* ] @@ -811,6 +913,7 @@ macro_rules! impl_benchmark { highest_range_values: &[u32], steps: &[u32], repeat: u32, + whitelist: &[Vec] ) -> Result, &'static str> { // Map the input to the selected benchmark. let extrinsic = sp_std::str::from_utf8(extrinsic) @@ -820,11 +923,16 @@ macro_rules! impl_benchmark { _ => return Err("Could not find extrinsic."), }; + // Add whitelist to DB + $crate::benchmarking::set_whitelist(whitelist.to_vec()); + // Warm up the DB $crate::benchmarking::commit_db(); $crate::benchmarking::wipe_db(); - let components = >::components(&selected_benchmark); + let components = < + SelectedBenchmark as $crate::BenchmarkingSetupInstance + >::components(&selected_benchmark); let mut results: Vec<$crate::BenchmarkResults> = Vec::new(); // Default number of steps for a component. @@ -867,7 +975,9 @@ macro_rules! impl_benchmark { // Run the benchmark `repeat` times. for _ in 0..repeat { // Set up the externalities environment for the setup we want to benchmark. - let closure_to_benchmark = >::instance(&selected_benchmark, &c)?; + let closure_to_benchmark = < + SelectedBenchmark as $crate::BenchmarkingSetupInstance + >::instance(&selected_benchmark, &c)?; // Set the block number to at least 1 so events are deposited. if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { @@ -878,13 +988,30 @@ macro_rules! impl_benchmark { // This will enable worst case scenario for reading from the database. $crate::benchmarking::commit_db(); + // Reset the read/write counter so we don't count operations in the setup process. + $crate::benchmarking::reset_read_write_count(); + // Time the extrinsic logic. - frame_support::debug::trace!(target: "benchmark", "Start Benchmark: {:?} {:?}", name, component_value); + frame_support::debug::trace!( + target: "benchmark", + "Start Benchmark: {:?} {:?}", name, component_value + ); + let start_extrinsic = $crate::benchmarking::current_time(); closure_to_benchmark()?; let finish_extrinsic = $crate::benchmarking::current_time(); let elapsed_extrinsic = finish_extrinsic - start_extrinsic; - frame_support::debug::trace!(target: "benchmark", "End Benchmark: {} ns", elapsed_extrinsic); + // Commit the changes to get proper write count + $crate::benchmarking::commit_db(); + frame_support::debug::trace!( + target: "benchmark", + "End Benchmark: {} ns", elapsed_extrinsic + ); + let read_write_count = $crate::benchmarking::read_write_count(); + frame_support::debug::trace!( + target: "benchmark", + "Read/Write Count {:?}", read_write_count + ); // Time the storage root recalculation. let start_storage_root = $crate::benchmarking::current_time(); @@ -892,7 +1019,15 @@ macro_rules! impl_benchmark { let finish_storage_root = $crate::benchmarking::current_time(); let elapsed_storage_root = finish_storage_root - start_storage_root; - results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root)); + results.push($crate::BenchmarkResults { + components: c.clone(), + extrinsic_time: elapsed_extrinsic, + storage_root_time: elapsed_storage_root, + reads: read_write_count.0, + repeat_reads: read_write_count.1, + writes: read_write_count.2, + repeat_writes: read_write_count.3, + }); // Wipe the DB back to the genesis state. $crate::benchmarking::wipe_db(); @@ -905,108 +1040,115 @@ macro_rules! impl_benchmark { } } -// This creates unit tests from the main benchmark macro. -// They run the benchmark using the `high` and `low` value for each component +// This creates a unit test for one benchmark of the main benchmark macro. +// It runs the benchmark using the `high` and `low` value for each component // and ensure that everything completes successfully. #[macro_export] -macro_rules! impl_benchmark_tests { +#[doc(hidden)] +macro_rules! impl_benchmark_test { ( + { $( $where_clause:tt )* } NO_INSTANCE - $( $name:ident ),* + $name:ident ) => { - $( - $crate::paste::item! { - fn [] () -> Result<(), &'static str> - where T: frame_system::Trait - { - let selected_benchmark = SelectedBenchmark::$name; - let components = >::components(&selected_benchmark); - - assert!( - components.len() != 0, - "You need to add components to your benchmark!", - ); - for (_, (name, low, high)) in components.iter().enumerate() { - // Test only the low and high value, assuming values in the middle won't break - for component_value in vec![low, high] { - // Select the max value for all the other components. - let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() - .enumerate() - .map(|(_, (n, _, h))| - if n == name { - (*n, *component_value) - } else { - (*n, *h) - } - ) - .collect(); - - // Set up the verification state - let closure_to_verify = >::verify(&selected_benchmark, &c)?; - - // Set the block number to at least 1 so events are deposited. - if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { - frame_system::Module::::set_block_number(1.into()); - } + $crate::paste::item! { + fn [] () -> Result<(), &'static str> + where T: frame_system::Trait, $( $where_clause )* + { + let selected_benchmark = SelectedBenchmark::$name; + let components = < + SelectedBenchmark as $crate::BenchmarkingSetup + >::components(&selected_benchmark); + + assert!( + components.len() != 0, + "You need to add components to your benchmark!", + ); + for (_, (name, low, high)) in components.iter().enumerate() { + // Test only the low and high value, assuming values in the middle won't break + for component_value in vec![low, high] { + // Select the max value for all the other components. + let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() + .enumerate() + .map(|(_, (n, _, h))| + if n == name { + (*n, *component_value) + } else { + (*n, *h) + } + ) + .collect(); - // Run verification - closure_to_verify()?; + // Set up the verification state + let closure_to_verify = < + SelectedBenchmark as $crate::BenchmarkingSetup + >::verify(&selected_benchmark, &c)?; - // Reset the state - $crate::benchmarking::wipe_db(); + // Set the block number to at least 1 so events are deposited. + if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { + frame_system::Module::::set_block_number(1.into()); } + + // Run verification + closure_to_verify()?; + + // Reset the state + $crate::benchmarking::wipe_db(); } - Ok(()) } + Ok(()) } - )* + } }; ( + { $( $where_clause:tt )* } INSTANCE - $( $name:ident ),* + $name:ident ) => { - $( - $crate::paste::item! { - fn [] () -> Result<(), &'static str> - where T: frame_system::Trait - { - let selected_benchmark = SelectedBenchmark::$name; - let components = >::components(&selected_benchmark); - - for (_, (name, low, high)) in components.iter().enumerate() { - // Test only the low and high value, assuming values in the middle won't break - for component_value in vec![low, high] { - // Select the max value for all the other components. - let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() - .enumerate() - .map(|(_, (n, _, h))| - if n == name { - (*n, *component_value) - } else { - (*n, *h) - } - ) - .collect(); - - // Set up the verification state - let closure_to_verify = >::verify(&selected_benchmark, &c)?; - - // Set the block number to at least 1 so events are deposited. - if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { - frame_system::Module::::set_block_number(1.into()); - } + $crate::paste::item! { + fn [] () -> Result<(), &'static str> + where T: frame_system::Trait, $( $where_clause )* + { + let selected_benchmark = SelectedBenchmark::$name; + let components = < + SelectedBenchmark as $crate::BenchmarkingSetupInstance + >::components(&selected_benchmark); + + for (_, (name, low, high)) in components.iter().enumerate() { + // Test only the low and high value, assuming values in the middle won't break + for component_value in vec![low, high] { + // Select the max value for all the other components. + let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() + .enumerate() + .map(|(_, (n, _, h))| + if n == name { + (*n, *component_value) + } else { + (*n, *h) + } + ) + .collect(); - // Run verification - closure_to_verify()?; + // Set up the verification state + let closure_to_verify = < + SelectedBenchmark as $crate::BenchmarkingSetupInstance + >::verify(&selected_benchmark, &c)?; - // Reset the state - $crate::benchmarking::wipe_db(); + // Set the block number to at least 1 so events are deposited. + if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { + frame_system::Module::::set_block_number(1.into()); } + + // Run verification + closure_to_verify()?; + + // Reset the state + $crate::benchmarking::wipe_db(); } - Ok(()) } + Ok(()) } - )* + } }; } @@ -1039,7 +1181,7 @@ macro_rules! impl_benchmark_tests { #[macro_export] macro_rules! add_benchmark { ( $params:ident, $batches:ident, $name:literal, $( $location:tt )* ) => ( - let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat) = $params; + let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat, whitelist) = $params; if &pallet[..] == &$name[..] || &pallet[..] == &b"*"[..] { if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] { for benchmark in $( $location )*::benchmarks().into_iter() { @@ -1050,6 +1192,7 @@ macro_rules! add_benchmark { &highest_range_values[..], &steps[..], repeat, + whitelist, )?, pallet: $name.to_vec(), benchmark: benchmark.to_vec(), @@ -1063,6 +1206,7 @@ macro_rules! add_benchmark { &highest_range_values[..], &steps[..], repeat, + whitelist, )?, pallet: $name.to_vec(), benchmark: benchmark.clone(), diff --git a/frame/benchmarking/src/tests.rs b/frame/benchmarking/src/tests.rs index 8418cb9081be80387d1df56df402c297ed51a66d..674d92eb856aa327ed9ba903500874918fea6985 100644 --- a/frame/benchmarking/src/tests.rs +++ b/frame/benchmarking/src/tests.rs @@ -30,13 +30,17 @@ use frame_support::{ use frame_system::{RawOrigin, ensure_signed, ensure_none}; decl_storage! { - trait Store for Module as Test { + trait Store for Module as Test where + ::OtherEvent: Into<::Event> + { Value get(fn value): Option; } } decl_module! { - pub struct Module for enum Call where origin: T::Origin { + pub struct Module for enum Call where + origin: T::Origin, ::OtherEvent: Into<::Event> + { #[weight = 0] fn set_value(origin, n: u32) -> DispatchResult { let _sender = ensure_signed(origin)?; @@ -56,17 +60,23 @@ impl_outer_origin! { pub enum Origin for Test where system = frame_system {} } -pub trait Trait { +pub trait OtherTrait { + type OtherEvent; +} + +pub trait Trait: OtherTrait where Self::OtherEvent: Into { type Event; type BlockNumber; type AccountId: 'static + Default + Decode; - type Origin: From> + Into, Self::Origin>>; + type Origin: From> + + Into, Self::Origin>>; } #[derive(Clone, Eq, PartialEq)] pub struct Test; impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -82,6 +92,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = (); type MaximumBlockLength = (); type AvailableBlockRatio = (); type Version = (); @@ -98,6 +109,10 @@ impl Trait for Test { type AccountId = u64; } +impl OtherTrait for Test { + type OtherEvent = (); +} + // This function basically just builds a genesis storage key/value store according to // our desired mockup. fn new_test_ext() -> sp_io::TestExternalities { @@ -105,6 +120,8 @@ fn new_test_ext() -> sp_io::TestExternalities { } benchmarks!{ + where_clause { where ::OtherEvent: Into<::Event> } + _ { // Define a common range for `b`. let b in 1 .. 1000 => (); @@ -154,13 +171,13 @@ benchmarks!{ #[test] fn benchmarks_macro_works() { // Check benchmark creation for `set_value`. - let selected_benchmark = SelectedBenchmark::set_value; + let selected = SelectedBenchmark::set_value; - let components = >::components(&selected_benchmark); + let components = >::components(&selected); assert_eq!(components, vec![(BenchmarkParameter::b, 1, 1000)]); let closure = >::instance( - &selected_benchmark, + &selected, &[(BenchmarkParameter::b, 1)], ).expect("failed to create closure"); @@ -172,12 +189,12 @@ fn benchmarks_macro_works() { #[test] fn benchmarks_macro_rename_works() { // Check benchmark creation for `other_dummy`. - let selected_benchmark = SelectedBenchmark::other_name; - let components = >::components(&selected_benchmark); + let selected = SelectedBenchmark::other_name; + let components = >::components(&selected); assert_eq!(components, vec![(BenchmarkParameter::b, 1, 1000)]); let closure = >::instance( - &selected_benchmark, + &selected, &[(BenchmarkParameter::b, 1)], ).expect("failed to create closure"); @@ -188,13 +205,13 @@ fn benchmarks_macro_rename_works() { #[test] fn benchmarks_macro_works_for_non_dispatchable() { - let selected_benchmark = SelectedBenchmark::sort_vector; + let selected = SelectedBenchmark::sort_vector; - let components = >::components(&selected_benchmark); + let components = >::components(&selected); assert_eq!(components, vec![(BenchmarkParameter::x, 1, 10000)]); let closure = >::instance( - &selected_benchmark, + &selected, &[(BenchmarkParameter::x, 1)], ).expect("failed to create closure"); @@ -204,10 +221,10 @@ fn benchmarks_macro_works_for_non_dispatchable() { #[test] fn benchmarks_macro_verify_works() { // Check postcondition for benchmark `set_value` is valid. - let selected_benchmark = SelectedBenchmark::set_value; + let selected = SelectedBenchmark::set_value; let closure = >::verify( - &selected_benchmark, + &selected, &[(BenchmarkParameter::b, 1)], ).expect("failed to create closure"); diff --git a/frame/benchmarking/src/utils.rs b/frame/benchmarking/src/utils.rs index 31ec3783cc06167f5a7e3f282894fbb117fdec0a..7f9d9121100e3b45dc78ae6497537335c6ee3919 100644 --- a/frame/benchmarking/src/utils.rs +++ b/frame/benchmarking/src/utils.rs @@ -44,7 +44,16 @@ pub struct BenchmarkBatch { /// Results from running benchmarks on a FRAME pallet. /// Contains duration of the function call in nanoseconds along with the benchmark parameters /// used for that benchmark result. -pub type BenchmarkResults = (Vec<(BenchmarkParameter, u32)>, u128, u128); +#[derive(Encode, Decode, Default, Clone, PartialEq, Debug)] +pub struct BenchmarkResults { + pub components: Vec<(BenchmarkParameter, u32)>, + pub extrinsic_time: u128, + pub storage_root_time: u128, + pub reads: u32, + pub repeat_reads: u32, + pub writes: u32, + pub repeat_writes: u32, +} sp_api::decl_runtime_apis! { /// Runtime api for benchmarking a FRAME runtime. @@ -83,6 +92,20 @@ pub trait Benchmarking { fn commit_db(&mut self) { self.commit() } + + /// Get the read/write count + fn read_write_count(&self) -> (u32, u32, u32, u32) { + self.read_write_count() + } + + /// Reset the read/write count + fn reset_read_write_count(&mut self) { + self.reset_read_write_count() + } + + fn set_whitelist(&mut self, new: Vec>) { + self.set_whitelist(new) + } } /// The pallet benchmarking trait. @@ -106,6 +129,7 @@ pub trait Benchmarking { highest_range_values: &[u32], steps: &[u32], repeat: u32, + whitelist: &[Vec] ) -> Result, &'static str>; } diff --git a/frame/collective/Cargo.toml b/frame/collective/Cargo.toml index 83728804004a5f77e9b0a373c235fc1447d1b8eb..c1b2f01089c59a4d61e265dc1165a253c86673c1 100644 --- a/frame/collective/Cargo.toml +++ b/frame/collective/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-collective" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,18 +13,18 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../benchmarking", optional = true } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } [dev-dependencies] hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } [features] default = ["std"] diff --git a/frame/collective/src/lib.rs b/frame/collective/src/lib.rs index 132320c7a3455a6faebbca49b1bf108093e79b0c..2be0241243524b0c3811e2e3518593740b34ebdb 100644 --- a/frame/collective/src/lib.rs +++ b/frame/collective/src/lib.rs @@ -165,11 +165,11 @@ decl_event! { Approved(Hash), /// A motion was not approved by the required threshold. Disapproved(Hash), - /// A motion was executed; `bool` is true if returned without error. + /// A motion was executed; result will be `Ok` if it returned without error. Executed(Hash, DispatchResult), - /// A single member did some action; `bool` is true if returned without error. + /// A single member did some action; result will be `Ok` if it returned without error. MemberExecuted(Hash, DispatchResult), - /// A proposal was closed after its duration was up. + /// A proposal was closed because its threshold was reached or after its duration was up. Closed(Hash, MemberCount, MemberCount), } } @@ -188,7 +188,7 @@ decl_error! { DuplicateVote, /// Members are already initialized! AlreadyInitialized, - /// The close call is made too early, before the end of the voting. + /// The close call was made too early, before the end of the voting. TooEarly, /// There can only be a maximum of `MaxProposals` active proposals. TooManyProposals, @@ -312,7 +312,7 @@ mod weight_for { .saturating_add(proposals.saturating_mul(490_000)) // P2 } - /// Calculate the weight for `close` without the call to `finalize_proposal`. + /// Calculate the weight for `close` without the call to `approve/disapprove_proposal`. pub(crate) fn close_without_finalize, I: Instance>( members: Weight, length: Weight, @@ -642,14 +642,14 @@ decl_module! { DispatchClass::Operational )] fn close(origin, - proposal: T::Hash, + proposal_hash: T::Hash, #[compact] index: ProposalIndex, #[compact] proposal_weight_bound: Weight, #[compact] length_bound: u32 ) -> DispatchResultWithPostInfo { let _ = ensure_signed(origin)?; - let voting = Self::voting(&proposal).ok_or(Error::::ProposalMissing)?; + let voting = Self::voting(&proposal_hash).ok_or(Error::::ProposalMissing)?; ensure!(voting.index == index, Error::::WrongIndex); let mut no_votes = voting.nays.len() as MemberCount; @@ -658,16 +658,24 @@ decl_module! { let approved = yes_votes >= voting.threshold; let disapproved = seats.saturating_sub(no_votes) < voting.threshold; // Allow (dis-)approving the proposal as soon as there are enough votes. - if approved || disapproved { - let (p, len) = Self::validate_and_get_proposal( - &proposal, + if approved { + let (proposal, len) = Self::validate_and_get_proposal( + &proposal_hash, length_bound, proposal_weight_bound )?; - let finalize_weight = Self::finalize_proposal(approved, seats, voting, proposal, p); + Self::deposit_event(RawEvent::Closed(proposal_hash, yes_votes, no_votes)); + let approve_weight = Self::do_approve_proposal(seats, voting, proposal_hash, proposal); return Ok(Some( weight_for::close_without_finalize::(seats.into(), len as Weight) - .saturating_add(finalize_weight) + .saturating_add(approve_weight) + ).into()); + } else if disapproved { + Self::deposit_event(RawEvent::Closed(proposal_hash, yes_votes, no_votes)); + let disapprove_weight = Self::do_disapprove_proposal(proposal_hash); + return Ok(Some( + weight_for::close_without_finalize::(seats.into(), 0) + .saturating_add(disapprove_weight) ).into()); } @@ -684,18 +692,51 @@ decl_module! { } let approved = yes_votes >= voting.threshold; - let (p, len) = Self::validate_and_get_proposal( - &proposal, - length_bound, - proposal_weight_bound - )?; - Self::deposit_event(RawEvent::Closed(proposal, yes_votes, no_votes)); - let finalize_weight = Self::finalize_proposal(approved, seats, voting, proposal, p); - Ok(Some( - weight_for::close_without_finalize::(seats.into(), len as Weight) - .saturating_add(T::DbWeight::get().reads(1)) // read `Prime` - .saturating_add(finalize_weight) - ).into()) + if approved { + let (proposal, len) = Self::validate_and_get_proposal( + &proposal_hash, + length_bound, + proposal_weight_bound + )?; + Self::deposit_event(RawEvent::Closed(proposal_hash, yes_votes, no_votes)); + let approve_weight = Self::do_approve_proposal(seats, voting, proposal_hash, proposal); + return Ok(Some( + weight_for::close_without_finalize::(seats.into(), len as Weight) + .saturating_add(T::DbWeight::get().reads(1)) // read `Prime` + .saturating_add(approve_weight) + ).into()); + } else { + Self::deposit_event(RawEvent::Closed(proposal_hash, yes_votes, no_votes)); + let disapprove_weight = Self::do_disapprove_proposal(proposal_hash); + return Ok(Some( + weight_for::close_without_finalize::(seats.into(), 0) + .saturating_add(T::DbWeight::get().reads(1)) // read `Prime` + .saturating_add(disapprove_weight) + ).into()); + } + } + + /// Disapprove a proposal, close, and remove it from the system, regardless of its current state. + /// + /// Must be called by the Root origin. + /// + /// Parameters: + /// * `proposal_hash`: The hash of the proposal that should be disapproved. + /// + /// # + /// Complexity: O(P) where P is the number of max proposals + /// Base Weight: .49 * P + /// DB Weight: + /// * Reads: Proposals + /// * Writes: Voting, Proposals, ProposalOf + /// # + #[weight = T::DbWeight::get().reads_writes(1, 3) // `Voting`, `Proposals`, `ProposalOf` + .saturating_add(490_000 * Weight::from(T::MaxProposals::get())) // P2 + ] + fn disapprove_proposal(origin, proposal_hash: T::Hash) -> DispatchResultWithPostInfo { + ensure_root(origin)?; + let actual_weight = Self::do_disapprove_proposal(proposal_hash); + Ok(Some(actual_weight).into()) } } } @@ -742,33 +783,38 @@ impl, I: Instance> Module { /// Two removals, one mutation. /// Computation and i/o `O(P)` where: /// - `P` is number of active proposals - fn finalize_proposal( - approved: bool, + fn do_approve_proposal( seats: MemberCount, voting: Votes, proposal_hash: T::Hash, proposal: >::Proposal, ) -> Weight { - let db = T::DbWeight::get(); let mut weight: Weight = 0; - if approved { - Self::deposit_event(RawEvent::Approved(proposal_hash)); + Self::deposit_event(RawEvent::Approved(proposal_hash)); + + let dispatch_weight = proposal.get_dispatch_info().weight; + let origin = RawOrigin::Members(voting.threshold, seats).into(); + let result = proposal.dispatch(origin); + Self::deposit_event( + RawEvent::Executed(proposal_hash, result.map(|_| ()).map_err(|e| e.error)) + ); + weight = weight.saturating_add( + // default to the dispatch info weight for safety + get_result_weight(result).unwrap_or(dispatch_weight) // P1 + ); + + let remove_proposal_weight = Self::remove_proposal(proposal_hash); + weight.saturating_add(remove_proposal_weight) + } - let dispatch_weight = proposal.get_dispatch_info().weight; - let origin = RawOrigin::Members(voting.threshold, seats).into(); - let result = proposal.dispatch(origin); - Self::deposit_event( - RawEvent::Executed(proposal_hash, result.map(|_| ()).map_err(|e| e.error)) - ); - weight = weight.saturating_add( - // default to the dispatch info weight for safety - get_result_weight(result).unwrap_or(dispatch_weight) // P1 - ); - } else { - // disapproved - Self::deposit_event(RawEvent::Disapproved(proposal_hash)); - } + fn do_disapprove_proposal(proposal_hash: T::Hash) -> Weight { + // disapproved + Self::deposit_event(RawEvent::Disapproved(proposal_hash)); + Self::remove_proposal(proposal_hash) + } + // Removes a proposal from the pallet, cleaning up votes and the vector of proposals. + fn remove_proposal(proposal_hash: T::Hash) -> Weight { // remove proposal and vote ProposalOf::::remove(&proposal_hash); Voting::::remove(&proposal_hash); @@ -776,7 +822,7 @@ impl, I: Instance> Module { proposals.retain(|h| h != &proposal_hash); proposals.len() + 1 // calculate weight based on original length }); - weight.saturating_add(db.reads_writes(1, 3)) // `Voting`, `Proposals`, `ProposalOf` + T::DbWeight::get().reads_writes(1, 3) // `Voting`, `Proposals`, `ProposalOf` .saturating_add(490_000 * num_proposals as Weight) // P2 } } @@ -969,10 +1015,11 @@ mod tests { pub const MaxProposals: u32 = 100; } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; - type Call = (); + type Call = Call; type Hash = H256; type Hashing = BlakeTwo256; type AccountId = u64; @@ -984,6 +1031,7 @@ mod tests { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -1050,19 +1098,21 @@ mod tests { fn close_works() { new_test_ext().execute_with(|| { let proposal = make_proposal(42); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); + let proposal_weight = proposal.get_dispatch_info().weight; let hash = BlakeTwo256::hash_of(&proposal); - assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), u32::max_value())); + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len)); assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true)); System::set_block_number(3); assert_noop!( - Collective::close(Origin::signed(4), hash.clone(), 0, Weight::max_value(), u32::max_value()), + Collective::close(Origin::signed(4), hash.clone(), 0, proposal_weight, proposal_len), Error::::TooEarly ); System::set_block_number(4); - assert_ok!(Collective::close(Origin::signed(4), hash.clone(), 0, Weight::max_value(), u32::max_value())); + assert_ok!(Collective::close(Origin::signed(4), hash.clone(), 0, proposal_weight, proposal_len)); let record = |event| EventRecord { phase: Phase::Initialization, event, topics: vec![] }; assert_eq!(System::events(), vec![ @@ -1075,21 +1125,39 @@ mod tests { } #[test] - fn proposal_weight_limit_works() { + fn proposal_weight_limit_works_on_approve() { new_test_ext().execute_with(|| { let proposal = Call::Collective(crate::Call::set_members(vec![1, 2, 3], None, MAX_MEMBERS)); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); + let proposal_weight = proposal.get_dispatch_info().weight; let hash = BlakeTwo256::hash_of(&proposal); - - assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), u32::max_value())); - assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true)); - + // Set 1 as prime voter + Prime::::set(Some(1)); + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len)); + // With 1's prime vote, this should pass System::set_block_number(4); - let proposal_weight = proposal.get_dispatch_info().weight; assert_noop!( - Collective::close(Origin::signed(4), hash.clone(), 0, proposal_weight - 100, u32::max_value()), + Collective::close(Origin::signed(4), hash.clone(), 0, proposal_weight - 100, proposal_len), Error::::WrongProposalWeight ); - assert_ok!(Collective::close(Origin::signed(4), hash.clone(), 0, Weight::max_value(), u32::max_value())); + assert_ok!(Collective::close(Origin::signed(4), hash.clone(), 0, proposal_weight, proposal_len)); + }) + } + + #[test] + fn proposal_weight_limit_ignored_on_disapprove() { + new_test_ext().execute_with(|| { + let proposal = Call::Collective(crate::Call::set_members(vec![1, 2, 3], None, MAX_MEMBERS)); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); + let proposal_weight = proposal.get_dispatch_info().weight; + let hash = BlakeTwo256::hash_of(&proposal); + + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len)); + // No votes, this proposal wont pass + System::set_block_number(4); + assert_ok!( + Collective::close(Origin::signed(4), hash.clone(), 0, proposal_weight - 100, proposal_len) + ); }) } @@ -1097,14 +1165,16 @@ mod tests { fn close_with_prime_works() { new_test_ext().execute_with(|| { let proposal = make_proposal(42); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); + let proposal_weight = proposal.get_dispatch_info().weight; let hash = BlakeTwo256::hash_of(&proposal); - assert_ok!(Collective::set_members(Origin::ROOT, vec![1, 2, 3], Some(3), MAX_MEMBERS)); + assert_ok!(Collective::set_members(Origin::root(), vec![1, 2, 3], Some(3), MAX_MEMBERS)); - assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), u32::max_value())); + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len)); assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true)); System::set_block_number(4); - assert_ok!(Collective::close(Origin::signed(4), hash.clone(), 0, Weight::max_value(), u32::max_value())); + assert_ok!(Collective::close(Origin::signed(4), hash.clone(), 0, proposal_weight, proposal_len)); let record = |event| EventRecord { phase: Phase::Initialization, event, topics: vec![] }; assert_eq!(System::events(), vec![ @@ -1120,14 +1190,16 @@ mod tests { fn close_with_voting_prime_works() { new_test_ext().execute_with(|| { let proposal = make_proposal(42); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); + let proposal_weight = proposal.get_dispatch_info().weight; let hash = BlakeTwo256::hash_of(&proposal); - assert_ok!(Collective::set_members(Origin::ROOT, vec![1, 2, 3], Some(1), MAX_MEMBERS)); + assert_ok!(Collective::set_members(Origin::root(), vec![1, 2, 3], Some(1), MAX_MEMBERS)); - assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), u32::max_value())); + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len)); assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true)); System::set_block_number(4); - assert_ok!(Collective::close(Origin::signed(4), hash.clone(), 0, Weight::max_value(), u32::max_value())); + assert_ok!(Collective::close(Origin::signed(4), hash.clone(), 0, proposal_weight, proposal_len)); let record = |event| EventRecord { phase: Phase::Initialization, event, topics: vec![] }; assert_eq!(System::events(), vec![ @@ -1144,9 +1216,10 @@ mod tests { fn removal_of_old_voters_votes_works() { new_test_ext().execute_with(|| { let proposal = make_proposal(42); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); let hash = BlakeTwo256::hash_of(&proposal); let end = 4; - assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), u32::max_value())); + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len)); assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true)); assert_eq!( Collective::voting(&hash), @@ -1159,8 +1232,9 @@ mod tests { ); let proposal = make_proposal(69); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); let hash = BlakeTwo256::hash_of(&proposal); - assert_ok!(Collective::propose(Origin::signed(2), 2, Box::new(proposal.clone()), u32::max_value())); + assert_ok!(Collective::propose(Origin::signed(2), 2, Box::new(proposal.clone()), proposal_len)); assert_ok!(Collective::vote(Origin::signed(3), hash.clone(), 1, false)); assert_eq!( Collective::voting(&hash), @@ -1178,29 +1252,31 @@ mod tests { fn removal_of_old_voters_votes_works_with_set_members() { new_test_ext().execute_with(|| { let proposal = make_proposal(42); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); let hash = BlakeTwo256::hash_of(&proposal); let end = 4; - assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), u32::max_value())); + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len)); assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true)); assert_eq!( Collective::voting(&hash), Some(Votes { index: 0, threshold: 3, ayes: vec![1, 2], nays: vec![], end }) ); - assert_ok!(Collective::set_members(Origin::ROOT, vec![2, 3, 4], None, MAX_MEMBERS)); + assert_ok!(Collective::set_members(Origin::root(), vec![2, 3, 4], None, MAX_MEMBERS)); assert_eq!( Collective::voting(&hash), Some(Votes { index: 0, threshold: 3, ayes: vec![2], nays: vec![], end }) ); let proposal = make_proposal(69); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); let hash = BlakeTwo256::hash_of(&proposal); - assert_ok!(Collective::propose(Origin::signed(2), 2, Box::new(proposal.clone()), u32::max_value())); + assert_ok!(Collective::propose(Origin::signed(2), 2, Box::new(proposal.clone()), proposal_len)); assert_ok!(Collective::vote(Origin::signed(3), hash.clone(), 1, false)); assert_eq!( Collective::voting(&hash), Some(Votes { index: 1, threshold: 2, ayes: vec![2], nays: vec![3], end }) ); - assert_ok!(Collective::set_members(Origin::ROOT, vec![2, 4], None, MAX_MEMBERS)); + assert_ok!(Collective::set_members(Origin::root(), vec![2, 4], None, MAX_MEMBERS)); assert_eq!( Collective::voting(&hash), Some(Votes { index: 1, threshold: 2, ayes: vec![2], nays: vec![], end }) @@ -1212,9 +1288,10 @@ mod tests { fn propose_works() { new_test_ext().execute_with(|| { let proposal = make_proposal(42); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); let hash = proposal.blake2_256().into(); let end = 4; - assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), u32::max_value())); + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len)); assert_eq!(Collective::proposals(), vec![hash]); assert_eq!(Collective::proposal_of(&hash), Some(proposal)); assert_eq!( @@ -1242,11 +1319,13 @@ mod tests { new_test_ext().execute_with(|| { for i in 0..MaxProposals::get() { let proposal = make_proposal(i as u64); - assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), u32::max_value())); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len)); } let proposal = make_proposal(MaxProposals::get() as u64 + 1); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); assert_noop!( - Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), u32::max_value()), + Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len), Error::::TooManyProposals ); }) @@ -1285,8 +1364,9 @@ mod tests { fn motions_ignoring_non_collective_proposals_works() { new_test_ext().execute_with(|| { let proposal = make_proposal(42); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); assert_noop!( - Collective::propose(Origin::signed(42), 3, Box::new(proposal.clone()), u32::max_value()), + Collective::propose(Origin::signed(42), 3, Box::new(proposal.clone()), proposal_len), Error::::NotMember ); }); @@ -1296,8 +1376,9 @@ mod tests { fn motions_ignoring_non_collective_votes_works() { new_test_ext().execute_with(|| { let proposal = make_proposal(42); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); let hash: H256 = proposal.blake2_256().into(); - assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), u32::max_value())); + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len)); assert_noop!( Collective::vote(Origin::signed(42), hash.clone(), 0, true), Error::::NotMember, @@ -1310,8 +1391,9 @@ mod tests { new_test_ext().execute_with(|| { System::set_block_number(3); let proposal = make_proposal(42); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); let hash: H256 = proposal.blake2_256().into(); - assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), u32::max_value())); + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len)); assert_noop!( Collective::vote(Origin::signed(2), hash.clone(), 1, true), Error::::WrongIndex, @@ -1323,9 +1405,10 @@ mod tests { fn motions_revoting_works() { new_test_ext().execute_with(|| { let proposal = make_proposal(42); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); let hash: H256 = proposal.blake2_256().into(); let end = 4; - assert_ok!(Collective::propose(Origin::signed(1), 2, Box::new(proposal.clone()), u32::max_value())); + assert_ok!(Collective::propose(Origin::signed(1), 2, Box::new(proposal.clone()), proposal_len)); assert_eq!( Collective::voting(&hash), Some(Votes { index: 0, threshold: 2, ayes: vec![1], nays: vec![], end }) @@ -1374,12 +1457,14 @@ mod tests { fn motions_reproposing_disapproved_works() { new_test_ext().execute_with(|| { let proposal = make_proposal(42); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); + let proposal_weight = proposal.get_dispatch_info().weight; let hash: H256 = proposal.blake2_256().into(); - assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), u32::max_value())); + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len)); assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, false)); - assert_ok!(Collective::close(Origin::signed(2), hash.clone(), 0, Weight::max_value(), u32::max_value())); + assert_ok!(Collective::close(Origin::signed(2), hash.clone(), 0, proposal_weight, proposal_len)); assert_eq!(Collective::proposals(), vec![]); - assert_ok!(Collective::propose(Origin::signed(1), 2, Box::new(proposal.clone()), u32::max_value())); + assert_ok!(Collective::propose(Origin::signed(1), 2, Box::new(proposal.clone()), proposal_len)); assert_eq!(Collective::proposals(), vec![hash]); }); } @@ -1388,10 +1473,12 @@ mod tests { fn motions_disapproval_works() { new_test_ext().execute_with(|| { let proposal = make_proposal(42); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); + let proposal_weight = proposal.get_dispatch_info().weight; let hash: H256 = proposal.blake2_256().into(); - assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), u32::max_value())); + assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()), proposal_len)); assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, false)); - assert_ok!(Collective::close(Origin::signed(2), hash.clone(), 0, Weight::max_value(), u32::max_value())); + assert_ok!(Collective::close(Origin::signed(2), hash.clone(), 0, proposal_weight, proposal_len)); assert_eq!(System::events(), vec![ EventRecord { @@ -1416,6 +1503,13 @@ mod tests { )), topics: vec![], }, + EventRecord { + phase: Phase::Initialization, + event: Event::collective_Instance1(RawEvent::Closed( + hex!["68eea8f20b542ec656c6ac2d10435ae3bd1729efc34d1354ab85af840aad2d35"].into(), 1, 1, + )), + topics: vec![], + }, EventRecord { phase: Phase::Initialization, event: Event::collective_Instance1(RawEvent::Disapproved( @@ -1431,10 +1525,12 @@ mod tests { fn motions_approval_works() { new_test_ext().execute_with(|| { let proposal = make_proposal(42); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); + let proposal_weight = proposal.get_dispatch_info().weight; let hash: H256 = proposal.blake2_256().into(); - assert_ok!(Collective::propose(Origin::signed(1), 2, Box::new(proposal.clone()), u32::max_value())); + assert_ok!(Collective::propose(Origin::signed(1), 2, Box::new(proposal.clone()), proposal_len)); assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true)); - assert_ok!(Collective::close(Origin::signed(2), hash.clone(), 0, Weight::max_value(), u32::max_value())); + assert_ok!(Collective::close(Origin::signed(2), hash.clone(), 0, proposal_weight, proposal_len)); assert_eq!(System::events(), vec![ EventRecord { @@ -1458,6 +1554,13 @@ mod tests { )), topics: vec![], }, + EventRecord { + phase: Phase::Initialization, + event: Event::collective_Instance1(RawEvent::Closed( + hex!["68eea8f20b542ec656c6ac2d10435ae3bd1729efc34d1354ab85af840aad2d35"].into(), 2, 0, + )), + topics: vec![], + }, EventRecord { phase: Phase::Initialization, event: Event::collective_Instance1(RawEvent::Approved( @@ -1476,4 +1579,53 @@ mod tests { ]); }); } + + #[test] + fn close_disapprove_does_not_care_about_weight_or_len() { + // This test confirms that if you close a proposal that would be disapproved, + // we do not care about the proposal length or proposal weight since it will + // not be read from storage or executed. + new_test_ext().execute_with(|| { + let proposal = make_proposal(42); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); + let hash: H256 = proposal.blake2_256().into(); + assert_ok!(Collective::propose(Origin::signed(1), 2, Box::new(proposal.clone()), proposal_len)); + // First we make the proposal succeed + assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true)); + // It will not close with bad weight/len information + assert_noop!( + Collective::close(Origin::signed(2), hash.clone(), 0, 0, 0), + Error::::WrongProposalLength, + ); + assert_noop!( + Collective::close(Origin::signed(2), hash.clone(), 0, 0, proposal_len), + Error::::WrongProposalWeight, + ); + // Now we make the proposal fail + assert_ok!(Collective::vote(Origin::signed(1), hash.clone(), 0, false)); + assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, false)); + // It can close even if the weight/len information is bad + assert_ok!(Collective::close(Origin::signed(2), hash.clone(), 0, 0, 0)); + }) + } + + #[test] + fn disapprove_proposal_works() { + new_test_ext().execute_with(|| { + let proposal = make_proposal(42); + let proposal_len: u32 = proposal.using_encoded(|p| p.len() as u32); + let hash: H256 = proposal.blake2_256().into(); + assert_ok!(Collective::propose(Origin::signed(1), 2, Box::new(proposal.clone()), proposal_len)); + // Proposal would normally succeed + assert_ok!(Collective::vote(Origin::signed(2), hash.clone(), 0, true)); + // But Root can disapprove and remove it anyway + assert_ok!(Collective::disapprove_proposal(Origin::root(), hash.clone())); + let record = |event| EventRecord { phase: Phase::Initialization, event, topics: vec![] }; + assert_eq!(System::events(), vec![ + record(Event::collective_Instance1(RawEvent::Proposed(1, 0, hash.clone(), 2))), + record(Event::collective_Instance1(RawEvent::Voted(2, hash.clone(), true, 2, 0))), + record(Event::collective_Instance1(RawEvent::Disapproved(hash.clone()))), + ]); + }) + } } diff --git a/frame/contracts/Cargo.toml b/frame/contracts/Cargo.toml index 0cbd731147528dccee653819c845bc59cb7ce3f9..348b8ff0e03e6edbb8cbce8e9d78e51658ab392e 100644 --- a/frame/contracts/Cargo.toml +++ b/frame/contracts/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-contracts" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -14,26 +14,26 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } pwasm-utils = { version = "0.12.0", default-features = false } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } parity-wasm = { version = "0.41.0", default-features = false } wasmi-validation = { version = "0.3.0", default-features = false } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-sandbox = { version = "0.8.0-alpha.8", default-features = false, path = "../../primitives/sandbox" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -pallet-contracts-primitives = { version = "2.0.0-alpha.8", default-features = false, path = "common" } -pallet-transaction-payment = { version = "2.0.0-alpha.8", default-features = false, path = "../transaction-payment" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-sandbox = { version = "0.8.0-rc4", default-features = false, path = "../../primitives/sandbox" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +pallet-contracts-primitives = { version = "2.0.0-rc4", default-features = false, path = "common" } [dev-dependencies] wabt = "0.9.2" assert_matches = "1.3.0" hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } -pallet-timestamp = { version = "2.0.0-alpha.8", path = "../timestamp" } -pallet-randomness-collective-flip = { version = "2.0.0-alpha.8", path = "../randomness-collective-flip" } +pretty_assertions = "0.6.1" +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } +pallet-timestamp = { version = "2.0.0-rc4", path = "../timestamp" } +pallet-randomness-collective-flip = { version = "2.0.0-rc4", path = "../randomness-collective-flip" } [features] default = ["std"] @@ -51,5 +51,4 @@ std = [ "pwasm-utils/std", "wasmi-validation/std", "pallet-contracts-primitives/std", - "pallet-transaction-payment/std", ] diff --git a/frame/contracts/common/Cargo.toml b/frame/contracts/common/Cargo.toml index 15f0b8a0e70df99f9be476c38b4d368a583610a4..e6e2bc653a21841acfcc7ba243620e1c3b014d14 100644 --- a/frame/contracts/common/Cargo.toml +++ b/frame/contracts/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-contracts-primitives" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,9 +13,9 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] # This crate should not rely on any of the frame primitives. -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/runtime" } [features] default = ["std"] diff --git a/frame/contracts/tests/caller_contract.wat b/frame/contracts/fixtures/caller_contract.wat similarity index 100% rename from frame/contracts/tests/caller_contract.wat rename to frame/contracts/fixtures/caller_contract.wat diff --git a/frame/contracts/tests/check_default_rent_allowance.wat b/frame/contracts/fixtures/check_default_rent_allowance.wat similarity index 100% rename from frame/contracts/tests/check_default_rent_allowance.wat rename to frame/contracts/fixtures/check_default_rent_allowance.wat diff --git a/frame/contracts/tests/crypto_hashes.wat b/frame/contracts/fixtures/crypto_hashes.wat similarity index 100% rename from frame/contracts/tests/crypto_hashes.wat rename to frame/contracts/fixtures/crypto_hashes.wat diff --git a/frame/contracts/tests/destroy_and_transfer.wat b/frame/contracts/fixtures/destroy_and_transfer.wat similarity index 100% rename from frame/contracts/tests/destroy_and_transfer.wat rename to frame/contracts/fixtures/destroy_and_transfer.wat diff --git a/frame/contracts/tests/drain.wat b/frame/contracts/fixtures/drain.wat similarity index 100% rename from frame/contracts/tests/drain.wat rename to frame/contracts/fixtures/drain.wat diff --git a/frame/contracts/tests/restoration.wat b/frame/contracts/fixtures/restoration.wat similarity index 84% rename from frame/contracts/tests/restoration.wat rename to frame/contracts/fixtures/restoration.wat index 4e11f97d5a2ccd722a44211082cbd523a9bc4737..07e11e9d38146b0edc2a897e60ab4042ad9179fa 100644 --- a/frame/contracts/tests/restoration.wat +++ b/frame/contracts/fixtures/restoration.wat @@ -1,6 +1,10 @@ (module (import "env" "ext_set_storage" (func $ext_set_storage (param i32 i32 i32))) - (import "env" "ext_restore_to" (func $ext_restore_to (param i32 i32 i32 i32 i32 i32 i32 i32))) + (import "env" "ext_restore_to" + (func $ext_restore_to + (param i32 i32 i32 i32 i32 i32 i32 i32) + ) + ) (import "env" "memory" (memory 1 1)) (func (export "call") @@ -47,8 +51,8 @@ ;; Code hash of SET_RENT (data (i32.const 264) - "\c2\1c\41\10\a5\22\d8\59\1c\4c\77\35\dd\2d\bf\a1" - "\13\0b\50\93\76\9b\92\31\97\b7\c5\74\26\aa\38\2a" + "\ab\d6\58\65\1e\83\6e\4a\18\0d\f2\6d\bc\42\ba\e9" + "\3d\64\76\e5\30\5b\33\46\bb\4d\43\99\38\21\ee\32" ) ;; Rent allowance diff --git a/frame/contracts/tests/return_from_start_fn.wat b/frame/contracts/fixtures/return_from_start_fn.wat similarity index 100% rename from frame/contracts/tests/return_from_start_fn.wat rename to frame/contracts/fixtures/return_from_start_fn.wat diff --git a/frame/contracts/tests/return_with_data.wat b/frame/contracts/fixtures/return_with_data.wat similarity index 100% rename from frame/contracts/tests/return_with_data.wat rename to frame/contracts/fixtures/return_with_data.wat diff --git a/frame/contracts/tests/run_out_of_gas.wat b/frame/contracts/fixtures/run_out_of_gas.wat similarity index 100% rename from frame/contracts/tests/run_out_of_gas.wat rename to frame/contracts/fixtures/run_out_of_gas.wat diff --git a/frame/contracts/tests/self_destruct.wat b/frame/contracts/fixtures/self_destruct.wat similarity index 100% rename from frame/contracts/tests/self_destruct.wat rename to frame/contracts/fixtures/self_destruct.wat diff --git a/frame/contracts/tests/self_destructing_constructor.wat b/frame/contracts/fixtures/self_destructing_constructor.wat similarity index 100% rename from frame/contracts/tests/self_destructing_constructor.wat rename to frame/contracts/fixtures/self_destructing_constructor.wat diff --git a/frame/contracts/fixtures/set_empty_storage.wat b/frame/contracts/fixtures/set_empty_storage.wat new file mode 100644 index 0000000000000000000000000000000000000000..d550eb2314b86d0dbb187887f23c8894a18f7348 --- /dev/null +++ b/frame/contracts/fixtures/set_empty_storage.wat @@ -0,0 +1,15 @@ +;; This module stores a KV pair into the storage +(module + (import "env" "ext_set_storage" (func $ext_set_storage (param i32 i32 i32))) + (import "env" "memory" (memory 16 16)) + + (func (export "call") + ) + (func (export "deploy") + (call $ext_set_storage + (i32.const 0) ;; Pointer to storage key + (i32.const 0) ;; Pointer to value + (i32.load (i32.const 0)) ;; Size of value + ) + ) +) diff --git a/frame/contracts/tests/set_rent.wat b/frame/contracts/fixtures/set_rent.wat similarity index 82% rename from frame/contracts/tests/set_rent.wat rename to frame/contracts/fixtures/set_rent.wat index d1affa0d7415f4257597cfac518b5cc7a72631e6..3e6bd491bc475604795796b1c2973169f38359ef 100644 --- a/frame/contracts/tests/set_rent.wat +++ b/frame/contracts/fixtures/set_rent.wat @@ -1,5 +1,5 @@ (module - (import "env" "ext_dispatch_call" (func $ext_dispatch_call (param i32 i32))) + (import "env" "ext_transfer" (func $ext_transfer (param i32 i32 i32 i32) (result i32))) (import "env" "ext_set_storage" (func $ext_set_storage (param i32 i32 i32))) (import "env" "ext_clear_storage" (func $ext_clear_storage (param i32))) (import "env" "ext_set_rent_allowance" (func $ext_set_rent_allowance (param i32 i32))) @@ -23,11 +23,13 @@ ) ) - ;; transfer 50 to ALICE + ;; transfer 50 to CHARLIE (func $call_2 - (call $ext_dispatch_call - (i32.const 68) - (i32.const 11) + (call $assert + (i32.eq + (call $ext_transfer (i32.const 68) (i32.const 8) (i32.const 76) (i32.const 8)) + (i32.const 0) + ) ) ) @@ -96,6 +98,9 @@ ;; Encoding of 10 in balance (data (i32.const 0) "\28") - ;; Encoding of call transfer 50 to CHARLIE - (data (i32.const 68) "\00\00\03\00\00\00\00\00\00\00\C8") + ;; encoding of Charlies's account id + (data (i32.const 68) "\03") + + ;; encoding of 50 balance + (data (i32.const 76) "\32") ) diff --git a/frame/contracts/tests/storage_size.wat b/frame/contracts/fixtures/storage_size.wat similarity index 100% rename from frame/contracts/tests/storage_size.wat rename to frame/contracts/fixtures/storage_size.wat diff --git a/frame/contracts/rpc/Cargo.toml b/frame/contracts/rpc/Cargo.toml index f8692de80a975658e6e14a2f19210f49d12205bd..35989a349049c6cb9eb1d3e42e3655d365fc39da 100644 --- a/frame/contracts/rpc/Cargo.toml +++ b/frame/contracts/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-contracts-rpc" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,18 +12,18 @@ description = "Node-specific RPC methods for interaction with contracts." targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0" } -jsonrpc-core = "14.0.3" -jsonrpc-core-client = "14.0.5" -jsonrpc-derive = "14.0.3" -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-rpc = { version = "2.0.0-alpha.8", path = "../../../primitives/rpc" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +jsonrpc-core = "14.2.0" +jsonrpc-core-client = "14.2.0" +jsonrpc-derive = "14.2.1" +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-rpc = { version = "2.0.0-rc4", path = "../../../primitives/rpc" } serde = { version = "1.0.101", features = ["derive"] } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.8", path = "../../../primitives/api" } -pallet-contracts-primitives = { version = "2.0.0-alpha.8", path = "../common" } -pallet-contracts-rpc-runtime-api = { version = "0.8.0-alpha.8", path = "./runtime-api" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-rc4", path = "../../../primitives/api" } +pallet-contracts-primitives = { version = "2.0.0-rc4", path = "../common" } +pallet-contracts-rpc-runtime-api = { version = "0.8.0-rc4", path = "./runtime-api" } [dev-dependencies] serde_json = "1.0.41" diff --git a/frame/contracts/rpc/runtime-api/Cargo.toml b/frame/contracts/rpc/runtime-api/Cargo.toml index e9d9e6fdfa2e291deb960d16ee42b544ccf32919..e97003c44d94bc4cea9f8bd85c0eafb7c1717d30 100644 --- a/frame/contracts/rpc/runtime-api/Cargo.toml +++ b/frame/contracts/rpc/runtime-api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-contracts-rpc-runtime-api" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,11 +12,11 @@ description = "Runtime API definition required by Contracts RPC extensions." targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-api = { version = "2.0.0-alpha.8", default-features = false, path = "../../../../primitives/api" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../../../primitives/runtime" } -pallet-contracts-primitives = { version = "2.0.0-alpha.8", default-features = false, path = "../../common" } +sp-api = { version = "2.0.0-rc4", default-features = false, path = "../../../../primitives/api" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../../../primitives/std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../../../primitives/runtime" } +pallet-contracts-primitives = { version = "2.0.0-rc4", default-features = false, path = "../../common" } [features] default = ["std"] diff --git a/frame/contracts/rpc/src/lib.rs b/frame/contracts/rpc/src/lib.rs index 89f43f42c3ad1b1512f02312742d7f1644140dc0..18496c13af9fa345faadb7ff749c0ada7e06dbe3 100644 --- a/frame/contracts/rpc/src/lib.rs +++ b/frame/contracts/rpc/src/lib.rs @@ -32,6 +32,7 @@ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, Header as HeaderT}, }; +use std::convert::TryInto; pub use self::gen_client::Client as ContractsClient; pub use pallet_contracts_rpc_runtime_api::{ @@ -80,7 +81,7 @@ pub struct CallRequest { origin: AccountId, dest: AccountId, value: Balance, - gas_limit: number::NumberOrHex, + gas_limit: number::NumberOrHex, input_data: Bytes, } @@ -203,9 +204,11 @@ where gas_limit, input_data, } = call_request; - let gas_limit = gas_limit.to_number().map_err(|e| Error { + + // Make sure that gas_limit fits into 64 bits. + let gas_limit: u64 = gas_limit.try_into().map_err(|_| Error { code: ErrorCode::InvalidParams, - message: e, + message: format!("{:?} doesn't fit in 64 bit unsigned value", gas_limit), data: None, })?; @@ -282,15 +285,30 @@ fn runtime_error_into_rpc_err(err: impl std::fmt::Debug) -> Error { #[cfg(test)] mod tests { use super::*; + use sp_core::U256; + + #[test] + fn call_request_should_serialize_deserialize_properly() { + type Req = CallRequest; + let req: Req = serde_json::from_str(r#" + { + "origin": "5CiPPseXPECbkjWCa6MnjNokrgYjMqmKndv2rSnekmSK2DjL", + "dest": "5DRakbLVnjVrW6niwLfHGW24EeCEvDAFGEXrtaYS5M4ynoom", + "value": 0, + "gasLimit": 1000000000000, + "inputData": "0x8c97db39" + } + "#).unwrap(); + assert_eq!(req.gas_limit.into_u256(), U256::from(0xe8d4a51000u64)); + } #[test] - fn should_serialize_deserialize_properly() { + fn result_should_serialize_deserialize_properly() { fn test(expected: &str) { let res: RpcContractExecResult = serde_json::from_str(expected).unwrap(); let actual = serde_json::to_string(&res).unwrap(); assert_eq!(actual, expected); } - test(r#"{"success":{"status":5,"data":"0x1234"}}"#); test(r#"{"error":null}"#); } diff --git a/frame/contracts/src/account_db.rs b/frame/contracts/src/account_db.rs deleted file mode 100644 index aae853d2ff9965c146eca1f816d6c59ed58c459d..0000000000000000000000000000000000000000 --- a/frame/contracts/src/account_db.rs +++ /dev/null @@ -1,388 +0,0 @@ -// Copyright 2018-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Auxiliaries to help with managing partial changes to accounts state. - -use super::{ - AliveContractInfo, BalanceOf, CodeHash, ContractInfo, ContractInfoOf, Trait, TrieId, - TrieIdGenerator, -}; -use crate::exec::StorageKey; -use sp_std::cell::RefCell; -use sp_std::collections::btree_map::{BTreeMap, Entry}; -use sp_std::prelude::*; -use sp_io::hashing::blake2_256; -use sp_runtime::traits::{Bounded, Zero}; -use frame_support::traits::{Currency, Get, Imbalance, SignedImbalance}; -use frame_support::{storage::child, StorageMap}; -use frame_system; - -// Note: we don't provide Option because we can't create -// the trie_id in the overlay, thus we provide an overlay on the fields -// specifically. -pub struct ChangeEntry { - /// If Some(_), then the account balance is modified to the value. If None and `reset` is false, - /// the balance unmodified. If None and `reset` is true, the balance is reset to 0. - balance: Option>, - /// If Some(_), then a contract is instantiated with the code hash. If None and `reset` is false, - /// then the contract code is unmodified. If None and `reset` is true, the contract is deleted. - code_hash: Option>, - /// If Some(_), then the rent allowance is set to the value. If None and `reset` is false, then - /// the rent allowance is unmodified. If None and `reset` is true, the contract is deleted. - rent_allowance: Option>, - storage: BTreeMap>>, - /// If true, indicates that the existing contract and all its storage entries should be removed - /// and replaced with the fields on this change entry. Otherwise, the fields on this change - /// entry are updates merged into the existing contract info and storage. - reset: bool, -} - -impl ChangeEntry { - fn balance(&self) -> Option> { - self.balance.or_else(|| { - if self.reset { - Some(>::zero()) - } else { - None - } - }) - } - - fn code_hash(&self) -> Option>> { - if self.reset { - Some(self.code_hash) - } else { - self.code_hash.map(Some) - } - } - - fn rent_allowance(&self) -> Option>> { - if self.reset { - Some(self.rent_allowance) - } else { - self.rent_allowance.map(Some) - } - } - - fn storage(&self, location: &StorageKey) -> Option>> { - let value = self.storage.get(location).cloned(); - if self.reset { - Some(value.unwrap_or(None)) - } else { - value - } - } -} - -// Cannot derive(Default) since it erroneously bounds T by Default. -impl Default for ChangeEntry { - fn default() -> Self { - ChangeEntry { - rent_allowance: Default::default(), - balance: Default::default(), - code_hash: Default::default(), - storage: Default::default(), - reset: false, - } - } -} - -pub type ChangeSet = BTreeMap<::AccountId, ChangeEntry>; - -pub trait AccountDb { - /// Account is used when overlayed otherwise trie_id must be provided. - /// This is for performance reason. - /// - /// Trie id is None iff account doesn't have an associated trie id in >. - /// Because DirectAccountDb bypass the lookup for this association. - fn get_storage(&self, account: &T::AccountId, trie_id: Option<&TrieId>, location: &StorageKey) -> Option>; - /// If account has an alive contract then return the code hash associated. - fn get_code_hash(&self, account: &T::AccountId) -> Option>; - /// If account has an alive contract then return the rent allowance associated. - fn get_rent_allowance(&self, account: &T::AccountId) -> Option>; - /// Returns false iff account has no alive contract nor tombstone. - fn contract_exists(&self, account: &T::AccountId) -> bool; - fn get_balance(&self, account: &T::AccountId) -> BalanceOf; - - fn commit(&mut self, change_set: ChangeSet); -} - -pub struct DirectAccountDb; -impl AccountDb for DirectAccountDb { - fn get_storage( - &self, - _account: &T::AccountId, - trie_id: Option<&TrieId>, - location: &StorageKey - ) -> Option> { - trie_id.and_then(|id| child::get_raw(&crate::child_trie_info(&id[..]), &blake2_256(location))) - } - fn get_code_hash(&self, account: &T::AccountId) -> Option> { - >::get(account).and_then(|i| i.as_alive().map(|i| i.code_hash)) - } - fn get_rent_allowance(&self, account: &T::AccountId) -> Option> { - >::get(account).and_then(|i| i.as_alive().map(|i| i.rent_allowance)) - } - fn contract_exists(&self, account: &T::AccountId) -> bool { - >::contains_key(account) - } - fn get_balance(&self, account: &T::AccountId) -> BalanceOf { - T::Currency::free_balance(account) - } - fn commit(&mut self, s: ChangeSet) { - let mut total_imbalance = SignedImbalance::zero(); - for (address, changed) in s.into_iter() { - if let Some(balance) = changed.balance() { - let imbalance = T::Currency::make_free_balance_be(&address, balance); - total_imbalance = total_imbalance.merge(imbalance); - } - - if changed.code_hash().is_some() - || changed.rent_allowance().is_some() - || !changed.storage.is_empty() - || changed.reset - { - let old_info = match >::get(&address) { - Some(ContractInfo::Alive(alive)) => Some(alive), - None => None, - // Cannot commit changes to tombstone contract - Some(ContractInfo::Tombstone(_)) => continue, - }; - - let mut new_info = match (changed.reset, old_info.clone(), changed.code_hash) { - // Existing contract is being modified. - (false, Some(info), _) => info, - // Existing contract is being removed. - (true, Some(info), None) => { - child::kill_storage(&info.child_trie_info()); - >::remove(&address); - continue; - } - // Existing contract is being replaced by a new one. - (true, Some(info), Some(code_hash)) => { - child::kill_storage(&info.child_trie_info()); - AliveContractInfo:: { - code_hash, - storage_size: T::StorageSizeOffset::get(), - trie_id: ::TrieIdGenerator::trie_id(&address), - deduct_block: >::block_number(), - rent_allowance: >::max_value(), - last_write: None, - } - } - // New contract is being instantiated. - (_, None, Some(code_hash)) => { - AliveContractInfo:: { - code_hash, - storage_size: T::StorageSizeOffset::get(), - trie_id: ::TrieIdGenerator::trie_id(&address), - deduct_block: >::block_number(), - rent_allowance: >::max_value(), - last_write: None, - } - } - // There is no existing at the address nor a new one to be instantiated. - (_, None, None) => continue, - }; - - if let Some(rent_allowance) = changed.rent_allowance { - new_info.rent_allowance = rent_allowance; - } - - if let Some(code_hash) = changed.code_hash { - new_info.code_hash = code_hash; - } - - if !changed.storage.is_empty() { - new_info.last_write = Some(>::block_number()); - } - - for (k, v) in changed.storage.into_iter() { - if let Some(value) = child::get_raw( - &new_info.child_trie_info(), - &blake2_256(&k), - ) { - new_info.storage_size -= value.len() as u32; - } - if let Some(value) = v { - new_info.storage_size += value.len() as u32; - child::put_raw(&new_info.child_trie_info(), &blake2_256(&k), &value[..]); - } else { - child::kill(&new_info.child_trie_info(), &blake2_256(&k)); - } - } - - if old_info - .map(|old_info| old_info != new_info) - .unwrap_or(true) - { - >::insert(&address, ContractInfo::Alive(new_info)); - } - } - } - - match total_imbalance { - // If we've detected a positive imbalance as a result of our contract-level machinations - // then it's indicative of a buggy contracts system. - // Panicking is far from ideal as it opens up a DoS attack on block validators, however - // it's a less bad option than allowing arbitrary value to be created. - SignedImbalance::Positive(ref p) if !p.peek().is_zero() => - panic!("contract subsystem resulting in positive imbalance!"), - _ => {} - } - } -} -pub struct OverlayAccountDb<'a, T: Trait + 'a> { - local: RefCell>, - underlying: &'a dyn AccountDb, -} -impl<'a, T: Trait> OverlayAccountDb<'a, T> { - pub fn new(underlying: &'a dyn AccountDb) -> OverlayAccountDb<'a, T> { - OverlayAccountDb { - local: RefCell::new(ChangeSet::new()), - underlying, - } - } - - pub fn into_change_set(self) -> ChangeSet { - self.local.into_inner() - } - - pub fn set_storage( - &mut self, - account: &T::AccountId, - location: StorageKey, - value: Option>, - ) { - self.local.borrow_mut() - .entry(account.clone()) - .or_insert(Default::default()) - .storage - .insert(location, value); - } - - /// Return an error if contract already exists (either if it is alive or tombstone) - pub fn instantiate_contract( - &mut self, - account: &T::AccountId, - code_hash: CodeHash, - ) -> Result<(), &'static str> { - if self.contract_exists(account) { - return Err("Alive contract or tombstone already exists"); - } - - let mut local = self.local.borrow_mut(); - let contract = local.entry(account.clone()).or_insert_with(|| Default::default()); - - contract.code_hash = Some(code_hash); - contract.rent_allowance = Some(>::max_value()); - - Ok(()) - } - - /// Mark a contract as deleted. - pub fn destroy_contract(&mut self, account: &T::AccountId) { - let mut local = self.local.borrow_mut(); - local.insert( - account.clone(), - ChangeEntry { - reset: true, - ..Default::default() - } - ); - } - - /// Assume contract exists - pub fn set_rent_allowance(&mut self, account: &T::AccountId, rent_allowance: BalanceOf) { - self.local - .borrow_mut() - .entry(account.clone()) - .or_insert(Default::default()) - .rent_allowance = Some(rent_allowance); - } - pub fn set_balance(&mut self, account: &T::AccountId, balance: BalanceOf) { - self.local - .borrow_mut() - .entry(account.clone()) - .or_insert(Default::default()) - .balance = Some(balance); - } -} - -impl<'a, T: Trait> AccountDb for OverlayAccountDb<'a, T> { - fn get_storage( - &self, - account: &T::AccountId, - trie_id: Option<&TrieId>, - location: &StorageKey - ) -> Option> { - self.local - .borrow() - .get(account) - .and_then(|changes| changes.storage(location)) - .unwrap_or_else(|| self.underlying.get_storage(account, trie_id, location)) - } - fn get_code_hash(&self, account: &T::AccountId) -> Option> { - self.local - .borrow() - .get(account) - .and_then(|changes| changes.code_hash()) - .unwrap_or_else(|| self.underlying.get_code_hash(account)) - } - fn get_rent_allowance(&self, account: &T::AccountId) -> Option> { - self.local - .borrow() - .get(account) - .and_then(|changes| changes.rent_allowance()) - .unwrap_or_else(|| self.underlying.get_rent_allowance(account)) - } - fn contract_exists(&self, account: &T::AccountId) -> bool { - self.local - .borrow() - .get(account) - .and_then(|changes| changes.code_hash().map(|code_hash| code_hash.is_some())) - .unwrap_or_else(|| self.underlying.contract_exists(account)) - } - fn get_balance(&self, account: &T::AccountId) -> BalanceOf { - self.local - .borrow() - .get(account) - .and_then(|changes| changes.balance()) - .unwrap_or_else(|| self.underlying.get_balance(account)) - } - fn commit(&mut self, s: ChangeSet) { - let mut local = self.local.borrow_mut(); - - for (address, changed) in s.into_iter() { - match local.entry(address) { - Entry::Occupied(e) => { - let mut value = e.into_mut(); - if changed.reset { - *value = changed; - } else { - value.balance = changed.balance.or(value.balance); - value.code_hash = changed.code_hash.or(value.code_hash); - value.rent_allowance = changed.rent_allowance.or(value.rent_allowance); - value.storage.extend(changed.storage.into_iter()); - } - } - Entry::Vacant(e) => { - e.insert(changed); - } - } - } - } -} diff --git a/frame/contracts/src/exec.rs b/frame/contracts/src/exec.rs index 9cc1c50260db9fed6581217e08107a61d4aafe6c..4e68aac61511c42de2200b0c5ac9fc38bbf0870b 100644 --- a/frame/contracts/src/exec.rs +++ b/frame/contracts/src/exec.rs @@ -15,20 +15,20 @@ // along with Substrate. If not, see . use super::{CodeHash, Config, ContractAddressFor, Event, RawEvent, Trait, - TrieId, BalanceOf, ContractInfo}; -use crate::account_db::{AccountDb, DirectAccountDb, OverlayAccountDb}; + TrieId, BalanceOf, ContractInfo, TrieIdGenerator}; use crate::gas::{Gas, GasMeter, Token}; use crate::rent; +use crate::storage; use sp_std::prelude::*; -use sp_runtime::traits::{Bounded, CheckedAdd, CheckedSub, Zero}; +use sp_runtime::traits::{Bounded, Zero, Convert}; use frame_support::{ storage::unhashed, dispatch::DispatchError, - traits::{WithdrawReason, Currency, Time, Randomness}, + traits::{ExistenceRequirement, Currency, Time, Randomness}, + weights::Weight, }; pub type AccountIdOf = ::AccountId; -pub type CallOf = ::Call; pub type MomentOf = <::Time as Time>::Moment; pub type SeedOf = ::Hash; pub type BlockNumberOf = ::BlockNumber; @@ -105,8 +105,8 @@ pub trait Ext { fn get_storage(&self, key: &StorageKey) -> Option>; /// Sets the storage entry by the given key to the specified value. If `value` is `None` then - /// the storage entry is deleted. Returns an Err if the value size is too large. - fn set_storage(&mut self, key: StorageKey, value: Option>) -> Result<(), &'static str>; + /// the storage entry is deleted. + fn set_storage(&mut self, key: StorageKey, value: Option>); /// Instantiate a contract from the given code. /// @@ -129,6 +129,12 @@ pub trait Ext { ) -> Result<(), DispatchError>; /// Transfer all funds to `beneficiary` and delete the contract. + /// + /// Since this function removes the self contract eagerly, if succeeded, no further actions should + /// be performed on this `Ext` instance. + /// + /// This function will fail if the same contract is present on the contract + /// call stack. fn terminate( &mut self, beneficiary: &AccountIdOf, @@ -144,17 +150,20 @@ pub trait Ext { input_data: Vec, ) -> ExecResult; - /// Notes a call dispatch. - fn note_dispatch_call(&mut self, call: CallOf); - - /// Notes a restoration request. - fn note_restore_to( + /// Restores the given destination contract sacrificing the current one. + /// + /// Since this function removes the self contract eagerly, if succeeded, no further actions should + /// be performed on this `Ext` instance. + /// + /// This function will fail if the same contract is present + /// on the contract call stack. + fn restore_to( &mut self, dest: AccountIdOf, code_hash: CodeHash, rent_allowance: BalanceOf, delta: Vec, - ); + ) -> Result<(), &'static str>; /// Returns a reference to the account id of the caller. fn caller(&self) -> &AccountIdOf; @@ -204,8 +213,8 @@ pub trait Ext { /// Returns `None` if the value doesn't exist. fn get_runtime_storage(&self, key: &[u8]) -> Option>; - /// Returns the price of one weight unit. - fn get_weight_price(&self) -> BalanceOf; + /// Returns the price for the specified amount of weight. + fn get_weight_price(&self, weight: Weight) -> BalanceOf; } /// Loader is a companion of the `Vm` trait. It loads an appropriate abstract @@ -261,43 +270,11 @@ impl Token for ExecFeeToken { } } -#[cfg_attr(any(feature = "std", test), derive(PartialEq, Eq, Clone))] -#[derive(sp_runtime::RuntimeDebug)] -pub enum DeferredAction { - DepositEvent { - /// A list of topics this event will be deposited with. - topics: Vec, - /// The event to deposit. - event: Event, - }, - DispatchRuntimeCall { - /// The account id of the contract who dispatched this call. - origin: T::AccountId, - /// The call to dispatch. - call: ::Call, - }, - RestoreTo { - /// The account id of the contract which is removed during the restoration and transfers - /// its storage to the restored contract. - donor: T::AccountId, - /// The account id of the restored contract. - dest: T::AccountId, - /// The code hash of the restored contract. - code_hash: CodeHash, - /// The initial rent allowance to set. - rent_allowance: BalanceOf, - /// The keys to delete upon restoration. - delta: Vec, - }, -} - pub struct ExecutionContext<'a, T: Trait + 'a, V, L> { pub caller: Option<&'a ExecutionContext<'a, T, V, L>>, pub self_account: T::AccountId, pub self_trie_id: Option, - pub overlay: OverlayAccountDb<'a, T>, pub depth: usize, - pub deferred: Vec>, pub config: &'a Config, pub vm: &'a V, pub loader: &'a L, @@ -320,9 +297,7 @@ where caller: None, self_trie_id: None, self_account: origin, - overlay: OverlayAccountDb::::new(&DirectAccountDb), depth: 0, - deferred: Vec::new(), config: &cfg, vm: &vm, loader: &loader, @@ -338,9 +313,7 @@ where caller: Some(self), self_trie_id: trie_id, self_account: dest, - overlay: OverlayAccountDb::new(&self.overlay), depth: self.depth + 1, - deferred: Vec::new(), config: self.config, vm: self.vm, loader: self.loader, @@ -349,23 +322,6 @@ where } } - /// Transfer balance to `dest` without calling any contract code. - pub fn transfer( - &mut self, - dest: T::AccountId, - value: BalanceOf, - gas_meter: &mut GasMeter - ) -> Result<(), DispatchError> { - transfer( - gas_meter, - TransferCause::Call, - &self.self_account.clone(), - &dest, - value, - self, - ) - } - /// Make a call to the specified address, optionally transferring some funds. pub fn call( &mut self, @@ -424,8 +380,8 @@ where // If code_hash is not none, then the destination account is a live contract, otherwise // it is a regular account since tombstone accounts have already been rejected. - match nested.overlay.get_code_hash(&dest) { - Some(dest_code_hash) => { + match storage::code_hash::(&dest) { + Ok(dest_code_hash) => { let executable = try_or_exec_error!( nested.loader.load_main(&dest_code_hash), input_data @@ -437,10 +393,9 @@ where input_data, gas_meter, )?; - Ok(output) } - None => Ok(ExecReturnValue { status: STATUS_SUCCESS, data: Vec::new() }), + Err(storage::ContractAbsentError) => Ok(ExecReturnValue { status: STATUS_SUCCESS, data: Vec::new() }), } }) } @@ -477,11 +432,20 @@ where ); // TrieId has not been generated yet and storage is empty since contract is new. - let dest_trie_id = None; + // + // Generate it now. + let dest_trie_id = ::TrieIdGenerator::trie_id(&dest); - let output = self.with_nested_context(dest.clone(), dest_trie_id, |nested| { + let output = self.with_nested_context(dest.clone(), Some(dest_trie_id), |nested| { try_or_exec_error!( - nested.overlay.instantiate_contract(&dest, code_hash.clone()), + storage::place_contract::( + &dest, + nested + .self_trie_id + .clone() + .expect("the nested context always has to have self_trie_id"), + code_hash.clone() + ), input_data ); @@ -512,7 +476,7 @@ where )?; // Error out if insufficient remaining balance. - if nested.overlay.get_balance(&dest) < nested.config.existential_deposit { + if T::Currency::free_balance(&dest) < nested.config.existential_deposit { return Err(ExecError { reason: "insufficient remaining balance".into(), buffer: output.data, @@ -520,10 +484,7 @@ where } // Deposit an instantiation event. - nested.deferred.push(DeferredAction::DepositEvent { - event: RawEvent::Instantiated(caller.clone(), dest.clone()), - topics: Vec::new(), - }); + deposit_event::(vec![], RawEvent::Instantiated(caller.clone(), dest.clone())); Ok(output) })?; @@ -531,32 +492,6 @@ where Ok((dest, output)) } - pub fn terminate( - &mut self, - beneficiary: &T::AccountId, - gas_meter: &mut GasMeter, - ) -> Result<(), DispatchError> { - let self_id = self.self_account.clone(); - let value = self.overlay.get_balance(&self_id); - if let Some(caller) = self.caller { - if caller.is_live(&self_id) { - return Err(DispatchError::Other( - "Cannot terminate a contract that is present on the call stack", - )); - } - } - transfer( - gas_meter, - TransferCause::Terminate, - &self_id, - beneficiary, - value, - self, - )?; - self.overlay.destroy_contract(&self_id); - Ok(()) - } - fn new_call_context<'b>( &'b mut self, caller: T::AccountId, @@ -573,22 +508,20 @@ where } } + /// Execute the given closure within a nested execution context. fn with_nested_context(&mut self, dest: T::AccountId, trie_id: Option, func: F) -> ExecResult where F: FnOnce(&mut ExecutionContext) -> ExecResult { - let (output, change_set, deferred) = { - let mut nested = self.nested(dest, trie_id); - let output = func(&mut nested)?; - (output, nested.overlay.into_change_set(), nested.deferred) - }; - - if output.is_success() { - self.overlay.commit(change_set); - self.deferred.extend(deferred); - } - - Ok(output) + use frame_support::storage::TransactionOutcome::*; + let mut nested = self.nested(dest, trie_id); + frame_support::storage::with_transaction(|| { + let output = func(&mut nested); + match output { + Ok(ref rv) if rv.is_success() => Commit(output), + _ => Rollback(output), + } + }) } /// Returns whether a contract, identified by address, is currently live in the execution @@ -676,48 +609,27 @@ fn transfer<'a, T: Trait, V: Vm, L: Loader>( Err("not enough gas to pay transfer fee")? } - // We allow balance to go below the existential deposit here: - let from_balance = ctx.overlay.get_balance(transactor); - let new_from_balance = match from_balance.checked_sub(&value) { - Some(b) => b, - None => Err("balance too low to send value")?, - }; - let to_balance = ctx.overlay.get_balance(dest); - if to_balance.is_zero() && value < ctx.config.existential_deposit { - Err("value too low to create account")? - } - // Only ext_terminate is allowed to bring the sender below the existential deposit - let required_balance = match cause { - Terminate => 0.into(), - _ => ctx.config.existential_deposit + let existence_requirement = match cause { + Terminate => ExistenceRequirement::AllowDeath, + _ => ExistenceRequirement::KeepAlive, }; - - T::Currency::ensure_can_withdraw( - transactor, - value, - WithdrawReason::Transfer.into(), - new_from_balance.checked_sub(&required_balance) - .ok_or("brings sender below existential deposit")?, - )?; - - let new_to_balance = match to_balance.checked_add(&value) { - Some(b) => b, - None => Err("destination balance too high to receive value")?, - }; - - if transactor != dest { - ctx.overlay.set_balance(transactor, new_from_balance); - ctx.overlay.set_balance(dest, new_to_balance); - ctx.deferred.push(DeferredAction::DepositEvent { - event: RawEvent::Transfer(transactor.clone(), dest.clone(), value), - topics: Vec::new(), - }); - } + T::Currency::transfer(transactor, dest, value, existence_requirement)?; Ok(()) } +/// A context that is active within a call. +/// +/// This context has some invariants that must be held at all times. Specifically: +///`ctx` always points to a context of an alive contract. That implies that it has an existent +/// `self_trie_id`. +/// +/// Be advised that there are brief time spans where these invariants could be invalidated. +/// For example, when a contract requests self-termination the contract is removed eagerly. That +/// implies that the control won't be returned to the contract anymore, but there is still some code +/// on the path of the return from that call context. Therefore, care must be taken in these +/// situations. struct CallContext<'a, 'b: 'a, T: Trait + 'b, V: Vm + 'b, L: Loader> { ctx: &'a mut ExecutionContext<'b, T, V, L>, caller: T::AccountId, @@ -735,20 +647,32 @@ where type T = T; fn get_storage(&self, key: &StorageKey) -> Option> { - self.ctx.overlay.get_storage(&self.ctx.self_account, self.ctx.self_trie_id.as_ref(), key) + let trie_id = self.ctx.self_trie_id.as_ref().expect( + "`ctx.self_trie_id` points to an alive contract within the `CallContext`;\ + it cannot be `None`;\ + expect can't fail;\ + qed", + ); + storage::read_contract_storage(trie_id, key) } - fn set_storage(&mut self, key: StorageKey, value: Option>) -> Result<(), &'static str> { - if let Some(ref value) = value { - if self.max_value_size() < value.len() as u32 { - return Err("value size exceeds maximum"); - } + fn set_storage(&mut self, key: StorageKey, value: Option>) { + let trie_id = self.ctx.self_trie_id.as_ref().expect( + "`ctx.self_trie_id` points to an alive contract within the `CallContext`;\ + it cannot be `None`;\ + expect can't fail;\ + qed", + ); + if let Err(storage::ContractAbsentError) = + storage::write_contract_storage::(&self.ctx.self_account, trie_id, &key, value) + { + panic!( + "the contract must be in the alive state within the `CallContext`;\ + the contract cannot be absent in storage; + write_contract_storage cannot return `None`; + qed" + ); } - - self.ctx - .overlay - .set_storage(&self.ctx.self_account, key, value); - Ok(()) } fn instantiate( @@ -767,7 +691,14 @@ where value: BalanceOf, gas_meter: &mut GasMeter, ) -> Result<(), DispatchError> { - self.ctx.transfer(to.clone(), value, gas_meter) + transfer( + gas_meter, + TransferCause::Call, + &self.ctx.self_account.clone(), + &to, + value, + self.ctx, + ) } fn terminate( @@ -775,7 +706,30 @@ where beneficiary: &AccountIdOf, gas_meter: &mut GasMeter, ) -> Result<(), DispatchError> { - self.ctx.terminate(beneficiary, gas_meter) + let self_id = self.ctx.self_account.clone(); + let value = T::Currency::free_balance(&self_id); + if let Some(caller_ctx) = self.ctx.caller { + if caller_ctx.is_live(&self_id) { + return Err(DispatchError::Other( + "Cannot terminate a contract that is present on the call stack", + )); + } + } + transfer( + gas_meter, + TransferCause::Terminate, + &self_id, + beneficiary, + value, + self.ctx, + )?; + let self_trie_id = self.ctx.self_trie_id.as_ref().expect( + "this function is only invoked by in the context of a contract;\ + a contract has a trie id;\ + this can't be None; qed", + ); + storage::destroy_contract::(&self_id, self_trie_id); + Ok(()) } fn call( @@ -788,27 +742,40 @@ where self.ctx.call(to.clone(), value, gas_meter, input_data) } - fn note_dispatch_call(&mut self, call: CallOf) { - self.ctx.deferred.push(DeferredAction::DispatchRuntimeCall { - origin: self.ctx.self_account.clone(), - call, - }); - } - - fn note_restore_to( + fn restore_to( &mut self, dest: AccountIdOf, code_hash: CodeHash, rent_allowance: BalanceOf, delta: Vec, - ) { - self.ctx.deferred.push(DeferredAction::RestoreTo { - donor: self.ctx.self_account.clone(), - dest, - code_hash, + ) -> Result<(), &'static str> { + if let Some(caller_ctx) = self.ctx.caller { + if caller_ctx.is_live(&self.ctx.self_account) { + return Err( + "Cannot perform restoration of a contract that is present on the call stack", + ); + } + } + + let result = crate::rent::restore_to::( + self.ctx.self_account.clone(), + dest.clone(), + code_hash.clone(), rent_allowance, delta, - }); + ); + if let Ok(_) = result { + deposit_event::( + vec![], + RawEvent::Restored( + self.ctx.self_account.clone(), + dest, + code_hash, + rent_allowance, + ), + ); + } + result } fn address(&self) -> &T::AccountId { @@ -820,7 +787,7 @@ where } fn balance(&self) -> BalanceOf { - self.ctx.overlay.get_balance(&self.ctx.self_account) + T::Currency::free_balance(&self.ctx.self_account) } fn value_transferred(&self) -> BalanceOf { @@ -844,18 +811,25 @@ where } fn deposit_event(&mut self, topics: Vec, data: Vec) { - self.ctx.deferred.push(DeferredAction::DepositEvent { + deposit_event::( topics, - event: RawEvent::ContractExecution(self.ctx.self_account.clone(), data), - }); + RawEvent::ContractExecution(self.ctx.self_account.clone(), data) + ); } fn set_rent_allowance(&mut self, rent_allowance: BalanceOf) { - self.ctx.overlay.set_rent_allowance(&self.ctx.self_account, rent_allowance) + if let Err(storage::ContractAbsentError) = + storage::set_rent_allowance::(&self.ctx.self_account, rent_allowance) + { + panic!( + "`self_account` points to an alive contract within the `CallContext`; + set_rent_allowance cannot return `Err`; qed" + ); + } } fn rent_allowance(&self) -> BalanceOf { - self.ctx.overlay.get_rent_allowance(&self.ctx.self_account) + storage::rent_allowance::(&self.ctx.self_account) .unwrap_or(>::max_value()) // Must never be triggered actually } @@ -869,38 +843,42 @@ where unhashed::get_raw(&key) } - fn get_weight_price(&self) -> BalanceOf { - use pallet_transaction_payment::Module as Payment; - use sp_runtime::SaturatedConversion; - let price = Payment::::weight_to_fee_with_adjustment::(1); - price.saturated_into() + fn get_weight_price(&self, weight: Weight) -> BalanceOf { + T::WeightPrice::convert(weight) } } +fn deposit_event( + topics: Vec, + event: Event, +) { + >::deposit_event_indexed( + &*topics, + ::Event::from(event).into(), + ) +} + /// These tests exercise the executive layer. /// /// In these tests the VM/loader are mocked. Instead of dealing with wasm bytecode they use simple closures. /// This allows you to tackle executive logic more thoroughly without writing a /// wasm VM code. -/// -/// Because it's the executive layer: -/// -/// - no gas meter setup and teardown logic. All balances are *AFTER* gas purchase. -/// - executive layer doesn't alter any storage! #[cfg(test)] mod tests { use super::{ - BalanceOf, ExecFeeToken, ExecutionContext, Ext, Loader, TransferFeeKind, TransferFeeToken, - Vm, ExecResult, RawEvent, DeferredAction, + BalanceOf, Event, ExecFeeToken, ExecResult, ExecutionContext, Ext, Loader, + RawEvent, TransferFeeKind, TransferFeeToken, Vm, }; use crate::{ - account_db::AccountDb, gas::GasMeter, tests::{ExtBuilder, Test}, + gas::GasMeter, tests::{ExtBuilder, Test, MetaEvent}, exec::{ExecReturnValue, ExecError, STATUS_SUCCESS}, CodeHash, Config, gas::Gas, + storage, }; - use std::{cell::RefCell, rc::Rc, collections::HashMap, marker::PhantomData}; - use assert_matches::assert_matches; + use crate::tests::test_utils::{place_contract, set_balance, get_balance}; use sp_runtime::DispatchError; + use assert_matches::assert_matches; + use std::{cell::RefCell, collections::HashMap, marker::PhantomData, rc::Rc}; const ALICE: u64 = 1; const BOB: u64 = 2; @@ -908,19 +886,14 @@ mod tests { const GAS_LIMIT: Gas = 10_000_000_000; - impl<'a, T, V, L> ExecutionContext<'a, T, V, L> - where T: crate::Trait - { - fn events(&self) -> Vec> { - self.deferred - .iter() - .filter(|action| match *action { - DeferredAction::DepositEvent { .. } => true, - _ => false, - }) - .cloned() - .collect() - } + fn events() -> Vec> { + >::events() + .into_iter() + .filter_map(|meta| match meta.event { + MetaEvent::contracts(contract_event) => Some(contract_event), + _ => None, + }) + .collect() } struct MockCtx<'a> { @@ -1029,7 +1002,7 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader); - ctx.overlay.instantiate_contract(&BOB, exec_ch).unwrap(); + place_contract(&BOB, exec_ch); assert_matches!( ctx.call(BOB, value, &mut gas_meter, data), @@ -1051,8 +1024,8 @@ mod tests { let loader = MockLoader::empty(); let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(origin, &cfg, &vm, &loader); - ctx.overlay.set_balance(&origin, 100); - ctx.overlay.set_balance(&dest, 0); + set_balance(&origin, 100); + set_balance(&dest, 0); let mut gas_meter = GasMeter::::new(GAS_LIMIT); @@ -1072,7 +1045,7 @@ mod tests { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(origin, &cfg, &vm, &loader); - ctx.overlay.set_balance(&origin, 100); + set_balance(&origin, 100); let mut gas_meter = GasMeter::::new(GAS_LIMIT); @@ -1097,8 +1070,8 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(origin, &cfg, &vm, &loader); - ctx.overlay.set_balance(&origin, 100); - ctx.overlay.set_balance(&dest, 0); + set_balance(&origin, 100); + set_balance(&dest, 0); let output = ctx.call( dest, @@ -1108,15 +1081,15 @@ mod tests { ).unwrap(); assert!(output.is_success()); - assert_eq!(ctx.overlay.get_balance(&origin), 45); - assert_eq!(ctx.overlay.get_balance(&dest), 55); + assert_eq!(get_balance(&origin), 45); + assert_eq!(get_balance(&dest), 55); }); } #[test] fn changes_are_reverted_on_failing_call() { - // This test verifies that a contract is able to transfer - // some funds to another account. + // This test verifies that changes are reverted on a call which fails (or equally, returns + // a non-zero status code). let origin = ALICE; let dest = BOB; @@ -1129,9 +1102,9 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(origin, &cfg, &vm, &loader); - ctx.overlay.instantiate_contract(&BOB, return_ch).unwrap(); - ctx.overlay.set_balance(&origin, 100); - ctx.overlay.set_balance(&dest, 0); + place_contract(&BOB, return_ch); + set_balance(&origin, 100); + set_balance(&dest, 0); let output = ctx.call( dest, @@ -1141,8 +1114,8 @@ mod tests { ).unwrap(); assert!(!output.is_success()); - assert_eq!(ctx.overlay.get_balance(&origin), 100); - assert_eq!(ctx.overlay.get_balance(&dest), 0); + assert_eq!(get_balance(&origin), 100); + assert_eq!(get_balance(&dest), 0); }); } @@ -1159,8 +1132,8 @@ mod tests { let loader = MockLoader::empty(); let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(origin, &cfg, &vm, &loader); - ctx.overlay.set_balance(&origin, 100); - ctx.overlay.set_balance(&dest, 0); + set_balance(&origin, 100); + set_balance(&dest, 0); let mut gas_meter = GasMeter::::new(GAS_LIMIT); @@ -1184,8 +1157,8 @@ mod tests { let loader = MockLoader::empty(); let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(origin, &cfg, &vm, &loader); - ctx.overlay.set_balance(&origin, 100); - ctx.overlay.set_balance(&dest, 15); + set_balance(&origin, 100); + set_balance(&dest, 15); let mut gas_meter = GasMeter::::new(GAS_LIMIT); @@ -1212,8 +1185,8 @@ mod tests { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(origin, &cfg, &vm, &loader); - ctx.overlay.set_balance(&origin, 100); - ctx.overlay.set_balance(&dest, 15); + set_balance(&origin, 100); + set_balance(&dest, 15); let mut gas_meter = GasMeter::::new(GAS_LIMIT); @@ -1244,7 +1217,7 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(origin, &cfg, &vm, &loader); - ctx.overlay.set_balance(&origin, 0); + set_balance(&origin, 0); let result = ctx.call( dest, @@ -1256,12 +1229,12 @@ mod tests { assert_matches!( result, Err(ExecError { - reason: DispatchError::Other("balance too low to send value"), + reason: DispatchError::Module { message: Some("InsufficientBalance"), .. }, buffer: _, }) ); - assert_eq!(ctx.overlay.get_balance(&origin), 0); - assert_eq!(ctx.overlay.get_balance(&dest), 0); + assert_eq!(get_balance(&origin), 0); + assert_eq!(get_balance(&dest), 0); }); } @@ -1281,7 +1254,7 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(origin, &cfg, &vm, &loader); - ctx.overlay.instantiate_contract(&BOB, return_ch).unwrap(); + place_contract(&BOB, return_ch); let result = ctx.call( dest, @@ -1312,7 +1285,7 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(origin, &cfg, &vm, &loader); - ctx.overlay.instantiate_contract(&BOB, return_ch).unwrap(); + place_contract(&BOB, return_ch); let result = ctx.call( dest, @@ -1340,7 +1313,7 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader); - ctx.overlay.instantiate_contract(&BOB, input_data_ch).unwrap(); + place_contract(&BOB, input_data_ch); let result = ctx.call( BOB, @@ -1366,7 +1339,7 @@ mod tests { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader); - ctx.overlay.set_balance(&ALICE, 100); + set_balance(&ALICE, 100); let result = ctx.instantiate( 1, @@ -1414,8 +1387,8 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader); - ctx.overlay.set_balance(&BOB, 1); - ctx.overlay.instantiate_contract(&BOB, recurse_ch).unwrap(); + set_balance(&BOB, 1); + place_contract(&BOB, recurse_ch); let result = ctx.call( BOB, @@ -1460,8 +1433,8 @@ mod tests { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(origin, &cfg, &vm, &loader); - ctx.overlay.instantiate_contract(&dest, bob_ch).unwrap(); - ctx.overlay.instantiate_contract(&CHARLIE, charlie_ch).unwrap(); + place_contract(&dest, bob_ch); + place_contract(&CHARLIE, charlie_ch); let result = ctx.call( dest, @@ -1501,8 +1474,8 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader); - ctx.overlay.instantiate_contract(&BOB, bob_ch).unwrap(); - ctx.overlay.instantiate_contract(&CHARLIE, charlie_ch).unwrap(); + place_contract(&BOB, bob_ch); + place_contract(&CHARLIE, charlie_ch); let result = ctx.call( BOB, @@ -1550,7 +1523,7 @@ mod tests { ExtBuilder::default().existential_deposit(15).build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader); - ctx.overlay.set_balance(&ALICE, 1000); + set_balance(&ALICE, 1000); let instantiated_contract_address = assert_matches!( ctx.instantiate( @@ -1564,16 +1537,9 @@ mod tests { // Check that the newly created account has the expected code hash and // there are instantiation event. - assert_eq!(ctx.overlay.get_code_hash(&instantiated_contract_address).unwrap(), dummy_ch); - assert_eq!(&ctx.events(), &[ - DeferredAction::DepositEvent { - event: RawEvent::Transfer(ALICE, instantiated_contract_address, 100), - topics: Vec::new(), - }, - DeferredAction::DepositEvent { - event: RawEvent::Instantiated(ALICE, instantiated_contract_address), - topics: Vec::new(), - } + assert_eq!(storage::code_hash::(&instantiated_contract_address).unwrap(), dummy_ch); + assert_eq!(&events(), &[ + RawEvent::Instantiated(ALICE, instantiated_contract_address) ]); }); } @@ -1590,7 +1556,7 @@ mod tests { ExtBuilder::default().existential_deposit(15).build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader); - ctx.overlay.set_balance(&ALICE, 1000); + set_balance(&ALICE, 1000); let instantiated_contract_address = assert_matches!( ctx.instantiate( @@ -1603,8 +1569,8 @@ mod tests { ); // Check that the account has not been created. - assert!(ctx.overlay.get_code_hash(&instantiated_contract_address).is_none()); - assert!(ctx.events().is_empty()); + assert!(storage::code_hash::(&instantiated_contract_address).is_err()); + assert!(events().is_empty()); }); } @@ -1635,9 +1601,9 @@ mod tests { ExtBuilder::default().existential_deposit(15).build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader); - ctx.overlay.set_balance(&ALICE, 1000); - ctx.overlay.set_balance(&BOB, 100); - ctx.overlay.instantiate_contract(&BOB, instantiator_ch).unwrap(); + set_balance(&ALICE, 1000); + set_balance(&BOB, 100); + place_contract(&BOB, instantiator_ch); assert_matches!( ctx.call(BOB, 20, &mut GasMeter::::new(GAS_LIMIT), vec![]), @@ -1648,20 +1614,9 @@ mod tests { // Check that the newly created account has the expected code hash and // there are instantiation event. - assert_eq!(ctx.overlay.get_code_hash(&instantiated_contract_address).unwrap(), dummy_ch); - assert_eq!(&ctx.events(), &[ - DeferredAction::DepositEvent { - event: RawEvent::Transfer(ALICE, BOB, 20), - topics: Vec::new(), - }, - DeferredAction::DepositEvent { - event: RawEvent::Transfer(BOB, instantiated_contract_address, 15), - topics: Vec::new(), - }, - DeferredAction::DepositEvent { - event: RawEvent::Instantiated(BOB, instantiated_contract_address), - topics: Vec::new(), - }, + assert_eq!(storage::code_hash::(&instantiated_contract_address).unwrap(), dummy_ch); + assert_eq!(&events(), &[ + RawEvent::Instantiated(BOB, instantiated_contract_address) ]); }); } @@ -1695,9 +1650,9 @@ mod tests { ExtBuilder::default().existential_deposit(15).build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader); - ctx.overlay.set_balance(&ALICE, 1000); - ctx.overlay.set_balance(&BOB, 100); - ctx.overlay.instantiate_contract(&BOB, instantiator_ch).unwrap(); + set_balance(&ALICE, 1000); + set_balance(&BOB, 100); + place_contract(&BOB, instantiator_ch); assert_matches!( ctx.call(BOB, 20, &mut GasMeter::::new(GAS_LIMIT), vec![]), @@ -1706,12 +1661,7 @@ mod tests { // The contract wasn't instantiated so we don't expect to see an instantiation // event here. - assert_eq!(&ctx.events(), &[ - DeferredAction::DepositEvent { - event: RawEvent::Transfer(ALICE, BOB, 20), - topics: Vec::new(), - }, - ]); + assert_eq!(&events(), &[]); }); } @@ -1732,7 +1682,7 @@ mod tests { .execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader); - ctx.overlay.set_balance(&ALICE, 1000); + set_balance(&ALICE, 1000); assert_matches!( ctx.instantiate( @@ -1748,7 +1698,7 @@ mod tests { ); assert_eq!( - &ctx.events(), + &events(), &[] ); }); @@ -1768,8 +1718,7 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader); - - ctx.overlay.set_balance(&ALICE, 100); + set_balance(&ALICE, 100); let result = ctx.instantiate( 1, diff --git a/frame/contracts/src/lib.rs b/frame/contracts/src/lib.rs index 10ca0441501babd05705160f11f2b4872260f79a..4db77a078e92d0aa8e3bde24df8bf14b76973b80 100644 --- a/frame/contracts/src/lib.rs +++ b/frame/contracts/src/lib.rs @@ -81,8 +81,7 @@ #[macro_use] mod gas; - -mod account_db; +mod storage; mod exec; mod wasm; mod rent; @@ -91,7 +90,6 @@ mod rent; mod tests; use crate::exec::ExecutionContext; -use crate::account_db::{AccountDb, DirectAccountDb}; use crate::wasm::{WasmLoader, WasmVm}; pub use crate::gas::{Gas, GasMeter}; @@ -102,24 +100,21 @@ use serde::{Serialize, Deserialize}; use sp_core::crypto::UncheckedFrom; use sp_std::{prelude::*, marker::PhantomData, fmt::Debug}; use codec::{Codec, Encode, Decode}; -use sp_io::hashing::blake2_256; use sp_runtime::{ traits::{ - Hash, StaticLookup, Zero, MaybeSerializeDeserialize, Member, + Hash, StaticLookup, Zero, MaybeSerializeDeserialize, Member, Convert, }, RuntimeDebug, }; -use frame_support::dispatch::{ - PostDispatchInfo, DispatchResult, Dispatchable, DispatchResultWithPostInfo -}; use frame_support::{ - Parameter, decl_module, decl_event, decl_storage, decl_error, - parameter_types, IsSubType, storage::child::{self, ChildInfo}, + decl_module, decl_event, decl_storage, decl_error, + parameter_types, storage::child::ChildInfo, + dispatch::{DispatchResult, DispatchResultWithPostInfo}, + traits::{OnUnbalanced, Currency, Get, Time, Randomness}, }; -use frame_support::traits::{OnUnbalanced, Currency, Get, Time, Randomness}; -use frame_support::weights::{FunctionOf, DispatchClass, Weight, GetDispatchInfo, Pays}; -use frame_system::{self as system, ensure_signed, RawOrigin, ensure_root}; +use frame_system::{self as system, ensure_signed, ensure_root}; use pallet_contracts_primitives::{RentProjection, ContractAccessError}; +use frame_support::weights::Weight; pub type CodeHash = ::Hash; pub type TrieId = Vec; @@ -129,11 +124,6 @@ pub trait ContractAddressFor { fn contract_address_for(code_hash: &CodeHash, data: &[u8], origin: &AccountId) -> AccountId; } -/// A function that returns the fee for dispatching a `Call`. -pub trait ComputeDispatchFee { - fn compute_dispatch_fee(call: &Call) -> Balance; -} - /// Information for managing an account and its sub trie abstraction. /// This is the required info to cache for an account #[derive(Encode, Decode, RuntimeDebug)] @@ -203,8 +193,15 @@ pub type AliveContractInfo = pub struct RawAliveContractInfo { /// Unique ID for the subtree encoded as a bytes vector. pub trie_id: TrieId, - /// The size of stored value in octet. + /// The total number of bytes used by this contract. + /// + /// It is a sum of each key-value pair stored by this contract. pub storage_size: u32, + /// The number of key-value pairs that have values of zero length. + /// The condition `empty_pair_count ≤ total_pair_count` always holds. + pub empty_pair_count: u32, + /// The total number of key-value pairs in storage of this contract. + pub total_pair_count: u32, /// The code associated with a given account. pub code_hash: CodeHash, /// Pay rent at most up to this value. @@ -248,6 +245,12 @@ where } } +impl From> for ContractInfo { + fn from(alive_info: AliveContractInfo) -> Self { + Self::Alive(alive_info) + } +} + /// Get a trie id (trie id must be unique and collision resistant depending upon its context). /// Note that it is different than encode because trie id should be collision resistant /// (being a proper unique identifier). @@ -284,9 +287,10 @@ where } } -pub type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; +pub type BalanceOf = + <::Currency as Currency<::AccountId>>::Balance; pub type NegativeImbalanceOf = - <::Currency as Currency<::AccountId>>::NegativeImbalance; + <::Currency as Currency<::AccountId>>::NegativeImbalance; parameter_types! { /// A reasonable default value for [`Trait::SignedClaimedHandicap`]. @@ -307,15 +311,12 @@ parameter_types! { pub const DefaultMaxValueSize: u32 = 16_384; } -pub trait Trait: frame_system::Trait + pallet_transaction_payment::Trait { +pub trait Trait: frame_system::Trait { type Time: Time; type Randomness: Randomness; - /// The outer call dispatch type. - type Call: - Parameter + - Dispatchable::Origin> + - IsSubType> + GetDispatchInfo; + /// The currency in which fees are paid and contract balances are held. + type Currency: Currency; /// The overarching event type. type Event: From> + Into<::Event>; @@ -338,8 +339,11 @@ pub trait Trait: frame_system::Trait + pallet_transaction_payment::Trait { /// The minimum amount required to generate a tombstone. type TombstoneDeposit: Get>; - /// Size of a contract at the time of instantiation. This is a simple way to ensure - /// that empty contracts eventually gets deleted. + /// A size offset for an contract. A just created account with untouched storage will have that + /// much of storage from the perspective of the state rent. + /// + /// This is a simple way to ensure that contracts with empty storage eventually get deleted by + /// making them pay rent. This creates an incentive to remove them early in order to save rent. type StorageSizeOffset: Get; /// Price of a byte of storage per one block interval. Should be greater than 0. @@ -363,6 +367,10 @@ pub trait Trait: frame_system::Trait + pallet_transaction_payment::Trait { /// The maximum size of a storage value in bytes. type MaxValueSize: Get; + + /// Used to answer contracts's queries regarding the current weight price. This is **not** + /// used to calculate the actual fee and is only for informational purposes. + type WeightPrice: Convert>; } /// Simple contract address determiner. @@ -420,8 +428,12 @@ decl_module! { /// The minimum amount required to generate a tombstone. const TombstoneDeposit: BalanceOf = T::TombstoneDeposit::get(); - /// Size of a contract at the time of instantiation. This is a simple way to ensure that - /// empty contracts eventually gets deleted. + /// A size offset for an contract. A just created account with untouched storage will have that + /// much of storage from the perspective of the state rent. + /// + /// This is a simple way to ensure that contracts with empty storage eventually get deleted + /// by making them pay rent. This creates an incentive to remove them early in order to save + /// rent. const StorageSizeOffset: u32 = T::StorageSizeOffset::get(); /// Price of a byte of storage per one block interval. Should be greater than 0. @@ -467,11 +479,7 @@ decl_module! { /// Stores the given binary Wasm code into the chain's storage and returns its `codehash`. /// You can instantiate contracts only with stored code. - #[weight = FunctionOf( - |args: (&Vec,)| Module::::calc_code_put_costs(args.0), - DispatchClass::Normal, - Pays::Yes - )] + #[weight = Module::::calc_code_put_costs(&code)] pub fn put_code( origin, code: Vec @@ -492,11 +500,7 @@ decl_module! { /// * If the account is a regular account, any value will be transferred. /// * If no account exists and the call value is not less than `existential_deposit`, /// a regular account will be created and any value will be transferred. - #[weight = FunctionOf( - |args: (&::Source, &BalanceOf, &Weight, &Vec)| *args.2, - DispatchClass::Normal, - Pays::Yes - )] + #[weight = *gas_limit] pub fn call( origin, dest: ::Source, @@ -524,11 +528,7 @@ decl_module! { /// after the execution is saved as the `code` of the account. That code will be invoked /// upon any call received by this account. /// - The contract is initialized. - #[weight = FunctionOf( - |args: (&BalanceOf, &Weight, &CodeHash, &Vec)| *args.1, - DispatchClass::Normal, - Pays::Yes - )] + #[weight = *gas_limit] pub fn instantiate( origin, #[compact] endowment: BalanceOf, @@ -610,12 +610,7 @@ impl Module { .get_alive() .ok_or(ContractAccessError::IsTombstone)?; - let maybe_value = AccountDb::::get_storage( - &DirectAccountDb, - &address, - Some(&contract_info.trie_id), - &key, - ); + let maybe_value = storage::read_contract_storage(&contract_info.trie_id, &key); Ok(maybe_value) } @@ -634,147 +629,13 @@ impl Module { fn execute_wasm( origin: T::AccountId, gas_meter: &mut GasMeter, - func: impl FnOnce(&mut ExecutionContext, &mut GasMeter) -> ExecResult + func: impl FnOnce(&mut ExecutionContext, &mut GasMeter) -> ExecResult, ) -> ExecResult { let cfg = Config::preload(); let vm = WasmVm::new(&cfg.schedule); let loader = WasmLoader::new(&cfg.schedule); let mut ctx = ExecutionContext::top_level(origin.clone(), &cfg, &vm, &loader); - - let result = func(&mut ctx, gas_meter); - - if result.as_ref().map(|output| output.is_success()).unwrap_or(false) { - // Commit all changes that made it thus far into the persistent storage. - DirectAccountDb.commit(ctx.overlay.into_change_set()); - } - - // Execute deferred actions. - ctx.deferred.into_iter().for_each(|deferred| { - use self::exec::DeferredAction::*; - match deferred { - DepositEvent { - topics, - event, - } => >::deposit_event_indexed( - &*topics, - ::Event::from(event).into(), - ), - DispatchRuntimeCall { - origin: who, - call, - } => { - let info = call.get_dispatch_info(); - let result = call.dispatch(RawOrigin::Signed(who.clone()).into()); - let post_info = match result { - Ok(post_info) => post_info, - Err(err) => err.post_info, - }; - gas_meter.refund(post_info.calc_unspent(&info)); - Self::deposit_event(RawEvent::Dispatched(who, result.is_ok())); - } - RestoreTo { - donor, - dest, - code_hash, - rent_allowance, - delta, - } => { - let result = Self::restore_to( - donor.clone(), dest.clone(), code_hash.clone(), rent_allowance.clone(), delta - ); - Self::deposit_event( - RawEvent::Restored(donor, dest, code_hash, rent_allowance, result.is_ok()) - ); - } - } - }); - - result - } - - fn restore_to( - origin: T::AccountId, - dest: T::AccountId, - code_hash: CodeHash, - rent_allowance: BalanceOf, - delta: Vec - ) -> DispatchResult { - let mut origin_contract = >::get(&origin) - .and_then(|c| c.get_alive()) - .ok_or(Error::::InvalidSourceContract)?; - - let current_block = >::block_number(); - - if origin_contract.last_write == Some(current_block) { - Err(Error::::InvalidContractOrigin)? - } - - let dest_tombstone = >::get(&dest) - .and_then(|c| c.get_tombstone()) - .ok_or(Error::::InvalidDestinationContract)?; - - let last_write = if !delta.is_empty() { - Some(current_block) - } else { - origin_contract.last_write - }; - - let key_values_taken = delta.iter() - .filter_map(|key| { - child::get_raw( - &origin_contract.child_trie_info(), - &blake2_256(key), - ).map(|value| { - child::kill( - &origin_contract.child_trie_info(), - &blake2_256(key), - ); - - (key, value) - }) - }) - .collect::>(); - - let tombstone = >::new( - // This operation is cheap enough because last_write (delta not included) - // is not this block as it has been checked earlier. - &child::root( - &origin_contract.child_trie_info(), - )[..], - code_hash, - ); - - if tombstone != dest_tombstone { - for (key, value) in key_values_taken { - child::put_raw( - &origin_contract.child_trie_info(), - &blake2_256(key), - &value, - ); - } - - return Err(Error::::InvalidTombstone.into()); - } - - origin_contract.storage_size -= key_values_taken.iter() - .map(|(_, value)| value.len() as u32) - .sum::(); - - >::remove(&origin); - >::insert(&dest, ContractInfo::Alive(RawAliveContractInfo { - trie_id: origin_contract.trie_id, - storage_size: origin_contract.storage_size, - code_hash, - rent_allowance, - deduct_block: current_block, - last_write, - })); - - let origin_free_balance = T::Currency::free_balance(&origin); - T::Currency::make_free_balance_be(&origin, >::zero()); - T::Currency::deposit_creating(&dest, origin_free_balance); - - Ok(()) + func(&mut ctx, gas_meter) } } @@ -785,9 +646,6 @@ decl_event! { ::AccountId, ::Hash { - /// Transfer happened `from` to `to` with given `value` as part of a `call` or `instantiate`. - Transfer(AccountId, AccountId, Balance), - /// Contract deployed by address at the specified address. Instantiated(AccountId, AccountId), @@ -799,7 +657,7 @@ decl_event! { /// - `tombstone`: `bool`: True if the evicted contract left behind a tombstone. Evicted(AccountId, bool), - /// Restoration for a contract has been initiated. + /// Restoration for a contract has been successful. /// /// # Params /// @@ -807,8 +665,7 @@ decl_event! { /// - `dest`: `AccountId`: Account ID of the restored contract /// - `code_hash`: `Hash`: Code hash of the restored contract /// - `rent_allowance: `Balance`: Rent allowance of the restored contract - /// - `success`: `bool`: True if the restoration was successful - Restored(AccountId, AccountId, Hash, Balance, bool), + Restored(AccountId, AccountId, Hash, Balance), /// Code with the specified hash has been stored. CodeStored(Hash), @@ -836,6 +693,8 @@ decl_storage! { /// The subtrie counter. pub AccountCounter: u64 = 0; /// The code associated with a given account. + /// + /// TWOX-NOTE: SAFE since `AccountId` is a secure hash. pub ContractInfoOf: map hasher(twox_64_concat) T::AccountId => Option>; } } diff --git a/frame/contracts/src/rent.rs b/frame/contracts/src/rent.rs index 1aa52fff314356e4758947e128139e92a4961c73..6afd85aa8eb425d2f967ed916af6d83dfe090659 100644 --- a/frame/contracts/src/rent.rs +++ b/frame/contracts/src/rent.rs @@ -18,8 +18,10 @@ use crate::{ AliveContractInfo, BalanceOf, ContractInfo, ContractInfoOf, Module, RawEvent, - TombstoneContractInfo, Trait, + TombstoneContractInfo, Trait, CodeHash, }; +use sp_std::prelude::*; +use sp_io::hashing::blake2_256; use frame_support::storage::child; use frame_support::traits::{Currency, ExistenceRequirement, Get, OnUnbalanced, WithdrawReason}; use frame_support::StorageMap; @@ -92,8 +94,13 @@ fn compute_fee_per_block( .checked_div(&T::RentDepositOffset::get()) .unwrap_or_else(Zero::zero); - let effective_storage_size = - >::from(contract.storage_size).saturating_sub(free_storage); + // For now, we treat every empty KV pair as if it was one byte long. + let empty_pairs_equivalent = contract.empty_pair_count; + + let effective_storage_size = >::from( + contract.storage_size + T::StorageSizeOffset::get() + empty_pairs_equivalent, + ) + .saturating_sub(free_storage); effective_storage_size .checked_mul(&T::RentByteFee::get()) @@ -391,3 +398,90 @@ pub fn compute_rent_projection( current_block_number + blocks_left, )) } + +/// Restores the destination account using the origin as prototype. +/// +/// The restoration will be performed iff: +/// - origin exists and is alive, +/// - the origin's storage is not written in the current block +/// - the restored account has tombstone +/// - the tombstone matches the hash of the origin storage root, and code hash. +/// +/// Upon succesful restoration, `origin` will be destroyed, all its funds are transferred to +/// the restored account. The restored account will inherit the last write block and its last +/// deduct block will be set to the current block. +pub fn restore_to( + origin: T::AccountId, + dest: T::AccountId, + code_hash: CodeHash, + rent_allowance: BalanceOf, + delta: Vec, +) -> Result<(), &'static str> { + let mut origin_contract = >::get(&origin) + .and_then(|c| c.get_alive()) + .ok_or("Cannot restore from inexisting or tombstone contract")?; + + let child_trie_info = origin_contract.child_trie_info(); + + let current_block = >::block_number(); + + if origin_contract.last_write == Some(current_block) { + return Err("Origin TrieId written in the current block"); + } + + let dest_tombstone = >::get(&dest) + .and_then(|c| c.get_tombstone()) + .ok_or("Cannot restore to inexisting or alive contract")?; + + let last_write = if !delta.is_empty() { + Some(current_block) + } else { + origin_contract.last_write + }; + + let key_values_taken = delta.iter() + .filter_map(|key| { + child::get_raw(&child_trie_info, &blake2_256(key)).map(|value| { + child::kill(&child_trie_info, &blake2_256(key)); + (key, value) + }) + }) + .collect::>(); + + let tombstone = >::new( + // This operation is cheap enough because last_write (delta not included) + // is not this block as it has been checked earlier. + &child::root(&child_trie_info)[..], + code_hash, + ); + + if tombstone != dest_tombstone { + for (key, value) in key_values_taken { + child::put_raw(&child_trie_info, &blake2_256(key), &value); + } + + return Err("Tombstones don't match"); + } + + origin_contract.storage_size -= key_values_taken.iter() + .map(|(_, value)| value.len() as u32) + .sum::(); + + >::remove(&origin); + >::insert(&dest, ContractInfo::Alive(AliveContractInfo:: { + trie_id: origin_contract.trie_id, + storage_size: origin_contract.storage_size, + empty_pair_count: origin_contract.empty_pair_count, + total_pair_count: origin_contract.total_pair_count, + code_hash, + rent_allowance, + deduct_block: current_block, + last_write, + })); + + let origin_free_balance = T::Currency::free_balance(&origin); + T::Currency::make_free_balance_be(&origin, >::zero()); + T::Currency::deposit_creating(&dest, origin_free_balance); + + Ok(()) +} diff --git a/frame/contracts/src/storage.rs b/frame/contracts/src/storage.rs new file mode 100644 index 0000000000000000000000000000000000000000..4c5ad892a967b1012060f4aaa682b3f1292a3d6c --- /dev/null +++ b/frame/contracts/src/storage.rs @@ -0,0 +1,195 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! This module contains routines for accessing and altering a contract related state. + +use crate::{ + exec::{AccountIdOf, StorageKey}, + AliveContractInfo, BalanceOf, CodeHash, ContractInfo, ContractInfoOf, Trait, TrieId, +}; +use sp_std::prelude::*; +use sp_io::hashing::blake2_256; +use sp_runtime::traits::Bounded; +use frame_support::{storage::child, StorageMap}; + +/// An error that means that the account requested either doesn't exist or represents a tombstone +/// account. +#[cfg_attr(test, derive(PartialEq, Eq, Debug))] +pub struct ContractAbsentError; + +/// Reads a storage kv pair of a contract. +/// +/// The read is performed from the `trie_id` only. The `address` is not necessary. If the contract +/// doesn't store under the given `key` `None` is returned. +pub fn read_contract_storage(trie_id: &TrieId, key: &StorageKey) -> Option> { + child::get_raw(&crate::child_trie_info(&trie_id), &blake2_256(key)) +} + +/// Update a storage entry into a contract's kv storage. +/// +/// If the `opt_new_value` is `None` then the kv pair is removed. +/// +/// This function also updates the bookkeeping info such as: number of total non-empty pairs a +/// contract owns, the last block the storage was written to, etc. That's why, in contrast to +/// `read_contract_storage`, this function also requires the `account` ID. +/// +/// If the contract specified by the id `account` doesn't exist `Err` is returned.` +pub fn write_contract_storage( + account: &AccountIdOf, + trie_id: &TrieId, + key: &StorageKey, + opt_new_value: Option>, +) -> Result<(), ContractAbsentError> { + let mut new_info = match >::get(account) { + Some(ContractInfo::Alive(alive)) => alive, + None | Some(ContractInfo::Tombstone(_)) => return Err(ContractAbsentError), + }; + + let hashed_key = blake2_256(key); + let child_trie_info = &crate::child_trie_info(&trie_id); + + // In order to correctly update the book keeping we need to fetch the previous + // value of the key-value pair. + // + // It might be a bit more clean if we had an API that supported getting the size + // of the value without going through the loading of it. But at the moment of + // writing, there is no such API. + // + // That's not a show stopper in any case, since the performance cost is + // dominated by the trie traversal anyway. + let opt_prev_value = child::get_raw(&child_trie_info, &hashed_key); + + // Update the total number of KV pairs and the number of empty pairs. + match (&opt_prev_value, &opt_new_value) { + (Some(prev_value), None) => { + new_info.total_pair_count -= 1; + if prev_value.is_empty() { + new_info.empty_pair_count -= 1; + } + }, + (None, Some(new_value)) => { + new_info.total_pair_count += 1; + if new_value.is_empty() { + new_info.empty_pair_count += 1; + } + }, + (Some(prev_value), Some(new_value)) => { + if prev_value.is_empty() { + new_info.empty_pair_count -= 1; + } + if new_value.is_empty() { + new_info.empty_pair_count += 1; + } + } + (None, None) => {} + } + + // Update the total storage size. + let prev_value_len = opt_prev_value + .as_ref() + .map(|old_value| old_value.len() as u32) + .unwrap_or(0); + let new_value_len = opt_new_value + .as_ref() + .map(|new_value| new_value.len() as u32) + .unwrap_or(0); + new_info.storage_size = new_info + .storage_size + .saturating_add(new_value_len) + .saturating_sub(prev_value_len); + + new_info.last_write = Some(>::block_number()); + >::insert(&account, ContractInfo::Alive(new_info)); + + // Finally, perform the change on the storage. + match opt_new_value { + Some(new_value) => child::put_raw(&child_trie_info, &hashed_key, &new_value[..]), + None => child::kill(&child_trie_info, &hashed_key), + } + + Ok(()) +} + +/// Returns the rent allowance set for the contract give by the account id. +pub fn rent_allowance( + account: &AccountIdOf, +) -> Result, ContractAbsentError> { + >::get(account) + .and_then(|i| i.as_alive().map(|i| i.rent_allowance)) + .ok_or(ContractAbsentError) +} + +/// Set the rent allowance for the contract given by the account id. +/// +/// Returns `Err` if the contract doesn't exist or is a tombstone. +pub fn set_rent_allowance( + account: &AccountIdOf, + rent_allowance: BalanceOf, +) -> Result<(), ContractAbsentError> { + >::mutate(account, |maybe_contract_info| match maybe_contract_info { + Some(ContractInfo::Alive(ref mut alive_info)) => { + alive_info.rent_allowance = rent_allowance; + Ok(()) + } + _ => Err(ContractAbsentError), + }) +} + +/// Returns the code hash of the contract specified by `account` ID. +pub fn code_hash(account: &AccountIdOf) -> Result, ContractAbsentError> { + >::get(account) + .and_then(|i| i.as_alive().map(|i| i.code_hash)) + .ok_or(ContractAbsentError) +} + +/// Creates a new contract descriptor in the storage with the given code hash at the given address. +/// +/// Returns `Err` if there is already a contract (or a tombstone) exists at the given address. +pub fn place_contract( + account: &AccountIdOf, + trie_id: TrieId, + ch: CodeHash, +) -> Result<(), &'static str> { + >::mutate(account, |maybe_contract_info| { + if maybe_contract_info.is_some() { + return Err("Alive contract or tombstone already exists"); + } + + *maybe_contract_info = Some( + AliveContractInfo:: { + code_hash: ch, + storage_size: 0, + trie_id, + deduct_block: >::block_number(), + rent_allowance: >::max_value(), + empty_pair_count: 0, + total_pair_count: 0, + last_write: None, + } + .into(), + ); + + Ok(()) + }) +} + +/// Removes the contract and all the storage associated with it. +/// +/// This function doesn't affect the account. +pub fn destroy_contract(address: &AccountIdOf, trie_id: &TrieId) { + >::remove(address); + child::kill_storage(&crate::child_trie_info(&trie_id)); +} diff --git a/frame/contracts/src/tests.rs b/frame/contracts/src/tests.rs index 218a5c99372d40d78ba6c68a099d89faf3f79200..5303375e016aed37b609c60db62144a86dafc2c8 100644 --- a/frame/contracts/src/tests.rs +++ b/frame/contracts/src/tests.rs @@ -14,33 +14,25 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -// TODO: #1417 Add more integration tests -// also remove the #![allow(unused)] below. - -#![allow(unused)] - use crate::{ - BalanceOf, ComputeDispatchFee, ContractAddressFor, ContractInfo, ContractInfoOf, GenesisConfig, - Module, RawAliveContractInfo, RawEvent, Trait, TrieId, TrieIdFromParentCounter, Schedule, - TrieIdGenerator, account_db::{AccountDb, DirectAccountDb, OverlayAccountDb}, - gas::Gas, + BalanceOf, ContractAddressFor, ContractInfo, ContractInfoOf, GenesisConfig, Module, + RawAliveContractInfo, RawEvent, Trait, TrieId, Schedule, TrieIdGenerator, gas::Gas, }; use assert_matches::assert_matches; use hex_literal::*; -use codec::{Decode, Encode, KeyedVec}; +use codec::Encode; use sp_runtime::{ - Perbill, BuildStorage, transaction_validity::{InvalidTransaction, ValidTransaction}, - traits::{BlakeTwo256, Hash, IdentityLookup, SignedExtension, Convert}, - testing::{Digest, DigestItem, Header, UintAuthorityId, H256}, + Perbill, + traits::{BlakeTwo256, Hash, IdentityLookup, Convert}, + testing::{Header, H256}, }; use frame_support::{ - assert_ok, assert_err, assert_err_ignore_postinfo, impl_outer_dispatch, impl_outer_event, - impl_outer_origin, parameter_types, - storage::child, StorageMap, StorageValue, traits::{Currency, Get}, - weights::{DispatchInfo, DispatchClass, Weight, PostDispatchInfo, Pays}, + assert_ok, assert_err_ignore_postinfo, impl_outer_dispatch, impl_outer_event, + impl_outer_origin, parameter_types, StorageMap, StorageValue, + traits::{Currency, Get}, + weights::{Weight, PostDispatchInfo}, }; -use std::{cell::RefCell, sync::atomic::{AtomicUsize, Ordering}}; -use sp_core::storage::well_known_keys; +use std::cell::RefCell; use frame_system::{self as system, EventRecord, Phase}; mod contracts { @@ -48,7 +40,7 @@ mod contracts { // needs to give a name for the current crate. // This hack is required for `impl_outer_event!`. pub use super::super::*; - use frame_support::impl_outer_event; + pub use frame_support::impl_outer_event; } use pallet_balances as balances; @@ -70,6 +62,34 @@ impl_outer_dispatch! { } } +pub mod test_utils { + use super::{Test, Balances}; + use crate::{ContractInfoOf, TrieIdGenerator, CodeHash}; + use crate::storage::{write_contract_storage, read_contract_storage}; + use crate::exec::StorageKey; + use frame_support::{StorageMap, traits::Currency}; + + pub fn set_storage(addr: &u64, key: &StorageKey, value: Option>) { + let contract_info = >::get(&addr).unwrap().get_alive().unwrap(); + write_contract_storage::(&1, &contract_info.trie_id, key, value).unwrap(); + } + pub fn get_storage(addr: &u64, key: &StorageKey) -> Option> { + let contract_info = >::get(&addr).unwrap().get_alive().unwrap(); + read_contract_storage(&contract_info.trie_id, key) + } + pub fn place_contract(address: &u64, code_hash: CodeHash) { + let trie_id = ::TrieIdGenerator::trie_id(address); + crate::storage::place_contract::(&address, trie_id, code_hash).unwrap() + } + pub fn set_balance(who: &u64, amount: u64) { + let imbalance = Balances::deposit_creating(who, amount); + drop(imbalance); + } + pub fn get_balance(who: &u64) -> u64 { + Balances::free_balance(who) + } +} + thread_local! { static EXISTENTIAL_DEPOSIT: RefCell = RefCell::new(0); } @@ -88,11 +108,12 @@ parameter_types! { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; type Hash = H256; - type Call = (); + type Call = Call; type Hashing = BlakeTwo256; type AccountId = u64; type Lookup = IdentityLookup; @@ -103,6 +124,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); @@ -147,18 +169,10 @@ impl Convert> for Test { } } -impl pallet_transaction_payment::Trait for Test { - type Currency = Balances; - type OnTransactionPayment = (); - type TransactionByteFee = TransactionByteFee; - type WeightToFee = Test; - type FeeMultiplierUpdate = (); -} - impl Trait for Test { type Time = Timestamp; type Randomness = Randomness; - type Call = Call; + type Currency = Balances; type DetermineContractAddress = DummyContractAddressFor; type Event = MetaEvent; type TrieIdGenerator = DummyTrieIdGenerator; @@ -171,6 +185,7 @@ impl Trait for Test { type SurchargeReward = SurchargeReward; type MaxDepth = MaxDepth; type MaxValueSize = MaxValueSize; + type WeightPrice = Self; } type Balances = pallet_balances::Module; @@ -189,8 +204,6 @@ impl ContractAddressFor for DummyContractAddressFor { pub struct DummyTrieIdGenerator; impl TrieIdGenerator for DummyTrieIdGenerator { fn trie_id(account_id: &u64) -> TrieId { - use sp_core::storage::well_known_keys; - let new_seed = super::AccountCounter::mutate(|v| { *v = v.wrapping_add(1); *v @@ -203,13 +216,6 @@ impl TrieIdGenerator for DummyTrieIdGenerator { } } -pub struct DummyComputeDispatchFee; -impl ComputeDispatchFee for DummyComputeDispatchFee { - fn compute_dispatch_fee(call: &Call) -> u64 { - 69 - } -} - const ALICE: u64 = 1; const BOB: u64 = 2; const CHARLIE: u64 = 3; @@ -253,14 +259,24 @@ impl ExtBuilder { } } -/// Generate Wasm binary and code hash from wabt source. -fn compile_module(wabt_module: &str) - -> Result<(Vec, ::Output), wabt::Error> - where T: frame_system::Trait +/// Load a given wasm module represented by a .wat file and returns a wasm binary contents along +/// with it's hash. +/// +/// The fixture files are located under the `fixtures/` directory. +fn compile_module( + fixture_name: &str, +) -> Result<(Vec, ::Output), wabt::Error> +where + T: frame_system::Trait, { - let wasm = wabt::wat2wasm(wabt_module)?; - let code_hash = T::Hashing::hash(&wasm); - Ok((wasm, code_hash)) + use std::fs; + + let fixture_path = ["fixtures/", fixture_name, ".wat"].concat(); + let module_wat_source = + fs::read_to_string(&fixture_path).expect(&format!("Unable to find {} fixture", fixture_name)); + let wasm_binary = wabt::wat2wasm(module_wat_source)?; + let code_hash = T::Hashing::hash(&wasm_binary); + Ok((wasm_binary, code_hash)) } // Perform a simple transfer to a non-existent account. @@ -268,7 +284,7 @@ fn compile_module(wabt_module: &str) #[test] fn returns_base_call_cost() { ExtBuilder::default().build().execute_with(|| { - Balances::deposit_creating(&ALICE, 100_000_000); + let _ = Balances::deposit_creating(&ALICE, 100_000_000); assert_eq!( Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, Vec::new()), @@ -283,6 +299,8 @@ fn returns_base_call_cost() { #[test] fn account_removal_does_not_remove_storage() { + use self::test_utils::{set_storage, get_storage}; + ExtBuilder::default().existential_deposit(100).build().execute_with(|| { let trie_id1 = ::TrieIdGenerator::trie_id(&1); let trie_id2 = ::TrieIdGenerator::trie_id(&2); @@ -291,63 +309,63 @@ fn account_removal_does_not_remove_storage() { // Set up two accounts with free balance above the existential threshold. { - Balances::deposit_creating(&1, 110); - ContractInfoOf::::insert(1, &ContractInfo::Alive(RawAliveContractInfo { + let alice_contract_info = ContractInfo::Alive(RawAliveContractInfo { trie_id: trie_id1.clone(), - storage_size: ::StorageSizeOffset::get(), + storage_size: 0, + empty_pair_count: 0, + total_pair_count: 0, deduct_block: System::block_number(), code_hash: H256::repeat_byte(1), rent_allowance: 40, last_write: None, - })); - - let mut overlay = OverlayAccountDb::::new(&DirectAccountDb); - overlay.set_storage(&1, key1.clone(), Some(b"1".to_vec())); - overlay.set_storage(&1, key2.clone(), Some(b"2".to_vec())); - DirectAccountDb.commit(overlay.into_change_set()); + }); + let _ = Balances::deposit_creating(&ALICE, 110); + ContractInfoOf::::insert(ALICE, &alice_contract_info); + set_storage(&ALICE, &key1, Some(b"1".to_vec())); + set_storage(&ALICE, &key2, Some(b"2".to_vec())); - Balances::deposit_creating(&2, 110); - ContractInfoOf::::insert(2, &ContractInfo::Alive(RawAliveContractInfo { + let bob_contract_info = ContractInfo::Alive(RawAliveContractInfo { trie_id: trie_id2.clone(), - storage_size: ::StorageSizeOffset::get(), + storage_size: 0, + empty_pair_count: 0, + total_pair_count: 0, deduct_block: System::block_number(), code_hash: H256::repeat_byte(2), rent_allowance: 40, last_write: None, - })); - - let mut overlay = OverlayAccountDb::::new(&DirectAccountDb); - overlay.set_storage(&2, key1.clone(), Some(b"3".to_vec())); - overlay.set_storage(&2, key2.clone(), Some(b"4".to_vec())); - DirectAccountDb.commit(overlay.into_change_set()); + }); + let _ = Balances::deposit_creating(&BOB, 110); + ContractInfoOf::::insert(BOB, &bob_contract_info); + set_storage(&BOB, &key1, Some(b"3".to_vec())); + set_storage(&BOB, &key2, Some(b"4".to_vec())); } - // Transfer funds from account 1 of such amount that after this transfer - // the balance of account 1 will be below the existential threshold. + // Transfer funds from ALICE account of such amount that after this transfer + // the balance of the ALICE account will be below the existential threshold. // // This does not remove the contract storage as we are not notified about a // account removal. This cannot happen in reality because a contract can only // remove itself by `ext_terminate`. There is no external event that can remove // the account appart from that. - assert_ok!(Balances::transfer(Origin::signed(1), 2, 20)); + assert_ok!(Balances::transfer(Origin::signed(ALICE), BOB, 20)); // Verify that no entries are removed. { assert_eq!( - >::get_storage(&DirectAccountDb, &1, Some(&trie_id1), key1), + get_storage(&ALICE, key1), Some(b"1".to_vec()) ); assert_eq!( - >::get_storage(&DirectAccountDb, &1, Some(&trie_id1), key2), + get_storage(&ALICE, key2), Some(b"2".to_vec()) ); assert_eq!( - >::get_storage(&DirectAccountDb, &2, Some(&trie_id2), key1), + get_storage(&BOB, key1), Some(b"3".to_vec()) ); assert_eq!( - >::get_storage(&DirectAccountDb, &2, Some(&trie_id2), key2), + get_storage(&BOB, key2), Some(b"4".to_vec()) ); } @@ -356,302 +374,86 @@ fn account_removal_does_not_remove_storage() { #[test] fn instantiate_and_call_and_deposit_event() { - let (wasm, code_hash) = compile_module::(&load_wasm("return_from_start_fn.wat")) - .unwrap(); - - ExtBuilder::default().existential_deposit(100).build().execute_with(|| { - Balances::deposit_creating(&ALICE, 1_000_000); - - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - - // Check at the end to get hash on error easily - let creation = Contracts::instantiate( - Origin::signed(ALICE), - 100, - GAS_LIMIT, - code_hash.into(), - vec![], - ); - - assert_eq!(System::events(), vec![ - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::CodeStored(code_hash.into())), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::system(frame_system::RawEvent::NewAccount(BOB)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::balances( - pallet_balances::RawEvent::Endowed(BOB, 100) - ), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::Transfer(ALICE, BOB, 100)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::ContractExecution(BOB, vec![1, 2, 3, 4])), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::Instantiated(ALICE, BOB)), - topics: vec![], - } - ]); - - assert_ok!(creation); - assert!(ContractInfoOf::::contains_key(BOB)); - }); -} - -#[test] -fn dispatch_call() { - // This test can fail due to the encoding changes. In case it becomes too annoying - // let's rewrite so as we use this module controlled call or we serialize it in runtime. - let encoded = Encode::encode(&Call::Balances(pallet_balances::Call::transfer(CHARLIE, 50))); - assert_eq!(&encoded[..], &hex!("00000300000000000000C8")[..]); - - let (wasm, code_hash) = compile_module::(&load_wasm("dispatch_call.wat")) - .unwrap(); - - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - Balances::deposit_creating(&ALICE, 1_000_000); - - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - - // Let's keep this assert even though it's redundant. If you ever need to update the - // wasm source this test will fail and will show you the actual hash. - assert_eq!(System::events(), vec![ - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::CodeStored(code_hash.into())), - topics: vec![], - }, - ]); - - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 100, - GAS_LIMIT, - code_hash.into(), - vec![], - )); - - assert_ok!(Contracts::call( - Origin::signed(ALICE), - BOB, // newly created account - 0, - GAS_LIMIT, - vec![], - )); - - assert_eq!(System::events(), vec![ - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::CodeStored(code_hash.into())), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::system(frame_system::RawEvent::NewAccount(BOB)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::balances( - pallet_balances::RawEvent::Endowed(BOB, 100) - ), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::Transfer(ALICE, BOB, 100)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::Instantiated(ALICE, BOB)), - topics: vec![], - }, - - // Dispatching the call. - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::system(frame_system::RawEvent::NewAccount(CHARLIE)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::balances( - pallet_balances::RawEvent::Endowed(CHARLIE, 50) - ), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::balances( - pallet_balances::RawEvent::Transfer(BOB, CHARLIE, 50) - ), - topics: vec![], - }, - - // Event emitted as a result of dispatch. - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::Dispatched(BOB, true)), - topics: vec![], - } - ]); - }); -} + let (wasm, code_hash) = compile_module::("return_from_start_fn").unwrap(); -#[test] -fn dispatch_call_not_dispatched_after_top_level_transaction_failure() { - // This test can fail due to the encoding changes. In case it becomes too annoying - // let's rewrite so as we use this module controlled call or we serialize it in runtime. - let encoded = Encode::encode(&Call::Balances(pallet_balances::Call::transfer(CHARLIE, 50))); - assert_eq!(&encoded[..], &hex!("00000300000000000000C8")[..]); - - let (wasm, code_hash) = compile_module::(&load_wasm("dispatch_call_then_trap.wat")) - .unwrap(); - - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - Balances::deposit_creating(&ALICE, 1_000_000); + ExtBuilder::default() + .existential_deposit(100) + .build() + .execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - // Let's keep this assert even though it's redundant. If you ever need to update the - // wasm source this test will fail and will show you the actual hash. - assert_eq!(System::events(), vec![ - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::CodeStored(code_hash.into())), - topics: vec![], - }, - ]); - - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 100, - GAS_LIMIT, - code_hash.into(), - vec![], - )); - - // Call the newly instantiated contract. The contract is expected to dispatch a call - // and then trap. - assert_err_ignore_postinfo!( - Contracts::call( + // Check at the end to get hash on error easily + let creation = Contracts::instantiate( Origin::signed(ALICE), - BOB, // newly created account - 0, + 100, GAS_LIMIT, + code_hash.into(), vec![], - ), - "contract trapped during execution" - ); - assert_eq!(System::events(), vec![ - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::CodeStored(code_hash.into())), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::system(frame_system::RawEvent::NewAccount(BOB)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::balances( - pallet_balances::RawEvent::Endowed(BOB, 100) - ), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::Transfer(ALICE, BOB, 100)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::Instantiated(ALICE, BOB)), - topics: vec![], - }, - // ABSENCE of events which would be caused by dispatched Balances::transfer call - ]); - }); + ); + + pretty_assertions::assert_eq!(System::events(), vec![ + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::contracts(RawEvent::CodeStored(code_hash.into())), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::system(frame_system::RawEvent::NewAccount(BOB)), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::balances( + pallet_balances::RawEvent::Endowed(BOB, 100) + ), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::balances( + pallet_balances::RawEvent::Transfer(ALICE, BOB, 100) + ), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::contracts(RawEvent::ContractExecution(BOB, vec![1, 2, 3, 4])), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::contracts(RawEvent::Instantiated(ALICE, BOB)), + topics: vec![], + } + ]); + + assert_ok!(creation); + assert!(ContractInfoOf::::contains_key(BOB)); + }); } #[test] fn run_out_of_gas() { - let (wasm, code_hash) = compile_module::(&load_wasm("run_out_of_gas.wat")) - .unwrap(); + let (wasm, code_hash) = compile_module::("run_out_of_gas").unwrap(); ExtBuilder::default() .existential_deposit(50) .build() .execute_with(|| { - Balances::deposit_creating(&ALICE, 1_000_000); + let _ = Balances::deposit_creating(&ALICE, 1_000_000); assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); @@ -695,60 +497,157 @@ fn test_set_rent_code_and_hash() { let encoded = Encode::encode(&Call::Balances(pallet_balances::Call::transfer(CHARLIE, 50))); assert_eq!(&encoded[..], &hex!("00000300000000000000C8")[..]); - let (wasm, code_hash) = compile_module::(&load_wasm("set_rent.wat")).unwrap(); + let (wasm, code_hash) = compile_module::("set_rent").unwrap(); - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - // If you ever need to update the wasm source this test will fail - // and will show you the actual hash. - assert_eq!(System::events(), vec![ - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::CodeStored(code_hash.into())), - topics: vec![], - }, - ]); - }); + // If you ever need to update the wasm source this test will fail + // and will show you the actual hash. + assert_eq!(System::events(), vec![ + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::contracts(RawEvent::CodeStored(code_hash.into())), + topics: vec![], + }, + ]); + }); } #[test] fn storage_size() { - let (wasm, code_hash) = compile_module::(&load_wasm("set_rent.wat")).unwrap(); + let (wasm, code_hash) = compile_module::("set_rent").unwrap(); // Storage size - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - // Create - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 30_000, - GAS_LIMIT, code_hash.into(), - ::Balance::from(1_000u32).encode() // rent allowance - )); - let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); - assert_eq!(bob_contract.storage_size, ::StorageSizeOffset::get() + 4); - - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::set_storage_4_byte())); - let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); - assert_eq!(bob_contract.storage_size, ::StorageSizeOffset::get() + 4 + 4); - - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::remove_storage_4_byte())); - let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); - assert_eq!(bob_contract.storage_size, ::StorageSizeOffset::get() + 4); - }); + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + // Create + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 30_000, + GAS_LIMIT, + code_hash.into(), + ::Balance::from(1_000u32).encode() // rent allowance + )); + let bob_contract = ContractInfoOf::::get(BOB) + .unwrap() + .get_alive() + .unwrap(); + assert_eq!( + bob_contract.storage_size, + 4 + ); + assert_eq!( + bob_contract.total_pair_count, + 1, + ); + assert_eq!( + bob_contract.empty_pair_count, + 0, + ); + + assert_ok!(Contracts::call( + Origin::signed(ALICE), + BOB, + 0, + GAS_LIMIT, + call::set_storage_4_byte() + )); + let bob_contract = ContractInfoOf::::get(BOB) + .unwrap() + .get_alive() + .unwrap(); + assert_eq!( + bob_contract.storage_size, + 4 + 4 + ); + assert_eq!( + bob_contract.total_pair_count, + 2, + ); + assert_eq!( + bob_contract.empty_pair_count, + 0, + ); + + assert_ok!(Contracts::call( + Origin::signed(ALICE), + BOB, + 0, + GAS_LIMIT, + call::remove_storage_4_byte() + )); + let bob_contract = ContractInfoOf::::get(BOB) + .unwrap() + .get_alive() + .unwrap(); + assert_eq!( + bob_contract.storage_size, + 4 + ); + assert_eq!( + bob_contract.total_pair_count, + 1, + ); + assert_eq!( + bob_contract.empty_pair_count, + 0, + ); + }); +} + +#[test] +fn empty_kv_pairs() { + let (wasm, code_hash) = compile_module::("set_empty_storage").unwrap(); + + ExtBuilder::default() + .build() + .execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 30_000, + GAS_LIMIT, + code_hash.into(), + vec![], + )); + let bob_contract = ContractInfoOf::::get(BOB) + .unwrap() + .get_alive() + .unwrap(); + + assert_eq!( + bob_contract.storage_size, + 0, + ); + assert_eq!( + bob_contract.total_pair_count, + 1, + ); + assert_eq!( + bob_contract.empty_pair_count, + 1, + ); + }); } fn initialize_block(number: u64) { @@ -763,75 +662,78 @@ fn initialize_block(number: u64) { #[test] fn deduct_blocks() { - let (wasm, code_hash) = compile_module::(&load_wasm("set_rent.wat")).unwrap(); - - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - // Create - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 30_000, - GAS_LIMIT, code_hash.into(), - ::Balance::from(1_000u32).encode() // rent allowance - )); - - // Check creation - let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); - assert_eq!(bob_contract.rent_allowance, 1_000); - - // Advance 4 blocks - initialize_block(5); - - // Trigger rent through call - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null())); - - // Check result - let rent = (8 + 4 - 3) // storage size = size_offset + deploy_set_storage - deposit_offset - * 4 // rent byte price - * 4; // blocks to rent - let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); - assert_eq!(bob_contract.rent_allowance, 1_000 - rent); - assert_eq!(bob_contract.deduct_block, 5); - assert_eq!(Balances::free_balance(BOB), 30_000 - rent); - - // Advance 7 blocks more - initialize_block(12); - - // Trigger rent through call - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null())); - - // Check result - let rent_2 = (8 + 4 - 2) // storage size = size_offset + deploy_set_storage - deposit_offset - * 4 // rent byte price - * 7; // blocks to rent - let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); - assert_eq!(bob_contract.rent_allowance, 1_000 - rent - rent_2); - assert_eq!(bob_contract.deduct_block, 12); - assert_eq!(Balances::free_balance(BOB), 30_000 - rent - rent_2); - - // Second call on same block should have no effect on rent - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null())); - - let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); - assert_eq!(bob_contract.rent_allowance, 1_000 - rent - rent_2); - assert_eq!(bob_contract.deduct_block, 12); - assert_eq!(Balances::free_balance(BOB), 30_000 - rent - rent_2); - }); + let (wasm, code_hash) = compile_module::("set_rent").unwrap(); + + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + // Create + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 30_000, + GAS_LIMIT, code_hash.into(), + ::Balance::from(1_000u32).encode() // rent allowance + )); + + // Check creation + let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); + assert_eq!(bob_contract.rent_allowance, 1_000); + + // Advance 4 blocks + initialize_block(5); + + // Trigger rent through call + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null())); + + // Check result + let rent = (8 + 4 - 3) // storage size = size_offset + deploy_set_storage - deposit_offset + * 4 // rent byte price + * 4; // blocks to rent + let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); + assert_eq!(bob_contract.rent_allowance, 1_000 - rent); + assert_eq!(bob_contract.deduct_block, 5); + assert_eq!(Balances::free_balance(BOB), 30_000 - rent); + + // Advance 7 blocks more + initialize_block(12); + + // Trigger rent through call + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null())); + + // Check result + let rent_2 = (8 + 4 - 2) // storage size = size_offset + deploy_set_storage - deposit_offset + * 4 // rent byte price + * 7; // blocks to rent + let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); + assert_eq!(bob_contract.rent_allowance, 1_000 - rent - rent_2); + assert_eq!(bob_contract.deduct_block, 12); + assert_eq!(Balances::free_balance(BOB), 30_000 - rent - rent_2); + + // Second call on same block should have no effect on rent + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null())); + + let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); + assert_eq!(bob_contract.rent_allowance, 1_000 - rent - rent_2); + assert_eq!(bob_contract.deduct_block, 12); + assert_eq!(Balances::free_balance(BOB), 30_000 - rent - rent_2); + }); } #[test] fn call_contract_removals() { removals(|| { // Call on already-removed account might fail, and this is fine. - Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null()); + let _ = Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null()); true }); } #[test] fn inherent_claim_surcharge_contract_removals() { - removals(|| Contracts::claim_surcharge(Origin::NONE, BOB, Some(ALICE)).is_ok()); + removals(|| Contracts::claim_surcharge(Origin::none(), BOB, Some(ALICE)).is_ok()); } #[test] @@ -842,10 +744,10 @@ fn signed_claim_surcharge_contract_removals() { #[test] fn claim_surcharge_malus() { // Test surcharge malus for inherent - claim_surcharge(4, || Contracts::claim_surcharge(Origin::NONE, BOB, Some(ALICE)).is_ok(), true); - claim_surcharge(3, || Contracts::claim_surcharge(Origin::NONE, BOB, Some(ALICE)).is_ok(), true); - claim_surcharge(2, || Contracts::claim_surcharge(Origin::NONE, BOB, Some(ALICE)).is_ok(), true); - claim_surcharge(1, || Contracts::claim_surcharge(Origin::NONE, BOB, Some(ALICE)).is_ok(), false); + claim_surcharge(4, || Contracts::claim_surcharge(Origin::none(), BOB, Some(ALICE)).is_ok(), true); + claim_surcharge(3, || Contracts::claim_surcharge(Origin::none(), BOB, Some(ALICE)).is_ok(), true); + claim_surcharge(2, || Contracts::claim_surcharge(Origin::none(), BOB, Some(ALICE)).is_ok(), true); + claim_surcharge(1, || Contracts::claim_surcharge(Origin::none(), BOB, Some(ALICE)).is_ok(), false); // Test surcharge malus for signed claim_surcharge(4, || Contracts::claim_surcharge(Origin::signed(ALICE), BOB, None).is_ok(), true); @@ -857,226 +759,281 @@ fn claim_surcharge_malus() { /// Claim surcharge with the given trigger_call at the given blocks. /// If `removes` is true then assert that the contract is a tombstone. fn claim_surcharge(blocks: u64, trigger_call: impl Fn() -> bool, removes: bool) { - let (wasm, code_hash) = compile_module::(&load_wasm("set_rent.wat")).unwrap(); - - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - // Create - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 100, - GAS_LIMIT, code_hash.into(), - ::Balance::from(1_000u32).encode() // rent allowance - )); - - // Advance blocks - initialize_block(blocks); - - // Trigger rent through call - assert!(trigger_call()); - - if removes { - assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); - } else { - assert!(ContractInfoOf::::get(BOB).unwrap().get_alive().is_some()); - } - }); -} + let (wasm, code_hash) = compile_module::("set_rent").unwrap(); -/// Test for all kind of removals for the given trigger: -/// * if balance is reached and balance > subsistence threshold -/// * if allowance is exceeded -/// * if balance is reached and balance < subsistence threshold -fn removals(trigger_call: impl Fn() -> bool) { - let (wasm, code_hash) = compile_module::(&load_wasm("set_rent.wat")).unwrap(); - - // Balance reached and superior to subsistence threshold - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - // Create - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm.clone())); - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 100, - GAS_LIMIT, code_hash.into(), - ::Balance::from(1_000u32).encode() // rent allowance - )); - - let subsistence_threshold = 50 /*existential_deposit*/ + 16 /*tombstone_deposit*/; - - // Trigger rent must have no effect - assert!(trigger_call()); - assert_eq!(ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap().rent_allowance, 1_000); - assert_eq!(Balances::free_balance(BOB), 100); - - // Advance blocks - initialize_block(10); - - // Trigger rent through call - assert!(trigger_call()); - assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); - assert_eq!(Balances::free_balance(BOB), subsistence_threshold); - - // Advance blocks - initialize_block(20); - - // Trigger rent must have no effect - assert!(trigger_call()); - assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); - assert_eq!(Balances::free_balance(BOB), subsistence_threshold); - }); + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + // Create + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 100, + GAS_LIMIT, code_hash.into(), + ::Balance::from(1_000u32).encode() // rent allowance + )); + + // Advance blocks + initialize_block(blocks); + + // Trigger rent through call + assert!(trigger_call()); + + if removes { + assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); + } else { + assert!(ContractInfoOf::::get(BOB).unwrap().get_alive().is_some()); + } + }); +} + +/// Test for all kind of removals for the given trigger: +/// * if balance is reached and balance > subsistence threshold +/// * if allowance is exceeded +/// * if balance is reached and balance < subsistence threshold +fn removals(trigger_call: impl Fn() -> bool) { + let (wasm, code_hash) = compile_module::("set_rent").unwrap(); + + // Balance reached and superior to subsistence threshold + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + // Create + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm.clone())); + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 100, + GAS_LIMIT, code_hash.into(), + ::Balance::from(1_000u32).encode() // rent allowance + )); + + let subsistence_threshold = 50 /*existential_deposit*/ + 16 /*tombstone_deposit*/; + + // Trigger rent must have no effect + assert!(trigger_call()); + assert_eq!(ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap().rent_allowance, 1_000); + assert_eq!(Balances::free_balance(BOB), 100); + + // Advance blocks + initialize_block(10); + + // Trigger rent through call + assert!(trigger_call()); + assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); + assert_eq!(Balances::free_balance(BOB), subsistence_threshold); + + // Advance blocks + initialize_block(20); + + // Trigger rent must have no effect + assert!(trigger_call()); + assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); + assert_eq!(Balances::free_balance(BOB), subsistence_threshold); + }); // Allowance exceeded - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - // Create - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm.clone())); - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 1_000, - GAS_LIMIT, code_hash.into(), - ::Balance::from(100u32).encode() // rent allowance - )); - - // Trigger rent must have no effect - assert!(trigger_call()); - assert_eq!(ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap().rent_allowance, 100); - assert_eq!(Balances::free_balance(BOB), 1_000); - - // Advance blocks - initialize_block(10); - - // Trigger rent through call - assert!(trigger_call()); - assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); - // Balance should be initial balance - initial rent_allowance - assert_eq!(Balances::free_balance(BOB), 900); - - // Advance blocks - initialize_block(20); - - // Trigger rent must have no effect - assert!(trigger_call()); - assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); - assert_eq!(Balances::free_balance(BOB), 900); - }); + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + // Create + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm.clone())); + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 1_000, + GAS_LIMIT, + code_hash.into(), + ::Balance::from(100u32).encode() // rent allowance + )); + + // Trigger rent must have no effect + assert!(trigger_call()); + assert_eq!( + ContractInfoOf::::get(BOB) + .unwrap() + .get_alive() + .unwrap() + .rent_allowance, + 100 + ); + assert_eq!(Balances::free_balance(BOB), 1_000); + + // Advance blocks + initialize_block(10); + + // Trigger rent through call + assert!(trigger_call()); + assert!(ContractInfoOf::::get(BOB) + .unwrap() + .get_tombstone() + .is_some()); + // Balance should be initial balance - initial rent_allowance + assert_eq!(Balances::free_balance(BOB), 900); + + // Advance blocks + initialize_block(20); + + // Trigger rent must have no effect + assert!(trigger_call()); + assert!(ContractInfoOf::::get(BOB) + .unwrap() + .get_tombstone() + .is_some()); + assert_eq!(Balances::free_balance(BOB), 900); + }); // Balance reached and inferior to subsistence threshold - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - // Create - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm.clone())); - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 50+Balances::minimum_balance(), - GAS_LIMIT, code_hash.into(), - ::Balance::from(1_000u32).encode() // rent allowance - )); - - // Trigger rent must have no effect - assert!(trigger_call()); - assert_eq!(ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap().rent_allowance, 1_000); - assert_eq!(Balances::free_balance(BOB), 50 + Balances::minimum_balance()); - - // Transfer funds - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::transfer())); - assert_eq!(ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap().rent_allowance, 1_000); - assert_eq!(Balances::free_balance(BOB), Balances::minimum_balance()); - - // Advance blocks - initialize_block(10); - - // Trigger rent through call - assert!(trigger_call()); - assert!(ContractInfoOf::::get(BOB).is_none()); - assert_eq!(Balances::free_balance(BOB), Balances::minimum_balance()); - - // Advance blocks - initialize_block(20); - - // Trigger rent must have no effect - assert!(trigger_call()); - assert!(ContractInfoOf::::get(BOB).is_none()); - assert_eq!(Balances::free_balance(BOB), Balances::minimum_balance()); - }); + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + // Create + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm.clone())); + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 50 + Balances::minimum_balance(), + GAS_LIMIT, + code_hash.into(), + ::Balance::from(1_000u32).encode() // rent allowance + )); + + // Trigger rent must have no effect + assert!(trigger_call()); + assert_eq!( + ContractInfoOf::::get(BOB) + .unwrap() + .get_alive() + .unwrap() + .rent_allowance, + 1_000 + ); + assert_eq!( + Balances::free_balance(BOB), + 50 + Balances::minimum_balance() + ); + + // Transfer funds + assert_ok!(Contracts::call( + Origin::signed(ALICE), + BOB, + 0, + GAS_LIMIT, + call::transfer() + )); + assert_eq!( + ContractInfoOf::::get(BOB) + .unwrap() + .get_alive() + .unwrap() + .rent_allowance, + 1_000 + ); + assert_eq!(Balances::free_balance(BOB), Balances::minimum_balance()); + + // Advance blocks + initialize_block(10); + + // Trigger rent through call + assert!(trigger_call()); + assert!(ContractInfoOf::::get(BOB).is_none()); + assert_eq!(Balances::free_balance(BOB), Balances::minimum_balance()); + + // Advance blocks + initialize_block(20); + + // Trigger rent must have no effect + assert!(trigger_call()); + assert!(ContractInfoOf::::get(BOB).is_none()); + assert_eq!(Balances::free_balance(BOB), Balances::minimum_balance()); + }); } #[test] fn call_removed_contract() { - let (wasm, code_hash) = compile_module::(&load_wasm("set_rent.wat")).unwrap(); + let (wasm, code_hash) = compile_module::("set_rent").unwrap(); // Balance reached and superior to subsistence threshold - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - // Create - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm.clone())); - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 100, - GAS_LIMIT, code_hash.into(), - ::Balance::from(1_000u32).encode() // rent allowance - )); - - // Calling contract should succeed. - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null())); - - // Advance blocks - initialize_block(10); - - // Calling contract should remove contract and fail. - assert_err_ignore_postinfo!( - Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null()), - "contract has been evicted" - ); - // Calling a contract that is about to evict shall emit an event. - assert_eq!(System::events(), vec![ - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::Evicted(BOB, true)), - topics: vec![], - }, - ]); + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + // Create + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm.clone())); + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 100, + GAS_LIMIT, code_hash.into(), + ::Balance::from(1_000u32).encode() // rent allowance + )); - // Subsequent contract calls should also fail. - assert_err_ignore_postinfo!( - Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null()), - "contract has been evicted" - ); - }) + // Calling contract should succeed. + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null())); + + // Advance blocks + initialize_block(10); + + // Calling contract should remove contract and fail. + assert_err_ignore_postinfo!( + Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null()), + "contract has been evicted" + ); + // Calling a contract that is about to evict shall emit an event. + assert_eq!(System::events(), vec![ + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::contracts(RawEvent::Evicted(BOB, true)), + topics: vec![], + }, + ]); + + // Subsequent contract calls should also fail. + assert_err_ignore_postinfo!( + Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null()), + "contract has been evicted" + ); + }) } #[test] fn default_rent_allowance_on_instantiate() { - let (wasm, code_hash) = compile_module::( - &load_wasm("check_default_rent_allowance.wat")).unwrap(); - - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - // Create - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 30_000, - GAS_LIMIT, - code_hash.into(), - vec![], - )); - - // Check creation - let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); - assert_eq!(bob_contract.rent_allowance, >::max_value()); - - // Advance blocks - initialize_block(5); - - // Trigger rent through call - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null())); - - // Check contract is still alive - let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive(); - assert!(bob_contract.is_some()) - }); + let (wasm, code_hash) = compile_module::("check_default_rent_allowance").unwrap(); + + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + // Create + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 30_000, + GAS_LIMIT, + code_hash.into(), + vec![], + )); + + // Check creation + let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); + assert_eq!(bob_contract.rent_allowance, >::max_value()); + + // Advance blocks + initialize_block(5); + + // Trigger rent through call + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null())); + + // Check contract is still alive + let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive(); + assert!(bob_contract.is_some()) + }); } #[test] @@ -1100,571 +1057,544 @@ fn restoration_success() { } fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: bool) { - let (set_rent_wasm, set_rent_code_hash) = - compile_module::(&load_wasm("set_rent.wat")).unwrap(); - let (restoration_wasm, restoration_code_hash) = - compile_module::(&load_wasm("restoration.wat")).unwrap(); - - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), restoration_wasm)); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), set_rent_wasm)); - - // If you ever need to update the wasm source this test will fail - // and will show you the actual hash. - assert_eq!(System::events(), vec![ - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::CodeStored(restoration_code_hash.into())), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::CodeStored(set_rent_code_hash.into())), - topics: vec![], - }, - ]); - - // Create an account with address `BOB` with code `CODE_SET_RENT`. - // The input parameter sets the rent allowance to 0. - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 30_000, - GAS_LIMIT, - set_rent_code_hash.into(), - ::Balance::from(0u32).encode() - )); - - // Check if `BOB` was created successfully and that the rent allowance is - // set to 0. - let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); - assert_eq!(bob_contract.rent_allowance, 0); - - if test_different_storage { - assert_ok!(Contracts::call( - Origin::signed(ALICE), - BOB, 0, GAS_LIMIT, - call::set_storage_4_byte()) - ); - } + let (set_rent_wasm, set_rent_code_hash) = compile_module::("set_rent").unwrap(); + let (restoration_wasm, restoration_code_hash) = compile_module::("restoration").unwrap(); - // Advance 4 blocks, to the 5th. - initialize_block(5); - - /// Preserve `BOB`'s code hash for later introspection. - let bob_code_hash = ContractInfoOf::::get(BOB).unwrap() - .get_alive().unwrap().code_hash; - // Call `BOB`, which makes it pay rent. Since the rent allowance is set to 0 - // we expect that it will get removed leaving tombstone. - assert_err_ignore_postinfo!( - Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null()), - "contract has been evicted" - ); - assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); - assert_eq!(System::events(), vec![ - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts( - RawEvent::Evicted(BOB.clone(), true) - ), - topics: vec![], - }, - ]); - - /// Create another account with the address `DJANGO` with `CODE_RESTORATION`. - /// - /// Note that we can't use `ALICE` for creating `DJANGO` so we create yet another - /// account `CHARLIE` and create `DJANGO` with it. - Balances::deposit_creating(&CHARLIE, 1_000_000); - assert_ok!(Contracts::instantiate( - Origin::signed(CHARLIE), - 30_000, - GAS_LIMIT, - restoration_code_hash.into(), - ::Balance::from(0u32).encode() - )); - - // Before performing a call to `DJANGO` save its original trie id. - let django_trie_id = ContractInfoOf::::get(DJANGO).unwrap() - .get_alive().unwrap().trie_id; - - if !test_restore_to_with_dirty_storage { - // Advance 1 block, to the 6th. - initialize_block(6); - } + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), restoration_wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), set_rent_wasm)); - // Perform a call to `DJANGO`. This should either perform restoration successfully or - // fail depending on the test parameters. - assert_ok!(Contracts::call( - Origin::signed(ALICE), - DJANGO, - 0, - GAS_LIMIT, - vec![], - )); - - if test_different_storage || test_restore_to_with_dirty_storage { - // Parametrization of the test imply restoration failure. Check that `DJANGO` aka - // restoration contract is still in place and also that `BOB` doesn't exist. - assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); - let django_contract = ContractInfoOf::::get(DJANGO).unwrap() - .get_alive().unwrap(); - assert_eq!(django_contract.storage_size, 16); - assert_eq!(django_contract.trie_id, django_trie_id); - assert_eq!(django_contract.deduct_block, System::block_number()); - match (test_different_storage, test_restore_to_with_dirty_storage) { - (true, false) => { - assert_eq!(System::events(), vec![ - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts( - RawEvent::Restored(DJANGO, BOB, bob_code_hash, 50, false) - ), - topics: vec![], - }, - ]); - } - (_, true) => { - assert_eq!(System::events(), vec![ - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::Evicted(BOB, true)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::system(frame_system::RawEvent::NewAccount(CHARLIE)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(CHARLIE, 1_000_000)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::system(frame_system::RawEvent::NewAccount(DJANGO)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(DJANGO, 30_000)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::Transfer(CHARLIE, DJANGO, 30_000)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::Instantiated(CHARLIE, DJANGO)), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: MetaEvent::contracts(RawEvent::Restored( - DJANGO, - BOB, - bob_code_hash, - 50, - false, - )), - topics: vec![], - }, - ]); - } - _ => unreachable!(), - } - } else { - // Here we expect that the restoration is succeeded. Check that the restoration - // contract `DJANGO` ceased to exist and that `BOB` returned back. - println!("{:?}", ContractInfoOf::::get(BOB)); - let bob_contract = ContractInfoOf::::get(BOB).unwrap() - .get_alive().unwrap(); - assert_eq!(bob_contract.rent_allowance, 50); - assert_eq!(bob_contract.storage_size, 12); - assert_eq!(bob_contract.trie_id, django_trie_id); - assert_eq!(bob_contract.deduct_block, System::block_number()); - assert!(ContractInfoOf::::get(DJANGO).is_none()); + // If you ever need to update the wasm source this test will fail + // and will show you the actual hash. assert_eq!(System::events(), vec![ EventRecord { phase: Phase::Initialization, - event: MetaEvent::system(system::RawEvent::KilledAccount(DJANGO)), + event: MetaEvent::system(frame_system::RawEvent::NewAccount(1)), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(1, 1_000_000)), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::contracts(RawEvent::CodeStored(restoration_code_hash.into())), topics: vec![], }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::contracts(RawEvent::CodeStored(set_rent_code_hash.into())), + topics: vec![], + }, + ]); + + // Create an account with address `BOB` with code `CODE_SET_RENT`. + // The input parameter sets the rent allowance to 0. + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 30_000, + GAS_LIMIT, + set_rent_code_hash.into(), + ::Balance::from(0u32).encode() + )); + + // Check if `BOB` was created successfully and that the rent allowance is + // set to 0. + let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); + assert_eq!(bob_contract.rent_allowance, 0); + + if test_different_storage { + assert_ok!(Contracts::call( + Origin::signed(ALICE), + BOB, 0, GAS_LIMIT, + call::set_storage_4_byte()) + ); + } + + // Advance 4 blocks, to the 5th. + initialize_block(5); + + // Call `BOB`, which makes it pay rent. Since the rent allowance is set to 0 + // we expect that it will get removed leaving tombstone. + assert_err_ignore_postinfo!( + Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null()), + "contract has been evicted" + ); + assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); + assert_eq!(System::events(), vec![ EventRecord { phase: Phase::Initialization, event: MetaEvent::contracts( - RawEvent::Restored(DJANGO, BOB, bob_contract.code_hash, 50, true) + RawEvent::Evicted(BOB.clone(), true) ), topics: vec![], }, ]); - } - }); + + // Create another account with the address `DJANGO` with `CODE_RESTORATION`. + // + // Note that we can't use `ALICE` for creating `DJANGO` so we create yet another + // account `CHARLIE` and create `DJANGO` with it. + let _ = Balances::deposit_creating(&CHARLIE, 1_000_000); + assert_ok!(Contracts::instantiate( + Origin::signed(CHARLIE), + 30_000, + GAS_LIMIT, + restoration_code_hash.into(), + ::Balance::from(0u32).encode() + )); + + // Before performing a call to `DJANGO` save its original trie id. + let django_trie_id = ContractInfoOf::::get(DJANGO).unwrap() + .get_alive().unwrap().trie_id; + + if !test_restore_to_with_dirty_storage { + // Advance 1 block, to the 6th. + initialize_block(6); + } + + // Perform a call to `DJANGO`. This should either perform restoration successfully or + // fail depending on the test parameters. + let perform_the_restoration = || { + Contracts::call( + Origin::signed(ALICE), + DJANGO, + 0, + GAS_LIMIT, + vec![], + ) + }; + + if test_different_storage || test_restore_to_with_dirty_storage { + // Parametrization of the test imply restoration failure. Check that `DJANGO` aka + // restoration contract is still in place and also that `BOB` doesn't exist. + + assert_err_ignore_postinfo!( + perform_the_restoration(), + "contract trapped during execution" + ); + + assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); + let django_contract = ContractInfoOf::::get(DJANGO).unwrap() + .get_alive().unwrap(); + assert_eq!(django_contract.storage_size, 8); + assert_eq!(django_contract.trie_id, django_trie_id); + assert_eq!(django_contract.deduct_block, System::block_number()); + match (test_different_storage, test_restore_to_with_dirty_storage) { + (true, false) => { + assert_eq!(System::events(), vec![]); + } + (_, true) => { + pretty_assertions::assert_eq!(System::events(), vec![ + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::contracts(RawEvent::Evicted(BOB, true)), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::system(frame_system::RawEvent::NewAccount(CHARLIE)), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(CHARLIE, 1_000_000)), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::system(frame_system::RawEvent::NewAccount(DJANGO)), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::balances(pallet_balances::RawEvent::Endowed(DJANGO, 30_000)), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::balances( + pallet_balances::RawEvent::Transfer(CHARLIE, DJANGO, 30_000) + ), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::contracts(RawEvent::Instantiated(CHARLIE, DJANGO)), + topics: vec![], + }, + ]); + } + _ => unreachable!(), + } + } else { + assert_ok!(perform_the_restoration()); + + // Here we expect that the restoration is succeeded. Check that the restoration + // contract `DJANGO` ceased to exist and that `BOB` returned back. + println!("{:?}", ContractInfoOf::::get(BOB)); + let bob_contract = ContractInfoOf::::get(BOB).unwrap() + .get_alive().unwrap(); + assert_eq!(bob_contract.rent_allowance, 50); + assert_eq!(bob_contract.storage_size, 4); + assert_eq!(bob_contract.trie_id, django_trie_id); + assert_eq!(bob_contract.deduct_block, System::block_number()); + assert!(ContractInfoOf::::get(DJANGO).is_none()); + assert_eq!(System::events(), vec![ + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::system(system::RawEvent::KilledAccount(DJANGO)), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: MetaEvent::contracts( + RawEvent::Restored(DJANGO, BOB, bob_contract.code_hash, 50) + ), + topics: vec![], + }, + ]); + } + }); } #[test] fn storage_max_value_limit() { - let (wasm, code_hash) = compile_module::(&load_wasm("storage_size.wat")) - .unwrap(); - - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - // Create - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 30_000, - GAS_LIMIT, - code_hash.into(), - vec![], - )); - - // Check creation - let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); - assert_eq!(bob_contract.rent_allowance, >::max_value()); - - // Call contract with allowed storage value. - assert_ok!(Contracts::call( - Origin::signed(ALICE), - BOB, - 0, - GAS_LIMIT, - Encode::encode(&self::MaxValueSize::get()), - )); - - // Call contract with too large a storage value. - assert_err_ignore_postinfo!( - Contracts::call( + let (wasm, code_hash) = compile_module::("storage_size").unwrap(); + + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + // Create + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 30_000, + GAS_LIMIT, + code_hash.into(), + vec![], + )); + + // Check creation + let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); + assert_eq!(bob_contract.rent_allowance, >::max_value()); + + // Call contract with allowed storage value. + assert_ok!(Contracts::call( Origin::signed(ALICE), BOB, 0, GAS_LIMIT, - Encode::encode(&(self::MaxValueSize::get() + 1)), - ), - "contract trapped during execution" - ); - }); + Encode::encode(&self::MaxValueSize::get()), + )); + + // Call contract with too large a storage value. + assert_err_ignore_postinfo!( + Contracts::call( + Origin::signed(ALICE), + BOB, + 0, + GAS_LIMIT, + Encode::encode(&(self::MaxValueSize::get() + 1)), + ), + "contract trapped during execution" + ); + }); } #[test] fn deploy_and_call_other_contract() { - let (callee_wasm, callee_code_hash) = - compile_module::(&load_wasm("return_with_data.wat")).unwrap(); - let (caller_wasm, caller_code_hash) = - compile_module::(&load_wasm("caller_contract.wat")).unwrap(); - - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - // Create - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), callee_wasm)); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), caller_wasm)); - - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 100_000, - GAS_LIMIT, - caller_code_hash.into(), - vec![], - )); - - // Call BOB contract, which attempts to instantiate and call the callee contract and - // makes various assertions on the results from those calls. - assert_ok!(Contracts::call( - Origin::signed(ALICE), - BOB, - 0, - GAS_LIMIT, - callee_code_hash.as_ref().to_vec(), - )); - }); + let (callee_wasm, callee_code_hash) = compile_module::("return_with_data").unwrap(); + let (caller_wasm, caller_code_hash) = compile_module::("caller_contract").unwrap(); + + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + // Create + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), callee_wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), caller_wasm)); + + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 100_000, + GAS_LIMIT, + caller_code_hash.into(), + vec![], + )); + + // Call BOB contract, which attempts to instantiate and call the callee contract and + // makes various assertions on the results from those calls. + assert_ok!(Contracts::call( + Origin::signed(ALICE), + BOB, + 0, + GAS_LIMIT, + callee_code_hash.as_ref().to_vec(), + )); + }); } #[test] fn cannot_self_destruct_through_draning() { - let (wasm, code_hash) = compile_module::(&load_wasm("drain.wat")).unwrap(); - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - - // Instantiate the BOB contract. - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 100_000, - GAS_LIMIT, - code_hash.into(), - vec![], - )); - - // Check that the BOB contract has been instantiated. - assert_matches!( - ContractInfoOf::::get(BOB), - Some(ContractInfo::Alive(_)) - ); + let (wasm, code_hash) = compile_module::("drain").unwrap(); + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - // Call BOB with no input data, forcing it to run until out-of-balance - // and eventually trapping because below existential deposit. - assert_err_ignore_postinfo!( - Contracts::call( + // Instantiate the BOB contract. + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), - BOB, - 0, + 100_000, GAS_LIMIT, + code_hash.into(), vec![], - ), - "contract trapped during execution" - ); - }); + )); + + // Check that the BOB contract has been instantiated. + assert_matches!( + ContractInfoOf::::get(BOB), + Some(ContractInfo::Alive(_)) + ); + + // Call BOB with no input data, forcing it to run until out-of-balance + // and eventually trapping because below existential deposit. + assert_err_ignore_postinfo!( + Contracts::call( + Origin::signed(ALICE), + BOB, + 0, + GAS_LIMIT, + vec![], + ), + "contract trapped during execution" + ); + }); } #[test] fn cannot_self_destruct_while_live() { - let (wasm, code_hash) = compile_module::(&load_wasm("self_destruct.wat")) - .unwrap(); - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - - // Instantiate the BOB contract. - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 100_000, - GAS_LIMIT, - code_hash.into(), - vec![], - )); - - // Check that the BOB contract has been instantiated. - assert_matches!( - ContractInfoOf::::get(BOB), - Some(ContractInfo::Alive(_)) - ); + let (wasm, code_hash) = compile_module::("self_destruct").unwrap(); + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - // Call BOB with input data, forcing it make a recursive call to itself to - // self-destruct, resulting in a trap. - assert_err_ignore_postinfo!( - Contracts::call( + // Instantiate the BOB contract. + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), - BOB, - 0, + 100_000, GAS_LIMIT, - vec![0], - ), - "contract trapped during execution" - ); + code_hash.into(), + vec![], + )); - // Check that BOB is still alive. - assert_matches!( - ContractInfoOf::::get(BOB), - Some(ContractInfo::Alive(_)) - ); - }); + // Check that the BOB contract has been instantiated. + assert_matches!( + ContractInfoOf::::get(BOB), + Some(ContractInfo::Alive(_)) + ); + + // Call BOB with input data, forcing it make a recursive call to itself to + // self-destruct, resulting in a trap. + assert_err_ignore_postinfo!( + Contracts::call( + Origin::signed(ALICE), + BOB, + 0, + GAS_LIMIT, + vec![0], + ), + "contract trapped during execution" + ); + + // Check that BOB is still alive. + assert_matches!( + ContractInfoOf::::get(BOB), + Some(ContractInfo::Alive(_)) + ); + }); } #[test] fn self_destruct_works() { - let (wasm, code_hash) = compile_module::(&load_wasm("self_destruct.wat")) - .unwrap(); - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - - // Instantiate the BOB contract. - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 100_000, - GAS_LIMIT, - code_hash.into(), - vec![], - )); - - // Check that the BOB contract has been instantiated. - assert_matches!( - ContractInfoOf::::get(BOB), - Some(ContractInfo::Alive(_)) - ); + let (wasm, code_hash) = compile_module::("self_destruct").unwrap(); + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - // Call BOB without input data which triggers termination. - assert_matches!( - Contracts::call( + // Instantiate the BOB contract. + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), - BOB, - 0, + 100_000, GAS_LIMIT, + code_hash.into(), vec![], - ), - Ok(_) - ); + )); - // Check that account is gone - assert!(ContractInfoOf::::get(BOB).is_none()); + // Check that the BOB contract has been instantiated. + assert_matches!( + ContractInfoOf::::get(BOB), + Some(ContractInfo::Alive(_)) + ); - // check that the beneficiary (django) got remaining balance - assert_eq!(Balances::free_balance(DJANGO), 100_000); - }); + // Call BOB without input data which triggers termination. + assert_matches!( + Contracts::call( + Origin::signed(ALICE), + BOB, + 0, + GAS_LIMIT, + vec![], + ), + Ok(_) + ); + + // Check that account is gone + assert!(ContractInfoOf::::get(BOB).is_none()); + + // check that the beneficiary (django) got remaining balance + assert_eq!(Balances::free_balance(DJANGO), 100_000); + }); } // This tests that one contract cannot prevent another from self-destructing by sending it // additional funds after it has been drained. #[test] fn destroy_contract_and_transfer_funds() { - let (callee_wasm, callee_code_hash) = - compile_module::(&load_wasm("self_destruct.wat")).unwrap(); - let (caller_wasm, caller_code_hash) = - compile_module::(&load_wasm("destroy_and_transfer.wat")).unwrap(); - - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - // Create - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), callee_wasm)); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), caller_wasm)); - - // This deploys the BOB contract, which in turn deploys the CHARLIE contract during - // construction. - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 200_000, - GAS_LIMIT, - caller_code_hash.into(), - callee_code_hash.as_ref().to_vec(), - )); - - // Check that the CHARLIE contract has been instantiated. - assert_matches!( - ContractInfoOf::::get(CHARLIE), - Some(ContractInfo::Alive(_)) - ); + let (callee_wasm, callee_code_hash) = compile_module::("self_destruct").unwrap(); + let (caller_wasm, caller_code_hash) = compile_module::("destroy_and_transfer").unwrap(); - // Call BOB, which calls CHARLIE, forcing CHARLIE to self-destruct. - assert_ok!(Contracts::call( - Origin::signed(ALICE), - BOB, - 0, - GAS_LIMIT, - CHARLIE.encode(), - )); - - // Check that CHARLIE has moved on to the great beyond (ie. died). - assert!(ContractInfoOf::::get(CHARLIE).is_none()); - }); -} + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + // Create + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), callee_wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), caller_wasm)); -#[test] -fn cannot_self_destruct_in_constructor() { - let (wasm, code_hash) = - compile_module::(&load_wasm("self_destructing_constructor.wat")).unwrap(); - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - - // Fail to instantiate the BOB because the call that is issued in the deploy - // function exhausts all balances which puts it below the existential deposit. - assert_err_ignore_postinfo!( - Contracts::instantiate( + // This deploys the BOB contract, which in turn deploys the CHARLIE contract during + // construction. + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), - 100_000, + 200_000, GAS_LIMIT, - code_hash.into(), - vec![], - ), - "contract trapped during execution" - ); - }); -} + caller_code_hash.into(), + callee_code_hash.as_ref().to_vec(), + )); -fn get_runtime_storage() { - let (wasm, code_hash) = compile_module::(&load_wasm("get_runtime_storage.wat")) - .unwrap(); - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - Balances::deposit_creating(&ALICE, 1_000_000); + // Check that the CHARLIE contract has been instantiated. + assert_matches!( + ContractInfoOf::::get(CHARLIE), + Some(ContractInfo::Alive(_)) + ); - frame_support::storage::unhashed::put_raw( - &[1, 2, 3, 4], - 0x14144020u32.to_le_bytes().to_vec().as_ref() - ); + // Call BOB, which calls CHARLIE, forcing CHARLIE to self-destruct. + assert_ok!(Contracts::call( + Origin::signed(ALICE), + BOB, + 0, + GAS_LIMIT, + CHARLIE.encode(), + )); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 100, - GAS_LIMIT, - code_hash.into(), - vec![], - )); - assert_ok!(Contracts::call( - Origin::signed(ALICE), - BOB, - 0, - GAS_LIMIT, - vec![], - )); - }); + // Check that CHARLIE has moved on to the great beyond (ie. died). + assert!(ContractInfoOf::::get(CHARLIE).is_none()); + }); } #[test] -fn crypto_hashes() { - let (wasm, code_hash) = compile_module::(&load_wasm("crypto_hashes.wat")).unwrap(); - - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); - - // Instantiate the CRYPTO_HASHES contract. - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 100_000, - GAS_LIMIT, - code_hash.into(), - vec![], - )); - // Perform the call. - let input = b"_DEAD_BEEF"; - use sp_io::hashing::*; - // Wraps a hash function into a more dynamic form usable for testing. - macro_rules! dyn_hash_fn { - ($name:ident) => { - Box::new(|input| $name(input).as_ref().to_vec().into_boxed_slice()) - }; - } - // All hash functions and their associated output byte lengths. - let test_cases: &[(Box Box<[u8]>>, usize)] = &[ - (dyn_hash_fn!(sha2_256), 32), - (dyn_hash_fn!(keccak_256), 32), - (dyn_hash_fn!(blake2_256), 32), - (dyn_hash_fn!(blake2_128), 16), - ]; - // Test the given hash functions for the input: "_DEAD_BEEF" - for (n, (hash_fn, expected_size)) in test_cases.iter().enumerate() { - // We offset data in the contract tables by 1. - let mut params = vec![(n + 1) as u8]; - params.extend_from_slice(input); - let result = >::bare_call( - ALICE, - BOB, - 0, - GAS_LIMIT, - params, - ).unwrap(); - assert_eq!(result.status, 0); - let expected = hash_fn(input.as_ref()); - assert_eq!(&result.data[..*expected_size], &*expected); - } - }) +fn cannot_self_destruct_in_constructor() { + let (wasm, code_hash) = compile_module::("self_destructing_constructor").unwrap(); + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); + + // Fail to instantiate the BOB because the call that is issued in the deploy + // function exhausts all balances which puts it below the existential deposit. + assert_err_ignore_postinfo!( + Contracts::instantiate( + Origin::signed(ALICE), + 100_000, + GAS_LIMIT, + code_hash.into(), + vec![], + ), + "contract trapped during execution" + ); + }); } -fn load_wasm(file_name: &str) -> String { - let path = ["tests/", file_name].concat(); - std::fs::read_to_string(&path).expect(&format!("Unable to read {} file", path)) +#[test] +fn crypto_hashes() { + let (wasm, code_hash) = compile_module::("crypto_hashes").unwrap(); + + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); + + // Instantiate the CRYPTO_HASHES contract. + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 100_000, + GAS_LIMIT, + code_hash.into(), + vec![], + )); + // Perform the call. + let input = b"_DEAD_BEEF"; + use sp_io::hashing::*; + // Wraps a hash function into a more dynamic form usable for testing. + macro_rules! dyn_hash_fn { + ($name:ident) => { + Box::new(|input| $name(input).as_ref().to_vec().into_boxed_slice()) + }; + } + // All hash functions and their associated output byte lengths. + let test_cases: &[(Box Box<[u8]>>, usize)] = &[ + (dyn_hash_fn!(sha2_256), 32), + (dyn_hash_fn!(keccak_256), 32), + (dyn_hash_fn!(blake2_256), 32), + (dyn_hash_fn!(blake2_128), 16), + ]; + // Test the given hash functions for the input: "_DEAD_BEEF" + for (n, (hash_fn, expected_size)) in test_cases.iter().enumerate() { + // We offset data in the contract tables by 1. + let mut params = vec![(n + 1) as u8]; + params.extend_from_slice(input); + let result = >::bare_call( + ALICE, + BOB, + 0, + GAS_LIMIT, + params, + ).unwrap(); + assert_eq!(result.status, 0); + let expected = hash_fn(input.as_ref()); + assert_eq!(&result.data[..*expected_size], &*expected); + } + }) } diff --git a/frame/contracts/src/wasm/mod.rs b/frame/contracts/src/wasm/mod.rs index cb69cd689b2657fa159eec447f1ed569cfc4a434..3d2f5b154ffd47f5295e915cb7115d7f2440432d 100644 --- a/frame/contracts/src/wasm/mod.rs +++ b/frame/contracts/src/wasm/mod.rs @@ -162,6 +162,7 @@ mod tests { use hex_literal::hex; use assert_matches::assert_matches; use sp_runtime::DispatchError; + use frame_support::weights::Weight; const GAS_LIMIT: Gas = 10_000_000_000; @@ -205,7 +206,6 @@ mod tests { instantiates: Vec, terminations: Vec, transfers: Vec, - dispatches: Vec, restores: Vec, // (topics, data) events: Vec<(Vec, Vec)>, @@ -229,11 +229,8 @@ mod tests { fn get_storage(&self, key: &StorageKey) -> Option> { self.storage.get(key).cloned() } - fn set_storage(&mut self, key: StorageKey, value: Option>) - -> Result<(), &'static str> - { + fn set_storage(&mut self, key: StorageKey, value: Option>) { *self.storage.entry(key).or_insert(Vec::new()) = value.unwrap_or(Vec::new()); - Ok(()) } fn instantiate( &mut self, @@ -301,22 +298,20 @@ mod tests { }); Ok(()) } - fn note_dispatch_call(&mut self, call: Call) { - self.dispatches.push(DispatchEntry(call)); - } - fn note_restore_to( + fn restore_to( &mut self, dest: u64, code_hash: H256, rent_allowance: u64, delta: Vec, - ) { + ) -> Result<(), &'static str> { self.restores.push(RestoreEntry { dest, code_hash, rent_allowance, delta, }); + Ok(()) } fn caller(&self) -> &u64 { &42 @@ -375,8 +370,8 @@ mod tests { ) ) } - fn get_weight_price(&self) -> BalanceOf { - 1312_u32.into() + fn get_weight_price(&self, weight: Weight) -> BalanceOf { + BalanceOf::::from(1312_u32).saturating_mul(weight.into()) } } @@ -386,9 +381,7 @@ mod tests { fn get_storage(&self, key: &[u8; 32]) -> Option> { (**self).get_storage(key) } - fn set_storage(&mut self, key: [u8; 32], value: Option>) - -> Result<(), &'static str> - { + fn set_storage(&mut self, key: [u8; 32], value: Option>) { (**self).set_storage(key, value) } fn instantiate( @@ -424,17 +417,14 @@ mod tests { ) -> ExecResult { (**self).call(to, value, gas_meter, input_data) } - fn note_dispatch_call(&mut self, call: Call) { - (**self).note_dispatch_call(call) - } - fn note_restore_to( + fn restore_to( &mut self, dest: u64, code_hash: H256, rent_allowance: u64, delta: Vec, - ) { - (**self).note_restore_to( + ) -> Result<(), &'static str> { + (**self).restore_to( dest, code_hash, rent_allowance, @@ -483,8 +473,8 @@ mod tests { fn get_runtime_storage(&self, key: &[u8]) -> Option> { (**self).get_runtime_storage(key) } - fn get_weight_price(&self) -> BalanceOf { - (**self).get_weight_price() + fn get_weight_price(&self, weight: Weight) -> BalanceOf { + (**self).get_weight_price(weight) } } @@ -1060,7 +1050,7 @@ mod tests { const CODE_GAS_PRICE: &str = r#" (module - (import "env" "ext_gas_price" (func $ext_gas_price)) + (import "env" "ext_gas_price" (func $ext_gas_price (param i64))) (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) (import "env" "memory" (memory 1 1)) @@ -1076,7 +1066,7 @@ mod tests { (func (export "call") ;; This stores the gas price in the scratch buffer - (call $ext_gas_price) + (call $ext_gas_price (i64.const 1)) ;; assert $ext_scratch_size == 8 (call $assert @@ -1241,44 +1231,6 @@ mod tests { ).unwrap(); } - const CODE_DISPATCH_CALL: &str = r#" -(module - (import "env" "ext_dispatch_call" (func $ext_dispatch_call (param i32 i32))) - (import "env" "memory" (memory 1 1)) - - (func (export "call") - (call $ext_dispatch_call - (i32.const 8) ;; Pointer to the start of encoded call buffer - (i32.const 13) ;; Length of the buffer - ) - ) - (func (export "deploy")) - - (data (i32.const 8) "\00\01\2A\00\00\00\00\00\00\00\E5\14\00") -) -"#; - - #[test] - fn dispatch_call() { - // This test can fail due to the encoding changes. In case it becomes too annoying - // let's rewrite so as we use this module controlled call or we serialize it in runtime. - - let mut mock_ext = MockExt::default(); - let _ = execute( - CODE_DISPATCH_CALL, - vec![], - &mut mock_ext, - &mut GasMeter::new(GAS_LIMIT), - ).unwrap(); - - assert_eq!( - &mock_ext.dispatches, - &[DispatchEntry( - Call::Balances(pallet_balances::Call::set_balance(42, 1337, 0)), - )] - ); - } - const CODE_RETURN_FROM_START_FN: &str = r#" (module (import "env" "ext_return" (func $ext_return (param i32 i32))) @@ -1886,103 +1838,4 @@ mod tests { assert_eq!(output, ExecReturnValue { status: 17, data: hex!("5566778899").to_vec() }); assert!(!output.is_success()); } - - const CODE_GET_RUNTIME_STORAGE: &str = r#" -(module - (import "env" "ext_get_runtime_storage" - (func $ext_get_runtime_storage (param i32 i32) (result i32)) - ) - (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) - (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) - (import "env" "ext_scratch_write" (func $ext_scratch_write (param i32 i32))) - (import "env" "memory" (memory 1 1)) - - (func (export "deploy")) - - (func $assert (param i32) - (block $ok - (br_if $ok - (get_local 0) - ) - (unreachable) - ) - ) - - (func $call (export "call") - ;; Load runtime storage for the first key and assert that it exists. - (call $assert - (i32.eq - (call $ext_get_runtime_storage - (i32.const 16) - (i32.const 4) - ) - (i32.const 0) - ) - ) - - ;; assert $ext_scratch_size == 4 - (call $assert - (i32.eq - (call $ext_scratch_size) - (i32.const 4) - ) - ) - - ;; copy contents of the scratch buffer into the contract's memory. - (call $ext_scratch_read - (i32.const 4) ;; Pointer in memory to the place where to copy. - (i32.const 0) ;; Offset from the start of the scratch buffer. - (i32.const 4) ;; Count of bytes to copy. - ) - - ;; assert that contents of the buffer is equal to the i32 value of 0x14144020. - (call $assert - (i32.eq - (i32.load - (i32.const 4) - ) - (i32.const 0x14144020) - ) - ) - - ;; Load the second key and assert that it doesn't exist. - (call $assert - (i32.eq - (call $ext_get_runtime_storage - (i32.const 20) - (i32.const 4) - ) - (i32.const 1) - ) - ) - ) - - ;; The first key, 4 bytes long. - (data (i32.const 16) "\01\02\03\04") - ;; The second key, 4 bytes long. - (data (i32.const 20) "\02\03\04\05") -) -"#; - - #[test] - fn get_runtime_storage() { - let mut gas_meter = GasMeter::new(GAS_LIMIT); - let mock_ext = MockExt::default(); - - // "\01\02\03\04" - Some(0x14144020) - // "\02\03\04\05" - None - *mock_ext.runtime_storage_keys.borrow_mut() = [ - ([1, 2, 3, 4].to_vec(), Some(0x14144020u32.to_le_bytes().to_vec())), - ([2, 3, 4, 5].to_vec().to_vec(), None), - ] - .iter() - .cloned() - .collect(); - let _ = execute( - CODE_GET_RUNTIME_STORAGE, - vec![], - mock_ext, - &mut gas_meter, - ).unwrap(); - } } diff --git a/frame/contracts/src/wasm/runtime.rs b/frame/contracts/src/wasm/runtime.rs index f87f5d1ef53cc962ee88cad097b6e524259d6f54..7b64117cd2314dc0ead88a4ac18e82697dd64a0b 100644 --- a/frame/contracts/src/wasm/runtime.rs +++ b/frame/contracts/src/wasm/runtime.rs @@ -32,7 +32,6 @@ use sp_io::hashing::{ blake2_128, sha2_256, }; -use frame_support::weights::GetDispatchInfo; /// The value returned from ext_call and ext_instantiate contract external functions if the call or /// instantiation traps. This value is chosen as if the execution does not trap, the return value @@ -51,6 +50,8 @@ enum SpecialTrap { /// Signals that a trap was generated in response to a succesful call to the /// `ext_terminate` host function. Termination, + /// Signals that a trap was generated because of a successful restoration. + Restoration, } /// Can only be used for one call. @@ -100,6 +101,12 @@ pub(crate) fn to_execution_result( data: Vec::new(), }) }, + Some(SpecialTrap::Restoration) => { + return Ok(ExecReturnValue { + status: STATUS_SUCCESS, + data: Vec::new(), + }) + } Some(SpecialTrap::OutOfGas) => { return Err(ExecError { reason: "ran out of gas during contract execution".into(), @@ -154,8 +161,6 @@ pub enum RuntimeToken { /// The given number of bytes is read from the sandbox memory and /// is returned as the return data buffer of the call. ReturnData(u32), - /// Dispatched a call with the given weight. - DispatchWithWeight(Gas), /// (topic_count, data_bytes): A buffer of the given size is posted as an event indexed with the /// given number of topics. DepositEvent(u32, u32), @@ -196,7 +201,6 @@ impl Token for RuntimeToken { data_and_topics_cost.checked_add(metadata.event_base_cost) ) }, - DispatchWithWeight(gas) => gas.checked_add(metadata.dispatch_base_cost), }; value.unwrap_or_else(|| Bounded::max_value()) @@ -387,7 +391,7 @@ define_env!(Env, , let mut key: StorageKey = [0; 32]; read_sandbox_memory_into_buf(ctx, key_ptr, &mut key)?; let value = Some(read_sandbox_memory(ctx, value_ptr, value_len)?); - ctx.ext.set_storage(key, value).map_err(|_| sp_sandbox::HostError)?; + ctx.ext.set_storage(key, value); Ok(()) }, @@ -399,7 +403,7 @@ define_env!(Env, , ext_clear_storage(ctx, key_ptr: u32) => { let mut key: StorageKey = [0; 32]; read_sandbox_memory_into_buf(ctx, key_ptr, &mut key)?; - ctx.ext.set_storage(key, None).map_err(|_| sp_sandbox::HostError)?; + ctx.ext.set_storage(key, None); Ok(()) }, @@ -688,12 +692,14 @@ define_env!(Env, , Ok(()) }, - // Stores the gas price for the current transaction into the scratch buffer. + // Stores the price for the specified amount of gas in scratch buffer. // // The data is encoded as T::Balance. The current contents of the scratch buffer are overwritten. - ext_gas_price(ctx) => { + // It is recommended to avoid specifying very small values for `gas` as the prices for a single + // gas can be smaller than one. + ext_gas_price(ctx, gas: u64) => { ctx.scratch_buf.clear(); - ctx.ext.get_weight_price().encode_to(&mut ctx.scratch_buf); + ctx.ext.get_weight_price(gas).encode_to(&mut ctx.scratch_buf); Ok(()) }, @@ -775,41 +781,18 @@ define_env!(Env, , Ok(()) }, - // Decodes the given buffer as a `T::Call` and adds it to the list - // of to-be-dispatched calls. + // Try to restore the given destination contract sacrificing the caller. // - // All calls made it to the top-level context will be dispatched before - // finishing the execution of the calling extrinsic. - ext_dispatch_call(ctx, call_ptr: u32, call_len: u32) => { - let call: <::T as Trait>::Call = - read_sandbox_memory_as(ctx, call_ptr, call_len)?; - - // We already deducted the len costs when reading from the sandbox. - // Bill on the actual weight of the dispatched call. - let info = call.get_dispatch_info(); - charge_gas( - &mut ctx.gas_meter, - ctx.schedule, - &mut ctx.special_trap, - RuntimeToken::DispatchWithWeight(info.weight) - )?; - - ctx.ext.note_dispatch_call(call); - - Ok(()) - }, - - // Record a request to restore the caller contract to the specified contract. + // This function will compute a tombstone hash from the caller's storage and the given code hash + // and if the hash matches the hash found in the tombstone at the specified address - kill + // the caller contract and restore the destination contract and set the specified `rent_allowance`. + // All caller's funds are transfered to the destination. // - // At the finalization stage, i.e. when all changes from the extrinsic that invoked this - // contract are committed, this function will compute a tombstone hash from the caller's - // storage and the given code hash and if the hash matches the hash found in the tombstone at - // the specified address - kill the caller contract and restore the destination contract and set - // the specified `rent_allowance`. All caller's funds are transferred to the destination. + // If there is no tombstone at the destination address, the hashes don't match or this contract + // instance is already present on the contract call stack, a trap is generated. // - // This function doesn't perform restoration right away but defers it to the end of the - // transaction. If there is no tombstone in the destination address or if the hashes don't match - // then restoration is cancelled and no changes are made. + // Otherwise, the destination contract is restored. This function is diverging and stops execution + // even on success. // // `dest_ptr`, `dest_len` - the pointer and the length of a buffer that encodes `T::AccountId` // with the address of the to be restored contract. @@ -857,14 +840,15 @@ define_env!(Env, , delta }; - ctx.ext.note_restore_to( + if let Ok(()) = ctx.ext.restore_to( dest, code_hash, rent_allowance, delta, - ); - - Ok(()) + ) { + ctx.special_trap = Some(SpecialTrap::Restoration); + } + Err(sp_sandbox::HostError) }, // Returns the size of the scratch buffer. @@ -993,32 +977,6 @@ define_env!(Env, , Ok(()) }, - // Retrieve the value under the given key from the **runtime** storage and return 0. - // If there is no entry under the given key then this function will return 1 and - // clear the scratch buffer. - // - // - key_ptr: the pointer into the linear memory where the requested value is placed. - // - key_len: the length of the key in bytes. - ext_get_runtime_storage(ctx, key_ptr: u32, key_len: u32) -> u32 => { - // Steal the scratch buffer so that we hopefully save an allocation for the `key_buf`. - read_sandbox_memory_into_scratch(ctx, key_ptr, key_len)?; - let key_buf = mem::replace(&mut ctx.scratch_buf, Vec::new()); - - match ctx.ext.get_runtime_storage(&key_buf) { - Some(value_buf) => { - // The given value exists. - ctx.scratch_buf = value_buf; - Ok(0) - } - None => { - // Put back the `key_buf` and allow its allocation to be reused. - ctx.scratch_buf = key_buf; - ctx.scratch_buf.clear(); - Ok(1) - } - } - }, - // Computes the SHA2 256-bit hash on the given input buffer. // // Returns the result directly into the given output buffer. diff --git a/frame/contracts/tests/dispatch_call.wat b/frame/contracts/tests/dispatch_call.wat deleted file mode 100644 index db0995bd6c79ad37a8b71b4db9d88a8682b308e8..0000000000000000000000000000000000000000 --- a/frame/contracts/tests/dispatch_call.wat +++ /dev/null @@ -1,14 +0,0 @@ -(module - (import "env" "ext_dispatch_call" (func $ext_dispatch_call (param i32 i32))) - (import "env" "memory" (memory 1 1)) - - (func (export "call") - (call $ext_dispatch_call - (i32.const 8) ;; Pointer to the start of encoded call buffer - (i32.const 11) ;; Length of the buffer - ) - ) - (func (export "deploy")) - - (data (i32.const 8) "\00\00\03\00\00\00\00\00\00\00\C8") -) diff --git a/frame/contracts/tests/dispatch_call_then_trap.wat b/frame/contracts/tests/dispatch_call_then_trap.wat deleted file mode 100644 index ce949d68236f39846f82434ac45c9b825cedd698..0000000000000000000000000000000000000000 --- a/frame/contracts/tests/dispatch_call_then_trap.wat +++ /dev/null @@ -1,15 +0,0 @@ -(module - (import "env" "ext_dispatch_call" (func $ext_dispatch_call (param i32 i32))) - (import "env" "memory" (memory 1 1)) - - (func (export "call") - (call $ext_dispatch_call - (i32.const 8) ;; Pointer to the start of encoded call buffer - (i32.const 11) ;; Length of the buffer - ) - (unreachable) ;; trap so that the top level transaction fails - ) - (func (export "deploy")) - - (data (i32.const 8) "\00\00\03\00\00\00\00\00\00\00\C8") -) diff --git a/frame/contracts/tests/get_runtime_storage.wat b/frame/contracts/tests/get_runtime_storage.wat deleted file mode 100644 index 6148f1c408c017c5096f055759d1ed66ba9b7104..0000000000000000000000000000000000000000 --- a/frame/contracts/tests/get_runtime_storage.wat +++ /dev/null @@ -1,74 +0,0 @@ -(module - (import "env" "ext_get_runtime_storage" - (func $ext_get_runtime_storage (param i32 i32) (result i32)) - ) - (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) - (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) - (import "env" "ext_scratch_write" (func $ext_scratch_write (param i32 i32))) - (import "env" "memory" (memory 1 1)) - - (func (export "deploy")) - - (func $assert (param i32) - (block $ok - (br_if $ok - (get_local 0) - ) - (unreachable) - ) - ) - - (func $call (export "call") - ;; Load runtime storage for the first key and assert that it exists. - (call $assert - (i32.eq - (call $ext_get_runtime_storage - (i32.const 16) - (i32.const 4) - ) - (i32.const 0) - ) - ) - - ;; assert $ext_scratch_size == 4 - (call $assert - (i32.eq - (call $ext_scratch_size) - (i32.const 4) - ) - ) - - ;; copy contents of the scratch buffer into the contract's memory. - (call $ext_scratch_read - (i32.const 4) ;; Pointer in memory to the place where to copy. - (i32.const 0) ;; Offset from the start of the scratch buffer. - (i32.const 4) ;; Count of bytes to copy. - ) - - ;; assert that contents of the buffer is equal to the i32 value of 0x14144020. - (call $assert - (i32.eq - (i32.load - (i32.const 4) - ) - (i32.const 0x14144020) - ) - ) - - ;; Load the second key and assert that it doesn't exist. - (call $assert - (i32.eq - (call $ext_get_runtime_storage - (i32.const 20) - (i32.const 4) - ) - (i32.const 1) - ) - ) - ) - - ;; The first key, 4 bytes long. - (data (i32.const 16) "\01\02\03\04") - ;; The second key, 4 bytes long. - (data (i32.const 20) "\02\03\04\05") -) diff --git a/frame/democracy/Cargo.toml b/frame/democracy/Cargo.toml index e5065614119d496c0c2c1d0e79e4fb268d267b06..9532be0e8ee2ddc0b996a3f91a3b17faac9a1bee 100644 --- a/frame/democracy/Cargo.toml +++ b/frame/democracy/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-democracy" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,20 +13,20 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../benchmarking", optional = true } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } -pallet-scheduler = { version = "2.0.0-alpha.8", path = "../scheduler" } -sp-storage = { version = "2.0.0-alpha.8", path = "../../primitives/storage" } -substrate-test-utils = { version = "2.0.0-alpha.8", path = "../../test-utils" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } +pallet-scheduler = { version = "2.0.0-rc4", path = "../scheduler" } +sp-storage = { version = "2.0.0-rc4", path = "../../primitives/storage" } +substrate-test-utils = { version = "2.0.0-rc4", path = "../../test-utils" } hex-literal = "0.2.1" [features] diff --git a/frame/democracy/src/benchmarking.rs b/frame/democracy/src/benchmarking.rs index 9fa619a994fa32aca63292b8cffaedc9c79653bb..ba3b9a0b13568aa0e6853fe81917f2da15a86208 100644 --- a/frame/democracy/src/benchmarking.rs +++ b/frame/democracy/src/benchmarking.rs @@ -22,7 +22,7 @@ use super::*; use frame_benchmarking::{benchmarks, account}; use frame_support::{ IterableStorageMap, - traits::{Currency, Get, EnsureOrigin, OnInitialize}, + traits::{Currency, Get, EnsureOrigin, OnInitialize, UnfilteredDispatchable}, }; use frame_system::{RawOrigin, Module as System, self, EventRecord}; use sp_runtime::traits::{Bounded, One}; @@ -30,7 +30,6 @@ use sp_runtime::traits::{Bounded, One}; use crate::Module as Democracy; const SEED: u32 = 0; -const MAX_USERS: u32 = 1000; const MAX_REFERENDUMS: u32 = 100; const MAX_PROPOSALS: u32 = 100; const MAX_SECONDERS: u32 = 100; @@ -75,13 +74,13 @@ fn add_referendum(n: u32) -> Result { 0.into(), ); let referendum_index: ReferendumIndex = ReferendumCount::get() - 1; - let _ = T::Scheduler::schedule_named( + T::Scheduler::schedule_named( (DEMOCRACY_ID, referendum_index).encode(), - 0.into(), + 1.into(), None, 63, Call::enact_proposal(proposal_hash, referendum_index).into(), - ); + ).map_err(|_| "failed to schedule named")?; Ok(referendum_index) } @@ -97,16 +96,6 @@ fn account_vote(b: BalanceOf) -> AccountVote> { } } -fn open_activate_proxy(u: u32) -> Result<(T::AccountId, T::AccountId), &'static str> { - let caller = funded_account::("caller", u); - let voter = funded_account::("voter", u); - - Democracy::::open_proxy(RawOrigin::Signed(caller.clone()).into(), voter.clone())?; - Democracy::::activate_proxy(RawOrigin::Signed(voter.clone()).into(), caller.clone())?; - - Ok((caller, voter)) -} - benchmarks! { _ { } @@ -215,70 +204,6 @@ benchmarks! { assert_eq!(tally.nays, 1000.into(), "changed vote was not recorded"); } - // Basically copy paste of `vote_new` - proxy_vote_new { - let r in 1 .. MAX_REFERENDUMS; - - let (caller, voter) = open_activate_proxy::(0)?; - let account_vote = account_vote::(100.into()); - - // Populate existing direct votes for the voter, they can vote on their own behalf - for i in 0 .. r { - let ref_idx = add_referendum::(i)?; - Democracy::::vote(RawOrigin::Signed(voter.clone()).into(), ref_idx, account_vote.clone())?; - } - - let referendum_index = add_referendum::(r)?; - - }: proxy_vote(RawOrigin::Signed(caller), referendum_index, account_vote) - verify { - let votes = match VotingOf::::get(&voter) { - Voting::Direct { votes, .. } => votes, - _ => return Err("Votes are not direct"), - }; - assert_eq!(votes.len(), (r + 1) as usize, "Vote was not recorded."); - } - - // Basically copy paste of `vote_existing` - proxy_vote_existing { - let r in 1 .. MAX_REFERENDUMS; - - let (caller, voter) = open_activate_proxy::(0)?; - let account_vote = account_vote::(100.into()); - - // We need to create existing direct votes - for i in 0 ..=r { - let ref_idx = add_referendum::(i)?; - Democracy::::vote(RawOrigin::Signed(voter.clone()).into(), ref_idx, account_vote.clone())?; - } - let votes = match VotingOf::::get(&voter) { - Voting::Direct { votes, .. } => votes, - _ => return Err("Votes are not direct"), - }; - assert_eq!(votes.len(), (r + 1) as usize, "Votes were not recorded."); - - // Change vote from aye to nay - let nay = Vote { aye: false, conviction: Conviction::Locked1x }; - let new_vote = AccountVote::Standard { vote: nay, balance: 1000.into() }; - let referendum_index = Democracy::::referendum_count() - 1; - - // This tests when a user changes a vote - }: proxy_vote(RawOrigin::Signed(caller.clone()), referendum_index, new_vote) - verify { - let votes = match VotingOf::::get(&voter) { - Voting::Direct { votes, .. } => votes, - _ => return Err("Votes are not direct"), - }; - assert_eq!(votes.len(), (r + 1) as usize, "Vote was incorrectly added"); - let referendum_info = Democracy::::referendum_info(referendum_index) - .ok_or("referendum doesn't exist")?; - let tally = match referendum_info { - ReferendumInfo::Ongoing(r) => r.tally, - _ => return Err("referendum not ongoing"), - }; - assert_eq!(tally.nays, 1000.into(), "changed vote was not recorded"); - } - emergency_cancel { let r in 1 .. MAX_REFERENDUMS; let origin = T::CancellationOrigin::successful_origin(); @@ -287,14 +212,14 @@ benchmarks! { for i in 0 .. r { let ref_idx = add_referendum::(i)?; let call = Call::::emergency_cancel(ref_idx); - call.dispatch(origin.clone())?; + call.dispatch_bypass_filter(origin.clone())?; } // Lets now measure one more let referendum_index = add_referendum::(r)?; let call = Call::::emergency_cancel(referendum_index); assert!(Democracy::::referendum_status(referendum_index).is_ok()); - }: { call.dispatch(origin)? } + }: { call.dispatch_bypass_filter(origin)? } verify { // Referendum has been canceled assert!(Democracy::::referendum_status(referendum_index).is_err()); @@ -314,7 +239,7 @@ benchmarks! { ); let call = Call::::external_propose(proposal_hash); - }: { call.dispatch(origin)? } + }: { call.dispatch_bypass_filter(origin)? } verify { // External proposal created ensure!(>::exists(), "External proposal didn't work"); @@ -326,7 +251,7 @@ benchmarks! { let origin = T::ExternalMajorityOrigin::successful_origin(); let proposal_hash = T::Hashing::hash_of(&p); let call = Call::::external_propose_majority(proposal_hash); - }: { call.dispatch(origin)? } + }: { call.dispatch_bypass_filter(origin)? } verify { // External proposal created ensure!(>::exists(), "External proposal didn't work"); @@ -338,7 +263,7 @@ benchmarks! { let origin = T::ExternalDefaultOrigin::successful_origin(); let proposal_hash = T::Hashing::hash_of(&p); let call = Call::::external_propose_default(proposal_hash); - }: { call.dispatch(origin)? } + }: { call.dispatch_bypass_filter(origin)? } verify { // External proposal created ensure!(>::exists(), "External proposal didn't work"); @@ -357,7 +282,7 @@ benchmarks! { let delay = 0; let call = Call::::fast_track(proposal_hash, voting_period.into(), delay.into()); - }: { call.dispatch(origin_fast_track)? } + }: { call.dispatch_bypass_filter(origin_fast_track)? } verify { assert_eq!(Democracy::::referendum_count(), 1, "referendum not created") } @@ -381,7 +306,7 @@ benchmarks! { let call = Call::::veto_external(proposal_hash); let origin = T::VetoOrigin::successful_origin(); ensure!(NextExternal::::get().is_some(), "no external proposal"); - }: { call.dispatch(origin)? } + }: { call.dispatch_bypass_filter(origin)? } verify { assert!(NextExternal::::get().is_none()); let (_, new_vetoers) = >::get(&proposal_hash).ok_or("no blacklist")?; @@ -422,7 +347,7 @@ benchmarks! { let origin = T::ExternalMajorityOrigin::successful_origin(); let proposal_hash = T::Hashing::hash_of(&r); let call = Call::::external_propose_majority(proposal_hash); - call.dispatch(origin)?; + call.dispatch_bypass_filter(origin)?; // External proposal created ensure!(>::exists(), "External proposal didn't work"); @@ -505,33 +430,6 @@ benchmarks! { } } - activate_proxy { - let u in 1 .. MAX_USERS; - - let caller: T::AccountId = funded_account::("caller", u); - let proxy: T::AccountId = funded_account::("proxy", u); - Democracy::::open_proxy(RawOrigin::Signed(proxy.clone()).into(), caller.clone())?; - }: _(RawOrigin::Signed(caller.clone()), proxy.clone()) - verify { - assert_eq!(Democracy::::proxy(proxy), Some(ProxyState::Active(caller))); - } - - close_proxy { - let u in 1 .. MAX_USERS; - let (caller, _) = open_activate_proxy::(u)?; - }: _(RawOrigin::Signed(caller.clone())) - verify { - assert_eq!(Democracy::::proxy(caller), None); - } - - deactivate_proxy { - let u in 1 .. MAX_USERS; - let (caller, voter) = open_activate_proxy::(u)?; - }: _(RawOrigin::Signed(voter.clone()), caller.clone()) - verify { - assert_eq!(Democracy::::proxy(caller), Some(ProxyState::Open(voter))); - } - delegate { let r in 1 .. MAX_REFERENDUMS; @@ -760,14 +658,6 @@ benchmarks! { assert_eq!(voting.locked_balance(), base_balance); } - open_proxy { - let u in 1 .. MAX_USERS; - - let caller: T::AccountId = funded_account::("caller", u); - let proxy: T::AccountId = funded_account::("proxy", u); - - }: _(RawOrigin::Signed(proxy), caller) - remove_vote { let r in 1 .. MAX_REFERENDUMS; @@ -831,131 +721,6 @@ benchmarks! { assert_eq!(votes.len(), (r - 1) as usize, "Vote was not removed"); } - // This is a copy of delegate benchmark, but with `open_activate_proxy` - proxy_delegate { - let r in 1 .. MAX_REFERENDUMS; - - let initial_balance: BalanceOf = 100.into(); - let delegated_balance: BalanceOf = 1000.into(); - - let (caller, voter) = open_activate_proxy::(0)?; - - // Voter will initially delegate to `old_delegate` - let old_delegate: T::AccountId = funded_account::("old_delegate", r); - Democracy::::delegate( - RawOrigin::Signed(voter.clone()).into(), - old_delegate.clone(), - Conviction::Locked1x, - delegated_balance, - )?; - let (target, balance) = match VotingOf::::get(&voter) { - Voting::Delegating { target, balance, .. } => (target, balance), - _ => return Err("Votes are not direct"), - }; - assert_eq!(target, old_delegate, "delegation target didn't work"); - assert_eq!(balance, delegated_balance, "delegation balance didn't work"); - // Voter will now switch to `new_delegate` - let new_delegate: T::AccountId = funded_account::("new_delegate", r); - let account_vote = account_vote::(initial_balance); - // We need to create existing direct votes for the `new_delegate` - for i in 0..r { - let ref_idx = add_referendum::(i)?; - Democracy::::vote(RawOrigin::Signed(new_delegate.clone()).into(), ref_idx, account_vote.clone())?; - } - let votes = match VotingOf::::get(&new_delegate) { - Voting::Direct { votes, .. } => votes, - _ => return Err("Votes are not direct"), - }; - assert_eq!(votes.len(), r as usize, "Votes were not recorded."); - }: _(RawOrigin::Signed(caller.clone()), new_delegate.clone(), Conviction::Locked1x, delegated_balance) - verify { - let (target, balance) = match VotingOf::::get(&voter) { - Voting::Delegating { target, balance, .. } => (target, balance), - _ => return Err("Votes are not direct"), - }; - assert_eq!(target, new_delegate, "delegation target didn't work"); - assert_eq!(balance, delegated_balance, "delegation balance didn't work"); - let delegations = match VotingOf::::get(&new_delegate) { - Voting::Direct { delegations, .. } => delegations, - _ => return Err("Votes are not direct"), - }; - assert_eq!(delegations.capital, delegated_balance, "delegation was not recorded."); - } - - // This is a copy of undelegate benchmark, but with `open_activate_proxy` - proxy_undelegate { - let r in 1 .. MAX_REFERENDUMS; - - let initial_balance: BalanceOf = 100.into(); - let delegated_balance: BalanceOf = 1000.into(); - - let (caller, voter) = open_activate_proxy::(0)?; - // Caller will delegate - let the_delegate: T::AccountId = funded_account::("delegate", r); - Democracy::::delegate( - RawOrigin::Signed(voter.clone()).into(), - the_delegate.clone(), - Conviction::Locked1x, - delegated_balance, - )?; - let (target, balance) = match VotingOf::::get(&voter) { - Voting::Delegating { target, balance, .. } => (target, balance), - _ => return Err("Votes are not direct"), - }; - assert_eq!(target, the_delegate, "delegation target didn't work"); - assert_eq!(balance, delegated_balance, "delegation balance didn't work"); - // We need to create votes direct votes for the `delegate` - let account_vote = account_vote::(initial_balance); - for i in 0..r { - let ref_idx = add_referendum::(i)?; - Democracy::::vote( - RawOrigin::Signed(the_delegate.clone()).into(), - ref_idx, - account_vote.clone() - )?; - } - let votes = match VotingOf::::get(&the_delegate) { - Voting::Direct { votes, .. } => votes, - _ => return Err("Votes are not direct"), - }; - assert_eq!(votes.len(), r as usize, "Votes were not recorded."); - }: _(RawOrigin::Signed(caller.clone())) - verify { - // Voting should now be direct - match VotingOf::::get(&voter) { - Voting::Direct { .. } => (), - _ => return Err("undelegation failed"), - } - } - - proxy_remove_vote { - let r in 1 .. MAX_REFERENDUMS; - - let (caller, voter) = open_activate_proxy::(0)?; - let account_vote = account_vote::(100.into()); - - for i in 0 .. r { - let ref_idx = add_referendum::(i)?; - Democracy::::vote(RawOrigin::Signed(voter.clone()).into(), ref_idx, account_vote.clone())?; - } - - let votes = match VotingOf::::get(&voter) { - Voting::Direct { votes, .. } => votes, - _ => return Err("Votes are not direct"), - }; - assert_eq!(votes.len(), r as usize, "Votes not created"); - - let referendum_index = r - 1; - - }: _(RawOrigin::Signed(caller.clone()), referendum_index) - verify { - let votes = match VotingOf::::get(&voter) { - Voting::Direct { votes, .. } => votes, - _ => return Err("Votes are not direct"), - }; - assert_eq!(votes.len(), (r - 1) as usize, "Vote was not removed"); - } - enact_proposal_execute { // Num of bytes in encoded proposal let b in 0 .. MAX_BYTES; @@ -1012,8 +777,6 @@ mod tests { assert_ok!(test_benchmark_second::()); assert_ok!(test_benchmark_vote_new::()); assert_ok!(test_benchmark_vote_existing::()); - assert_ok!(test_benchmark_proxy_vote_new::()); - assert_ok!(test_benchmark_proxy_vote_existing::()); assert_ok!(test_benchmark_emergency_cancel::()); assert_ok!(test_benchmark_external_propose::()); assert_ok!(test_benchmark_external_propose_majority::()); @@ -1025,10 +788,6 @@ mod tests { assert_ok!(test_benchmark_on_initialize_external::()); assert_ok!(test_benchmark_on_initialize_public::()); assert_ok!(test_benchmark_on_initialize_no_launch_no_maturing::()); - assert_ok!(test_benchmark_open_proxy::()); - assert_ok!(test_benchmark_activate_proxy::()); - assert_ok!(test_benchmark_close_proxy::()); - assert_ok!(test_benchmark_deactivate_proxy::()); assert_ok!(test_benchmark_delegate::()); assert_ok!(test_benchmark_undelegate::()); assert_ok!(test_benchmark_clear_public_proposals::()); @@ -1039,9 +798,6 @@ mod tests { assert_ok!(test_benchmark_unlock_set::()); assert_ok!(test_benchmark_remove_vote::()); assert_ok!(test_benchmark_remove_other_vote::()); - assert_ok!(test_benchmark_proxy_delegate::()); - assert_ok!(test_benchmark_proxy_undelegate::()); - assert_ok!(test_benchmark_proxy_remove_vote::()); assert_ok!(test_benchmark_enact_proposal_execute::()); assert_ok!(test_benchmark_enact_proposal_slash::()); }); diff --git a/frame/democracy/src/lib.rs b/frame/democracy/src/lib.rs index e0f1ec9b5c707dac4dfde47fbb6ce24386427b42..79cc136d458752de9e116484e2eda70983abefe6 100644 --- a/frame/democracy/src/lib.rs +++ b/frame/democracy/src/lib.rs @@ -52,8 +52,6 @@ //! account or an external origin) suggests that the system adopt. //! - **Referendum:** A proposal that is in the process of being voted on for //! either acceptance or rejection as a change to the system. -//! - **Proxy:** An account that has full voting power on behalf of a separate "Stash" account -//! that holds the funds. //! - **Delegation:** The act of granting your voting power to the decisions of another account for //! up to a certain conviction. //! @@ -93,23 +91,13 @@ //! - `reap_vote` - Remove some account's expired votes. //! - `unlock` - Redetermine the account's balance lock, potentially making tokens available. //! -//! Proxy administration: -//! - `activate_proxy` - Activates a proxy that is already open to the sender. -//! - `close_proxy` - Clears the proxy status, called by the proxy. -//! - `deactivate_proxy` - Deactivates a proxy back to the open status, called by the stash. -//! - `open_proxy` - Opens a proxy account on behalf of the sender. -//! -//! Proxy actions: -//! - `proxy_vote` - Votes in a referendum on behalf of a stash account. -//! - `proxy_unvote` - Cancel a previous vote, done on behalf of the voter by a proxy. -//! - `proxy_delegate` - Delegate voting power, done on behalf of the voter by a proxy. -//! - `proxy_undelegate` - Stop delegating voting power, done on behalf of the voter by a proxy. -//! //! Preimage actions: //! - `note_preimage` - Registers the preimage for an upcoming proposal, requires //! a deposit that is returned once the proposal is enacted. +//! - `note_preimage_operational` - same but provided by `T::OperationalPreimageOrigin`. //! - `note_imminent_preimage` - Registers the preimage for an upcoming proposal. //! Does not require a deposit, but the proposal must be in the dispatch queue. +//! - `note_imminent_preimage_operational` - same but provided by `T::OperationalPreimageOrigin`. //! - `reap_preimage` - Removes the preimage for an expired proposal. Will only //! work under the condition that it's the same account that noted it and //! after the voting period, OR it's a different account after the enactment period. @@ -172,7 +160,6 @@ use sp_runtime::{ use codec::{Encode, Decode, Input}; use frame_support::{ decl_module, decl_storage, decl_event, decl_error, ensure, Parameter, - storage::IterableStorageMap, weights::{Weight, DispatchClass}, traits::{ Currency, ReservableCurrency, LockableCurrency, WithdrawReason, LockIdentifier, Get, @@ -189,7 +176,7 @@ mod types; pub use vote_threshold::{Approved, VoteThreshold}; pub use vote::{Vote, AccountVote, Voting}; pub use conviction::Conviction; -pub use types::{ReferendumInfo, ReferendumStatus, ProxyState, Tally, UnvoteScope, Delegations}; +pub use types::{ReferendumInfo, ReferendumStatus, Tally, UnvoteScope, Delegations}; #[cfg(test)] mod tests; @@ -285,6 +272,9 @@ pub trait Trait: frame_system::Trait + Sized { /// The amount of balance that must be deposited per byte of preimage stored. type PreimageByteDeposit: Get>; + /// An origin that can provide a preimage using operational extrinsics. + type OperationalPreimageOrigin: EnsureOrigin; + /// Handler for the unbalanced reduction when slashing a preimage deposit. type Slash: OnUnbalanced>; @@ -339,6 +329,8 @@ decl_storage! { /// The public proposals. Unsorted. The second item is the proposal's hash. pub PublicProps get(fn public_props): Vec<(PropIndex, T::Hash, T::AccountId)>; /// Those who have locked a deposit. + /// + /// TWOX-NOTE: Safe, as increasing integer keys are safe. pub DepositOf get(fn deposit_of): map hasher(twox_64_concat) PropIndex => Option<(Vec, BalanceOf)>; @@ -357,22 +349,22 @@ decl_storage! { pub LowestUnbaked get(fn lowest_unbaked) build(|_| 0 as ReferendumIndex): ReferendumIndex; /// Information concerning any given referendum. + /// + /// TWOX-NOTE: SAFE as indexes are not under an attacker’s control. pub ReferendumInfoOf get(fn referendum_info): map hasher(twox_64_concat) ReferendumIndex => Option>>; /// All votes for a particular voter. We store the balance for the number of votes that we /// have recorded. The second item is the total amount of delegations, that will be added. + /// + /// TWOX-NOTE: SAFE as `AccountId`s are crypto hashes anyway. pub VotingOf: map hasher(twox_64_concat) T::AccountId => Voting, T::AccountId, T::BlockNumber>; - /// Who is able to vote for whom. Value is the fund-holding account, key is the - /// vote-transaction-sending account. - // TODO: Refactor proxy into its own pallet. - // https://github.com/paritytech/substrate/issues/5322 - pub Proxy get(fn proxy): map hasher(twox_64_concat) T::AccountId => Option>; - /// Accounts for which there are locks in action which may be removed at some point in the /// future. The value is the block number at which the lock expires and may be removed. + /// + /// TWOX-NOTE: OK ― `AccountId` is a secure hash. pub Locks get(fn locks): map hasher(twox_64_concat) T::AccountId => Option; /// True if the last referendum tabled was submitted externally. False if it was a public @@ -452,8 +444,6 @@ decl_error! { ValueLow, /// Proposal does not exist ProposalMissing, - /// Not a proxy - NotProxy, /// Unknown index BadIndex, /// Cannot cancel the same proposal twice @@ -470,10 +460,6 @@ decl_error! { NoProposal, /// Identity may not veto a proposal twice AlreadyVetoed, - /// Already a proxy - AlreadyProxy, - /// Wrong proxy - WrongProxy, /// Not delegated NotDelegated, /// Preimage already noted @@ -496,12 +482,6 @@ decl_error! { NotLocked, /// The lock on the account to be unlocked has not yet expired. NotExpired, - /// A proxy-pairing was attempted to an account that was not open. - NotOpen, - /// A proxy-pairing was attempted to an account that was open to another account. - WrongOpen, - /// A proxy-de-pairing was attempted to an account that was not active. - NotActive, /// The given account did not vote on the referendum. NotVoter, /// The actor has no permission to conduct the action. @@ -542,7 +522,7 @@ mod weight_for { /// - Db writes per votes: `ReferendumInfoOf` /// - Base Weight: 65.78 + 8.229 * R µs // NOTE: weight must cover an incorrect voting of origin with 100 votes. - pub(crate) fn delegate(votes: Weight) -> Weight { + pub fn delegate(votes: Weight) -> Weight { T::DbWeight::get().reads_writes(votes.saturating_add(3), votes.saturating_add(3)) .saturating_add(66_000_000) .saturating_add(votes.saturating_mul(8_100_000)) @@ -554,31 +534,36 @@ mod weight_for { /// - Db reads per votes: `ReferendumInfoOf` /// - Db writes per votes: `ReferendumInfoOf` /// - Base Weight: 33.29 + 8.104 * R µs - pub(crate) fn undelegate(votes: Weight) -> Weight { + pub fn undelegate(votes: Weight) -> Weight { T::DbWeight::get().reads_writes(votes.saturating_add(2), votes.saturating_add(2)) .saturating_add(33_000_000) .saturating_add(votes.saturating_mul(8_000_000)) } - /// Calculate the weight for `proxy_delegate`. - /// same as `delegate with additional: - /// - Db reads: `Proxy`, `proxy account` - /// - Db writes: `proxy account` - /// - Base Weight: 68.61 + 8.039 * R µs - pub(crate) fn proxy_delegate(votes: Weight) -> Weight { - T::DbWeight::get().reads_writes(votes.saturating_add(5), votes.saturating_add(4)) - .saturating_add(69_000_000) - .saturating_add(votes.saturating_mul(8_000_000)) + /// Calculate the weight for `note_preimage`. + /// # + /// - Complexity: `O(E)` with E size of `encoded_proposal` (protected by a required deposit). + /// - Db reads: `Preimages` + /// - Db writes: `Preimages` + /// - Base Weight: 37.93 + .004 * b µs + /// # + pub fn note_preimage(encoded_proposal_len: Weight) -> Weight { + T::DbWeight::get().reads_writes(1, 1) + .saturating_add(38_000_000) + .saturating_add(encoded_proposal_len.saturating_mul(4_000)) } - /// Calculate the weight for `proxy_undelegate`. - /// same as `undelegate with additional: - /// Db reads: `Proxy` - /// Base Weight: 39 + 7.958 * R µs - pub(crate) fn proxy_undelegate(votes: Weight) -> Weight { - T::DbWeight::get().reads_writes(votes.saturating_add(3), votes.saturating_add(2)) - .saturating_add(40_000_000) - .saturating_add(votes.saturating_mul(8_000_000)) + /// Calculate the weight for `note_imminent_preimage`. + /// # + /// - Complexity: `O(E)` with E size of `encoded_proposal` (protected by a required deposit). + /// - Db reads: `Preimages` + /// - Db writes: `Preimages` + /// - Base Weight: 28.04 + .003 * b µs + /// # + pub fn note_imminent_preimage(encoded_proposal_len: Weight) -> Weight { + T::DbWeight::get().reads_writes(1, 1) + .saturating_add(28_000_000) + .saturating_add(encoded_proposal_len.saturating_mul(3_000)) } } @@ -611,23 +596,10 @@ decl_module! { /// The amount of balance that must be deposited per byte of preimage stored. const PreimageByteDeposit: BalanceOf = T::PreimageByteDeposit::get(); - fn deposit_event() = default; - - fn on_runtime_upgrade() -> Weight { - if let None = StorageVersion::get() { - StorageVersion::put(Releases::V1); - - DepositOf::::translate::< - (BalanceOf, Vec), _ - >(|_, (balance, accounts)| { - Some((accounts, balance)) - }); + /// The maximum number of votes for an account. + const MaxVotes: u32 = T::MaxVotes::get(); - T::MaximumBlockWeight::get() - } else { - T::DbWeight::get().reads(1) - } - } + fn deposit_event() = default; /// Propose a sensitive action to be taken. /// @@ -722,34 +694,6 @@ decl_module! { Self::try_vote(&who, ref_index, vote) } - /// Vote in a referendum on behalf of a stash. If `vote.is_aye()`, the vote is to enact - /// the proposal; otherwise it is a vote to keep the status quo. - /// - /// The dispatch origin of this call must be _Signed_. - /// - /// - `ref_index`: The index of the referendum to proxy vote for. - /// - `vote`: The vote configuration. - /// - /// # - /// - Complexity: `O(R)` where R is the number of referendums the proxy has voted on. - /// weight is charged as if maximum votes. - /// - Db reads: `ReferendumInfoOf`, `VotingOf`, `balances locks`, `Proxy`, `proxy account` - /// - Db writes: `ReferendumInfoOf`, `VotingOf`, `balances locks` - /// ------------ - /// - Base Weight: - /// - Proxy Vote New: 54.35 + .344 * R µs - /// - Proxy Vote Existing: 54.35 + .35 * R µs - /// # - #[weight = 55_000_000 + 350_000 * Weight::from(T::MaxVotes::get()) + T::DbWeight::get().reads_writes(5, 3)] - fn proxy_vote(origin, - #[compact] ref_index: ReferendumIndex, - vote: AccountVote>, - ) -> DispatchResult { - let who = ensure_signed(origin)?; - let voter = Self::proxy(who).and_then(|a| a.as_active()).ok_or(Error::::NotProxy)?; - Self::try_vote(&voter, ref_index, vote) - } - /// Schedule an emergency cancellation of a referendum. Cannot happen twice to the same /// referendum. /// @@ -984,86 +928,6 @@ decl_module! { }) } - /// Specify a proxy that is already open to us. Called by the stash. - /// - /// NOTE: Used to be called `set_proxy`. - /// - /// The dispatch origin of this call must be _Signed_. - /// - /// - `proxy`: The account that will be activated as proxy. - /// - /// # - /// - Complexity: `O(1)` - /// - Db reads: `Proxy` - /// - Db writes: `Proxy` - /// - Base Weight: 7.972 µs - /// # - #[weight = 8_000_000 + T::DbWeight::get().reads_writes(1, 1)] - fn activate_proxy(origin, proxy: T::AccountId) { - let who = ensure_signed(origin)?; - Proxy::::try_mutate(&proxy, |a| match a.take() { - None => Err(Error::::NotOpen), - Some(ProxyState::Active(_)) => Err(Error::::AlreadyProxy), - Some(ProxyState::Open(x)) if &x == &who => { - *a = Some(ProxyState::Active(who)); - Ok(()) - } - Some(ProxyState::Open(_)) => Err(Error::::WrongOpen), - })?; - } - - /// Clear the proxy. Called by the proxy. - /// - /// NOTE: Used to be called `resign_proxy`. - /// - /// The dispatch origin of this call must be _Signed_. - /// - /// # - /// - Complexity: `O(1)` - /// - Db reads: `Proxy`, `sender account` - /// - Db writes: `Proxy`, `sender account` - /// - Base Weight: 15.41 µs - /// # - #[weight = 16_000_000 + T::DbWeight::get().reads_writes(1, 1)] - fn close_proxy(origin) { - let who = ensure_signed(origin)?; - Proxy::::mutate(&who, |a| { - if a.is_some() { - system::Module::::dec_ref(&who); - } - *a = None; - }); - } - - /// Deactivate the proxy, but leave open to this account. Called by the stash. - /// - /// The proxy must already be active. - /// - /// NOTE: Used to be called `remove_proxy`. - /// - /// The dispatch origin of this call must be _Signed_. - /// - /// - `proxy`: The account that will be deactivated as proxy. - /// - /// # - /// - Complexity: `O(1)` - /// - Db reads: `Proxy` - /// - Db writes: `Proxy` - /// - Base Weight: 8.03 µs - /// # - #[weight = 8_000_000 + T::DbWeight::get().reads_writes(1, 1)] - fn deactivate_proxy(origin, proxy: T::AccountId) { - let who = ensure_signed(origin)?; - Proxy::::try_mutate(&proxy, |a| match a.take() { - None | Some(ProxyState::Open(_)) => Err(Error::::NotActive), - Some(ProxyState::Active(x)) if &x == &who => { - *a = Some(ProxyState::Open(who)); - Ok(()) - } - Some(ProxyState::Active(_)) => Err(Error::::WrongProxy), - })?; - } - /// Delegate the voting power (with some given conviction) of the sending account. /// /// The balance delegated is locked for as long as it's delegated, and thereafter for the @@ -1157,33 +1021,21 @@ decl_module! { /// Emits `PreimageNoted`. /// /// # - /// - Complexity: `O(E)` with E size of `encoded_proposal` (protected by a required deposit). - /// - Db reads: `Preimages` - /// - Db writes: `Preimages` - /// - Base Weight: 37.93 + .004 * b µs + /// see `weight_for::note_preimage` /// # - #[weight = 38_000_000 + 4_000 * Weight::from(encoded_proposal.len() as u32) - + T::DbWeight::get().reads_writes(1, 1)] + #[weight = weight_for::note_preimage::((encoded_proposal.len() as u32).into())] fn note_preimage(origin, encoded_proposal: Vec) { - let who = ensure_signed(origin)?; - let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); - ensure!(!>::contains_key(&proposal_hash), Error::::DuplicatePreimage); - - let deposit = >::from(encoded_proposal.len() as u32) - .saturating_mul(T::PreimageByteDeposit::get()); - T::Currency::reserve(&who, deposit)?; - - let now = >::block_number(); - let a = PreimageStatus::Available { - data: encoded_proposal, - provider: who.clone(), - deposit, - since: now, - expiry: None, - }; - >::insert(proposal_hash, a); + Self::note_preimage_inner(ensure_signed(origin)?, encoded_proposal)?; + } - Self::deposit_event(RawEvent::PreimageNoted(proposal_hash, who, deposit)); + /// Same as `note_preimage` but origin is `OperationalPreimageOrigin`. + #[weight = ( + weight_for::note_preimage::((encoded_proposal.len() as u32).into()), + DispatchClass::Operational, + )] + fn note_preimage_operational(origin, encoded_proposal: Vec) { + let who = T::OperationalPreimageOrigin::ensure_origin(origin)?; + Self::note_preimage_inner(who, encoded_proposal)?; } /// Register the preimage for an upcoming proposal. This requires the proposal to be @@ -1196,32 +1048,21 @@ decl_module! { /// Emits `PreimageNoted`. /// /// # - /// - Complexity: `O(E)` with E size of `encoded_proposal` (protected by a required deposit). - /// - Db reads: `Preimages` - /// - Db writes: `Preimages` - /// - Base Weight: 28.04 + .003 * b µs + /// see `weight_for::note_preimage` /// # - #[weight = 28_000_000 + 3_000 * Weight::from(encoded_proposal.len() as u32) - + T::DbWeight::get().reads_writes(1, 1)] + #[weight = weight_for::note_imminent_preimage::((encoded_proposal.len() as u32).into())] fn note_imminent_preimage(origin, encoded_proposal: Vec) { - let who = ensure_signed(origin)?; - let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); - Self::check_pre_image_is_missing(proposal_hash)?; - let status = Preimages::::get(&proposal_hash).ok_or(Error::::NotImminent)?; - let expiry = status.to_missing_expiry().ok_or(Error::::DuplicatePreimage)?; - - let now = >::block_number(); - let free = >::zero(); - let a = PreimageStatus::Available { - data: encoded_proposal, - provider: who.clone(), - deposit: Zero::zero(), - since: now, - expiry: Some(expiry), - }; - >::insert(proposal_hash, a); + Self::note_imminent_preimage_inner(ensure_signed(origin)?, encoded_proposal)?; + } - Self::deposit_event(RawEvent::PreimageNoted(proposal_hash, who, free)); + /// Same as `note_imminent_preimage` but origin is `OperationalPreimageOrigin`. + #[weight = ( + weight_for::note_imminent_preimage::((encoded_proposal.len() as u32).into()), + DispatchClass::Operational, + )] + fn note_imminent_preimage_operational(origin, encoded_proposal: Vec) { + let who = T::OperationalPreimageOrigin::ensure_origin(origin)?; + Self::note_imminent_preimage_inner(who, encoded_proposal)?; } /// Remove an expired proposal preimage and collect the deposit. @@ -1293,33 +1134,6 @@ decl_module! { Self::update_lock(&target); } - /// Become a proxy. - /// - /// This must be called prior to a later `activate_proxy`. - /// - /// Origin must be a Signed. - /// - /// - `target`: The account whose votes will later be proxied. - /// - /// `close_proxy` must be called before the account can be destroyed. - /// - /// # - /// - Complexity: O(1) - /// - Db reads: `Proxy`, `proxy account` - /// - Db writes: `Proxy`, `proxy account` - /// - Base Weight: 14.86 µs - /// # - #[weight = 15_000_000 + T::DbWeight::get().reads_writes(2, 2)] - fn open_proxy(origin, target: T::AccountId) { - let who = ensure_signed(origin)?; - Proxy::::mutate(&who, |a| { - if a.is_none() { - system::Module::::inc_ref(&who); - } - *a = Some(ProxyState::Open(target)); - }); - } - /// Remove a vote for a referendum. /// /// If: @@ -1386,94 +1200,6 @@ decl_module! { Ok(()) } - /// Delegate the voting power (with some given conviction) of a proxied account. - /// - /// The balance delegated is locked for as long as it's delegated, and thereafter for the - /// time appropriate for the conviction's lock period. - /// - /// The dispatch origin of this call must be _Signed_, and the signing account must have - /// been set as the proxy account for `target`. - /// - /// - `target`: The account whole voting power shall be delegated and whose balance locked. - /// This account must either: - /// - be delegating already; or - /// - have no voting activity (if there is, then it will need to be removed/consolidated - /// through `reap_vote` or `unvote`). - /// - `to`: The account whose voting the `target` account's voting power will follow. - /// - `conviction`: The conviction that will be attached to the delegated votes. When the - /// account is undelegated, the funds will be locked for the corresponding period. - /// - `balance`: The amount of the account's balance to be used in delegating. This must - /// not be more than the account's current balance. - /// - /// Emits `Delegated`. - /// - /// # - /// same as `delegate with additional: - /// - Db reads: `Proxy`, `proxy account` - /// - Db writes: `proxy account` - /// - Base Weight: 68.61 + 8.039 * R µs - /// # - #[weight = weight_for::proxy_delegate::(T::MaxVotes::get().into())] - pub fn proxy_delegate(origin, - to: T::AccountId, - conviction: Conviction, - balance: BalanceOf, - ) -> DispatchResultWithPostInfo { - let who = ensure_signed(origin)?; - let target = Self::proxy(who).and_then(|a| a.as_active()).ok_or(Error::::NotProxy)?; - let votes = Self::try_delegate(target, to, conviction, balance)?; - - Ok(Some(weight_for::proxy_delegate::(votes.into())).into()) - } - - /// Undelegate the voting power of a proxied account. - /// - /// Tokens may be unlocked following once an amount of time consistent with the lock period - /// of the conviction with which the delegation was issued. - /// - /// The dispatch origin of this call must be _Signed_ and the signing account must be a - /// proxy for some other account which is currently delegating. - /// - /// Emits `Undelegated`. - /// - /// # - /// same as `undelegate with additional: - /// Db reads: `Proxy` - /// Base Weight: 39 + 7.958 * R µs - /// # - #[weight = weight_for::proxy_undelegate::(T::MaxVotes::get().into())] - fn proxy_undelegate(origin) -> DispatchResultWithPostInfo { - let who = ensure_signed(origin)?; - let target = Self::proxy(who).and_then(|a| a.as_active()).ok_or(Error::::NotProxy)?; - let votes = Self::try_undelegate(target)?; - - Ok(Some(weight_for::proxy_undelegate::(votes.into())).into()) - } - - /// Remove a proxied vote for a referendum. - /// - /// Exactly equivalent to `remove_vote` except that it operates on the account that the - /// sender is a proxy for. - /// - /// The dispatch origin of this call must be _Signed_ and the signing account must be a - /// proxy for some other account which has a registered vote for the referendum of `index`. - /// - /// - `index`: The index of referendum of the vote to be removed. - /// - /// # - /// - `O(R + log R)` where R is the number of referenda that `target` has voted on. - /// Weight is calculated for the maximum number of vote. - /// - Db reads: `ReferendumInfoOf`, `VotingOf`, `Proxy` - /// - Db writes: `ReferendumInfoOf`, `VotingOf` - /// - Base Weight: 26.35 + .36 * R µs - /// # - #[weight = 26_000_000 + 360_000 * Weight::from(T::MaxVotes::get()) + T::DbWeight::get().reads_writes(2, 3)] - fn proxy_remove_vote(origin, index: ReferendumIndex) -> DispatchResult { - let who = ensure_signed(origin)?; - let target = Self::proxy(who).and_then(|a| a.as_active()).ok_or(Error::::NotProxy)?; - Self::try_remove_vote(&target, index, UnvoteScope::Any) - } - /// Enact a proposal from a referendum. For now we just make the weight be the maximum. #[weight = T::MaximumBlockWeight::get()] fn enact_proposal(origin, proposal_hash: T::Hash, index: ReferendumIndex) -> DispatchResult { @@ -1517,16 +1243,6 @@ impl Module { // Exposed mutables. - #[cfg(feature = "std")] - pub fn force_proxy(stash: T::AccountId, proxy: T::AccountId) { - Proxy::::mutate(&proxy, |o| { - if o.is_none() { - system::Module::::inc_ref(&proxy); - } - *o = Some(ProxyState::Active(stash)) - }) - } - /// Start a referendum. pub fn internal_start_referendum( proposal_hash: T::Hash, @@ -2030,6 +1746,53 @@ impl Module { Ok(len) } + + // See `note_preimage` + fn note_preimage_inner(who: T::AccountId, encoded_proposal: Vec) -> DispatchResult { + let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); + ensure!(!>::contains_key(&proposal_hash), Error::::DuplicatePreimage); + + let deposit = >::from(encoded_proposal.len() as u32) + .saturating_mul(T::PreimageByteDeposit::get()); + T::Currency::reserve(&who, deposit)?; + + let now = >::block_number(); + let a = PreimageStatus::Available { + data: encoded_proposal, + provider: who.clone(), + deposit, + since: now, + expiry: None, + }; + >::insert(proposal_hash, a); + + Self::deposit_event(RawEvent::PreimageNoted(proposal_hash, who, deposit)); + + Ok(()) + } + + // See `note_imminent_preimage` + fn note_imminent_preimage_inner(who: T::AccountId, encoded_proposal: Vec) -> DispatchResult { + let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); + Self::check_pre_image_is_missing(proposal_hash)?; + let status = Preimages::::get(&proposal_hash).ok_or(Error::::NotImminent)?; + let expiry = status.to_missing_expiry().ok_or(Error::::DuplicatePreimage)?; + + let now = >::block_number(); + let free = >::zero(); + let a = PreimageStatus::Available { + data: encoded_proposal, + provider: who.clone(), + deposit: Zero::zero(), + since: now, + expiry: Some(expiry), + }; + >::insert(proposal_hash, a); + + Self::deposit_event(RawEvent::PreimageNoted(proposal_hash, who, free)); + + Ok(()) + } } /// Decode `Compact` from the trie at given key. diff --git a/frame/democracy/src/tests.rs b/frame/democracy/src/tests.rs index d46214a1699c88c3a0fc25a6c5f84c5162966154..b92f4bd0760f62cb957945ef4962c809477ff03c 100644 --- a/frame/democracy/src/tests.rs +++ b/frame/democracy/src/tests.rs @@ -22,7 +22,8 @@ use std::cell::RefCell; use codec::Encode; use frame_support::{ impl_outer_origin, impl_outer_dispatch, assert_noop, assert_ok, parameter_types, - impl_outer_event, ord_parameter_types, traits::{Contains, OnInitialize}, weights::Weight, + impl_outer_event, ord_parameter_types, traits::{Contains, OnInitialize, Filter}, + weights::Weight, }; use sp_core::H256; use sp_runtime::{ @@ -38,11 +39,9 @@ mod external_proposing; mod fast_tracking; mod lock_voting; mod preimage; -mod proxying; mod public_proposals; mod scheduling; mod voting; -mod migration; mod decoders; const AYE: Vote = Vote { aye: true, conviction: Conviction::None }; @@ -75,6 +74,14 @@ impl_outer_event! { } } +// Test that a fitlered call can be dispatched. +pub struct BaseFilter; +impl Filter for BaseFilter { + fn filter(call: &Call) -> bool { + !matches!(call, &Call::Balances(pallet_balances::Call::set_balance(..))) + } +} + // Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. #[derive(Clone, Eq, PartialEq, Debug)] pub struct Test; @@ -85,6 +92,7 @@ parameter_types! { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = BaseFilter; type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -100,6 +108,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -109,7 +118,7 @@ impl frame_system::Trait for Test { type OnKilledAccount = (); } parameter_types! { - pub const MaximumSchedulerWeight: Weight = Perbill::from_percent(80) * MaximumBlockWeight::get(); + pub MaximumSchedulerWeight: Weight = Perbill::from_percent(80) * MaximumBlockWeight::get(); } impl pallet_scheduler::Trait for Test { type Event = Event; @@ -186,6 +195,7 @@ impl super::Trait for Test { type InstantAllowed = InstantAllowed; type Scheduler = Scheduler; type MaxVotes = MaxVotes; + type OperationalPreimageOrigin = EnsureSignedBy; } pub fn new_test_ext() -> sp_io::TestExternalities { @@ -199,6 +209,12 @@ pub fn new_test_ext() -> sp_io::TestExternalities { ext } +/// Execute the function two times, with `true` and with `false`. +pub fn new_test_ext_execute_with_cond(execute: impl FnOnce(bool) -> () + Clone) { + new_test_ext().execute_with(|| (execute.clone())(false)); + new_test_ext().execute_with(|| execute(true)); +} + type System = frame_system::Module; type Balances = pallet_balances::Module; type Scheduler = pallet_scheduler::Module; @@ -217,6 +233,14 @@ fn set_balance_proposal(value: u64) -> Vec { Call::Balances(pallet_balances::Call::set_balance(42, value, 0)).encode() } +#[test] +fn set_balance_proposal_is_correctly_filtered_out() { + for i in 0..10 { + let call = Call::decode(&mut &set_balance_proposal(i)[..]).unwrap(); + assert!(!::BaseCallFilter::filter(&call)); + } +} + fn set_balance_proposal_hash(value: u64) -> H256 { BlakeTwo256::hash(&set_balance_proposal(value)[..]) } diff --git a/frame/democracy/src/tests/cancellation.rs b/frame/democracy/src/tests/cancellation.rs index e75fd281091633d996e4518bd1bd2de5f4e3caaf..4221865a3e5b0cda7ced129ee0f60e31e728705e 100644 --- a/frame/democracy/src/tests/cancellation.rs +++ b/frame/democracy/src/tests/cancellation.rs @@ -29,7 +29,7 @@ fn cancel_referendum_should_work() { 0 ); assert_ok!(Democracy::vote(Origin::signed(1), r, aye(1))); - assert_ok!(Democracy::cancel_referendum(Origin::ROOT, r.into())); + assert_ok!(Democracy::cancel_referendum(Origin::root(), r.into())); next_block(); next_block(); @@ -53,8 +53,8 @@ fn cancel_queued_should_work() { assert!(pallet_scheduler::Agenda::::get(6)[0].is_some()); - assert_noop!(Democracy::cancel_queued(Origin::ROOT, 1), Error::::ProposalMissing); - assert_ok!(Democracy::cancel_queued(Origin::ROOT, 0)); + assert_noop!(Democracy::cancel_queued(Origin::root(), 1), Error::::ProposalMissing); + assert_ok!(Democracy::cancel_queued(Origin::root(), 0)); assert!(pallet_scheduler::Agenda::::get(6)[0].is_none()); }); } diff --git a/frame/democracy/src/tests/migration.rs b/frame/democracy/src/tests/migration.rs deleted file mode 100644 index cab8f7f5c936c433237d4aaa7e952ada6933da4c..0000000000000000000000000000000000000000 --- a/frame/democracy/src/tests/migration.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! The tests for migration. - -use super::*; -use frame_support::{storage::migration, Hashable, traits::OnRuntimeUpgrade}; -use substrate_test_utils::assert_eq_uvec; - -#[test] -fn migration() { - new_test_ext().execute_with(|| { - for i in 0..3 { - let k = i.twox_64_concat(); - let v: (BalanceOf, Vec) = (i * 1000, vec![i]); - migration::put_storage_value(b"Democracy", b"DepositOf", &k, v); - } - StorageVersion::kill(); - - Democracy::on_runtime_upgrade(); - - assert_eq!(StorageVersion::get(), Some(Releases::V1)); - assert_eq_uvec!( - DepositOf::::iter().collect::>(), - vec![ - (0, (vec![0u64], >::from(0u32))), - (1, (vec![1u64], >::from(1000u32))), - (2, (vec![2u64], >::from(2000u32))), - ] - ); - }) -} diff --git a/frame/democracy/src/tests/preimage.rs b/frame/democracy/src/tests/preimage.rs index 094cde86d0b7b492c6562b052ce7c79272bfd439..4100a6a6b6375e7208f8105bcfdb000d10ba8ade 100644 --- a/frame/democracy/src/tests/preimage.rs +++ b/frame/democracy/src/tests/preimage.rs @@ -39,13 +39,14 @@ fn missing_preimage_should_fail() { #[test] fn preimage_deposit_should_be_required_and_returned() { - new_test_ext().execute_with(|| { + new_test_ext_execute_with_cond(|operational| { // fee of 100 is too much. PREIMAGE_BYTE_DEPOSIT.with(|v| *v.borrow_mut() = 100); assert_noop!( - Democracy::note_preimage(Origin::signed(6), vec![0; 500]), - BalancesError::::InsufficientBalance, - ); + if operational { Democracy::note_preimage_operational(Origin::signed(6), vec![0; 500]) } + else { Democracy::note_preimage(Origin::signed(6), vec![0; 500]) }, + BalancesError::::InsufficientBalance, + ); // fee of 1 is reasonable. PREIMAGE_BYTE_DEPOSIT.with(|v| *v.borrow_mut() = 1); let r = Democracy::inject_referendum( @@ -69,17 +70,20 @@ fn preimage_deposit_should_be_required_and_returned() { #[test] fn preimage_deposit_should_be_reapable_earlier_by_owner() { - new_test_ext().execute_with(|| { + new_test_ext_execute_with_cond(|operational| { PREIMAGE_BYTE_DEPOSIT.with(|v| *v.borrow_mut() = 1); - assert_ok!(Democracy::note_preimage(Origin::signed(6), set_balance_proposal(2))); + assert_ok!( + if operational { Democracy::note_preimage_operational(Origin::signed(6), set_balance_proposal(2)) } + else { Democracy::note_preimage(Origin::signed(6), set_balance_proposal(2)) } + ); assert_eq!(Balances::reserved_balance(6), 12); next_block(); assert_noop!( - Democracy::reap_preimage(Origin::signed(6), set_balance_proposal_hash(2), u32::max_value()), - Error::::TooEarly - ); + Democracy::reap_preimage(Origin::signed(6), set_balance_proposal_hash(2), u32::max_value()), + Error::::TooEarly + ); next_block(); assert_ok!(Democracy::reap_preimage(Origin::signed(6), set_balance_proposal_hash(2), u32::max_value())); @@ -90,14 +94,17 @@ fn preimage_deposit_should_be_reapable_earlier_by_owner() { #[test] fn preimage_deposit_should_be_reapable() { - new_test_ext().execute_with(|| { + new_test_ext_execute_with_cond(|operational| { assert_noop!( Democracy::reap_preimage(Origin::signed(5), set_balance_proposal_hash(2), u32::max_value()), Error::::PreimageMissing ); PREIMAGE_BYTE_DEPOSIT.with(|v| *v.borrow_mut() = 1); - assert_ok!(Democracy::note_preimage(Origin::signed(6), set_balance_proposal(2))); + assert_ok!( + if operational { Democracy::note_preimage_operational(Origin::signed(6), set_balance_proposal(2)) } + else { Democracy::note_preimage(Origin::signed(6), set_balance_proposal(2)) } + ); assert_eq!(Balances::reserved_balance(6), 12); next_block(); @@ -118,7 +125,7 @@ fn preimage_deposit_should_be_reapable() { #[test] fn noting_imminent_preimage_for_free_should_work() { - new_test_ext().execute_with(|| { + new_test_ext_execute_with_cond(|operational| { PREIMAGE_BYTE_DEPOSIT.with(|v| *v.borrow_mut() = 1); let r = Democracy::inject_referendum( @@ -130,14 +137,15 @@ fn noting_imminent_preimage_for_free_should_work() { assert_ok!(Democracy::vote(Origin::signed(1), r, aye(1))); assert_noop!( - Democracy::note_imminent_preimage(Origin::signed(7), set_balance_proposal(2)), - Error::::NotImminent - ); + if operational { Democracy::note_imminent_preimage_operational(Origin::signed(6), set_balance_proposal(2)) } + else { Democracy::note_imminent_preimage(Origin::signed(6), set_balance_proposal(2)) }, + Error::::NotImminent + ); next_block(); // Now we're in the dispatch queue it's all good. - assert_ok!(Democracy::note_imminent_preimage(Origin::signed(7), set_balance_proposal(2))); + assert_ok!(Democracy::note_imminent_preimage(Origin::signed(6), set_balance_proposal(2))); next_block(); diff --git a/frame/democracy/src/tests/proxying.rs b/frame/democracy/src/tests/proxying.rs deleted file mode 100644 index 2e39528e75a3b9206328f76b19b959c752e69506..0000000000000000000000000000000000000000 --- a/frame/democracy/src/tests/proxying.rs +++ /dev/null @@ -1,105 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! The tests for functionality concerning proxying. - -use super::*; - -#[test] -fn proxy_should_work() { - new_test_ext().execute_with(|| { - assert_eq!(Democracy::proxy(10), None); - assert!(System::allow_death(&10)); - - assert_noop!(Democracy::activate_proxy(Origin::signed(1), 10), Error::::NotOpen); - - assert_ok!(Democracy::open_proxy(Origin::signed(10), 1)); - assert!(!System::allow_death(&10)); - assert_eq!(Democracy::proxy(10), Some(ProxyState::Open(1))); - - assert_noop!(Democracy::activate_proxy(Origin::signed(2), 10), Error::::WrongOpen); - assert_ok!(Democracy::activate_proxy(Origin::signed(1), 10)); - assert_eq!(Democracy::proxy(10), Some(ProxyState::Active(1))); - - // Can't set when already set. - assert_noop!(Democracy::activate_proxy(Origin::signed(2), 10), Error::::AlreadyProxy); - - // But this works because 11 isn't proxying. - assert_ok!(Democracy::open_proxy(Origin::signed(11), 2)); - assert_ok!(Democracy::activate_proxy(Origin::signed(2), 11)); - assert_eq!(Democracy::proxy(10), Some(ProxyState::Active(1))); - assert_eq!(Democracy::proxy(11), Some(ProxyState::Active(2))); - - // 2 cannot fire 1's proxy: - assert_noop!(Democracy::deactivate_proxy(Origin::signed(2), 10), Error::::WrongProxy); - - // 1 deactivates their proxy: - assert_ok!(Democracy::deactivate_proxy(Origin::signed(1), 10)); - assert_eq!(Democracy::proxy(10), Some(ProxyState::Open(1))); - // but the proxy account cannot be killed until the proxy is closed. - assert!(!System::allow_death(&10)); - - // and then 10 closes it completely: - assert_ok!(Democracy::close_proxy(Origin::signed(10))); - assert_eq!(Democracy::proxy(10), None); - assert!(System::allow_death(&10)); - - // 11 just closes without 2's "permission". - assert_ok!(Democracy::close_proxy(Origin::signed(11))); - assert_eq!(Democracy::proxy(11), None); - assert!(System::allow_death(&11)); - }); -} - -#[test] -fn voting_and_removing_votes_should_work_with_proxy() { - new_test_ext().execute_with(|| { - System::set_block_number(0); - assert_ok!(propose_set_balance_and_note(1, 2, 1)); - - fast_forward_to(2); - let r = 0; - assert_ok!(Democracy::open_proxy(Origin::signed(10), 1)); - assert_ok!(Democracy::activate_proxy(Origin::signed(1), 10)); - - assert_ok!(Democracy::proxy_vote(Origin::signed(10), r, aye(1))); - assert_eq!(tally(r), Tally { ayes: 1, nays: 0, turnout: 10 }); - - assert_ok!(Democracy::proxy_remove_vote(Origin::signed(10), r)); - assert_eq!(tally(r), Tally { ayes: 0, nays: 0, turnout: 0 }); - }); -} - -#[test] -fn delegation_and_undelegation_should_work_with_proxy() { - new_test_ext().execute_with(|| { - System::set_block_number(0); - assert_ok!(propose_set_balance_and_note(1, 2, 1)); - fast_forward_to(2); - let r = 0; - assert_ok!(Democracy::open_proxy(Origin::signed(10), 1)); - assert_ok!(Democracy::activate_proxy(Origin::signed(1), 10)); - assert_ok!(Democracy::vote(Origin::signed(2), r, aye(2))); - - assert_ok!(Democracy::proxy_delegate(Origin::signed(10), 2, Conviction::None, 10)); - assert_eq!(tally(r), Tally { ayes: 3, nays: 0, turnout: 30 }); - - assert_ok!(Democracy::proxy_undelegate(Origin::signed(10))); - assert_eq!(tally(r), Tally { ayes: 2, nays: 0, turnout: 20 }); - }); -} - diff --git a/frame/democracy/src/types.rs b/frame/democracy/src/types.rs index efd52361f52a8cd55e11ac0fa9746dd683da1394..8ee0838f8a36d14fb841b488f0055d452f1b6b18 100644 --- a/frame/democracy/src/types.rs +++ b/frame/democracy/src/types.rs @@ -197,24 +197,6 @@ impl ReferendumInfo { - /// Account is open to becoming a proxy but is not yet assigned. - Open(AccountId), - /// Account is actively being a proxy. - Active(AccountId), -} - -impl ProxyState { - pub (crate) fn as_active(self) -> Option { - match self { - ProxyState::Active(a) => Some(a), - ProxyState::Open(_) => None, - } - } -} - /// Whether an `unvote` operation is able to make actions that are not strictly always in the /// interest of an account. pub enum UnvoteScope { diff --git a/frame/elections-phragmen/Cargo.toml b/frame/elections-phragmen/Cargo.toml index fc7c6c7993a59a0bb7b2a7acb8e423c9ab595704..afbd53d3da0919885dfb3ed39a3ca503088b96a6 100644 --- a/frame/elections-phragmen/Cargo.toml +++ b/frame/elections-phragmen/Cargo.toml @@ -1,32 +1,32 @@ [package] name = "pallet-elections-phragmen" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" -description = "FRAME election pallet for PHRAGMEN" +description = "FRAME pallet based on seq-Phragmén election method." [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-phragmen = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/phragmen" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../benchmarking", optional = true } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-npos-elections = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/npos-elections" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-io = { version = "2.0.0-alpha.8", path = "../../primitives/io" } +sp-io = { version = "2.0.0-rc4", path = "../../primitives/io" } hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -substrate-test-utils = { version = "2.0.0-alpha.8", path = "../../test-utils" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +substrate-test-utils = { version = "2.0.0-rc4", path = "../../test-utils" } [features] default = ["std"] @@ -35,7 +35,7 @@ std = [ "codec/std", "frame-support/std", "sp-runtime/std", - "sp-phragmen/std", + "sp-npos-elections/std", "frame-system/std", "sp-std/std", ] diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs index 4e2ae09afafca423c6d24c9eefa157a51745aa0a..9436a15d5c749416f1f936486942420c7260ccc6 100644 --- a/frame/elections-phragmen/src/lib.rs +++ b/frame/elections-phragmen/src/lib.rs @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! # Phragmen Election Module. +//! # Phragmén Election Module. //! //! An election module based on sequential phragmen. //! @@ -100,7 +100,7 @@ use frame_support::{ ContainsLengthBound, } }; -use sp_phragmen::{build_support_map, ExtendedBalance, VoteWeight, PhragmenResult}; +use sp_npos_elections::{build_support_map, ExtendedBalance, VoteWeight, ElectionResult}; use frame_system::{self as system, ensure_signed, ensure_root}; mod benchmarking; @@ -197,6 +197,8 @@ decl_storage! { pub ElectionRounds get(fn election_rounds): u32 = Zero::zero(); /// Votes and locked stake of a particular voter. + /// + /// TWOX-NOTE: SAFE as `AccountId` is a crypto hash pub Voting get(fn voting): map hasher(twox_64_concat) T::AccountId => (BalanceOf, Vec); /// The present candidate list. Sorted based on account-id. A current member or runner-up @@ -243,7 +245,6 @@ decl_storage! { } decl_error! { - /// Error for the elections-phragmen module. pub enum Error for Module { /// Cannot vote when no candidates or members exist. UnableToVote, @@ -608,7 +609,7 @@ decl_module! { /// the outgoing member is slashed. /// /// If a runner-up is available, then the best runner-up will be removed and replaces the - /// outgoing member. Otherwise, a new phragmen round is started. + /// outgoing member. Otherwise, a new phragmen election is started. /// /// Note that this does not affect the designated block number of the next election. /// @@ -838,13 +839,10 @@ impl Module { /// Run the phragmen election with all required side processes and state updates. /// - /// Calls the appropriate `ChangeMembers` function variant internally. + /// Calls the appropriate [`ChangeMembers`] function variant internally. /// - /// # - /// #### State /// Reads: O(C + V*E) where C = candidates, V voters and E votes per voter exits. /// Writes: O(M + R) with M desired members and R runners_up. - /// # fn do_phragmen() { let desired_seats = Self::desired_members() as usize; let desired_runners_up = Self::desired_runners_up() as usize; @@ -874,14 +872,14 @@ impl Module { let voters_and_votes = Voting::::iter() .map(|(voter, (stake, targets))| { (voter, to_votes(stake), targets) }) .collect::>(); - let maybe_phragmen_result = sp_phragmen::elect::( + let maybe_phragmen_result = sp_npos_elections::seq_phragmen::( num_to_elect, 0, candidates, voters_and_votes.clone(), ); - if let Some(PhragmenResult { winners, assignments }) = maybe_phragmen_result { + if let Some(ElectionResult { winners, assignments }) = maybe_phragmen_result { let old_members_ids = >::take().into_iter() .map(|(m, _)| m) .collect::>(); @@ -905,7 +903,7 @@ impl Module { // exposed candidates, cleaning any previous members, and so on. For now, in favour of // readability and veracity, we keep it simple. - let staked_assignments = sp_phragmen::assignment_ratio_to_staked( + let staked_assignments = sp_npos_elections::assignment_ratio_to_staked( assignments, stake_of, ); @@ -1072,10 +1070,11 @@ mod tests { } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; - type Call = (); + type Call = Call; type Hash = H256; type Hashing = BlakeTwo256; type AccountId = u64; @@ -1087,6 +1086,7 @@ mod tests { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -1377,11 +1377,8 @@ mod tests { // historical note: helper function was created in a period of time in which the API of vote // call was changing. Currently it is a wrapper for the original call and does not do much. // Nonetheless, totally harmless. - if let Origin::system(frame_system::RawOrigin::Signed(_account)) = origin { - Elections::vote(origin, votes, stake) - } else { - panic!("vote origin must be signed"); - } + ensure_signed(origin.clone()).expect("vote origin must be signed"); + Elections::vote(origin, votes, stake) } fn votes_of(who: &u64) -> Vec { @@ -2359,7 +2356,7 @@ mod tests { assert_ok!(submit_candidacy(Origin::signed(3))); assert_ok!(vote(Origin::signed(3), vec![3], 30)); - assert_ok!(Elections::remove_member(Origin::ROOT, 4, false)); + assert_ok!(Elections::remove_member(Origin::root(), 4, false)); assert_eq!(balances(&4), (35, 2)); // slashed assert_eq!(Elections::election_rounds(), 2); // new election round @@ -2382,7 +2379,7 @@ mod tests { // no replacement yet. assert_err_with_weight!( - Elections::remove_member(Origin::ROOT, 4, true), + Elections::remove_member(Origin::root(), 4, true), Error::::InvalidReplacement, Some(6000000), ); @@ -2404,7 +2401,7 @@ mod tests { // there is a replacement! and this one needs a weight refund. assert_err_with_weight!( - Elections::remove_member(Origin::ROOT, 4, false), + Elections::remove_member(Origin::root(), 4, false), Error::::InvalidReplacement, Some(6000000) // only thing that matters for now is that it is NOT the full block. ); @@ -2563,7 +2560,7 @@ mod tests { Elections::end_block(System::block_number()); assert_eq!(Elections::members_ids(), vec![2, 4]); - assert_ok!(Elections::remove_member(Origin::ROOT, 2, true)); + assert_ok!(Elections::remove_member(Origin::root(), 2, true)); assert_eq!(Elections::members_ids(), vec![4, 5]); }); } diff --git a/frame/elections/Cargo.toml b/frame/elections/Cargo.toml index 0eb99ad94ed35fd573869fdd6c5c95c893981b71..b7914d66fd70bf44d38563275e4451af566c5570 100644 --- a/frame/elections/Cargo.toml +++ b/frame/elections/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-elections" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,17 +13,17 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } [dev-dependencies] hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } [features] default = ["std"] diff --git a/frame/elections/src/lib.rs b/frame/elections/src/lib.rs index b87db45909cedadbd98de3c1d3cfb1764376d71f..171a2dbb8ba91050b79266ddd71016e166250b0f 100644 --- a/frame/elections/src/lib.rs +++ b/frame/elections/src/lib.rs @@ -237,16 +237,25 @@ decl_storage! { // bit-wise manner. In order to get a human-readable representation (`Vec`), use // [`all_approvals_of`]. Furthermore, each vector of scalars is chunked with the cap of // `APPROVAL_SET_SIZE`. + /// + /// TWOX-NOTE: SAFE as `AccountId` is a crypto hash and `SetIndex` is not + /// attacker-controlled. pub ApprovalsOf get(fn approvals_of): map hasher(twox_64_concat) (T::AccountId, SetIndex) => Vec; /// The vote index and list slot that the candidate `who` was registered or `None` if they /// are not currently registered. + /// + /// TWOX-NOTE: SAFE as `AccountId` is a crypto hash. pub RegisterInfoOf get(fn candidate_reg_info): map hasher(twox_64_concat) T::AccountId => Option<(VoteIndex, u32)>; /// Basic information about a voter. + /// + /// TWOX-NOTE: SAFE as `AccountId` is a crypto hash. pub VoterInfoOf get(fn voter_info): map hasher(twox_64_concat) T::AccountId => Option>>; /// The present voter list (chunked and capped at [`VOTER_SET_SIZE`]). + /// + /// TWOX-NOTE: OKAY ― `SetIndex` is not user-controlled data. pub Voters get(fn voters): map hasher(twox_64_concat) SetIndex => Vec>; /// the next free set to store a voter in. This will keep growing. pub NextVoterSet get(fn next_nonfull_voter_set): SetIndex = 0; @@ -265,10 +274,6 @@ decl_storage! { /// of each entry; It may be the direct summed approval stakes, or a weighted version of it. /// Sorted from low to high. pub Leaderboard get(fn leaderboard): Option, T::AccountId)> >; - - /// Who is able to vote for whom. Value is the fund-holding account, key is the - /// vote-transaction-sending account. - pub Proxy get(fn proxy): map hasher(blake2_128_concat) T::AccountId => Option; } } @@ -283,8 +288,6 @@ decl_error! { CannotReapPresenting, /// Cannot reap during grace period. ReapGrace, - /// Not a proxy. - NotProxy, /// Invalid reporter index. InvalidReporterIndex, /// Invalid target index. @@ -421,23 +424,6 @@ decl_module! { Self::do_set_approvals(who, votes, index, hint, value) } - /// Set candidate approvals from a proxy. Approval slots stay valid as long as candidates in - /// those slots are registered. - /// - /// # - /// - Same as `set_approvals` with one additional storage read. - /// # - #[weight = 2_500_000_000] - fn proxy_set_approvals(origin, - votes: Vec, - #[compact] index: VoteIndex, - hint: SetIndex, - #[compact] value: BalanceOf, - ) -> DispatchResult { - let who = Self::proxy(ensure_signed(origin)?).ok_or(Error::::NotProxy)?; - Self::do_set_approvals(who, votes, index, hint, value) - } - /// Remove a voter. For it not to be a bond-consuming no-op, all approved candidate indices /// must now be either unregistered or registered to a candidate that registered the slot /// after the voter gave their last approval set. diff --git a/frame/elections/src/mock.rs b/frame/elections/src/mock.rs index c433f8d0369ec25fc58c793a351404b97fb0600e..b0be542ab75a24ebbc3aee5ca262de70668515af 100644 --- a/frame/elections/src/mock.rs +++ b/frame/elections/src/mock.rs @@ -39,8 +39,9 @@ parameter_types! { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; - type Call = (); + type Call = Call; type Index = u64; type BlockNumber = u64; type Hash = H256; @@ -54,6 +55,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/elections/src/tests.rs b/frame/elections/src/tests.rs index 590266f7fe2cdce7d0608cee493fa06efced7453..247b6272524b1c64073e4a6157af1b2a5e2bd3a3 100644 --- a/frame/elections/src/tests.rs +++ b/frame/elections/src/tests.rs @@ -671,7 +671,7 @@ fn retracting_active_voter_should_slash_reporter() { assert_ok!(Elections::end_block(System::block_number())); System::set_block_number(8); - assert_ok!(Elections::set_desired_seats(Origin::ROOT, 3)); + assert_ok!(Elections::set_desired_seats(Origin::root(), 3)); assert_ok!(Elections::end_block(System::block_number())); System::set_block_number(10); @@ -868,45 +868,6 @@ fn election_voting_should_work() { }); } -#[test] -fn election_proxy_voting_should_work() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); - - >::insert(11, 1); - >::insert(12, 2); - >::insert(13, 3); - >::insert(14, 4); - assert_ok!( - Elections::proxy_set_approvals(Origin::signed(11), vec![true], 0, 0, 10) - ); - assert_ok!( - Elections::proxy_set_approvals(Origin::signed(14), vec![true], 0, 1, 40) - ); - - assert_eq!(Elections::all_approvals_of(&1), vec![true]); - assert_eq!(Elections::all_approvals_of(&4), vec![true]); - assert_eq!(voter_ids(), vec![1, 4]); - - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 1)); - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 2)); - - assert_ok!( - Elections::proxy_set_approvals(Origin::signed(12), vec![false, true], 0, 2, 20) - ); - assert_ok!( - Elections::proxy_set_approvals(Origin::signed(13), vec![false, true], 0, 3, 30) - ); - - assert_eq!(Elections::all_approvals_of(&1), vec![true]); - assert_eq!(Elections::all_approvals_of(&4), vec![true]); - assert_eq!(Elections::all_approvals_of(&2), vec![false, true]); - assert_eq!(Elections::all_approvals_of(&3), vec![false, true]); - - assert_eq!(voter_ids(), vec![1, 4, 2, 3]); - }); -} - #[test] fn election_simple_tally_should_work() { ExtBuilder::default().build().execute_with(|| { @@ -1284,7 +1245,7 @@ fn election_second_tally_should_use_runners_up() { System::set_block_number(8); assert_ok!(Elections::set_approvals(Origin::signed(6), vec![false, false, true, false], 1, 0, 60)); - assert_ok!(Elections::set_desired_seats(Origin::ROOT, 3)); + assert_ok!(Elections::set_desired_seats(Origin::root(), 3)); assert_ok!(Elections::end_block(System::block_number())); System::set_block_number(10); diff --git a/frame/evm/Cargo.toml b/frame/evm/Cargo.toml index 124ee92ca8c174ed64de3b9ee1863565164f569b..8b030be4b7a54c6b39a7f0d8f57e01f1197ea557 100644 --- a/frame/evm/Cargo.toml +++ b/frame/evm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-evm" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,15 +13,15 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -pallet-timestamp = { version = "2.0.0-alpha.8", default-features = false, path = "../timestamp" } -pallet-balances = { version = "2.0.0-alpha.8", default-features = false, path = "../balances" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +pallet-timestamp = { version = "2.0.0-rc4", default-features = false, path = "../timestamp" } +pallet-balances = { version = "2.0.0-rc4", default-features = false, path = "../balances" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } primitive-types = { version = "0.7.0", default-features = false, features = ["rlp"] } rlp = { version = "0.4", default-features = false } evm = { version = "0.16", default-features = false } diff --git a/frame/evm/src/backend.rs b/frame/evm/src/backend.rs index c610f24bb1db2d763979e978dd183c5719a731b9..09f31d8aeba83392e038b6f4ffbba3ff6c4f71fa 100644 --- a/frame/evm/src/backend.rs +++ b/frame/evm/src/backend.rs @@ -5,6 +5,7 @@ use serde::{Serialize, Deserialize}; use codec::{Encode, Decode}; use sp_core::{U256, H256, H160}; use sp_runtime::traits::UniqueSaturatedInto; +use frame_support::traits::Get; use frame_support::storage::{StorageMap, StorageDoubleMap}; use sha3::{Keccak256, Digest}; use evm::backend::{Backend as BackendT, ApplyBackend, Apply}; @@ -91,7 +92,7 @@ impl<'vicinity, T: Trait> BackendT for Backend<'vicinity, T> { } fn chain_id(&self) -> U256 { - U256::from(sp_io::misc::chain_id()) + U256::from(T::ChainId::get()) } fn exists(&self, _address: H160) -> bool { diff --git a/frame/evm/src/lib.rs b/frame/evm/src/lib.rs index 1c38e22917b9b18b5e0757b684008e9c2a74ba8f..eebdc66b38f81adc35164df769363ed10b560b15 100644 --- a/frame/evm/src/lib.rs +++ b/frame/evm/src/lib.rs @@ -25,8 +25,12 @@ mod backend; pub use crate::backend::{Account, Log, Vicinity, Backend}; use sp_std::{vec::Vec, marker::PhantomData}; +#[cfg(feature = "std")] +use codec::{Encode, Decode}; +#[cfg(feature = "std")] +use serde::{Serialize, Deserialize}; use frame_support::{ensure, decl_module, decl_storage, decl_event, decl_error}; -use frame_support::weights::{Weight, DispatchClass, FunctionOf, Pays}; +use frame_support::weights::Weight; use frame_support::traits::{Currency, WithdrawReason, ExistenceRequirement, Get}; use frame_system::{self as system, ensure_signed}; use sp_runtime::ModuleId; @@ -114,6 +118,15 @@ impl Precompiles for () { } } +/// Substrate system chain ID. +pub struct SystemChainId; + +impl Get for SystemChainId { + fn get() -> u64 { + sp_io::misc::chain_id() + } +} + static ISTANBUL_CONFIG: Config = Config::istanbul(); /// EVM module trait @@ -130,6 +143,8 @@ pub trait Trait: frame_system::Trait + pallet_timestamp::Trait { type Event: From> + Into<::Event>; /// Precompiles associated with this EVM engine. type Precompiles: Precompiles; + /// Chain ID of EVM. + type ChainId: Get; /// EVM config used in the module. fn config() -> &'static Config { @@ -137,11 +152,43 @@ pub trait Trait: frame_system::Trait + pallet_timestamp::Trait { } } +#[cfg(feature = "std")] +#[derive(Clone, Eq, PartialEq, Encode, Decode, Debug, Serialize, Deserialize)] +/// Account definition used for genesis block construction. +pub struct GenesisAccount { + /// Account nonce. + pub nonce: U256, + /// Account balance. + pub balance: U256, + /// Full account storage. + pub storage: std::collections::BTreeMap, + /// Account code. + pub code: Vec, +} + decl_storage! { trait Store for Module as EVM { - Accounts get(fn accounts) config(): map hasher(blake2_128_concat) H160 => Account; - AccountCodes: map hasher(blake2_128_concat) H160 => Vec; - AccountStorages: double_map hasher(blake2_128_concat) H160, hasher(blake2_128_concat) H256 => H256; + Accounts get(fn accounts): map hasher(blake2_128_concat) H160 => Account; + AccountCodes get(fn account_codes): map hasher(blake2_128_concat) H160 => Vec; + AccountStorages get(fn account_storages): + double_map hasher(blake2_128_concat) H160, hasher(blake2_128_concat) H256 => H256; + } + + add_extra_genesis { + config(accounts): std::collections::BTreeMap; + build(|config: &GenesisConfig| { + for (address, account) in &config.accounts { + Accounts::insert(address, Account { + balance: account.balance, + nonce: account.nonce, + }); + AccountCodes::insert(address, &account.code); + + for (index, value) in &account.storage { + AccountStorages::insert(address, index, value); + } + } + }); } } @@ -238,12 +285,7 @@ decl_module! { } /// Issue an EVM call operation. This is similar to a message call transaction in Ethereum. - #[weight = FunctionOf( - |(_, _, _, gas_limit, gas_price, _): (&H160, &Vec, &U256, &u32, &U256, &Option)| - (*gas_price).saturated_into::().saturating_mul(*gas_limit as Weight), - DispatchClass::Normal, - Pays::Yes, - )] + #[weight = (*gas_price).saturated_into::().saturating_mul(*gas_limit as Weight)] fn call( origin, target: H160, @@ -253,33 +295,25 @@ decl_module! { gas_price: U256, nonce: Option, ) -> DispatchResult { + ensure!(gas_price >= T::FeeCalculator::min_gas_price(), Error::::GasPriceTooLow); + let sender = ensure_signed(origin)?; let source = T::ConvertAccountId::convert_account_id(&sender); - Self::execute_evm( + Self::execute_call( source, + target, + input, value, gas_limit, gas_price, nonce, - |executor| ((), executor.transact_call( - source, - target, - value, - input, - gas_limit as usize, - )), ).map_err(Into::into) } /// Issue an EVM create operation. This is similar to a contract creation transaction in /// Ethereum. - #[weight = FunctionOf( - |(_, _, gas_limit, gas_price, _): (&Vec, &U256, &u32, &U256, &Option)| - (*gas_price).saturated_into::().saturating_mul(*gas_limit as Weight), - DispatchClass::Normal, - Pays::Yes, - )] + #[weight = (*gas_price).saturated_into::().saturating_mul(*gas_limit as Weight)] fn create( origin, init: Vec, @@ -288,25 +322,18 @@ decl_module! { gas_price: U256, nonce: Option, ) -> DispatchResult { + ensure!(gas_price >= T::FeeCalculator::min_gas_price(), Error::::GasPriceTooLow); + let sender = ensure_signed(origin)?; let source = T::ConvertAccountId::convert_account_id(&sender); - let create_address = Self::execute_evm( + let create_address = Self::execute_create( source, + init, value, gas_limit, gas_price, - nonce, - |executor| { - (executor.create_address( - evm::CreateScheme::Legacy { caller: source }, - ), executor.transact_create( - source, - value, - init, - gas_limit as usize, - )) - }, + nonce )?; Module::::deposit_event(Event::::Created(create_address)); @@ -314,12 +341,7 @@ decl_module! { } /// Issue an EVM create2 operation. - #[weight = FunctionOf( - |(_, _, _, gas_limit, gas_price, _): (&Vec, &H256, &U256, &u32, &U256, &Option)| - (*gas_price).saturated_into::().saturating_mul(*gas_limit as Weight), - DispatchClass::Normal, - Pays::Yes, - )] + #[weight = (*gas_price).saturated_into::().saturating_mul(*gas_limit as Weight)] fn create2( origin, init: Vec, @@ -329,27 +351,19 @@ decl_module! { gas_price: U256, nonce: Option, ) -> DispatchResult { + ensure!(gas_price >= T::FeeCalculator::min_gas_price(), Error::::GasPriceTooLow); + let sender = ensure_signed(origin)?; let source = T::ConvertAccountId::convert_account_id(&sender); - let code_hash = H256::from_slice(Keccak256::digest(&init).as_slice()); - let create_address = Self::execute_evm( + let create_address = Self::execute_create2( source, + init, + salt, value, gas_limit, gas_price, - nonce, - |executor| { - (executor.create_address( - evm::CreateScheme::Create2 { caller: source, code_hash, salt }, - ), executor.transact_create2( - source, - value, - init, - salt, - gas_limit as usize, - )) - }, + nonce )?; Module::::deposit_event(Event::::Created(create_address)); @@ -391,6 +405,91 @@ impl Module { AccountStorages::remove_prefix(address); } + /// Execute a create transaction on behalf of given sender. + pub fn execute_create( + source: H160, + init: Vec, + value: U256, + gas_limit: u32, + gas_price: U256, + nonce: Option + ) -> Result> { + Self::execute_evm( + source, + value, + gas_limit, + gas_price, + nonce, + |executor| { + (executor.create_address( + evm::CreateScheme::Legacy { caller: source }, + ), executor.transact_create( + source, + value, + init, + gas_limit as usize, + )) + }, + ) + } + + /// Execute a create2 transaction on behalf of a given sender. + pub fn execute_create2( + source: H160, + init: Vec, + salt: H256, + value: U256, + gas_limit: u32, + gas_price: U256, + nonce: Option + ) -> Result> { + let code_hash = H256::from_slice(Keccak256::digest(&init).as_slice()); + Self::execute_evm( + source, + value, + gas_limit, + gas_price, + nonce, + |executor| { + (executor.create_address( + evm::CreateScheme::Create2 { caller: source, code_hash, salt }, + ), executor.transact_create2( + source, + value, + init, + salt, + gas_limit as usize, + )) + }, + ) + } + + /// Execute a call transaction on behalf of a given sender. + pub fn execute_call( + source: H160, + target: H160, + input: Vec, + value: U256, + gas_limit: u32, + gas_price: U256, + nonce: Option, + ) -> Result<(), Error> { + Self::execute_evm( + source, + value, + gas_limit, + gas_price, + nonce, + |executor| ((), executor.transact_call( + source, + target, + value, + input, + gas_limit as usize, + )), + ) + } + /// Execute an EVM operation. fn execute_evm( source: H160, @@ -402,8 +501,6 @@ impl Module { ) -> Result> where F: FnOnce(&mut StackExecutor>) -> (R, ExitReason), { - ensure!(gas_price >= T::FeeCalculator::min_gas_price(), Error::::GasPriceTooLow); - let vicinity = Vicinity { gas_price, origin: source, diff --git a/frame/example-offchain-worker/Cargo.toml b/frame/example-offchain-worker/Cargo.toml index 5953240a4b64d8e8ebef02cf24cdbf7fb7cea963..50d398a512268d83e09066c5e9d509b28a9bd247 100644 --- a/frame/example-offchain-worker/Cargo.toml +++ b/frame/example-offchain-worker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-example-offchain-worker" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Unlicense" @@ -12,14 +12,14 @@ description = "FRAME example pallet for offchain worker" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } serde = { version = "1.0.101", optional = true } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } lite-json = { version = "0.1", default-features = false } [features] diff --git a/frame/example-offchain-worker/src/tests.rs b/frame/example-offchain-worker/src/tests.rs index d94a98906770007985725a6993a5446b46b4c488..b300809f4107b18a1304a6c7fc4314de2b1cea55 100644 --- a/frame/example-offchain-worker/src/tests.rs +++ b/frame/example-offchain-worker/src/tests.rs @@ -54,6 +54,7 @@ parameter_types! { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Call = (); type Index = u64; @@ -69,6 +70,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -152,6 +154,52 @@ fn should_make_http_call_and_parse_result() { }); } +#[test] +fn knows_how_to_mock_several_http_calls() { + let (offchain, state) = testing::TestOffchainExt::new(); + let mut t = sp_io::TestExternalities::default(); + t.register_extension(OffchainExt::new(offchain)); + + { + let mut state = state.write(); + state.expect_request(testing::PendingRequest { + method: "GET".into(), + uri: "https://min-api.cryptocompare.com/data/price?fsym=BTC&tsyms=USD".into(), + response: Some(br#"{"USD": 1}"#.to_vec()), + sent: true, + ..Default::default() + }); + + state.expect_request(testing::PendingRequest { + method: "GET".into(), + uri: "https://min-api.cryptocompare.com/data/price?fsym=BTC&tsyms=USD".into(), + response: Some(br#"{"USD": 2}"#.to_vec()), + sent: true, + ..Default::default() + }); + + state.expect_request(testing::PendingRequest { + method: "GET".into(), + uri: "https://min-api.cryptocompare.com/data/price?fsym=BTC&tsyms=USD".into(), + response: Some(br#"{"USD": 3}"#.to_vec()), + sent: true, + ..Default::default() + }); + } + + + t.execute_with(|| { + let price1 = Example::fetch_price().unwrap(); + let price2 = Example::fetch_price().unwrap(); + let price3 = Example::fetch_price().unwrap(); + + assert_eq!(price1, 100); + assert_eq!(price2, 200); + assert_eq!(price3, 300); + }) + +} + #[test] fn should_submit_signed_transaction_on_chain() { const PHRASE: &str = "news slush supreme milk chapter athlete soap sausage put clutch what kitten"; @@ -317,7 +365,7 @@ fn should_submit_raw_unsigned_transaction_on_chain() { } fn price_oracle_response(state: &mut testing::OffchainState) { - state.expect_request(0, testing::PendingRequest { + state.expect_request(testing::PendingRequest { method: "GET".into(), uri: "https://min-api.cryptocompare.com/data/price?fsym=BTC&tsyms=USD".into(), response: Some(br#"{"USD": 155.23}"#.to_vec()), diff --git a/frame/example/Cargo.toml b/frame/example/Cargo.toml index 16b05180bb8aefa81c44ef096319aba0d02122b0..cf09a3d4b296020564afeec2daf4dc46e1878231 100644 --- a/frame/example/Cargo.toml +++ b/frame/example/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-example" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Unlicense" @@ -13,18 +13,18 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -pallet-balances = { version = "2.0.0-alpha.8", default-features = false, path = "../balances" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +pallet-balances = { version = "2.0.0-rc4", default-features = false, path = "../balances" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../benchmarking", optional = true } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core", default-features = false } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core", default-features = false } [features] default = ["std"] diff --git a/frame/example/src/lib.rs b/frame/example/src/lib.rs index 23f21e30dca88a274dd2c82cddf1e4eee949d6e4..00e0d78d9b6dd2c165b4ada08bd3e54c5187e21e 100644 --- a/frame/example/src/lib.rs +++ b/frame/example/src/lib.rs @@ -256,7 +256,7 @@ use sp_std::marker::PhantomData; use frame_support::{ - dispatch::DispatchResult, decl_module, decl_storage, decl_event, + dispatch::{DispatchResult, IsSubType}, decl_module, decl_storage, decl_event, weights::{DispatchClass, ClassifyDispatch, WeighData, Weight, PaysFee, Pays}, }; use sp_std::prelude::*; @@ -609,14 +609,13 @@ impl sp_std::fmt::Debug for WatchDummy { } } -impl SignedExtension for WatchDummy { +impl SignedExtension for WatchDummy +where + ::Call: IsSubType>, +{ const IDENTIFIER: &'static str = "WatchDummy"; type AccountId = T::AccountId; - // Note that this could also be assigned to the top-level call enum. It is passed into the - // Balances Pallet directly and since `Trait: pallet_balances::Trait`, you could also use `T::Call`. - // In that case, you would have had access to all call variants and could match on variants from - // other pallets. - type Call = Call; + type Call = ::Call; type AdditionalSigned = (); type Pre = (); @@ -635,8 +634,8 @@ impl SignedExtension for WatchDummy { } // check for `set_dummy` - match call { - Call::set_dummy(..) => { + match call.is_sub_type() { + Some(Call::set_dummy(..)) => { sp_runtime::print("set_dummy was received."); let mut valid_tx = ValidTransaction::default(); @@ -711,8 +710,8 @@ mod tests { use super::*; use frame_support::{ - assert_ok, impl_outer_origin, parameter_types, weights::{DispatchInfo, GetDispatchInfo}, - traits::{OnInitialize, OnFinalize} + assert_ok, impl_outer_origin, parameter_types, impl_outer_dispatch, + weights::{DispatchInfo, GetDispatchInfo}, traits::{OnInitialize, OnFinalize} }; use sp_core::H256; // The testing primitives are very useful for avoiding having to work with signatures @@ -727,6 +726,12 @@ mod tests { pub enum Origin for Test where system = frame_system {} } + impl_outer_dispatch! { + pub enum OuterCall for Test where origin: Origin { + self::Example, + } + } + // For testing the pallet, we construct most of a mock runtime. This means // first constructing a configuration type (`Test`) which `impl`s each of the // configuration traits of pallets we want to use. @@ -739,11 +744,12 @@ mod tests { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; type Hash = H256; - type Call = (); + type Call = OuterCall; type Hashing = BlakeTwo256; type AccountId = u64; type Lookup = IdentityLookup; @@ -754,6 +760,7 @@ mod tests { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -826,7 +833,7 @@ mod tests { #[test] fn signed_ext_watch_dummy_works() { new_test_ext().execute_with(|| { - let call = >::set_dummy(10); + let call = >::set_dummy(10).into(); let info = DispatchInfo::default(); assert_eq!( diff --git a/frame/executive/Cargo.toml b/frame/executive/Cargo.toml index cb0efafac17117723c8a3197d4a5114524568cbb..52225d9824042561fbeb9a26d6942591334d6390 100644 --- a/frame/executive/Cargo.toml +++ b/frame/executive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-executive" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,23 +12,23 @@ description = "FRAME executives engine" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } serde = { version = "1.0.101", optional = true } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-tracing = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/tracing" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-tracing = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/tracing" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } [dev-dependencies] hex-literal = "0.2.1" -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-io ={ path = "../../primitives/io", version = "2.0.0-alpha.8"} -pallet-indices = { version = "2.0.0-alpha.8", path = "../indices" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } -pallet-transaction-payment = { version = "2.0.0-alpha.8", path = "../transaction-payment" } -sp-version = { version = "2.0.0-alpha.8", path = "../../primitives/version" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-io ={ version = "2.0.0-rc4", path = "../../primitives/io" } +pallet-indices = { version = "2.0.0-rc4", path = "../indices" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } +pallet-transaction-payment = { version = "2.0.0-rc4", path = "../transaction-payment" } +sp-version = { version = "2.0.0-rc4", path = "../../primitives/version" } [features] default = ["std"] diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index fcef03883ba423df27545071f347a86cb8ae4a79..9b0e4eab0299996227c1d701810bd0171a63ca5d 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -119,6 +119,7 @@ use sp_std::{prelude::*, marker::PhantomData}; use frame_support::{ storage::StorageValue, weights::{GetDispatchInfo, DispatchInfo, DispatchClass}, traits::{OnInitialize, OnFinalize, OnRuntimeUpgrade, OffchainWorker}, + dispatch::PostDispatchInfo, }; use sp_runtime::{ generic::Digest, ApplyExtrinsicResult, @@ -174,7 +175,7 @@ where CheckedOf: Applyable + GetDispatchInfo, - CallOf: Dispatchable, + CallOf: Dispatchable, OriginOf: From>, UnsignedValidator: ValidateUnsigned>, { @@ -200,7 +201,7 @@ where CheckedOf: Applyable + GetDispatchInfo, - CallOf: Dispatchable, + CallOf: Dispatchable, OriginOf: From>, UnsignedValidator: ValidateUnsigned>, { @@ -368,9 +369,9 @@ where let dispatch_info = xt.get_dispatch_info(); let r = Applyable::apply::(xt, &dispatch_info, encoded_len)?; - >::note_applied_extrinsic(&r, encoded_len as u32, dispatch_info); + >::note_applied_extrinsic(&r, dispatch_info); - Ok(r) + Ok(r.map(|_| ()).map_err(|e| e.error)) } fn final_checks(header: &System::Header) { @@ -453,12 +454,12 @@ mod tests { use sp_core::H256; use sp_runtime::{ generic::Era, Perbill, DispatchError, testing::{Digest, Header, Block}, - traits::{Header as HeaderT, BlakeTwo256, IdentityLookup, Convert, ConvertInto}, + traits::{Header as HeaderT, BlakeTwo256, IdentityLookup}, transaction_validity::{InvalidTransaction, UnknownTransaction, TransactionValidityError}, }; use frame_support::{ impl_outer_event, impl_outer_origin, parameter_types, impl_outer_dispatch, - weights::{Weight, RuntimeDbWeight}, + weights::{Weight, RuntimeDbWeight, IdentityFee, WeightToFeePolynomial}, traits::{Currency, LockIdentifier, LockableCurrency, WithdrawReasons, WithdrawReason}, }; use frame_system::{self as system, Call as SystemCall, ChainContext, LastRuntimeUpgradeInfo}; @@ -544,6 +545,7 @@ mod tests { }; } impl frame_system::Trait for Runtime { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type Call = Call; @@ -559,6 +561,7 @@ mod tests { type DbWeight = DbWeight; type BlockExecutionWeight = BlockExecutionWeight; type ExtrinsicBaseWeight = ExtrinsicBaseWeight; + type MaximumExtrinsicWeight = MaximumBlockWeight; type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = RuntimeVersion; @@ -587,7 +590,7 @@ mod tests { type Currency = Balances; type OnTransactionPayment = (); type TransactionByteFee = TransactionByteFee; - type WeightToFee = ConvertInto; + type WeightToFee = IdentityFee; type FeeMultiplierUpdate = (); } impl custom::Trait for Runtime {} @@ -673,7 +676,8 @@ mod tests { }.assimilate_storage(&mut t).unwrap(); let xt = TestXt::new(Call::Balances(BalancesCall::transfer(2, 69)), sign_extra(1, 0, 0)); let weight = xt.get_dispatch_info().weight + ::ExtrinsicBaseWeight::get(); - let fee: Balance = ::WeightToFee::convert(weight); + let fee: Balance + = ::WeightToFee::calc(&weight); let mut t = sp_io::TestExternalities::new(t); t.execute_with(|| { Executive::initialize_block(&Header::new( @@ -705,7 +709,7 @@ mod tests { header: Header { parent_hash: [69u8; 32].into(), number: 1, - state_root: hex!("409fb5a14aeb8b8c59258b503396a56dee45a0ee28a78de3e622db957425e275").into(), + state_root: hex!("e8ff7b3dd4375f6f3a76e24a1999e2a7be2d15b353e49ac94ace1eae3e80eb87").into(), extrinsics_root: hex!("03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314").into(), digest: Digest { logs: vec![], }, }, @@ -786,7 +790,7 @@ mod tests { Digest::default(), )); // Base block execution weight + `on_initialize` weight from the custom module. - assert_eq!(>::all_extrinsics_weight().total(), base_block_weight); + assert_eq!(>::block_weight().total(), base_block_weight); for nonce in 0..=num_to_exhaust_block { let xt = TestXt::new( @@ -796,7 +800,7 @@ mod tests { if nonce != num_to_exhaust_block { assert!(res.is_ok()); assert_eq!( - >::all_extrinsics_weight().total(), + >::block_weight().total(), //--------------------- on_initialize + block_execution + extrinsic_base weight (encoded_len + 5) * (nonce + 1) + base_block_weight, ); @@ -816,7 +820,18 @@ mod tests { let len = xt.clone().encode().len() as u32; let mut t = new_test_ext(1); t.execute_with(|| { - assert_eq!(>::all_extrinsics_weight().total(), 0); + // Block execution weight + on_initialize weight from custom module + let base_block_weight = 175 + ::BlockExecutionWeight::get(); + + Executive::initialize_block(&Header::new( + 1, + H256::default(), + H256::default(), + [69u8; 32].into(), + Digest::default(), + )); + + assert_eq!(>::block_weight().total(), base_block_weight); assert_eq!(>::all_extrinsics_len(), 0); assert!(Executive::apply_extrinsic(xt.clone()).unwrap().is_ok()); @@ -824,16 +839,28 @@ mod tests { assert!(Executive::apply_extrinsic(x2.clone()).unwrap().is_ok()); // default weight for `TestXt` == encoded length. + let extrinsic_weight = len as Weight + ::ExtrinsicBaseWeight::get(); assert_eq!( - >::all_extrinsics_weight().total(), - 3 * (len as Weight + ::ExtrinsicBaseWeight::get()), + >::block_weight().total(), + base_block_weight + 3 * extrinsic_weight, ); assert_eq!(>::all_extrinsics_len(), 3 * len); let _ = >::finalize(); - - assert_eq!(>::all_extrinsics_weight().total(), 0); + // All extrinsics length cleaned on `System::finalize` assert_eq!(>::all_extrinsics_len(), 0); + + // New Block + Executive::initialize_block(&Header::new( + 2, + H256::default(), + H256::default(), + [69u8; 32].into(), + Digest::default(), + )); + + // Block weight cleaned up on `System::initialize` + assert_eq!(>::block_weight().total(), base_block_weight); }); } @@ -869,7 +896,8 @@ mod tests { ); let weight = xt.get_dispatch_info().weight + ::ExtrinsicBaseWeight::get(); - let fee: Balance = ::WeightToFee::convert(weight); + let fee: Balance = + ::WeightToFee::calc(&weight); Executive::initialize_block(&Header::new( 1, H256::default(), @@ -904,7 +932,7 @@ mod tests { // NOTE: might need updates over time if new weights are introduced. // For now it only accounts for the base block execution weight and // the `on_initialize` weight defined in the custom test module. - assert_eq!(>::all_extrinsics_weight().total(), 175 + 10); + assert_eq!(>::block_weight().total(), 175 + 10); }) } diff --git a/frame/finality-tracker/Cargo.toml b/frame/finality-tracker/Cargo.toml index b0bf31304cadbafabfeb54c94647b1ac1eac7533..f9922af84e119d2a59a6855ce7f6767e1219094b 100644 --- a/frame/finality-tracker/Cargo.toml +++ b/frame/finality-tracker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-finality-tracker" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -15,18 +15,18 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", default-features = false, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/inherents" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-finality-tracker = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/finality-tracker" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/inherents" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-finality-tracker = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/finality-tracker" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } impl-trait-for-tuples = "0.1.3" [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } [features] default = ["std"] diff --git a/frame/finality-tracker/src/lib.rs b/frame/finality-tracker/src/lib.rs index e5065cd91711830d688853bb663aaaf28e8813a3..e5ed9574e5b7846413f36649bf79315af3cb10f5 100644 --- a/frame/finality-tracker/src/lib.rs +++ b/frame/finality-tracker/src/lib.rs @@ -252,6 +252,7 @@ mod tests { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -267,6 +268,7 @@ mod tests { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); @@ -336,10 +338,7 @@ mod tests { &Default::default(), Default::default(), ); - assert_ok!(FinalityTracker::dispatch( - Call::final_hint(i-1), - Origin::NONE, - )); + assert_ok!(FinalityTracker::final_hint(Origin::none(), i - 1)); FinalityTracker::on_finalize(i); let hdr = System::finalize(); parent_hash = hdr.hash(); diff --git a/frame/generic-asset/Cargo.toml b/frame/generic-asset/Cargo.toml index 97c37fae3736bae07a66005fcfee5a3cc228ddbc..f39a45837850d5aced6fa6195e5d0146324f251f 100644 --- a/frame/generic-asset/Cargo.toml +++ b/frame/generic-asset/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-generic-asset" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Centrality Developers "] edition = "2018" license = "Apache-2.0" @@ -13,15 +13,15 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } [dev-dependencies] -sp-io ={ version = "2.0.0-alpha.8", path = "../../primitives/io" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } +sp-io ={ version = "2.0.0-rc4", path = "../../primitives/io" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } [features] default = ["std"] diff --git a/frame/generic-asset/src/lib.rs b/frame/generic-asset/src/lib.rs index f2507669e537866bc58a54eb1451c4d4ddbc185c..7d24f89d7016c75584d4490f1323b26aeb892e7f 100644 --- a/frame/generic-asset/src/lib.rs +++ b/frame/generic-asset/src/lib.rs @@ -157,7 +157,7 @@ use codec::{Decode, Encode, HasCompact, Input, Output, Error as CodecError}; use sp_runtime::{RuntimeDebug, DispatchResult, DispatchError}; use sp_runtime::traits::{ CheckedAdd, CheckedSub, MaybeSerializeDeserialize, Member, One, Saturating, AtLeast32Bit, - Zero, Bounded, + Zero, Bounded, AtLeast32BitUnsigned }; use sp_std::prelude::*; @@ -165,8 +165,9 @@ use sp_std::{cmp, result, fmt::Debug}; use frame_support::{ decl_event, decl_module, decl_storage, ensure, decl_error, traits::{ - Currency, ExistenceRequirement, Imbalance, LockIdentifier, LockableCurrency, ReservableCurrency, - SignedImbalance, WithdrawReason, WithdrawReasons, TryDrop, BalanceStatus, + Currency, ExistenceRequirement, Imbalance, LockIdentifier, LockableCurrency, + ReservableCurrency, SignedImbalance, WithdrawReason, WithdrawReasons, TryDrop, + BalanceStatus, }, Parameter, StorageMap, }; @@ -178,25 +179,15 @@ mod tests; pub use self::imbalances::{NegativeImbalance, PositiveImbalance}; pub trait Trait: frame_system::Trait { - type Balance: Parameter - + Member - + AtLeast32Bit - + Default - + Copy - + MaybeSerializeDeserialize - + Debug; + type Balance: Parameter + Member + AtLeast32BitUnsigned + Default + Copy + Debug + + MaybeSerializeDeserialize; type AssetId: Parameter + Member + AtLeast32Bit + Default + Copy; type Event: From> + Into<::Event>; } pub trait Subtrait: frame_system::Trait { - type Balance: Parameter - + Member - + AtLeast32Bit - + Default - + Copy - + MaybeSerializeDeserialize - + Debug; + type Balance: Parameter + Member + AtLeast32BitUnsigned + Default + Copy + Debug + + MaybeSerializeDeserialize; type AssetId: Parameter + Member + AtLeast32Bit + Default + Copy; } @@ -442,16 +433,22 @@ pub struct BalanceLock { decl_storage! { trait Store for Module as GenericAsset { /// Total issuance of a given asset. + /// + /// TWOX-NOTE: `AssetId` is trusted. pub TotalIssuance get(fn total_issuance) build(|config: &GenesisConfig| { let issuance = config.initial_balance * (config.endowed_accounts.len() as u32).into(); config.assets.iter().map(|id| (id.clone(), issuance)).collect::>() }): map hasher(twox_64_concat) T::AssetId => T::Balance; /// The free balance of a given asset under an account. + /// + /// TWOX-NOTE: `AssetId` is trusted. pub FreeBalance: double_map hasher(twox_64_concat) T::AssetId, hasher(blake2_128_concat) T::AccountId => T::Balance; /// The reserved balance of a given asset under an account. + /// + /// TWOX-NOTE: `AssetId` is trusted. pub ReservedBalance: double_map hasher(twox_64_concat) T::AssetId, hasher(blake2_128_concat) T::AccountId => T::Balance; @@ -459,6 +456,8 @@ decl_storage! { pub NextAssetId get(fn next_asset_id) config(): T::AssetId; /// Permission options for a given asset. + /// + /// TWOX-NOTE: `AssetId` is trusted. pub Permissions get(fn get_permission): map hasher(twox_64_concat) T::AssetId => PermissionVersions; @@ -1112,6 +1111,7 @@ impl PartialEq for ElevatedTrait { } impl Eq for ElevatedTrait {} impl frame_system::Trait for ElevatedTrait { + type BaseCallFilter = T::BaseCallFilter; type Origin = T::Origin; type Call = T::Call; type Index = T::Index; @@ -1127,6 +1127,7 @@ impl frame_system::Trait for ElevatedTrait { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = T::MaximumBlockWeight; type MaximumBlockLength = T::MaximumBlockLength; type AvailableBlockRatio = T::AvailableBlockRatio; type Version = T::Version; diff --git a/frame/generic-asset/src/mock.rs b/frame/generic-asset/src/mock.rs index 3e3bd892d56880f2a3d512e1120cf71590f513c0..a928c9d67b2bd2f0a108d9aa14c1398138d72dcc 100644 --- a/frame/generic-asset/src/mock.rs +++ b/frame/generic-asset/src/mock.rs @@ -46,6 +46,7 @@ parameter_types! { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -60,6 +61,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type BlockHashCount = BlockHashCount; diff --git a/frame/generic-asset/src/tests.rs b/frame/generic-asset/src/tests.rs index d5c0a877dfe75aba4236a2130372a80b06f74a6c..a094f69ba1fc581f0138574e06c202c404050cd6 100644 --- a/frame/generic-asset/src/tests.rs +++ b/frame/generic-asset/src/tests.rs @@ -598,7 +598,7 @@ fn create_reserved_should_create_a_default_account_with_the_balance_given() { let created_asset_id = 9; let created_account_id = 0; - assert_ok!(GenericAsset::create_reserved(Origin::ROOT, created_asset_id, options)); + assert_ok!(GenericAsset::create_reserved(Origin::root(), created_asset_id, options)); // Tests for side effects. assert_eq!(>::get(created_asset_id), expected_total_issuance); diff --git a/frame/grandpa/Cargo.toml b/frame/grandpa/Cargo.toml index 3ee12a4aab756b78b70079bf9af096bca4ecdf2b..0f2477d50e8a94eda0a60982ab44559d2a873e98 100644 --- a/frame/grandpa/Cargo.toml +++ b/frame/grandpa/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-grandpa" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,28 +13,28 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-application-crypto = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/application-crypto" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -sp-finality-grandpa = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/finality-grandpa" } -sp-session = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/session" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-staking = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/staking" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -pallet-session = { version = "2.0.0-alpha.8", default-features = false, path = "../session" } -pallet-finality-tracker = { version = "2.0.0-alpha.8", default-features = false, path = "../finality-tracker" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-application-crypto = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/application-crypto" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-finality-grandpa = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/finality-grandpa" } +sp-session = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/session" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/staking" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +pallet-session = { version = "2.0.0-rc4", default-features = false, path = "../session" } +pallet-finality-tracker = { version = "2.0.0-rc4", default-features = false, path = "../finality-tracker" } [dev-dependencies] grandpa = { package = "finality-grandpa", version = "0.12.3", features = ["derive-codec"] } -sp-io = { version = "2.0.0-alpha.8", path = "../../primitives/io" } -sp-keyring = { version = "2.0.0-alpha.8", path = "../../primitives/keyring" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } -pallet-offences = { version = "2.0.0-alpha.8", path = "../offences" } -pallet-staking = { version = "2.0.0-alpha.8", path = "../staking" } -pallet-staking-reward-curve = { version = "2.0.0-alpha.8", path = "../staking/reward-curve" } -pallet-timestamp = { version = "2.0.0-alpha.8", path = "../timestamp" } +sp-io = { version = "2.0.0-rc4", path = "../../primitives/io" } +sp-keyring = { version = "2.0.0-rc4", path = "../../primitives/keyring" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } +pallet-offences = { version = "2.0.0-rc4", path = "../offences" } +pallet-staking = { version = "2.0.0-rc4", path = "../staking" } +pallet-staking-reward-curve = { version = "2.0.0-rc4", path = "../staking/reward-curve" } +pallet-timestamp = { version = "2.0.0-rc4", path = "../timestamp" } [features] default = ["std"] diff --git a/frame/grandpa/src/equivocation.rs b/frame/grandpa/src/equivocation.rs index bcdc93ff34e5b18adeaeb8b7ab8243fd5a188711..f9c4d4f8da2ef5ee2d4ed2eceea14ab2c2d6cd56 100644 --- a/frame/grandpa/src/equivocation.rs +++ b/frame/grandpa/src/equivocation.rs @@ -145,7 +145,7 @@ where // validate equivocation proof (check votes are different and // signatures are valid). - if let Err(_) = sp_finality_grandpa::check_equivocation_proof(equivocation_proof.clone()) { + if !sp_finality_grandpa::check_equivocation_proof(equivocation_proof.clone()) { return Err(ReportEquivocationValidityError::InvalidEquivocationProof.into()); } diff --git a/frame/grandpa/src/lib.rs b/frame/grandpa/src/lib.rs index 43b6ba0b2fb84d8596aa916b042d89f636bad87d..3432c1102008ba06a8474cff48eaa687c1db0df0 100644 --- a/frame/grandpa/src/lib.rs +++ b/frame/grandpa/src/lib.rs @@ -212,6 +212,8 @@ decl_storage! { /// A mapping from grandpa set ID to the index of the *most recent* session for which its /// members were responsible. + /// + /// TWOX-NOTE: `SetId` is not under user control. SetIdSession get(fn session_for_set): map hasher(twox_64_concat) SetId => Option; } add_extra_genesis { diff --git a/frame/grandpa/src/mock.rs b/frame/grandpa/src/mock.rs index 13608db42aa495e5a63db28840ce31d8e31f6c43..0f3122c860dd03f817c0473147560cdebbc34d17 100644 --- a/frame/grandpa/src/mock.rs +++ b/frame/grandpa/src/mock.rs @@ -94,6 +94,7 @@ parameter_types! { } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -109,6 +110,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -228,12 +230,18 @@ impl staking::Trait for Test { type Call = Call; type UnsignedPriority = StakingUnsignedPriority; type MaxIterations = (); + type MinSolutionScoreBump = (); +} + +parameter_types! { + pub OffencesWeightSoftLimit: Weight = Perbill::from_percent(60) * MaximumBlockWeight::get(); } impl offences::Trait for Test { type Event = TestEvent; type IdentificationTuple = session::historical::IdentificationTuple; type OnOffenceHandler = Staking; + type WeightSoftLimit = OffencesWeightSoftLimit; } impl Trait for Test { diff --git a/frame/grandpa/src/tests.rs b/frame/grandpa/src/tests.rs index e15021733ffa054a39ade8c5292c9dda0ec1c099..2337e00e8d20336a786a0941f7dede621dcfd053 100644 --- a/frame/grandpa/src/tests.rs +++ b/frame/grandpa/src/tests.rs @@ -25,7 +25,7 @@ use codec::{Decode, Encode}; use fg_primitives::ScheduledChange; use frame_support::{ assert_err, assert_ok, - traits::{Currency, OnFinalize}, + traits::{Currency, OnFinalize, UnfilteredDispatchable}, }; use frame_system::{EventRecord, Phase}; use sp_core::H256; @@ -376,7 +376,7 @@ fn report_equivocation_current_set_works() { // report the equivocation and the tx should be dispatched successfully let inner = report_equivocation(equivocation_proof, key_owner_proof).unwrap(); - assert_ok!(Grandpa::dispatch(inner, Origin::signed(1))); + assert_ok!(inner.dispatch_bypass_filter(Origin::signed(1))); start_era(2); @@ -457,7 +457,7 @@ fn report_equivocation_old_set_works() { // report the equivocation using the key ownership proof generated on // the old set, the tx should be dispatched successfully let inner = report_equivocation(equivocation_proof, key_owner_proof).unwrap(); - assert_ok!(Grandpa::dispatch(inner, Origin::signed(1))); + assert_ok!(inner.dispatch_bypass_filter(Origin::signed(1))); start_era(3); diff --git a/frame/identity/Cargo.toml b/frame/identity/Cargo.toml index 9d36a235e0a062a5f4dbe5017098275681099ba1..8dcfd5bd2d11927d2a4d2329281b01367a246d51 100644 --- a/frame/identity/Cargo.toml +++ b/frame/identity/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-identity" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,18 +13,18 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } enumflags2 = { version = "0.6.2" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../benchmarking", optional = true } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } [features] default = ["std"] diff --git a/frame/identity/src/lib.rs b/frame/identity/src/lib.rs index 37ed8f8672a15f65ce39e343335ba370c497685a..19b23a644d64a9e28efc4e654b7a0affcf4fea77 100644 --- a/frame/identity/src/lib.rs +++ b/frame/identity/src/lib.rs @@ -78,7 +78,7 @@ use frame_support::{ traits::{Currency, ReservableCurrency, OnUnbalanced, Get, BalanceStatus, EnsureOrigin}, weights::Weight, }; -use frame_system::{self as system, ensure_signed, ensure_root}; +use frame_system::{self as system, ensure_signed}; mod benchmarking; @@ -389,6 +389,8 @@ pub struct RegistrarInfo< decl_storage! { trait Store for Module as Identity { /// Information that is pertinent to identify the entity behind an account. + /// + /// TWOX-NOTE: OK ― `AccountId` is a secure hash. pub IdentityOf get(fn identity): map hasher(twox_64_concat) T::AccountId => Option>>; @@ -400,6 +402,8 @@ decl_storage! { /// Alternative "sub" identities of this account. /// /// The first item is the deposit, the second is a vector of the accounts. + /// + /// TWOX-NOTE: OK ― `AccountId` is a secure hash. pub SubsOf get(fn subs_of): map hasher(twox_64_concat) T::AccountId => (BalanceOf, Vec); @@ -618,7 +622,7 @@ decl_module! { /// Add a registrar to the system. /// - /// The dispatch origin for this call must be `RegistrarOrigin` or `Root`. + /// The dispatch origin for this call must be `T::RegistrarOrigin`. /// /// - `account`: the account of the registrar. /// @@ -631,9 +635,7 @@ decl_module! { /// # #[weight = weight_for::add_registrar::(T::MaxRegistrars::get().into()) ] fn add_registrar(origin, account: T::AccountId) -> DispatchResultWithPostInfo { - T::RegistrarOrigin::try_origin(origin) - .map(|_| ()) - .or_else(ensure_root)?; + T::RegistrarOrigin::ensure_origin(origin)?; let (i, registrar_count) = >::try_mutate( |registrars| -> Result<(RegistrarIndex, usize), DispatchError> { @@ -1085,7 +1087,7 @@ decl_module! { /// `Slash`. Verification request deposits are not returned; they should be cancelled /// manually using `cancel_request`. /// - /// The dispatch origin for this call must be _Root_ or match `T::ForceOrigin`. + /// The dispatch origin for this call must match `T::ForceOrigin`. /// /// - `target`: the account whose identity the judgement is upon. This must be an account /// with a registered identity. @@ -1104,9 +1106,7 @@ decl_module! { T::MaxAdditionalFields::get().into(), // X )] fn kill_identity(origin, target: ::Source) -> DispatchResultWithPostInfo { - T::ForceOrigin::try_origin(origin) - .map(|_| ()) - .or_else(ensure_root)?; + T::ForceOrigin::ensure_origin(origin)?; // Figure out who we're meant to be clearing. let target = T::Lookup::lookup(target)?; @@ -1151,7 +1151,7 @@ mod tests { ord_parameter_types, }; use sp_core::H256; - use frame_system::EnsureSignedBy; + use frame_system::{EnsureSignedBy, EnsureOneOf, EnsureRoot}; // The testing primitives are very useful for avoiding having to work with signatures // or public keys. `u64` is used as the `AccountId` and no `Signature`s are required. use sp_runtime::{ @@ -1174,6 +1174,7 @@ mod tests { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -1189,6 +1190,7 @@ mod tests { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -1219,6 +1221,16 @@ mod tests { pub const One: u64 = 1; pub const Two: u64 = 2; } + type EnsureOneOrRoot = EnsureOneOf< + u64, + EnsureRoot, + EnsureSignedBy + >; + type EnsureTwoOrRoot = EnsureOneOf< + u64, + EnsureRoot, + EnsureSignedBy + >; impl Trait for Test { type Event = (); type Currency = Balances; @@ -1229,8 +1241,8 @@ mod tests { type MaxSubAccounts = MaxSubAccounts; type MaxAdditionalFields = MaxAdditionalFields; type MaxRegistrars = MaxRegistrars; - type RegistrarOrigin = EnsureSignedBy; - type ForceOrigin = EnsureSignedBy; + type RegistrarOrigin = EnsureOneOrRoot; + type ForceOrigin = EnsureTwoOrRoot; } type System = frame_system::Module; type Balances = pallet_balances::Module; @@ -1430,7 +1442,7 @@ mod tests { new_test_ext().execute_with(|| { assert_ok!(Identity::set_identity(Origin::signed(10), ten())); assert_ok!(Identity::set_subs(Origin::signed(10), vec![(20, Data::Raw(vec![40; 1]))])); - assert_ok!(Identity::kill_identity(Origin::ROOT, 10)); + assert_ok!(Identity::kill_identity(Origin::signed(2), 10)); assert_eq!(Balances::free_balance(10), 80); assert!(Identity::super_of(20).is_none()); }); diff --git a/frame/im-online/Cargo.toml b/frame/im-online/Cargo.toml index 7c93b25e35949f11f6b9bebf22e9fd423f6929cc..7324342ec8afbb0aaf81de299a4e3e2098d0f1b1 100644 --- a/frame/im-online/Cargo.toml +++ b/frame/im-online/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-im-online" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,20 +12,20 @@ description = "FRAME's I'm online pallet" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/application-crypto" } -pallet-authorship = { version = "2.0.0-alpha.8", default-features = false, path = "../authorship" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } +sp-application-crypto = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/application-crypto" } +pallet-authorship = { version = "2.0.0-rc4", default-features = false, path = "../authorship" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } serde = { version = "1.0.101", optional = true } -pallet-session = { version = "2.0.0-alpha.8", default-features = false, path = "../session" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-staking = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/staking" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +pallet-session = { version = "2.0.0-rc4", default-features = false, path = "../session" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/staking" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../benchmarking", optional = true } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } [features] default = ["std", "pallet-session/historical"] diff --git a/frame/im-online/src/benchmarking.rs b/frame/im-online/src/benchmarking.rs index 63457168b3638fda25b0dc1a1d6b2145f99c1c22..92d9b9d5a5364524065b263af8e422628b847420 100644 --- a/frame/im-online/src/benchmarking.rs +++ b/frame/im-online/src/benchmarking.rs @@ -24,8 +24,9 @@ use super::*; use frame_system::RawOrigin; use frame_benchmarking::benchmarks; use sp_core::offchain::{OpaquePeerId, OpaqueMultiaddr}; -use sp_runtime::traits::{ValidateUnsigned, Zero, Dispatchable}; +use sp_runtime::traits::{ValidateUnsigned, Zero}; use sp_runtime::transaction_validity::TransactionSource; +use frame_support::traits::UnfilteredDispatchable; use crate::Module as ImOnline; @@ -85,7 +86,7 @@ benchmarks! { let call = Call::heartbeat(input_heartbeat, signature); }: { ImOnline::::validate_unsigned(TransactionSource::InBlock, &call)?; - call.dispatch(RawOrigin::None.into())?; + call.dispatch_bypass_filter(RawOrigin::None.into())?; } } diff --git a/frame/im-online/src/lib.rs b/frame/im-online/src/lib.rs index c1c93910ece22181360db201a57e54e785a78beb..ddbbb52bd2cc7470237da53d4744d8dbb71cb52f 100644 --- a/frame/im-online/src/lib.rs +++ b/frame/im-online/src/lib.rs @@ -82,7 +82,7 @@ use pallet_session::historical::IdentificationTuple; use sp_runtime::{ offchain::storage::StorageValueRef, RuntimeDebug, - traits::{Convert, Member, Saturating, AtLeast32Bit}, Perbill, + traits::{Convert, Member, Saturating, AtLeast32BitUnsigned}, Perbill, transaction_validity::{ TransactionValidity, ValidTransaction, InvalidTransaction, TransactionSource, TransactionPriority, @@ -160,7 +160,7 @@ struct HeartbeatStatus { pub sent_at: BlockNumber, } -impl HeartbeatStatus { +impl HeartbeatStatus { /// Returns true if heartbeat has been recently sent. /// /// Parameters: diff --git a/frame/im-online/src/mock.rs b/frame/im-online/src/mock.rs index 9d67e78eeff98dab1700348d3766b957e96ce6a3..d313646b289356e08bff8ba13effa2d66711a055 100644 --- a/frame/im-online/src/mock.rs +++ b/frame/im-online/src/mock.rs @@ -104,6 +104,7 @@ parameter_types! { } impl frame_system::Trait for Runtime { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -119,6 +120,7 @@ impl frame_system::Trait for Runtime { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/im-online/src/tests.rs b/frame/im-online/src/tests.rs index 7619781b68df0a46303434a0c8bf596853b5ad14..835d8440e6d5b0e0f16caf750d841361a3012edc 100644 --- a/frame/im-online/src/tests.rs +++ b/frame/im-online/src/tests.rs @@ -135,7 +135,7 @@ fn heartbeat( e @ _ => <&'static str>::from(e), })?; ImOnline::heartbeat( - Origin::system(frame_system::RawOrigin::None), + Origin::none(), heartbeat, signature, ) diff --git a/frame/indices/Cargo.toml b/frame/indices/Cargo.toml index 7a747ed294a240c1b1d2ea87bd7e544dd0100fd1..3ec8ea363bc9a764a2406dd2798aff5ed48d4682 100644 --- a/frame/indices/Cargo.toml +++ b/frame/indices/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-indices" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,17 +13,19 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-keyring = { version = "2.0.0-alpha.8", optional = true, path = "../../primitives/keyring" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-keyring = { version = "2.0.0-rc4", optional = true, path = "../../primitives/keyring" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } + +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } [features] default = ["std"] @@ -38,3 +40,7 @@ std = [ "sp-runtime/std", "frame-system/std", ] +runtime-benchmarks = [ + "frame-benchmarking", + "frame-support/runtime-benchmarks", +] diff --git a/frame/indices/src/benchmarking.rs b/frame/indices/src/benchmarking.rs new file mode 100644 index 0000000000000000000000000000000000000000..a6b543bb43f4abac986c537ddc1fedf258d27e54 --- /dev/null +++ b/frame/indices/src/benchmarking.rs @@ -0,0 +1,124 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Benchmarks for Indices Pallet + +#![cfg(feature = "runtime-benchmarks")] + +use super::*; +use frame_system::RawOrigin; +use frame_benchmarking::{benchmarks, account}; +use sp_runtime::traits::Bounded; + +use crate::Module as Indices; + +const SEED: u32 = 0; + +benchmarks! { + _ { } + + claim { + // Index being claimed + let i in 0 .. 1000; + let account_index = T::AccountIndex::from(i); + let caller: T::AccountId = account("caller", 0, SEED); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + }: _(RawOrigin::Signed(caller.clone()), account_index) + verify { + assert_eq!(Accounts::::get(account_index).unwrap().0, caller); + } + + transfer { + // Index being claimed + let i in 0 .. 1000; + let account_index = T::AccountIndex::from(i); + // Setup accounts + let caller: T::AccountId = account("caller", 0, SEED); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + let recipient: T::AccountId = account("recipient", i, SEED); + T::Currency::make_free_balance_be(&recipient, BalanceOf::::max_value()); + // Claim the index + Indices::::claim(RawOrigin::Signed(caller.clone()).into(), account_index)?; + }: _(RawOrigin::Signed(caller.clone()), recipient.clone(), account_index) + verify { + assert_eq!(Accounts::::get(account_index).unwrap().0, recipient); + } + + free { + // Index being claimed + let i in 0 .. 1000; + let account_index = T::AccountIndex::from(i); + // Setup accounts + let caller: T::AccountId = account("caller", 0, SEED); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + // Claim the index + Indices::::claim(RawOrigin::Signed(caller.clone()).into(), account_index)?; + }: _(RawOrigin::Signed(caller.clone()), account_index) + verify { + assert_eq!(Accounts::::get(account_index), None); + } + + force_transfer { + // Index being claimed + let i in 0 .. 1000; + let account_index = T::AccountIndex::from(i); + // Setup accounts + let original: T::AccountId = account("original", 0, SEED); + T::Currency::make_free_balance_be(&original, BalanceOf::::max_value()); + let recipient: T::AccountId = account("recipient", i, SEED); + T::Currency::make_free_balance_be(&recipient, BalanceOf::::max_value()); + // Claim the index + Indices::::claim(RawOrigin::Signed(original).into(), account_index)?; + }: _(RawOrigin::Root, recipient.clone(), account_index, false) + verify { + assert_eq!(Accounts::::get(account_index).unwrap().0, recipient); + } + + freeze { + // Index being claimed + let i in 0 .. 1000; + let account_index = T::AccountIndex::from(i); + // Setup accounts + let caller: T::AccountId = account("caller", 0, SEED); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + // Claim the index + Indices::::claim(RawOrigin::Signed(caller.clone()).into(), account_index)?; + }: _(RawOrigin::Signed(caller.clone()), account_index) + verify { + assert_eq!(Accounts::::get(account_index).unwrap().2, true); + } + + // TODO in another PR: lookup and unlookup trait weights (not critical) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mock::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_claim::()); + assert_ok!(test_benchmark_transfer::()); + assert_ok!(test_benchmark_free::()); + assert_ok!(test_benchmark_force_transfer::()); + assert_ok!(test_benchmark_freeze::()); + }); + } +} diff --git a/frame/indices/src/lib.rs b/frame/indices/src/lib.rs index 77a73a21aca481ba60618259a7d62e9e09984a68..e58112403f628341aad8205e0b6bd6085efeab01 100644 --- a/frame/indices/src/lib.rs +++ b/frame/indices/src/lib.rs @@ -28,12 +28,14 @@ use sp_runtime::traits::{ use frame_support::{Parameter, decl_module, decl_error, decl_event, decl_storage, ensure}; use frame_support::dispatch::DispatchResult; use frame_support::traits::{Currency, ReservableCurrency, Get, BalanceStatus::Reserved}; +use frame_support::weights::constants::WEIGHT_PER_MICROS; use frame_system::{ensure_signed, ensure_root}; use self::address::Address as RawAddress; mod mock; pub mod address; mod tests; +mod benchmarking; pub type Address = RawAddress<::AccountId, ::AccountIndex>; type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; @@ -60,9 +62,9 @@ decl_storage! { pub Accounts build(|config: &GenesisConfig| config.indices.iter() .cloned() - .map(|(a, b)| (a, (b, Zero::zero()))) + .map(|(a, b)| (a, (b, Zero::zero(), false))) .collect::>() - ): map hasher(blake2_128_concat) T::AccountIndex => Option<(T::AccountId, BalanceOf)>; + ): map hasher(blake2_128_concat) T::AccountIndex => Option<(T::AccountId, BalanceOf, bool)>; } add_extra_genesis { config(indices): Vec<(T::AccountIndex, T::AccountId)>; @@ -78,6 +80,8 @@ decl_event!( IndexAssigned(AccountId, AccountIndex), /// A account index has been freed up (unassigned). IndexFreed(AccountIndex), + /// A account index has been frozen to its current account ID. + IndexFrozen(AccountIndex, AccountId), } ); @@ -91,6 +95,8 @@ decl_error! { InUse, /// The source and destination accounts are identical. NotTransfer, + /// The index is permanent and may not be freed/changed. + Permanent, } } @@ -113,14 +119,17 @@ decl_module! { /// - One storage mutation (codec `O(1)`). /// - One reserve operation. /// - One event. + /// ------------------- + /// - Base Weight: 28.69 µs + /// - DB Weight: 1 Read/Write (Accounts) /// # - #[weight = 0] + #[weight = T::DbWeight::get().reads_writes(1, 1) + 30 * WEIGHT_PER_MICROS] fn claim(origin, index: T::AccountIndex) { let who = ensure_signed(origin)?; Accounts::::try_mutate(index, |maybe_value| { ensure!(maybe_value.is_none(), Error::::InUse); - *maybe_value = Some((who.clone(), T::Deposit::get())); + *maybe_value = Some((who.clone(), T::Deposit::get(), false)); T::Currency::reserve(&who, T::Deposit::get()) })?; Self::deposit_event(RawEvent::IndexAssigned(who, index)); @@ -141,17 +150,23 @@ decl_module! { /// - One storage mutation (codec `O(1)`). /// - One transfer operation. /// - One event. + /// ------------------- + /// - Base Weight: 33.74 µs + /// - DB Weight: + /// - Reads: Indices Accounts, System Account (recipient) + /// - Writes: Indices Accounts, System Account (recipient) /// # - #[weight = 0] + #[weight = T::DbWeight::get().reads_writes(2, 2) + 35 * WEIGHT_PER_MICROS] fn transfer(origin, new: T::AccountId, index: T::AccountIndex) { let who = ensure_signed(origin)?; ensure!(who != new, Error::::NotTransfer); Accounts::::try_mutate(index, |maybe_value| -> DispatchResult { - let (account, amount) = maybe_value.take().ok_or(Error::::NotAssigned)?; + let (account, amount, perm) = maybe_value.take().ok_or(Error::::NotAssigned)?; + ensure!(!perm, Error::::Permanent); ensure!(&account == &who, Error::::NotOwner); let lost = T::Currency::repatriate_reserved(&who, &new, amount, Reserved)?; - *maybe_value = Some((new.clone(), amount.saturating_sub(lost))); + *maybe_value = Some((new.clone(), amount.saturating_sub(lost), false)); Ok(()) })?; Self::deposit_event(RawEvent::IndexAssigned(new, index)); @@ -172,13 +187,17 @@ decl_module! { /// - One storage mutation (codec `O(1)`). /// - One reserve operation. /// - One event. + /// ------------------- + /// - Base Weight: 25.53 µs + /// - DB Weight: 1 Read/Write (Accounts) /// # - #[weight = 0] + #[weight = T::DbWeight::get().reads_writes(1, 1) + 25 * WEIGHT_PER_MICROS] fn free(origin, index: T::AccountIndex) { let who = ensure_signed(origin)?; Accounts::::try_mutate(index, |maybe_value| -> DispatchResult { - let (account, amount) = maybe_value.take().ok_or(Error::::NotAssigned)?; + let (account, amount, perm) = maybe_value.take().ok_or(Error::::NotAssigned)?; + ensure!(!perm, Error::::Permanent); ensure!(&account == &who, Error::::NotOwner); T::Currency::unreserve(&who, amount); Ok(()) @@ -193,6 +212,7 @@ decl_module! { /// /// - `index`: the index to be (re-)assigned. /// - `new`: the new owner of the index. This function is a no-op if it is equal to sender. + /// - `freeze`: if set to `true`, will freeze the index so it cannot be transferred. /// /// Emits `IndexAssigned` if successful. /// @@ -201,19 +221,57 @@ decl_module! { /// - One storage mutation (codec `O(1)`). /// - Up to one reserve operation. /// - One event. + /// ------------------- + /// - Base Weight: 26.83 µs + /// - DB Weight: + /// - Reads: Indices Accounts, System Account (original owner) + /// - Writes: Indices Accounts, System Account (original owner) /// # - #[weight = 0] - fn force_transfer(origin, new: T::AccountId, index: T::AccountIndex) { + #[weight = T::DbWeight::get().reads_writes(2, 2) + 25 * WEIGHT_PER_MICROS] + fn force_transfer(origin, new: T::AccountId, index: T::AccountIndex, freeze: bool) { ensure_root(origin)?; Accounts::::mutate(index, |maybe_value| { - if let Some((account, amount)) = maybe_value.take() { + if let Some((account, amount, _)) = maybe_value.take() { T::Currency::unreserve(&account, amount); } - *maybe_value = Some((new.clone(), Zero::zero())); + *maybe_value = Some((new.clone(), Zero::zero(), freeze)); }); Self::deposit_event(RawEvent::IndexAssigned(new, index)); } + + /// Freeze an index so it will always point to the sender account. This consumes the deposit. + /// + /// The dispatch origin for this call must be _Signed_ and the signing account must have a + /// non-frozen account `index`. + /// + /// - `index`: the index to be frozen in place. + /// + /// Emits `IndexFrozen` if successful. + /// + /// # + /// - `O(1)`. + /// - One storage mutation (codec `O(1)`). + /// - Up to one slash operation. + /// - One event. + /// ------------------- + /// - Base Weight: 30.86 µs + /// - DB Weight: 1 Read/Write (Accounts) + /// # + #[weight = T::DbWeight::get().reads_writes(1, 1) + 30 * WEIGHT_PER_MICROS] + fn freeze(origin, index: T::AccountIndex) { + let who = ensure_signed(origin)?; + + Accounts::::try_mutate(index, |maybe_value| -> DispatchResult { + let (account, amount, perm) = maybe_value.take().ok_or(Error::::NotAssigned)?; + ensure!(!perm, Error::::Permanent); + ensure!(&account == &who, Error::::NotOwner); + T::Currency::slash_reserved(&who, amount); + *maybe_value = Some((account, Zero::zero(), true)); + Ok(()) + })?; + Self::deposit_event(RawEvent::IndexFrozen(index, who)); + } } } diff --git a/frame/indices/src/mock.rs b/frame/indices/src/mock.rs index 62f6c93caeb9b7f65aebc17c083c82f026cb8e2b..da30c129c39491349adb099b54e1b1c486aad408 100644 --- a/frame/indices/src/mock.rs +++ b/frame/indices/src/mock.rs @@ -50,6 +50,7 @@ parameter_types! { } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Call = (); type Index = u64; @@ -65,6 +66,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/indices/src/tests.rs b/frame/indices/src/tests.rs index 7f416afbd3392b87bcf93ec09025d09a6508ae1b..e288871d5530739f07aa0212d9721f681d1c1912 100644 --- a/frame/indices/src/tests.rs +++ b/frame/indices/src/tests.rs @@ -48,6 +48,20 @@ fn freeing_should_work() { }); } +#[test] +fn freezing_should_work() { + new_test_ext().execute_with(|| { + assert_ok!(Indices::claim(Some(1).into(), 0)); + assert_noop!(Indices::freeze(Some(1).into(), 1), Error::::NotAssigned); + assert_noop!(Indices::freeze(Some(2).into(), 0), Error::::NotOwner); + assert_ok!(Indices::freeze(Some(1).into(), 0)); + assert_noop!(Indices::freeze(Some(1).into(), 0), Error::::Permanent); + + assert_noop!(Indices::free(Some(1).into(), 0), Error::::Permanent); + assert_noop!(Indices::transfer(Some(1).into(), 2, 0), Error::::Permanent); + }); +} + #[test] fn indexing_lookup_should_work() { new_test_ext().execute_with(|| { @@ -87,7 +101,7 @@ fn transfer_index_on_accounts_should_work() { fn force_transfer_index_on_preowned_should_work() { new_test_ext().execute_with(|| { assert_ok!(Indices::claim(Some(1).into(), 0)); - assert_ok!(Indices::force_transfer(Origin::ROOT, 3, 0)); + assert_ok!(Indices::force_transfer(Origin::root(), 3, 0, false)); assert_eq!(Balances::reserved_balance(1), 0); assert_eq!(Balances::reserved_balance(3), 0); assert_eq!(Indices::lookup_index(0), Some(3)); @@ -97,7 +111,7 @@ fn force_transfer_index_on_preowned_should_work() { #[test] fn force_transfer_index_on_free_should_work() { new_test_ext().execute_with(|| { - assert_ok!(Indices::force_transfer(Origin::ROOT, 3, 0)); + assert_ok!(Indices::force_transfer(Origin::root(), 3, 0, false)); assert_eq!(Balances::reserved_balance(3), 0); assert_eq!(Indices::lookup_index(0), Some(3)); }); diff --git a/frame/membership/Cargo.toml b/frame/membership/Cargo.toml index 538c63acbbb223af5d6f9ae761090ab47e4572ed..5df5d4ad6e3b8f3dc732635ce3610602799b5070 100644 --- a/frame/membership/Cargo.toml +++ b/frame/membership/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-membership" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,15 +13,15 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } [features] default = ["std"] diff --git a/frame/membership/src/lib.rs b/frame/membership/src/lib.rs index 9dd1c8ecfc79e64173631506a34193533dd578f9..c8563b52f81f9526bfa6afaf97f3a8fdf8910964 100644 --- a/frame/membership/src/lib.rs +++ b/frame/membership/src/lib.rs @@ -26,9 +26,9 @@ use sp_std::prelude::*; use frame_support::{ decl_module, decl_storage, decl_event, decl_error, - traits::{ChangeMembers, InitializeMembers, EnsureOrigin}, + traits::{ChangeMembers, InitializeMembers, EnsureOrigin, Contains}, }; -use frame_system::{self as system, ensure_root, ensure_signed}; +use frame_system::{self as system, ensure_signed}; pub trait Trait: frame_system::Trait { /// The overarching event type. @@ -117,12 +117,10 @@ decl_module! { /// Add a member `who` to the set. /// - /// May only be called from `AddOrigin` or root. + /// May only be called from `T::AddOrigin`. #[weight = 50_000_000] - fn add_member(origin, who: T::AccountId) { - T::AddOrigin::try_origin(origin) - .map(|_| ()) - .or_else(ensure_root)?; + pub fn add_member(origin, who: T::AccountId) { + T::AddOrigin::ensure_origin(origin)?; let mut members = >::get(); let location = members.binary_search(&who).err().ok_or(Error::::AlreadyMember)?; @@ -136,12 +134,10 @@ decl_module! { /// Remove a member `who` from the set. /// - /// May only be called from `RemoveOrigin` or root. + /// May only be called from `T::RemoveOrigin`. #[weight = 50_000_000] - fn remove_member(origin, who: T::AccountId) { - T::RemoveOrigin::try_origin(origin) - .map(|_| ()) - .or_else(ensure_root)?; + pub fn remove_member(origin, who: T::AccountId) { + T::RemoveOrigin::ensure_origin(origin)?; let mut members = >::get(); let location = members.binary_search(&who).ok().ok_or(Error::::NotMember)?; @@ -156,14 +152,12 @@ decl_module! { /// Swap out one member `remove` for another `add`. /// - /// May only be called from `SwapOrigin` or root. + /// May only be called from `T::SwapOrigin`. /// /// Prime membership is *not* passed from `remove` to `add`, if extant. #[weight = 50_000_000] - fn swap_member(origin, remove: T::AccountId, add: T::AccountId) { - T::SwapOrigin::try_origin(origin) - .map(|_| ()) - .or_else(ensure_root)?; + pub fn swap_member(origin, remove: T::AccountId, add: T::AccountId) { + T::SwapOrigin::ensure_origin(origin)?; if remove == add { return Ok(()) } @@ -187,12 +181,10 @@ decl_module! { /// Change the membership to a new set, disregarding the existing membership. Be nice and /// pass `members` pre-sorted. /// - /// May only be called from `ResetOrigin` or root. + /// May only be called from `T::ResetOrigin`. #[weight = 50_000_000] - fn reset_members(origin, members: Vec) { - T::ResetOrigin::try_origin(origin) - .map(|_| ()) - .or_else(ensure_root)?; + pub fn reset_members(origin, members: Vec) { + T::ResetOrigin::ensure_origin(origin)?; let mut members = members; members.sort(); @@ -212,7 +204,7 @@ decl_module! { /// /// Prime membership is passed from the origin account to `new`, if extant. #[weight = 50_000_000] - fn change_key(origin, new: T::AccountId) { + pub fn change_key(origin, new: T::AccountId) { let remove = ensure_signed(origin)?; if remove != new { @@ -239,22 +231,22 @@ decl_module! { } /// Set the prime member. Must be a current member. + /// + /// May only be called from `T::PrimeOrigin`. #[weight = 50_000_000] - fn set_prime(origin, who: T::AccountId) { - T::PrimeOrigin::try_origin(origin) - .map(|_| ()) - .or_else(ensure_root)?; + pub fn set_prime(origin, who: T::AccountId) { + T::PrimeOrigin::ensure_origin(origin)?; Self::members().binary_search(&who).ok().ok_or(Error::::NotMember)?; Prime::::put(&who); T::MembershipChanged::set_prime(Some(who)); } /// Remove the prime member if it exists. + /// + /// May only be called from `T::PrimeOrigin`. #[weight = 50_000_000] - fn clear_prime(origin) { - T::PrimeOrigin::try_origin(origin) - .map(|_| ()) - .or_else(ensure_root)?; + pub fn clear_prime(origin) { + T::PrimeOrigin::ensure_origin(origin)?; Prime::::kill(); T::MembershipChanged::set_prime(None); } @@ -272,6 +264,16 @@ impl, I: Instance> Module { } } +impl, I: Instance> Contains for Module { + fn sorted_members() -> Vec { + Self::members() + } + + fn count() -> usize { + Members::::decode_len().unwrap_or(0) + } +} + #[cfg(test)] mod tests { use super::*; @@ -303,6 +305,7 @@ mod tests { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -318,6 +321,7 @@ mod tests { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/metadata/Cargo.toml b/frame/metadata/Cargo.toml index bbf2839341b34febf444b83518445a4efb3f5e52..ae9cf736e957f2d626c2819e3786a51eff3f4bba 100644 --- a/frame/metadata/Cargo.toml +++ b/frame/metadata/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-metadata" -version = "11.0.0-alpha.8" +version = "11.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,10 +12,10 @@ description = "Decodable variant of the RuntimeMetadata." targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } [features] default = ["std"] diff --git a/frame/multisig/Cargo.toml b/frame/multisig/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..d0b79bf4e329e245e8dc98bbcf3bce0862d0b230 --- /dev/null +++ b/frame/multisig/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "pallet-multisig" +version = "2.0.0-rc4" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME multi-signature dispatch pallet" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +serde = { version = "1.0.101", optional = true } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } + +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } + +[dev-dependencies] +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } + +[features] +default = ["std"] +std = [ + "serde", + "codec/std", + "sp-runtime/std", + "frame-support/std", + "frame-system/std", + "sp-io/std", + "sp-std/std" +] +runtime-benchmarks = [ + "frame-benchmarking", + "frame-support/runtime-benchmarks", +] diff --git a/frame/multisig/src/benchmarking.rs b/frame/multisig/src/benchmarking.rs new file mode 100644 index 0000000000000000000000000000000000000000..8113d179cd1576c43dfb0418dbcf94327f63b44b --- /dev/null +++ b/frame/multisig/src/benchmarking.rs @@ -0,0 +1,289 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Benchmarks for Multisig Pallet + +#![cfg(feature = "runtime-benchmarks")] + +use super::*; +use frame_system::RawOrigin; +use frame_benchmarking::{benchmarks, account}; +use sp_runtime::traits::Bounded; +use core::convert::TryInto; + +use crate::Module as Multisig; + +const SEED: u32 = 0; + +fn setup_multi(s: u32, z: u32) + -> Result<(Vec, Vec), &'static str> +{ + let mut signatories: Vec = Vec::new(); + for i in 0 .. s { + let signatory = account("signatory", i, SEED); + // Give them some balance for a possible deposit + let balance = BalanceOf::::max_value(); + T::Currency::make_free_balance_be(&signatory, balance); + signatories.push(signatory); + } + signatories.sort(); + // Must first convert to outer call type. + let call: ::Call = frame_system::Call::::remark(vec![0; z as usize]).into(); + let call_data = call.encode(); + return Ok((signatories, call_data)) +} + +benchmarks! { + _ { } + + as_multi_threshold_1 { + // Transaction Length + let z in 0 .. 10_000; + let max_signatories = T::MaxSignatories::get().into(); + let (mut signatories, _) = setup_multi::(max_signatories, z)?; + let call: ::Call = frame_system::Call::::remark(vec![0; z as usize]).into(); + let call_hash = call.using_encoded(blake2_256); + let multi_account_id = Multisig::::multi_account_id(&signatories, 1); + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + }: _(RawOrigin::Signed(caller.clone()), signatories, Box::new(call)) + verify { + // If the benchmark resolves, then the call was dispatched successfully. + } + + as_multi_create { + // Signatories, need at least 2 total people + let s in 2 .. T::MaxSignatories::get() as u32; + // Transaction Length + let z in 0 .. 10_000; + let (mut signatories, call) = setup_multi::(s, z)?; + let call_hash = blake2_256(&call); + let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + }: as_multi(RawOrigin::Signed(caller), s as u16, signatories, None, call, false, 0) + verify { + assert!(Multisigs::::contains_key(multi_account_id, call_hash)); + } + + as_multi_create_store { + // Signatories, need at least 2 total people + let s in 2 .. T::MaxSignatories::get() as u32; + // Transaction Length + let z in 0 .. 10_000; + let (mut signatories, call) = setup_multi::(s, z)?; + let call_hash = blake2_256(&call); + let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + }: as_multi(RawOrigin::Signed(caller), s as u16, signatories, None, call, true, 0) + verify { + assert!(Multisigs::::contains_key(multi_account_id, call_hash)); + assert!(Calls::::contains_key(call_hash)); + } + + as_multi_approve { + // Signatories, need at least 3 people (so we don't complete the multisig) + let s in 3 .. T::MaxSignatories::get() as u32; + // Transaction Length + let z in 0 .. 10_000; + let (mut signatories, call) = setup_multi::(s, z)?; + let call_hash = blake2_256(&call); + let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); + let mut signatories2 = signatories.clone(); + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + // before the call, get the timepoint + let timepoint = Multisig::::timepoint(); + // Create the multi, storing for worst case + Multisig::::as_multi(RawOrigin::Signed(caller).into(), s as u16, signatories, None, call.clone(), true, 0)?; + let caller2 = signatories2.remove(0); + }: as_multi(RawOrigin::Signed(caller2), s as u16, signatories2, Some(timepoint), call, false, 0) + verify { + let multisig = Multisigs::::get(multi_account_id, call_hash).ok_or("multisig not created")?; + assert_eq!(multisig.approvals.len(), 2); + } + + as_multi_complete { + // Signatories, need at least 2 people + let s in 2 .. T::MaxSignatories::get() as u32; + // Transaction Length + let z in 0 .. 10_000; + let (mut signatories, call) = setup_multi::(s, z)?; + let call_hash = blake2_256(&call); + let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); + let mut signatories2 = signatories.clone(); + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + // before the call, get the timepoint + let timepoint = Multisig::::timepoint(); + // Create the multi, storing it for worst case + Multisig::::as_multi(RawOrigin::Signed(caller).into(), s as u16, signatories, None, call.clone(), true, 0)?; + // Everyone except the first person approves + for i in 1 .. s - 1 { + let mut signatories_loop = signatories2.clone(); + let caller_loop = signatories_loop.remove(i as usize); + let o = RawOrigin::Signed(caller_loop).into(); + Multisig::::as_multi(o, s as u16, signatories_loop, Some(timepoint), call.clone(), false, 0)?; + } + let caller2 = signatories2.remove(0); + assert!(Multisigs::::contains_key(&multi_account_id, call_hash)); + }: as_multi(RawOrigin::Signed(caller2), s as u16, signatories2, Some(timepoint), call, false, Weight::max_value()) + verify { + assert!(!Multisigs::::contains_key(&multi_account_id, call_hash)); + } + + approve_as_multi_create { + // Signatories, need at least 2 people + let s in 2 .. T::MaxSignatories::get() as u32; + // Transaction Length + let z in 0 .. 10_000; + let (mut signatories, call) = setup_multi::(s, z)?; + let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + let call_hash = blake2_256(&call); + // Create the multi + }: approve_as_multi(RawOrigin::Signed(caller), s as u16, signatories, None, call_hash, 0) + verify { + assert!(Multisigs::::contains_key(multi_account_id, call_hash)); + } + + approve_as_multi_approve { + // Signatories, need at least 2 people + let s in 2 .. T::MaxSignatories::get() as u32; + // Transaction Length + let z in 0 .. 10_000; + let (mut signatories, call) = setup_multi::(s, z)?; + let mut signatories2 = signatories.clone(); + let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + let call_hash = blake2_256(&call); + // before the call, get the timepoint + let timepoint = Multisig::::timepoint(); + // Create the multi + Multisig::::as_multi( + RawOrigin::Signed(caller.clone()).into(), + s as u16, + signatories, + None, + call.clone(), + false, + 0 + )?; + let caller2 = signatories2.remove(0); + }: approve_as_multi(RawOrigin::Signed(caller2), s as u16, signatories2, Some(timepoint), call_hash, 0) + verify { + let multisig = Multisigs::::get(multi_account_id, call_hash).ok_or("multisig not created")?; + assert_eq!(multisig.approvals.len(), 2); + } + + approve_as_multi_complete { + // Signatories, need at least 2 people + let s in 2 .. T::MaxSignatories::get() as u32; + // Transaction Length + let z in 0 .. 10_000; + let (mut signatories, call) = setup_multi::(s, z)?; + let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); + let mut signatories2 = signatories.clone(); + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + let call_hash = blake2_256(&call); + // before the call, get the timepoint + let timepoint = Multisig::::timepoint(); + // Create the multi + Multisig::::as_multi(RawOrigin::Signed(caller).into(), s as u16, signatories, None, call.clone(), true, 0)?; + // Everyone except the first person approves + for i in 1 .. s - 1 { + let mut signatories_loop = signatories2.clone(); + let caller_loop = signatories_loop.remove(i as usize); + let o = RawOrigin::Signed(caller_loop).into(); + Multisig::::as_multi(o, s as u16, signatories_loop, Some(timepoint), call.clone(), false, 0)?; + } + let caller2 = signatories2.remove(0); + assert!(Multisigs::::contains_key(&multi_account_id, call_hash)); + }: approve_as_multi( + RawOrigin::Signed(caller2), + s as u16, + signatories2, + Some(timepoint), + call_hash, + Weight::max_value() + ) + verify { + assert!(!Multisigs::::contains_key(multi_account_id, call_hash)); + } + + cancel_as_multi { + // Signatories, need at least 2 people + let s in 2 .. T::MaxSignatories::get() as u32; + // Transaction Length + let z in 0 .. 10_000; + let (mut signatories, call) = setup_multi::(s, z)?; + let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + let call_hash = blake2_256(&call); + let timepoint = Multisig::::timepoint(); + // Create the multi + let o = RawOrigin::Signed(caller.clone()).into(); + Multisig::::as_multi(o, s as u16, signatories.clone(), None, call.clone(), true, 0)?; + assert!(Multisigs::::contains_key(&multi_account_id, call_hash)); + }: _(RawOrigin::Signed(caller), s as u16, signatories, timepoint, call_hash) + verify { + assert!(!Multisigs::::contains_key(multi_account_id, call_hash)); + } + + cancel_as_multi_store { + // Signatories, need at least 2 people + let s in 2 .. T::MaxSignatories::get() as u32; + // Transaction Length + let z in 0 .. 10_000; + let (mut signatories, call) = setup_multi::(s, z)?; + let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); + let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + let call_hash = blake2_256(&call); + let timepoint = Multisig::::timepoint(); + // Create the multi + let o = RawOrigin::Signed(caller.clone()).into(); + Multisig::::as_multi(o, s as u16, signatories.clone(), None, call.clone(), true, 0)?; + assert!(Multisigs::::contains_key(&multi_account_id, call_hash)); + assert!(Calls::::contains_key(call_hash)); + }: cancel_as_multi(RawOrigin::Signed(caller), s as u16, signatories, timepoint, call_hash) + verify { + assert!(!Multisigs::::contains_key(&multi_account_id, call_hash)); + assert!(!Calls::::contains_key(call_hash)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_as_multi_threshold_1::()); + assert_ok!(test_benchmark_as_multi_create::()); + assert_ok!(test_benchmark_as_multi_create_store::()); + assert_ok!(test_benchmark_as_multi_approve::()); + assert_ok!(test_benchmark_as_multi_complete::()); + assert_ok!(test_benchmark_approve_as_multi_create::()); + assert_ok!(test_benchmark_approve_as_multi_approve::()); + assert_ok!(test_benchmark_approve_as_multi_complete::()); + assert_ok!(test_benchmark_cancel_as_multi::()); + assert_ok!(test_benchmark_cancel_as_multi_store::()); + }); + } +} diff --git a/frame/multisig/src/lib.rs b/frame/multisig/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..bcea34f9b36fa224071c91b854ff82823c8d54c8 --- /dev/null +++ b/frame/multisig/src/lib.rs @@ -0,0 +1,700 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Multisig Module +//! A module for doing multisig dispatch. +//! +//! - [`multisig::Trait`](./trait.Trait.html) +//! - [`Call`](./enum.Call.html) +//! +//! ## Overview +//! +//! This module contains functionality for multi-signature dispatch, a (potentially) stateful +//! operation, allowing multiple signed +//! origins (accounts) to coordinate and dispatch a call from a well-known origin, derivable +//! deterministically from the set of account IDs and the threshold number of accounts from the +//! set that must approve it. In the case that the threshold is just one then this is a stateless +//! operation. This is useful for multisig wallets where cryptographic threshold signatures are +//! not available or desired. +//! +//! ## Interface +//! +//! ### Dispatchable Functions +//! +//! * `as_multi` - Approve and if possible dispatch a call from a composite origin formed from a +//! number of signed origins. +//! * `approve_as_multi` - Approve a call from a composite origin. +//! * `cancel_as_multi` - Cancel a call from a composite origin. +//! +//! [`Call`]: ./enum.Call.html +//! [`Trait`]: ./trait.Trait.html + +// Ensure we're `no_std` when compiling for Wasm. +#![cfg_attr(not(feature = "std"), no_std)] + +use sp_std::prelude::*; +use codec::{Encode, Decode}; +use sp_io::hashing::blake2_256; +use frame_support::{decl_module, decl_event, decl_error, decl_storage, Parameter, ensure, RuntimeDebug}; +use frame_support::{traits::{Get, ReservableCurrency, Currency}, + weights::{Weight, GetDispatchInfo, constants::{WEIGHT_PER_NANOS, WEIGHT_PER_MICROS}}, + dispatch::{DispatchResultWithPostInfo, DispatchErrorWithPostInfo, PostDispatchInfo}, +}; +use frame_system::{self as system, ensure_signed, RawOrigin}; +use sp_runtime::{DispatchError, DispatchResult, traits::{Dispatchable, Zero}}; + +mod tests; +mod benchmarking; + +type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; + +/// Configuration trait. +pub trait Trait: frame_system::Trait { + /// The overarching event type. + type Event: From> + Into<::Event>; + + /// The overarching call type. + type Call: Parameter + Dispatchable + + GetDispatchInfo + From>; + + /// The currency mechanism. + type Currency: ReservableCurrency; + + /// The base amount of currency needed to reserve for creating a multisig execution or to store + /// a dispatch call for later. + /// + /// This is held for an additional storage item whose value size is + /// `4 + sizeof((BlockNumber, Balance, AccountId))` bytes and whose key size is + /// `32 + sizeof(AccountId)` bytes. + type DepositBase: Get>; + + /// The amount of currency needed per unit threshold when creating a multisig execution. + /// + /// This is held for adding 32 bytes more into a pre-existing storage value. + type DepositFactor: Get>; + + /// The maximum amount of signatories allowed in the multisig. + type MaxSignatories: Get; +} + +/// A global extrinsic index, formed as the extrinsic index within a block, together with that +/// block's height. This allows a transaction in which a multisig operation of a particular +/// composite was created to be uniquely identified. +#[derive(Copy, Clone, Eq, PartialEq, Encode, Decode, Default, RuntimeDebug)] +pub struct Timepoint { + /// The height of the chain at the point in time. + height: BlockNumber, + /// The index of the extrinsic at the point in time. + index: u32, +} + +/// An open multisig operation. +#[derive(Clone, Eq, PartialEq, Encode, Decode, Default, RuntimeDebug)] +pub struct Multisig { + /// The extrinsic when the multisig operation was opened. + when: Timepoint, + /// The amount held in reserve of the `depositor`, to be returned once the operation ends. + deposit: Balance, + /// The account who opened it (i.e. the first to approve it). + depositor: AccountId, + /// The approvals achieved so far, including the depositor. Always sorted. + approvals: Vec, +} + +decl_storage! { + trait Store for Module as Multisig { + /// The set of open multisig operations. + pub Multisigs: double_map + hasher(twox_64_concat) T::AccountId, hasher(blake2_128_concat) [u8; 32] + => Option, T::AccountId>>; + + pub Calls: map hasher(identity) [u8; 32] => Option<(Vec, T::AccountId, BalanceOf)>; + } +} + +decl_error! { + pub enum Error for Module { + /// Threshold must be 2 or greater. + MinimumThreshold, + /// Call is already approved by this signatory. + AlreadyApproved, + /// Call doesn't need any (more) approvals. + NoApprovalsNeeded, + /// There are too few signatories in the list. + TooFewSignatories, + /// There are too many signatories in the list. + TooManySignatories, + /// The signatories were provided out of order; they should be ordered. + SignatoriesOutOfOrder, + /// The sender was contained in the other signatories; it shouldn't be. + SenderInSignatories, + /// Multisig operation not found when attempting to cancel. + NotFound, + /// Only the account that originally created the multisig is able to cancel it. + NotOwner, + /// No timepoint was given, yet the multisig operation is already underway. + NoTimepoint, + /// A different timepoint was given to the multisig operation that is underway. + WrongTimepoint, + /// A timepoint was given, yet no multisig operation is underway. + UnexpectedTimepoint, + /// The maximum weight information provided was too low. + WeightTooLow, + /// The data to be stored is already stored. + AlreadyStored, + } +} + +decl_event! { + /// Events type. + pub enum Event where + AccountId = ::AccountId, + BlockNumber = ::BlockNumber, + CallHash = [u8; 32] + { + /// A new multisig operation has begun. First param is the account that is approving, + /// second is the multisig account, third is hash of the call. + NewMultisig(AccountId, AccountId, CallHash), + /// A multisig operation has been approved by someone. First param is the account that is + /// approving, third is the multisig account, fourth is hash of the call. + MultisigApproval(AccountId, Timepoint, AccountId, CallHash), + /// A multisig operation has been executed. First param is the account that is + /// approving, third is the multisig account, fourth is hash of the call to be executed. + MultisigExecuted(AccountId, Timepoint, AccountId, CallHash, DispatchResult), + /// A multisig operation has been cancelled. First param is the account that is + /// cancelling, third is the multisig account, fourth is hash of the call. + MultisigCancelled(AccountId, Timepoint, AccountId, CallHash), + } +} + +mod weight_of { + use super::*; + + /// - Base Weight: 33.72 + 0.002 * Z µs + /// - DB Weight: None + /// - Plus Call Weight + pub fn as_multi_threshold_1( + call_len: usize, + call_weight: Weight, + ) -> Weight { + (34 * WEIGHT_PER_MICROS) + .saturating_add((2 * WEIGHT_PER_NANOS).saturating_mul(call_len as Weight)) + .saturating_add(call_weight) + } + + /// - Base Weight: + /// - Create: 38.82 + 0.121 * S + .001 * Z µs + /// - Create w/ Store: 54.22 + 0.120 * S + .003 * Z µs + /// - Approve: 29.86 + 0.143 * S + .001 * Z µs + /// - Complete: 39.55 + 0.267 * S + .002 * Z µs + /// - DB Weight: + /// - Reads: Multisig Storage, [Caller Account], Calls, Depositor Account + /// - Writes: Multisig Storage, [Caller Account], Calls, Depositor Account + /// - Plus Call Weight + pub fn as_multi( + sig_len: usize, + call_len: usize, + call_weight: Weight, + calls_write: bool, + refunded: bool, + ) -> Weight { + call_weight + .saturating_add(55 * WEIGHT_PER_MICROS) + .saturating_add((250 * WEIGHT_PER_NANOS).saturating_mul(sig_len as Weight)) + .saturating_add((3 * WEIGHT_PER_NANOS).saturating_mul(call_len as Weight)) + .saturating_add(T::DbWeight::get().reads_writes(1, 1)) // Multisig read/write + .saturating_add(T::DbWeight::get().reads(1)) // Calls read + .saturating_add(T::DbWeight::get().writes(calls_write.into())) // Calls write + .saturating_add(T::DbWeight::get().reads_writes(refunded.into(), refunded.into())) // Deposit refunded + } +} + +enum CallOrHash { + Call(Vec, bool), + Hash([u8; 32]), +} + +decl_module! { + pub struct Module for enum Call where origin: T::Origin { + type Error = Error; + + /// Deposit one of this module's events by using the default implementation. + fn deposit_event() = default; + + /// Immediately dispatch a multi-signature call using a single approval from the caller. + /// + /// The dispatch origin for this call must be _Signed_. + /// + /// - `other_signatories`: The accounts (other than the sender) who are part of the + /// multi-signature, but do not participate in the approval process. + /// - `call`: The call to be executed. + /// + /// Result is equivalent to the dispatched result. + /// + /// # + /// O(Z + C) where Z is the length of the call and C its execution weight. + /// ------------------------------- + /// - Base Weight: 33.72 + 0.002 * Z µs + /// - DB Weight: None + /// - Plus Call Weight + /// # + #[weight = ( + weight_of::as_multi_threshold_1::( + call.using_encoded(|c| c.len()), + call.get_dispatch_info().weight + ), + call.get_dispatch_info().class, + )] + fn as_multi_threshold_1(origin, + other_signatories: Vec, + call: Box<::Call>, + ) -> DispatchResultWithPostInfo { + let who = ensure_signed(origin)?; + let max_sigs = T::MaxSignatories::get() as usize; + ensure!(!other_signatories.is_empty(), Error::::TooFewSignatories); + let other_signatories_len = other_signatories.len(); + ensure!(other_signatories_len < max_sigs, Error::::TooManySignatories); + let signatories = Self::ensure_sorted_and_insert(other_signatories, who.clone())?; + + let id = Self::multi_account_id(&signatories, 1); + + let call_len = call.using_encoded(|c| c.len()); + let result = call.dispatch(RawOrigin::Signed(id.clone()).into()); + + result.map(|post_dispatch_info| post_dispatch_info.actual_weight + .map(|actual_weight| weight_of::as_multi_threshold_1::( + call_len, + actual_weight, + )) + .into() + ).map_err(|err| match err.post_info.actual_weight { + Some(actual_weight) => { + let weight_used = weight_of::as_multi_threshold_1::( + call_len, + actual_weight, + ); + let post_info = Some(weight_used).into(); + let error = err.error.into(); + DispatchErrorWithPostInfo { post_info, error } + }, + None => err, + }) + } + + /// Register approval for a dispatch to be made from a deterministic composite account if + /// approved by a total of `threshold - 1` of `other_signatories`. + /// + /// If there are enough, then dispatch the call. + /// + /// Payment: `DepositBase` will be reserved if this is the first approval, plus + /// `threshold` times `DepositFactor`. It is returned once this dispatch happens or + /// is cancelled. + /// + /// The dispatch origin for this call must be _Signed_. + /// + /// - `threshold`: The total number of approvals for this dispatch before it is executed. + /// - `other_signatories`: The accounts (other than the sender) who can approve this + /// dispatch. May not be empty. + /// - `maybe_timepoint`: If this is the first approval, then this must be `None`. If it is + /// not the first approval, then it must be `Some`, with the timepoint (block number and + /// transaction index) of the first approval transaction. + /// - `call`: The call to be executed. + /// + /// NOTE: Unless this is the final approval, you will generally want to use + /// `approve_as_multi` instead, since it only requires a hash of the call. + /// + /// Result is equivalent to the dispatched result if `threshold` is exactly `1`. Otherwise + /// on success, result is `Ok` and the result from the interior call, if it was executed, + /// may be found in the deposited `MultisigExecuted` event. + /// + /// # + /// - `O(S + Z + Call)`. + /// - Up to one balance-reserve or unreserve operation. + /// - One passthrough operation, one insert, both `O(S)` where `S` is the number of + /// signatories. `S` is capped by `MaxSignatories`, with weight being proportional. + /// - One call encode & hash, both of complexity `O(Z)` where `Z` is tx-len. + /// - One encode & hash, both of complexity `O(S)`. + /// - Up to one binary search and insert (`O(logS + S)`). + /// - I/O: 1 read `O(S)`, up to 1 mutate `O(S)`. Up to one remove. + /// - One event. + /// - The weight of the `call`. + /// - Storage: inserts one item, value size bounded by `MaxSignatories`, with a + /// deposit taken for its lifetime of + /// `DepositBase + threshold * DepositFactor`. + /// ------------------------------- + /// - Base Weight: + /// - Create: 41.89 + 0.118 * S + .002 * Z µs + /// - Create w/ Store: 53.57 + 0.119 * S + .003 * Z µs + /// - Approve: 31.39 + 0.136 * S + .002 * Z µs + /// - Complete: 39.94 + 0.26 * S + .002 * Z µs + /// - DB Weight: + /// - Reads: Multisig Storage, [Caller Account], Calls (if `store_call`) + /// - Writes: Multisig Storage, [Caller Account], Calls (if `store_call`) + /// - Plus Call Weight + /// # + #[weight = weight_of::as_multi::( + other_signatories.len(), + call.len(), + *max_weight, + true, // assume worst case: calls write + true, // assume worst case: refunded + )] + fn as_multi(origin, + threshold: u16, + other_signatories: Vec, + maybe_timepoint: Option>, + call: Vec, + store_call: bool, + max_weight: Weight, + ) -> DispatchResultWithPostInfo { + let who = ensure_signed(origin)?; + Self::operate(who, threshold, other_signatories, maybe_timepoint, CallOrHash::Call(call, store_call), max_weight) + } + + /// Register approval for a dispatch to be made from a deterministic composite account if + /// approved by a total of `threshold - 1` of `other_signatories`. + /// + /// Payment: `DepositBase` will be reserved if this is the first approval, plus + /// `threshold` times `DepositFactor`. It is returned once this dispatch happens or + /// is cancelled. + /// + /// The dispatch origin for this call must be _Signed_. + /// + /// - `threshold`: The total number of approvals for this dispatch before it is executed. + /// - `other_signatories`: The accounts (other than the sender) who can approve this + /// dispatch. May not be empty. + /// - `maybe_timepoint`: If this is the first approval, then this must be `None`. If it is + /// not the first approval, then it must be `Some`, with the timepoint (block number and + /// transaction index) of the first approval transaction. + /// - `call_hash`: The hash of the call to be executed. + /// + /// NOTE: If this is the final approval, you will want to use `as_multi` instead. + /// + /// # + /// - `O(S)`. + /// - Up to one balance-reserve or unreserve operation. + /// - One passthrough operation, one insert, both `O(S)` where `S` is the number of + /// signatories. `S` is capped by `MaxSignatories`, with weight being proportional. + /// - One encode & hash, both of complexity `O(S)`. + /// - Up to one binary search and insert (`O(logS + S)`). + /// - I/O: 1 read `O(S)`, up to 1 mutate `O(S)`. Up to one remove. + /// - One event. + /// - Storage: inserts one item, value size bounded by `MaxSignatories`, with a + /// deposit taken for its lifetime of + /// `DepositBase + threshold * DepositFactor`. + /// ---------------------------------- + /// - Base Weight: + /// - Create: 44.71 + 0.088 * S + /// - Approve: 31.48 + 0.116 * S + /// - DB Weight: + /// - Read: Multisig Storage, [Caller Account] + /// - Write: Multisig Storage, [Caller Account] + /// # + #[weight = weight_of::as_multi::( + other_signatories.len(), + 0, // call_len is zero in this case + *max_weight, + true, // assume worst case: calls write + true, // assume worst case: refunded + )] + fn approve_as_multi(origin, + threshold: u16, + other_signatories: Vec, + maybe_timepoint: Option>, + call_hash: [u8; 32], + max_weight: Weight, + ) -> DispatchResultWithPostInfo { + let who = ensure_signed(origin)?; + Self::operate(who, threshold, other_signatories, maybe_timepoint, CallOrHash::Hash(call_hash), max_weight) + } + + /// Cancel a pre-existing, on-going multisig transaction. Any deposit reserved previously + /// for this operation will be unreserved on success. + /// + /// The dispatch origin for this call must be _Signed_. + /// + /// - `threshold`: The total number of approvals for this dispatch before it is executed. + /// - `other_signatories`: The accounts (other than the sender) who can approve this + /// dispatch. May not be empty. + /// - `timepoint`: The timepoint (block number and transaction index) of the first approval + /// transaction for this dispatch. + /// - `call_hash`: The hash of the call to be executed. + /// + /// # + /// - `O(S)`. + /// - Up to one balance-reserve or unreserve operation. + /// - One passthrough operation, one insert, both `O(S)` where `S` is the number of + /// signatories. `S` is capped by `MaxSignatories`, with weight being proportional. + /// - One encode & hash, both of complexity `O(S)`. + /// - One event. + /// - I/O: 1 read `O(S)`, one remove. + /// - Storage: removes one item. + /// ---------------------------------- + /// - Base Weight: 36.07 + 0.124 * S + /// - DB Weight: + /// - Read: Multisig Storage, [Caller Account], Refund Account, Calls + /// - Write: Multisig Storage, [Caller Account], Refund Account, Calls + /// # + #[weight = T::DbWeight::get().reads_writes(3, 3) + .saturating_add(36 * WEIGHT_PER_MICROS) + .saturating_add((other_signatories.len() as Weight).saturating_mul(100 * WEIGHT_PER_NANOS)) + ] + fn cancel_as_multi(origin, + threshold: u16, + other_signatories: Vec, + timepoint: Timepoint, + call_hash: [u8; 32], + ) -> DispatchResult { + let who = ensure_signed(origin)?; + ensure!(threshold >= 2, Error::::MinimumThreshold); + let max_sigs = T::MaxSignatories::get() as usize; + ensure!(!other_signatories.is_empty(), Error::::TooFewSignatories); + ensure!(other_signatories.len() < max_sigs, Error::::TooManySignatories); + let signatories = Self::ensure_sorted_and_insert(other_signatories, who.clone())?; + + let id = Self::multi_account_id(&signatories, threshold); + + let m = >::get(&id, call_hash) + .ok_or(Error::::NotFound)?; + ensure!(m.when == timepoint, Error::::WrongTimepoint); + ensure!(m.depositor == who, Error::::NotOwner); + + let _ = T::Currency::unreserve(&m.depositor, m.deposit); + >::remove(&id, &call_hash); + Self::clear_call(&call_hash); + + Self::deposit_event(RawEvent::MultisigCancelled(who, timepoint, id, call_hash)); + Ok(()) + } + } +} + +impl Module { + /// Derive a multi-account ID from the sorted list of accounts and the threshold that are + /// required. + /// + /// NOTE: `who` must be sorted. If it is not, then you'll get the wrong answer. + pub fn multi_account_id(who: &[T::AccountId], threshold: u16) -> T::AccountId { + let entropy = (b"modlpy/utilisuba", who, threshold).using_encoded(blake2_256); + T::AccountId::decode(&mut &entropy[..]).unwrap_or_default() + } + + fn operate( + who: T::AccountId, + threshold: u16, + other_signatories: Vec, + maybe_timepoint: Option>, + call_or_hash: CallOrHash, + max_weight: Weight, + ) -> DispatchResultWithPostInfo { + ensure!(threshold >= 2, Error::::MinimumThreshold); + let max_sigs = T::MaxSignatories::get() as usize; + ensure!(!other_signatories.is_empty(), Error::::TooFewSignatories); + let other_signatories_len = other_signatories.len(); + ensure!(other_signatories_len < max_sigs, Error::::TooManySignatories); + let signatories = Self::ensure_sorted_and_insert(other_signatories, who.clone())?; + + let id = Self::multi_account_id(&signatories, threshold); + + // Threshold > 1; this means it's a multi-step operation. We extract the `call_hash`. + let (call_hash, call_len, maybe_call, store) = match call_or_hash { + CallOrHash::Call(call, should_store) => { + let call_hash = blake2_256(&call); + let call_len = call.len(); + (call_hash, call_len, Some(call), should_store) + } + CallOrHash::Hash(h) => (h, 0, None, false), + }; + + // Branch on whether the operation has already started or not. + if let Some(mut m) = >::get(&id, call_hash) { + // Yes; ensure that the timepoint exists and agrees. + let timepoint = maybe_timepoint.ok_or(Error::::NoTimepoint)?; + ensure!(m.when == timepoint, Error::::WrongTimepoint); + + // Ensure that either we have not yet signed or that it is at threshold. + let mut approvals = m.approvals.len() as u16; + // We only bother with the approval if we're below threshold. + let maybe_pos = m.approvals.binary_search(&who).err().filter(|_| approvals < threshold); + // Bump approvals if not yet voted and the vote is needed. + if maybe_pos.is_some() { approvals += 1; } + + // We only bother fetching/decoding call if we know that we're ready to execute. + let maybe_approved_call = if approvals >= threshold { + Self::get_call(&call_hash, maybe_call.as_ref().map(|c| c.as_ref())) + } else { None }; + + if let Some(call) = maybe_approved_call { + // verify weight + ensure!(call.get_dispatch_info().weight <= max_weight, Error::::WeightTooLow); + + // Clean up storage before executing call to avoid an possibility of reentrancy + // attack. + >::remove(&id, call_hash); + Self::clear_call(&call_hash); + T::Currency::unreserve(&m.depositor, m.deposit); + + let result = call.dispatch(RawOrigin::Signed(id.clone()).into()); + Self::deposit_event(RawEvent::MultisigExecuted( + who, timepoint, id, call_hash, result.map(|_| ()).map_err(|e| e.error) + )); + Ok(get_result_weight(result).map(|actual_weight| weight_of::as_multi::( + other_signatories_len, + call_len, + actual_weight, + true, // Call is removed + true, // User is refunded + )).into()) + } else { + // We cannot dispatch the call now; either it isn't available, or it is, but we + // don't have threshold approvals even with our signature. + + // Store the call if desired. + let stored = if let Some(data) = maybe_call.filter(|_| store) { + Self::store_call_and_reserve(who.clone(), &call_hash, data, BalanceOf::::zero())?; + true + } else { + false + }; + + if let Some(pos) = maybe_pos { + // Record approval. + m.approvals.insert(pos, who.clone()); + >::insert(&id, call_hash, m); + Self::deposit_event(RawEvent::MultisigApproval(who, timepoint, id, call_hash)); + } else { + // If we already approved and didn't store the Call, then this was useless and + // we report an error. + ensure!(stored, Error::::AlreadyApproved); + } + + // Call is not made, so the actual weight does not include call + Ok(Some(weight_of::as_multi::( + other_signatories_len, + call_len, + 0, + stored, // Call stored? + false, // No refund + )).into()) + } + } else { + // Not yet started; there should be no timepoint given. + ensure!(maybe_timepoint.is_none(), Error::::UnexpectedTimepoint); + + // Just start the operation by recording it in storage. + let deposit = T::DepositBase::get() + T::DepositFactor::get() * threshold.into(); + + // Store the call if desired. + let stored = if let Some(data) = maybe_call.filter(|_| store) { + Self::store_call_and_reserve(who.clone(), &call_hash, data, deposit)?; + true + } else { + T::Currency::reserve(&who, deposit)?; + false + }; + + >::insert(&id, call_hash, Multisig { + when: Self::timepoint(), + deposit, + depositor: who.clone(), + approvals: vec![who.clone()], + }); + Self::deposit_event(RawEvent::NewMultisig(who, id, call_hash)); + // Call is not made, so we can return that weight + return Ok(Some(weight_of::as_multi::( + other_signatories_len, + call_len, + 0, + stored, // Call stored? + false, // No refund + )).into()) + } + } + + /// Place a call's encoded data in storage, reserving funds as appropriate. + /// + /// We store `data` here because storing `call` would result in needing another `.encode`. + /// + /// Returns a `bool` indicating whether the data did end up being stored. + fn store_call_and_reserve(who: T::AccountId, hash: &[u8; 32], data: Vec, other_deposit: BalanceOf) + -> DispatchResult + { + ensure!(!Calls::::contains_key(hash), Error::::AlreadyStored); + let deposit = other_deposit + T::DepositBase::get() + + T::DepositFactor::get() * BalanceOf::::from(((data.len() + 31) / 32) as u32); + T::Currency::reserve(&who, deposit)?; + Calls::::insert(&hash, (data, who, deposit)); + Ok(()) + } + + /// Attempt to decode and return the call, provided by the user or from storage. + fn get_call(hash: &[u8; 32], maybe_known: Option<&[u8]>) -> Option<::Call> { + maybe_known.map_or_else(|| { + Calls::::get(hash).and_then(|(data, ..)| { + Decode::decode(&mut &data[..]).ok() + }) + }, |data| { + Decode::decode(&mut &data[..]).ok() + }) + } + + /// Attempt to remove a call from storage, returning any deposit on it to the owner. + fn clear_call(hash: &[u8; 32]) { + if let Some((_, who, deposit)) = Calls::::take(hash) { + T::Currency::unreserve(&who, deposit); + } + } + + /// The current `Timepoint`. + pub fn timepoint() -> Timepoint { + Timepoint { + height: >::block_number(), + index: >::extrinsic_index().unwrap_or_default(), + } + } + + /// Check that signatories is sorted and doesn't contain sender, then insert sender. + fn ensure_sorted_and_insert(other_signatories: Vec, who: T::AccountId) + -> Result, DispatchError> + { + let mut signatories = other_signatories; + let mut maybe_last = None; + let mut index = 0; + for item in signatories.iter() { + if let Some(last) = maybe_last { + ensure!(last < item, Error::::SignatoriesOutOfOrder); + } + if item <= &who { + ensure!(item != &who, Error::::SenderInSignatories); + index += 1; + } + maybe_last = Some(item); + } + signatories.insert(index, who); + Ok(signatories) + } +} + +/// Return the weight of a dispatch call result as an `Option`. +/// +/// Will return the weight regardless of what the state of the result is. +fn get_result_weight(result: DispatchResultWithPostInfo) -> Option { + match result { + Ok(post_info) => post_info.actual_weight, + Err(err) => err.post_info.actual_weight, + } +} diff --git a/frame/multisig/src/tests.rs b/frame/multisig/src/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..4911ca90cf33d937eb4487d54f82c67f5749d571 --- /dev/null +++ b/frame/multisig/src/tests.rs @@ -0,0 +1,574 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for Multisig Pallet + +#![cfg(test)] + +use super::*; + +use frame_support::{ + assert_ok, assert_noop, impl_outer_origin, parameter_types, impl_outer_dispatch, + weights::Weight, impl_outer_event, traits::Filter, +}; +use sp_core::H256; +use sp_runtime::{Perbill, traits::{BlakeTwo256, IdentityLookup}, testing::Header}; +use crate as multisig; + +impl_outer_origin! { + pub enum Origin for Test where system = frame_system {} +} + +impl_outer_event! { + pub enum TestEvent for Test { + system, + pallet_balances, + multisig, + } +} +impl_outer_dispatch! { + pub enum Call for Test where origin: Origin { + frame_system::System, + pallet_balances::Balances, + multisig::Multisig, + } +} + +// For testing the pallet, we construct most of a mock runtime. This means +// first constructing a configuration type (`Test`) which `impl`s each of the +// configuration traits of pallets we want to use. +#[derive(Clone, Eq, PartialEq)] +pub struct Test; +parameter_types! { + pub const BlockHashCount: u64 = 250; + pub const MaximumBlockWeight: Weight = 1024; + pub const MaximumBlockLength: u32 = 2 * 1024; + pub const AvailableBlockRatio: Perbill = Perbill::one(); +} +impl frame_system::Trait for Test { + type BaseCallFilter = TestBaseCallFilter; + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Call = Call; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = TestEvent; + type BlockHashCount = BlockHashCount; + type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; + type MaximumBlockLength = MaximumBlockLength; + type AvailableBlockRatio = AvailableBlockRatio; + type Version = (); + type ModuleToIndex = (); + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); +} +parameter_types! { + pub const ExistentialDeposit: u64 = 1; +} +impl pallet_balances::Trait for Test { + type Balance = u64; + type Event = TestEvent; + type DustRemoval = (); + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; +} +parameter_types! { + pub const DepositBase: u64 = 1; + pub const DepositFactor: u64 = 1; + pub const MaxSignatories: u16 = 3; +} +pub struct TestBaseCallFilter; +impl Filter for TestBaseCallFilter { + fn filter(c: &Call) -> bool { + match *c { + Call::Balances(_) => true, + // Needed for benchmarking + Call::System(frame_system::Call::remark(_)) => true, + _ => false, + } + } +} +impl Trait for Test { + type Event = TestEvent; + type Call = Call; + type Currency = Balances; + type DepositBase = DepositBase; + type DepositFactor = DepositFactor; + type MaxSignatories = MaxSignatories; +} +type System = frame_system::Module; +type Balances = pallet_balances::Module; +type Multisig = Module; + +use pallet_balances::Call as BalancesCall; +use pallet_balances::Error as BalancesError; + +pub fn new_test_ext() -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + pallet_balances::GenesisConfig:: { + balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 2)], + }.assimilate_storage(&mut t).unwrap(); + let mut ext = sp_io::TestExternalities::new(t); + ext.execute_with(|| System::set_block_number(1)); + ext +} + +fn last_event() -> TestEvent { + system::Module::::events().pop().map(|e| e.event).expect("Event expected") +} + +fn expect_event>(e: E) { + assert_eq!(last_event(), e.into()); +} + +fn now() -> Timepoint { + Multisig::timepoint() +} + +#[test] +fn multisig_deposit_is_taken_and_returned() { + new_test_ext().execute_with(|| { + let multi = Multisig::multi_account_id(&[1, 2, 3][..], 2); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Call::Balances(BalancesCall::transfer(6, 15)); + let call_weight = call.get_dispatch_info().weight; + let data = call.encode(); + assert_ok!(Multisig::as_multi(Origin::signed(1), 2, vec![2, 3], None, data.clone(), false, 0)); + assert_eq!(Balances::free_balance(1), 2); + assert_eq!(Balances::reserved_balance(1), 3); + + assert_ok!(Multisig::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), data, false, call_weight)); + assert_eq!(Balances::free_balance(1), 5); + assert_eq!(Balances::reserved_balance(1), 0); + }); +} + +#[test] +fn multisig_deposit_is_taken_and_returned_with_call_storage() { + new_test_ext().execute_with(|| { + let multi = Multisig::multi_account_id(&[1, 2, 3][..], 2); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Call::Balances(BalancesCall::transfer(6, 15)); + let call_weight = call.get_dispatch_info().weight; + let data = call.encode(); + let hash = blake2_256(&data); + assert_ok!(Multisig::as_multi(Origin::signed(1), 2, vec![2, 3], None, data, true, 0)); + assert_eq!(Balances::free_balance(1), 0); + assert_eq!(Balances::reserved_balance(1), 5); + + assert_ok!(Multisig::approve_as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), hash, call_weight)); + assert_eq!(Balances::free_balance(1), 5); + assert_eq!(Balances::reserved_balance(1), 0); + }); +} + +#[test] +fn multisig_deposit_is_taken_and_returned_with_alt_call_storage() { + new_test_ext().execute_with(|| { + let multi = Multisig::multi_account_id(&[1, 2, 3][..], 3); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Call::Balances(BalancesCall::transfer(6, 15)); + let call_weight = call.get_dispatch_info().weight; + let data = call.encode(); + let hash = blake2_256(&data); + + assert_ok!(Multisig::approve_as_multi(Origin::signed(1), 3, vec![2, 3], None, hash.clone(), 0)); + assert_eq!(Balances::free_balance(1), 1); + assert_eq!(Balances::reserved_balance(1), 4); + + assert_ok!(Multisig::as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), data, true, 0)); + assert_eq!(Balances::free_balance(2), 3); + assert_eq!(Balances::reserved_balance(2), 2); + assert_eq!(Balances::free_balance(1), 1); + assert_eq!(Balances::reserved_balance(1), 4); + + assert_ok!(Multisig::approve_as_multi(Origin::signed(3), 3, vec![1, 2], Some(now()), hash, call_weight)); + assert_eq!(Balances::free_balance(1), 5); + assert_eq!(Balances::reserved_balance(1), 0); + assert_eq!(Balances::free_balance(2), 5); + assert_eq!(Balances::reserved_balance(2), 0); + }); +} + +#[test] +fn cancel_multisig_returns_deposit() { + new_test_ext().execute_with(|| { + let call = Call::Balances(BalancesCall::transfer(6, 15)).encode(); + let hash = blake2_256(&call); + assert_ok!(Multisig::approve_as_multi(Origin::signed(1), 3, vec![2, 3], None, hash.clone(), 0)); + assert_ok!(Multisig::approve_as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), hash.clone(), 0)); + assert_eq!(Balances::free_balance(1), 6); + assert_eq!(Balances::reserved_balance(1), 4); + assert_ok!( + Multisig::cancel_as_multi(Origin::signed(1), 3, vec![2, 3], now(), hash.clone()), + ); + assert_eq!(Balances::free_balance(1), 10); + assert_eq!(Balances::reserved_balance(1), 0); + }); +} + +#[test] +fn timepoint_checking_works() { + new_test_ext().execute_with(|| { + let multi = Multisig::multi_account_id(&[1, 2, 3][..], 2); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Call::Balances(BalancesCall::transfer(6, 15)).encode(); + let hash = blake2_256(&call); + + assert_noop!( + Multisig::approve_as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), hash.clone(), 0), + Error::::UnexpectedTimepoint, + ); + + assert_ok!(Multisig::approve_as_multi(Origin::signed(1), 2, vec![2, 3], None, hash, 0)); + + assert_noop!( + Multisig::as_multi(Origin::signed(2), 2, vec![1, 3], None, call.clone(), false, 0), + Error::::NoTimepoint, + ); + let later = Timepoint { index: 1, .. now() }; + assert_noop!( + Multisig::as_multi(Origin::signed(2), 2, vec![1, 3], Some(later), call.clone(), false, 0), + Error::::WrongTimepoint, + ); + }); +} + +#[test] +fn multisig_2_of_3_works_with_call_storing() { + new_test_ext().execute_with(|| { + let multi = Multisig::multi_account_id(&[1, 2, 3][..], 2); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Call::Balances(BalancesCall::transfer(6, 15)); + let call_weight = call.get_dispatch_info().weight; + let data = call.encode(); + let hash = blake2_256(&data); + assert_ok!(Multisig::as_multi(Origin::signed(1), 2, vec![2, 3], None, data, true, 0)); + assert_eq!(Balances::free_balance(6), 0); + + assert_ok!(Multisig::approve_as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), hash, call_weight)); + assert_eq!(Balances::free_balance(6), 15); + }); +} + +#[test] +fn multisig_2_of_3_works() { + new_test_ext().execute_with(|| { + let multi = Multisig::multi_account_id(&[1, 2, 3][..], 2); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Call::Balances(BalancesCall::transfer(6, 15)); + let call_weight = call.get_dispatch_info().weight; + let data = call.encode(); + let hash = blake2_256(&data); + assert_ok!(Multisig::approve_as_multi(Origin::signed(1), 2, vec![2, 3], None, hash, 0)); + assert_eq!(Balances::free_balance(6), 0); + + assert_ok!(Multisig::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), data, false, call_weight)); + assert_eq!(Balances::free_balance(6), 15); + }); +} + +#[test] +fn multisig_3_of_3_works() { + new_test_ext().execute_with(|| { + let multi = Multisig::multi_account_id(&[1, 2, 3][..], 3); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Call::Balances(BalancesCall::transfer(6, 15)); + let call_weight = call.get_dispatch_info().weight; + let data = call.encode(); + let hash = blake2_256(&data); + assert_ok!(Multisig::approve_as_multi(Origin::signed(1), 3, vec![2, 3], None, hash.clone(), 0)); + assert_ok!(Multisig::approve_as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), hash.clone(), 0)); + assert_eq!(Balances::free_balance(6), 0); + + assert_ok!(Multisig::as_multi(Origin::signed(3), 3, vec![1, 2], Some(now()), data, false, call_weight)); + assert_eq!(Balances::free_balance(6), 15); + }); +} + +#[test] +fn cancel_multisig_works() { + new_test_ext().execute_with(|| { + let call = Call::Balances(BalancesCall::transfer(6, 15)).encode(); + let hash = blake2_256(&call); + assert_ok!(Multisig::approve_as_multi(Origin::signed(1), 3, vec![2, 3], None, hash.clone(), 0)); + assert_ok!(Multisig::approve_as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), hash.clone(), 0)); + assert_noop!( + Multisig::cancel_as_multi(Origin::signed(2), 3, vec![1, 3], now(), hash.clone()), + Error::::NotOwner, + ); + assert_ok!( + Multisig::cancel_as_multi(Origin::signed(1), 3, vec![2, 3], now(), hash.clone()), + ); + }); +} + +#[test] +fn cancel_multisig_with_call_storage_works() { + new_test_ext().execute_with(|| { + let call = Call::Balances(BalancesCall::transfer(6, 15)).encode(); + let hash = blake2_256(&call); + assert_ok!(Multisig::as_multi(Origin::signed(1), 3, vec![2, 3], None, call, true, 0)); + assert_eq!(Balances::free_balance(1), 4); + assert_ok!(Multisig::approve_as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), hash.clone(), 0)); + assert_noop!( + Multisig::cancel_as_multi(Origin::signed(2), 3, vec![1, 3], now(), hash.clone()), + Error::::NotOwner, + ); + assert_ok!( + Multisig::cancel_as_multi(Origin::signed(1), 3, vec![2, 3], now(), hash.clone()), + ); + assert_eq!(Balances::free_balance(1), 10); + }); +} + +#[test] +fn cancel_multisig_with_alt_call_storage_works() { + new_test_ext().execute_with(|| { + let call = Call::Balances(BalancesCall::transfer(6, 15)).encode(); + let hash = blake2_256(&call); + assert_ok!(Multisig::approve_as_multi(Origin::signed(1), 3, vec![2, 3], None, hash.clone(), 0)); + assert_eq!(Balances::free_balance(1), 6); + assert_ok!(Multisig::as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), call, true, 0)); + assert_eq!(Balances::free_balance(2), 8); + assert_ok!(Multisig::cancel_as_multi(Origin::signed(1), 3, vec![2, 3], now(), hash)); + assert_eq!(Balances::free_balance(1), 10); + assert_eq!(Balances::free_balance(2), 10); + }); +} + +#[test] +fn multisig_2_of_3_as_multi_works() { + new_test_ext().execute_with(|| { + let multi = Multisig::multi_account_id(&[1, 2, 3][..], 2); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Call::Balances(BalancesCall::transfer(6, 15)); + let call_weight = call.get_dispatch_info().weight; + let data = call.encode(); + assert_ok!(Multisig::as_multi(Origin::signed(1), 2, vec![2, 3], None, data.clone(), false, 0)); + assert_eq!(Balances::free_balance(6), 0); + + assert_ok!(Multisig::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), data, false, call_weight)); + assert_eq!(Balances::free_balance(6), 15); + }); +} + +#[test] +fn multisig_2_of_3_as_multi_with_many_calls_works() { + new_test_ext().execute_with(|| { + let multi = Multisig::multi_account_id(&[1, 2, 3][..], 2); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call1 = Call::Balances(BalancesCall::transfer(6, 10)); + let call1_weight = call1.get_dispatch_info().weight; + let data1 = call1.encode(); + let call2 = Call::Balances(BalancesCall::transfer(7, 5)); + let call2_weight = call2.get_dispatch_info().weight; + let data2 = call2.encode(); + + assert_ok!(Multisig::as_multi(Origin::signed(1), 2, vec![2, 3], None, data1.clone(), false, 0)); + assert_ok!(Multisig::as_multi(Origin::signed(2), 2, vec![1, 3], None, data2.clone(), false, 0)); + assert_ok!(Multisig::as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), data1, false, call1_weight)); + assert_ok!(Multisig::as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), data2, false, call2_weight)); + + assert_eq!(Balances::free_balance(6), 10); + assert_eq!(Balances::free_balance(7), 5); + }); +} + +#[test] +fn multisig_2_of_3_cannot_reissue_same_call() { + new_test_ext().execute_with(|| { + let multi = Multisig::multi_account_id(&[1, 2, 3][..], 2); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Call::Balances(BalancesCall::transfer(6, 10)); + let call_weight = call.get_dispatch_info().weight; + let data = call.encode(); + let hash = blake2_256(&data); + assert_ok!(Multisig::as_multi(Origin::signed(1), 2, vec![2, 3], None, data.clone(), false, 0)); + assert_ok!(Multisig::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), data.clone(), false, call_weight)); + assert_eq!(Balances::free_balance(multi), 5); + + assert_ok!(Multisig::as_multi(Origin::signed(1), 2, vec![2, 3], None, data.clone(), false, 0)); + assert_ok!(Multisig::as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), data.clone(), false, call_weight)); + + let err = DispatchError::from(BalancesError::::InsufficientBalance).stripped(); + expect_event(RawEvent::MultisigExecuted(3, now(), multi, hash, Err(err))); + }); +} + +#[test] +fn minimum_threshold_check_works() { + new_test_ext().execute_with(|| { + let call = Call::Balances(BalancesCall::transfer(6, 15)).encode(); + assert_noop!( + Multisig::as_multi(Origin::signed(1), 0, vec![2], None, call.clone(), false, 0), + Error::::MinimumThreshold, + ); + assert_noop!( + Multisig::as_multi(Origin::signed(1), 1, vec![2], None, call.clone(), false, 0), + Error::::MinimumThreshold, + ); + }); +} + +#[test] +fn too_many_signatories_fails() { + new_test_ext().execute_with(|| { + let call = Call::Balances(BalancesCall::transfer(6, 15)).encode(); + assert_noop!( + Multisig::as_multi(Origin::signed(1), 2, vec![2, 3, 4], None, call.clone(), false, 0), + Error::::TooManySignatories, + ); + }); +} + +#[test] +fn duplicate_approvals_are_ignored() { + new_test_ext().execute_with(|| { + let call = Call::Balances(BalancesCall::transfer(6, 15)).encode(); + let hash = blake2_256(&call); + assert_ok!(Multisig::approve_as_multi(Origin::signed(1), 2, vec![2, 3], None, hash.clone(), 0)); + assert_noop!( + Multisig::approve_as_multi(Origin::signed(1), 2, vec![2, 3], Some(now()), hash.clone(), 0), + Error::::AlreadyApproved, + ); + assert_ok!(Multisig::approve_as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), hash.clone(), 0)); + assert_noop!( + Multisig::approve_as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), hash.clone(), 0), + Error::::AlreadyApproved, + ); + }); +} + +#[test] +fn multisig_1_of_3_works() { + new_test_ext().execute_with(|| { + let multi = Multisig::multi_account_id(&[1, 2, 3][..], 1); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Call::Balances(BalancesCall::transfer(6, 15)).encode(); + let hash = blake2_256(&call); + assert_noop!( + Multisig::approve_as_multi(Origin::signed(1), 1, vec![2, 3], None, hash.clone(), 0), + Error::::MinimumThreshold, + ); + assert_noop!( + Multisig::as_multi(Origin::signed(1), 1, vec![2, 3], None, call.clone(), false, 0), + Error::::MinimumThreshold, + ); + let boxed_call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); + assert_ok!(Multisig::as_multi_threshold_1(Origin::signed(1), vec![2, 3], boxed_call)); + + assert_eq!(Balances::free_balance(6), 15); + }); +} + +#[test] +fn multisig_filters() { + new_test_ext().execute_with(|| { + let call = Box::new(Call::System(frame_system::Call::set_code(vec![]))); + assert_noop!( + Multisig::as_multi_threshold_1(Origin::signed(1), vec![2], call.clone()), + DispatchError::BadOrigin, + ); + }); +} + +#[test] +fn weight_check_works() { + new_test_ext().execute_with(|| { + let multi = Multisig::multi_account_id(&[1, 2, 3][..], 2); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Call::Balances(BalancesCall::transfer(6, 15)); + let data = call.encode(); + assert_ok!(Multisig::as_multi(Origin::signed(1), 2, vec![2, 3], None, data.clone(), false, 0)); + assert_eq!(Balances::free_balance(6), 0); + + assert_noop!( + Multisig::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), data, false, 0), + Error::::WeightTooLow, + ); + }); +} + +#[test] +fn multisig_handles_no_preimage_after_all_approve() { + // This test checks the situation where everyone approves a multi-sig, but no-one provides the call data. + // In the end, any of the multisig callers can approve again with the call data and the call will go through. + new_test_ext().execute_with(|| { + let multi = Multisig::multi_account_id(&[1, 2, 3][..], 3); + assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); + assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); + + let call = Call::Balances(BalancesCall::transfer(6, 15)); + let call_weight = call.get_dispatch_info().weight; + let data = call.encode(); + let hash = blake2_256(&data); + assert_ok!(Multisig::approve_as_multi(Origin::signed(1), 3, vec![2, 3], None, hash.clone(), 0)); + assert_ok!(Multisig::approve_as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), hash.clone(), 0)); + assert_ok!(Multisig::approve_as_multi(Origin::signed(3), 3, vec![1, 2], Some(now()), hash.clone(), 0)); + assert_eq!(Balances::free_balance(6), 0); + + assert_ok!(Multisig::as_multi(Origin::signed(3), 3, vec![1, 2], Some(now()), data, false, call_weight)); + assert_eq!(Balances::free_balance(6), 15); + }); +} diff --git a/frame/nicks/Cargo.toml b/frame/nicks/Cargo.toml index 24fd06e454afd50ed1b11c68596ad4e4c64f0acb..143e5b198e15861101d77f5e9daf7aebb3ade915 100644 --- a/frame/nicks/Cargo.toml +++ b/frame/nicks/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-nicks" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,16 +13,16 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } [features] default = ["std"] diff --git a/frame/nicks/src/lib.rs b/frame/nicks/src/lib.rs index 205544cdd2e3b471d442bca6957338c9448e7992..93c60819410adb96150cf3e2277572457008911e 100644 --- a/frame/nicks/src/lib.rs +++ b/frame/nicks/src/lib.rs @@ -47,7 +47,7 @@ use frame_support::{ decl_module, decl_event, decl_storage, ensure, decl_error, traits::{Currency, EnsureOrigin, ReservableCurrency, OnUnbalanced, Get}, }; -use frame_system::{self as system, ensure_signed, ensure_root}; +use frame_system::{self as system, ensure_signed}; type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; type NegativeImbalanceOf = <::Currency as Currency<::AccountId>>::NegativeImbalance; @@ -187,7 +187,7 @@ decl_module! { /// Fails if `who` has not been named. The deposit is dealt with through `T::Slashed` /// imbalance handler. /// - /// The dispatch origin for this call must be _Root_ or match `T::ForceOrigin`. + /// The dispatch origin for this call must match `T::ForceOrigin`. /// /// # /// - O(1). @@ -197,9 +197,7 @@ decl_module! { /// # #[weight = 70_000_000] fn kill_name(origin, target: ::Source) { - T::ForceOrigin::try_origin(origin) - .map(|_| ()) - .or_else(ensure_root)?; + T::ForceOrigin::ensure_origin(origin)?; // Figure out who we're meant to be clearing. let target = T::Lookup::lookup(target)?; @@ -215,7 +213,7 @@ decl_module! { /// /// No length checking is done on the name. /// - /// The dispatch origin for this call must be _Root_ or match `T::ForceOrigin`. + /// The dispatch origin for this call must match `T::ForceOrigin`. /// /// # /// - O(1). @@ -225,9 +223,7 @@ decl_module! { /// # #[weight = 70_000_000] fn force_name(origin, target: ::Source, name: Vec) { - T::ForceOrigin::try_origin(origin) - .map(|_| ()) - .or_else(ensure_root)?; + T::ForceOrigin::ensure_origin(origin)?; let target = T::Lookup::lookup(target)?; let deposit = >::get(&target).map(|x| x.1).unwrap_or_else(Zero::zero); @@ -270,6 +266,7 @@ mod tests { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -285,6 +282,7 @@ mod tests { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/offences/Cargo.toml b/frame/offences/Cargo.toml index 4b5fbb9dbe9b6d1d0df0e66728c0ac67b17ed3f2..74487ba16394340e628046332c95181b769daa42 100644 --- a/frame/offences/Cargo.toml +++ b/frame/offences/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-offences" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,18 +12,18 @@ description = "FRAME offences pallet" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -pallet-balances = { version = "2.0.0-alpha.8", default-features = false, path = "../balances" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } +pallet-balances = { version = "2.0.0-rc4", default-features = false, path = "../balances" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } serde = { version = "1.0.101", optional = true } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-staking = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/staking" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/staking" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } [dev-dependencies] -sp-io = { version = "2.0.0-alpha.8", path = "../../primitives/io" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } +sp-io = { version = "2.0.0-rc4", path = "../../primitives/io" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } [features] default = ["std"] diff --git a/frame/offences/benchmarking/Cargo.toml b/frame/offences/benchmarking/Cargo.toml index f18865a45ab829568b2a437998c01a76559645ec..b942a98baa2c38b03eab74ef8b0c67bf725bba0f 100644 --- a/frame/offences/benchmarking/Cargo.toml +++ b/frame/offences/benchmarking/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-offences-benchmarking" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,29 +12,28 @@ description = "FRAME offences pallet benchmarking" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../../benchmarking" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../../system" } -pallet-babe = { version = "2.0.0-alpha.8", default-features = false, path = "../../babe" } -pallet-balances = { version = "2.0.0-alpha.8", default-features = false, path = "../../balances" } -pallet-grandpa = { version = "2.0.0-alpha.8", default-features = false, path = "../../grandpa" } -pallet-im-online = { version = "2.0.0-alpha.8", default-features = false, path = "../../im-online" } -pallet-offences = { version = "2.0.0-alpha.8", default-features = false, features = ["runtime-benchmarks"], path = "../../offences" } -pallet-session = { version = "2.0.0-alpha.8", default-features = false, path = "../../session" } -pallet-staking = { version = "2.0.0-alpha.8", default-features = false, features = ["runtime-benchmarks"], path = "../../staking" } -sp-io = { path = "../../../primitives/io", default-features = false, version = "2.0.0-alpha.8"} -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/runtime" } -sp-staking = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/staking" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/std" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../../benchmarking" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../../system" } +pallet-babe = { version = "2.0.0-rc4", default-features = false, path = "../../babe" } +pallet-balances = { version = "2.0.0-rc4", default-features = false, path = "../../balances" } +pallet-grandpa = { version = "2.0.0-rc4", default-features = false, path = "../../grandpa" } +pallet-im-online = { version = "2.0.0-rc4", default-features = false, path = "../../im-online" } +pallet-offences = { version = "2.0.0-rc4", default-features = false, features = ["runtime-benchmarks"], path = "../../offences" } +pallet-session = { version = "2.0.0-rc4", default-features = false, path = "../../session" } +pallet-staking = { version = "2.0.0-rc4", default-features = false, features = ["runtime-benchmarks"], path = "../../staking" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/runtime" } +sp-staking = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/staking" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/std" } [dev-dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } -pallet-staking-reward-curve = { version = "2.0.0-alpha.8", path = "../../staking/reward-curve" } -pallet-timestamp = { version = "2.0.0-alpha.8", path = "../../timestamp" } +codec = { package = "parity-scale-codec", version = "1.3.1", features = ["derive"] } +pallet-staking-reward-curve = { version = "2.0.0-rc4", path = "../../staking/reward-curve" } +pallet-timestamp = { version = "2.0.0-rc4", path = "../../timestamp" } serde = { version = "1.0.101" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-io ={ path = "../../../primitives/io", version = "2.0.0-alpha.8"} +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-io = { version = "2.0.0-rc4", path = "../../../primitives/io" } [features] default = ["std"] @@ -52,5 +51,4 @@ std = [ "sp-runtime/std", "sp-staking/std", "sp-std/std", - "sp-io/std", ] diff --git a/frame/offences/benchmarking/src/mock.rs b/frame/offences/benchmarking/src/mock.rs index 76cd01769039943f5b8fa82a1bd81879446b665d..90ad7eeb3cfcb1f3f4450e9cbc0cfb02fdbafabb 100644 --- a/frame/offences/benchmarking/src/mock.rs +++ b/frame/offences/benchmarking/src/mock.rs @@ -20,7 +20,10 @@ #![cfg(test)] use super::*; -use frame_support::parameter_types; +use frame_support::{ + parameter_types, + weights::{Weight, constants::WEIGHT_PER_SECOND}, +}; use frame_system as system; use sp_runtime::{ SaturatedConversion, @@ -34,7 +37,12 @@ type AccountIndex = u32; type BlockNumber = u64; type Balance = u64; +parameter_types! { + pub const MaximumBlockWeight: Weight = 2 * WEIGHT_PER_SECOND; +} + impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = AccountIndex; type BlockNumber = BlockNumber; @@ -46,7 +54,7 @@ impl frame_system::Trait for Test { type Header = sp_runtime::testing::Header; type Event = Event; type BlockHashCount = (); - type MaximumBlockWeight = (); + type MaximumBlockWeight = MaximumBlockWeight; type DbWeight = (); type AvailableBlockRatio = (); type MaximumBlockLength = (); @@ -57,6 +65,7 @@ impl frame_system::Trait for Test { type OnKilledAccount = (Balances,); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = (); } parameter_types! { pub const ExistentialDeposit: Balance = 10; @@ -168,6 +177,7 @@ impl pallet_staking::Trait for Test { type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; type UnsignedPriority = (); type MaxIterations = (); + type MinSolutionScoreBump = (); } impl pallet_im_online::Trait for Test { @@ -178,10 +188,15 @@ impl pallet_im_online::Trait for Test { type UnsignedPriority = (); } +parameter_types! { + pub OffencesWeightSoftLimit: Weight = Perbill::from_percent(60) * MaximumBlockWeight::get(); +} + impl pallet_offences::Trait for Test { type Event = Event; type IdentificationTuple = pallet_session::historical::IdentificationTuple; type OnOffenceHandler = Staking; + type WeightSoftLimit = OffencesWeightSoftLimit; } impl frame_system::offchain::SendTransactionTypes for Test where Call: From { diff --git a/frame/offences/src/lib.rs b/frame/offences/src/lib.rs index dd1b052811dae48be3af1ab078f1388f4b9cab44..a42f09697e3b46c3ef6e75f488411a55ee6c9f39 100644 --- a/frame/offences/src/lib.rs +++ b/frame/offences/src/lib.rs @@ -28,9 +28,10 @@ mod tests; use sp_std::vec::Vec; use frame_support::{ decl_module, decl_event, decl_storage, Parameter, debug, + traits::Get, weights::Weight, }; -use sp_runtime::{traits::Hash, Perbill}; +use sp_runtime::{traits::{Hash, Zero}, Perbill}; use sp_staking::{ SessionIndex, offence::{Offence, ReportOffence, Kind, OnOffenceHandler, OffenceDetails, OffenceError}, @@ -58,7 +59,11 @@ pub trait Trait: frame_system::Trait { /// Full identification of the validator. type IdentificationTuple: Parameter + Ord; /// A handler called for every offence report. - type OnOffenceHandler: OnOffenceHandler; + type OnOffenceHandler: OnOffenceHandler; + /// The a soft limit on maximum weight that may be consumed while dispatching deferred offences in + /// `on_initialize`. + /// Note it's going to be exceeded before we stop adding to it, so it has to be set conservatively. + type WeightSoftLimit: Get; } decl_storage! { @@ -102,23 +107,39 @@ decl_module! { fn on_initialize(now: T::BlockNumber) -> Weight { // only decode storage if we can actually submit anything again. - if T::OnOffenceHandler::can_report() { - >::mutate(|deferred| { - // keep those that fail to be reported again. An error log is emitted here; this - // should not happen if staking's `can_report` is implemented properly. - deferred.retain(|(o, p, s)| { - T::OnOffenceHandler::on_offence(&o, &p, *s).map_err(|_| { - debug::native::error!( - target: "pallet-offences", - "re-submitting a deferred slash returned Err at {}. This should not happen with pallet-staking", - now, - ); - }).is_err() - }) - }) + if !T::OnOffenceHandler::can_report() { + return 0; } - 0 + let limit = T::WeightSoftLimit::get(); + let mut consumed = Weight::zero(); + + >::mutate(|deferred| { + deferred.retain(|(offences, perbill, session)| { + if consumed >= limit { + true + } else { + // keep those that fail to be reported again. An error log is emitted here; this + // should not happen if staking's `can_report` is implemented properly. + match T::OnOffenceHandler::on_offence(&offences, &perbill, *session) { + Ok(weight) => { + consumed += weight; + false + }, + Err(_) => { + debug::native::error!( + target: "pallet-offences", + "re-submitting a deferred slash returned Err at {}. This should not happen with pallet-staking", + now, + ); + true + }, + } + } + }) + }); + + consumed } } } diff --git a/frame/offences/src/mock.rs b/frame/offences/src/mock.rs index 3a407654a2246b931a06519c0e2fa2c2c5694dde..6c89072a0f5bd9d88455541aaec4f259e8fd0308 100644 --- a/frame/offences/src/mock.rs +++ b/frame/offences/src/mock.rs @@ -32,7 +32,7 @@ use sp_runtime::traits::{IdentityLookup, BlakeTwo256}; use sp_core::H256; use frame_support::{ impl_outer_origin, impl_outer_event, parameter_types, StorageMap, StorageDoubleMap, - weights::Weight, + weights::{Weight, constants::{WEIGHT_PER_SECOND, RocksDbWeight}}, }; use frame_system as system; @@ -45,20 +45,23 @@ pub struct OnOffenceHandler; thread_local! { pub static ON_OFFENCE_PERBILL: RefCell> = RefCell::new(Default::default()); pub static CAN_REPORT: RefCell = RefCell::new(true); + pub static OFFENCE_WEIGHT: RefCell = RefCell::new(Default::default()); } -impl offence::OnOffenceHandler for OnOffenceHandler { +impl + offence::OnOffenceHandler for OnOffenceHandler +{ fn on_offence( _offenders: &[OffenceDetails], slash_fraction: &[Perbill], _offence_session: SessionIndex, - ) -> Result<(), ()> { - if >::can_report() { + ) -> Result { + if >::can_report() { ON_OFFENCE_PERBILL.with(|f| { *f.borrow_mut() = slash_fraction.to_vec(); }); - Ok(()) + Ok(OFFENCE_WEIGHT.with(|w| *w.borrow())) } else { Err(()) } @@ -79,16 +82,21 @@ pub fn with_on_offence_fractions) -> R>(f: F) -> }) } +pub fn set_offence_weight(new: Weight) { + OFFENCE_WEIGHT.with(|w| *w.borrow_mut() = new); +} + // Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. #[derive(Clone, PartialEq, Eq, Debug)] pub struct Runtime; parameter_types! { pub const BlockHashCount: u64 = 250; - pub const MaximumBlockWeight: Weight = 1024; + pub const MaximumBlockWeight: Weight = 2 * WEIGHT_PER_SECOND; pub const MaximumBlockLength: u32 = 2 * 1024; pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Runtime { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -101,9 +109,10 @@ impl frame_system::Trait for Runtime { type Event = TestEvent; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; - type DbWeight = (); + type DbWeight = RocksDbWeight; type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -113,10 +122,15 @@ impl frame_system::Trait for Runtime { type OnKilledAccount = (); } +parameter_types! { + pub OffencesWeightSoftLimit: Weight = Perbill::from_percent(60) * MaximumBlockWeight::get(); +} + impl Trait for Runtime { type Event = TestEvent; type IdentificationTuple = u64; type OnOffenceHandler = OnOffenceHandler; + type WeightSoftLimit = OffencesWeightSoftLimit; } mod offences { diff --git a/frame/offences/src/tests.rs b/frame/offences/src/tests.rs index b05fee1790099a02bbc474ac2b8b51c0bee6fae2..0fb6620b7d81210e7fbe7f8781e376d7dfcee593 100644 --- a/frame/offences/src/tests.rs +++ b/frame/offences/src/tests.rs @@ -22,7 +22,7 @@ use super::*; use crate::mock::{ Offences, System, Offence, TestEvent, KIND, new_test_ext, with_on_offence_fractions, - offence_reports, set_can_report, + offence_reports, set_can_report, set_offence_weight, }; use sp_runtime::Perbill; use frame_support::traits::OnInitialize; @@ -265,3 +265,48 @@ fn should_queue_and_resubmit_rejected_offence() { assert_eq!(Offences::deferred_offences().len(), 0); }) } + +#[test] +fn weight_soft_limit_is_used() { + new_test_ext().execute_with(|| { + set_can_report(false); + // Only 2 can fit in one block + set_offence_weight(::WeightSoftLimit::get() / 2); + + // Queue 3 offences + // #1 + let offence = Offence { + validator_set_count: 5, + time_slot: 42, + offenders: vec![5], + }; + Offences::report_offence(vec![], offence).unwrap(); + // #2 + let offence = Offence { + validator_set_count: 5, + time_slot: 62, + offenders: vec![5], + }; + Offences::report_offence(vec![], offence).unwrap(); + // #3 + let offence = Offence { + validator_set_count: 5, + time_slot: 72, + offenders: vec![5], + }; + Offences::report_offence(vec![], offence).unwrap(); + // 3 are queued + assert_eq!(Offences::deferred_offences().len(), 3); + + // Allow reporting + set_can_report(true); + + Offences::on_initialize(3); + // Two are completed, one is left in the queue + assert_eq!(Offences::deferred_offences().len(), 1); + + Offences::on_initialize(4); + // All are done now + assert_eq!(Offences::deferred_offences().len(), 0); + }) +} diff --git a/frame/proxy/Cargo.toml b/frame/proxy/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..07e2abac3173e95f1d2a5571954de2028d5e0784 --- /dev/null +++ b/frame/proxy/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "pallet-proxy" +version = "2.0.0-rc4" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME proxying pallet" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +serde = { version = "1.0.101", optional = true } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } + +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } + +[dev-dependencies] +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } +pallet-utility = { version = "2.0.0-rc4", path = "../utility" } + +[features] +default = ["std"] +std = [ + "serde", + "codec/std", + "sp-runtime/std", + "frame-support/std", + "frame-system/std", + "sp-std/std", + "sp-io/std" +] +runtime-benchmarks = [ + "frame-benchmarking", + "frame-support/runtime-benchmarks", +] diff --git a/frame/proxy/src/benchmarking.rs b/frame/proxy/src/benchmarking.rs new file mode 100644 index 0000000000000000000000000000000000000000..3cbe517dfd7da782cb69f0079037d035c1c09680 --- /dev/null +++ b/frame/proxy/src/benchmarking.rs @@ -0,0 +1,108 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Benchmarks for Proxy Pallet + +#![cfg(feature = "runtime-benchmarks")] + +use super::*; +use frame_system::RawOrigin; +use frame_benchmarking::{benchmarks, account}; +use sp_runtime::traits::Bounded; +use crate::Module as Proxy; + +const SEED: u32 = 0; + +fn add_proxies(n: u32, maybe_who: Option) -> Result<(), &'static str> { + let caller = maybe_who.unwrap_or_else(|| account("caller", 0, SEED)); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + for i in 0..n { + Proxy::::add_proxy( + RawOrigin::Signed(caller.clone()).into(), + account("target", i, SEED), + T::ProxyType::default() + )?; + } + Ok(()) +} + +benchmarks! { + _ { + let p in 1 .. (T::MaxProxies::get() - 1).into() => add_proxies::(p, None)?; + } + + proxy { + let p in ...; + // In this case the caller is the "target" proxy + let caller: T::AccountId = account("target", p - 1, SEED); + // ... and "real" is the traditional caller. This is not a typo. + let real: T::AccountId = account("caller", 0, SEED); + let call: ::Call = frame_system::Call::::remark(vec![]).into(); + }: _(RawOrigin::Signed(caller), real, Some(T::ProxyType::default()), Box::new(call)) + + add_proxy { + let p in ...; + let caller: T::AccountId = account("caller", 0, SEED); + }: _(RawOrigin::Signed(caller), account("target", T::MaxProxies::get().into(), SEED), T::ProxyType::default()) + + remove_proxy { + let p in ...; + let caller: T::AccountId = account("caller", 0, SEED); + }: _(RawOrigin::Signed(caller), account("target", 0, SEED), T::ProxyType::default()) + + remove_proxies { + let p in ...; + let caller: T::AccountId = account("caller", 0, SEED); + }: _(RawOrigin::Signed(caller)) + + anonymous { + let p in ...; + }: _(RawOrigin::Signed(account("caller", 0, SEED)), T::ProxyType::default(), 0) + + kill_anonymous { + let p in 0 .. (T::MaxProxies::get() - 2).into(); + + let caller: T::AccountId = account("caller", 0, SEED); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + Module::::anonymous(RawOrigin::Signed(account("caller", 0, SEED)).into(), T::ProxyType::default(), 0)?; + let height = system::Module::::block_number(); + let ext_index = system::Module::::extrinsic_index().unwrap_or(0); + let anon = Module::::anonymous_account(&caller, &T::ProxyType::default(), 0, None); + + add_proxies::(p, Some(anon.clone()))?; + + }: _(RawOrigin::Signed(anon), caller, T::ProxyType::default(), 0, height, ext_index) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_proxy::()); + assert_ok!(test_benchmark_add_proxy::()); + assert_ok!(test_benchmark_remove_proxy::()); + assert_ok!(test_benchmark_remove_proxies::()); + assert_ok!(test_benchmark_anonymous::()); + assert_ok!(test_benchmark_kill_anonymous::()); + }); + } +} diff --git a/frame/proxy/src/lib.rs b/frame/proxy/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..fb72fa895389f24a8f90179ed4091c2a145551fa --- /dev/null +++ b/frame/proxy/src/lib.rs @@ -0,0 +1,393 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Proxy Module +//! A module allowing accounts to give permission to other accounts to dispatch types of calls from +//! their signed origin. +//! +//! - [`proxy::Trait`](./trait.Trait.html) +//! - [`Call`](./enum.Call.html) +//! +//! ## Overview +//! +//! ## Interface +//! +//! ### Dispatchable Functions +//! +//! [`Call`]: ./enum.Call.html +//! [`Trait`]: ./trait.Trait.html + +// Ensure we're `no_std` when compiling for Wasm. +#![cfg_attr(not(feature = "std"), no_std)] + +use sp_std::prelude::*; +use codec::{Encode, Decode}; +use sp_io::hashing::blake2_256; +use sp_runtime::{DispatchResult, traits::{Dispatchable, Zero}}; +use sp_runtime::traits::Member; +use frame_support::{ + decl_module, decl_event, decl_error, decl_storage, Parameter, ensure, traits::{ + Get, ReservableCurrency, Currency, InstanceFilter, + OriginTrait, IsType, + }, weights::{GetDispatchInfo, constants::{WEIGHT_PER_MICROS, WEIGHT_PER_NANOS}}, + dispatch::{PostDispatchInfo, IsSubType}, +}; +use frame_system::{self as system, ensure_signed}; + +mod tests; +mod benchmarking; + +type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; + +/// Configuration trait. +pub trait Trait: frame_system::Trait { + /// The overarching event type. + type Event: From> + Into<::Event>; + + /// The overarching call type. + type Call: Parameter + Dispatchable + + GetDispatchInfo + From> + IsSubType> + + IsType<::Call>; + + /// The currency mechanism. + type Currency: ReservableCurrency; + + /// A kind of proxy; specified with the proxy and passed in to the `IsProxyable` fitler. + /// The instance filter determines whether a given call may be proxied under this type. + /// + /// IMPORTANT: `Default` must be provided and MUST BE the the *most permissive* value. + type ProxyType: Parameter + Member + Ord + PartialOrd + InstanceFilter<::Call> + + Default; + + /// The base amount of currency needed to reserve for creating a proxy. + /// + /// This is held for an additional storage item whose value size is + /// `sizeof(Balance)` bytes and whose key size is `sizeof(AccountId)` bytes. + type ProxyDepositBase: Get>; + + /// The amount of currency needed per proxy added. + /// + /// This is held for adding 32 bytes plus an instance of `ProxyType` more into a pre-existing + /// storage value. + type ProxyDepositFactor: Get>; + + /// The maximum amount of proxies allowed for a single account. + type MaxProxies: Get; +} + +decl_storage! { + trait Store for Module as Proxy { + /// The set of account proxies. Maps the account which has delegated to the accounts + /// which are being delegated to, together with the amount held on deposit. + pub Proxies: map hasher(twox_64_concat) T::AccountId => (Vec<(T::AccountId, T::ProxyType)>, BalanceOf); + } +} + +decl_error! { + pub enum Error for Module { + /// There are too many proxies registered. + TooMany, + /// Proxy registration not found. + NotFound, + /// Sender is not a proxy of the account to be proxied. + NotProxy, + /// A call which is incompatible with the proxy type's filter was attempted. + Unproxyable, + /// Account is already a proxy. + Duplicate, + /// Call may not be made by proxy because it may escalate its privileges. + NoPermission, + } +} + +decl_event! { + /// Events type. + pub enum Event where + AccountId = ::AccountId, + ProxyType = ::ProxyType + { + /// A proxy was executed correctly, with the given result. + ProxyExecuted(DispatchResult), + /// Anonymous account (first parameter) has been created by new proxy (second) with given + /// disambiguation index and proxy type. + AnonymousCreated(AccountId, AccountId, ProxyType, u16), + } +} + +decl_module! { + pub struct Module for enum Call where origin: T::Origin { + type Error = Error; + + /// Deposit one of this module's events by using the default implementation. + fn deposit_event() = default; + + /// The base amount of currency needed to reserve for creating a proxy. + const ProxyDepositBase: BalanceOf = T::ProxyDepositBase::get(); + + /// The amount of currency needed per proxy added. + const ProxyDepositFactor: BalanceOf = T::ProxyDepositFactor::get(); + + /// The maximum amount of proxies allowed for a single account. + const MaxProxies: u16 = T::MaxProxies::get(); + + /// Dispatch the given `call` from an account that the sender is authorised for through + /// `add_proxy`. + /// + /// The dispatch origin for this call must be _Signed_. + /// + /// Parameters: + /// - `real`: The account that the proxy will make a call on behalf of. + /// - `force_proxy_type`: Specify the exact proxy type to be used and checked for this call. + /// - `call`: The call to be made by the `real` account. + /// + /// # + /// P is the number of proxies the user has + /// - Base weight: 19.87 + .141 * P µs + /// - DB weight: 1 storage read. + /// - Plus the weight of the `call` + /// # + #[weight = { + let di = call.get_dispatch_info(); + (T::DbWeight::get().reads(1) + .saturating_add(di.weight) + .saturating_add(20 * WEIGHT_PER_MICROS) + .saturating_add((140 * WEIGHT_PER_NANOS).saturating_mul(T::MaxProxies::get().into())), + di.class) + }] + fn proxy(origin, + real: T::AccountId, + force_proxy_type: Option, + call: Box<::Call> + ) { + let who = ensure_signed(origin)?; + let (_, proxy_type) = Proxies::::get(&real).0.into_iter() + .find(|x| &x.0 == &who && force_proxy_type.as_ref().map_or(true, |y| &x.1 == y)) + .ok_or(Error::::NotProxy)?; + + // This is a freshly authenticated new account, the origin restrictions doesn't apply. + let mut origin: T::Origin = frame_system::RawOrigin::Signed(real).into(); + origin.add_filter(move |c: &::Call| { + let c = ::Call::from_ref(c); + match c.is_sub_type() { + Some(Call::add_proxy(_, ref pt)) | Some(Call::remove_proxy(_, ref pt)) + if !proxy_type.is_superset(&pt) => false, + Some(Call::remove_proxies(..)) | Some(Call::kill_anonymous(..)) + if proxy_type != T::ProxyType::default() => false, + _ => proxy_type.filter(c) + } + }); + let e = call.dispatch(origin); + Self::deposit_event(RawEvent::ProxyExecuted(e.map(|_| ()).map_err(|e| e.error))); + } + + /// Register a proxy account for the sender that is able to make calls on its behalf. + /// + /// The dispatch origin for this call must be _Signed_. + /// + /// Parameters: + /// - `proxy`: The account that the `caller` would like to make a proxy. + /// - `proxy_type`: The permissions allowed for this proxy account. + /// + /// # + /// P is the number of proxies the user has + /// - Base weight: 17.48 + .176 * P µs + /// - DB weight: 1 storage read and write. + /// # + #[weight = T::DbWeight::get().reads_writes(1, 1) + .saturating_add(18 * WEIGHT_PER_MICROS) + .saturating_add((200 * WEIGHT_PER_NANOS).saturating_mul(T::MaxProxies::get().into())) + ] + fn add_proxy(origin, proxy: T::AccountId, proxy_type: T::ProxyType) -> DispatchResult { + let who = ensure_signed(origin)?; + Proxies::::try_mutate(&who, |(ref mut proxies, ref mut deposit)| { + ensure!(proxies.len() < T::MaxProxies::get() as usize, Error::::TooMany); + let typed_proxy = (proxy, proxy_type); + let i = proxies.binary_search(&typed_proxy).err().ok_or(Error::::Duplicate)?; + proxies.insert(i, typed_proxy); + let new_deposit = T::ProxyDepositBase::get() + + T::ProxyDepositFactor::get() * (proxies.len() as u32).into(); + if new_deposit > *deposit { + T::Currency::reserve(&who, new_deposit - *deposit)?; + } else if new_deposit < *deposit { + T::Currency::unreserve(&who, *deposit - new_deposit); + } + *deposit = new_deposit; + Ok(()) + }) + } + + /// Unregister a proxy account for the sender. + /// + /// The dispatch origin for this call must be _Signed_. + /// + /// Parameters: + /// - `proxy`: The account that the `caller` would like to remove as a proxy. + /// - `proxy_type`: The permissions currently enabled for the removed proxy account. + /// + /// # + /// P is the number of proxies the user has + /// - Base weight: 14.37 + .164 * P µs + /// - DB weight: 1 storage read and write. + /// # + #[weight = T::DbWeight::get().reads_writes(1, 1) + .saturating_add(14 * WEIGHT_PER_MICROS) + .saturating_add((160 * WEIGHT_PER_NANOS).saturating_mul(T::MaxProxies::get().into())) + ] + fn remove_proxy(origin, proxy: T::AccountId, proxy_type: T::ProxyType) -> DispatchResult { + let who = ensure_signed(origin)?; + Proxies::::try_mutate_exists(&who, |x| { + let (mut proxies, old_deposit) = x.take().ok_or(Error::::NotFound)?; + let typed_proxy = (proxy, proxy_type); + let i = proxies.binary_search(&typed_proxy).ok().ok_or(Error::::NotFound)?; + proxies.remove(i); + let new_deposit = if proxies.is_empty() { + BalanceOf::::zero() + } else { + T::ProxyDepositBase::get() + T::ProxyDepositFactor::get() * (proxies.len() as u32).into() + }; + if new_deposit > old_deposit { + T::Currency::reserve(&who, new_deposit - old_deposit)?; + } else if new_deposit < old_deposit { + T::Currency::unreserve(&who, old_deposit - new_deposit); + } + if !proxies.is_empty() { + *x = Some((proxies, new_deposit)) + } + Ok(()) + }) + } + + /// Unregister all proxy accounts for the sender. + /// + /// The dispatch origin for this call must be _Signed_. + /// + /// WARNING: This may be called on accounts created by `anonymous`, however if done, then + /// the unreserved fees will be inaccessible. **All access to this account will be lost.** + /// + /// # + /// P is the number of proxies the user has + /// - Base weight: 13.73 + .129 * P µs + /// - DB weight: 1 storage read and write. + /// # + #[weight = T::DbWeight::get().reads_writes(1, 1) + .saturating_add(14 * WEIGHT_PER_MICROS) + .saturating_add((130 * WEIGHT_PER_NANOS).saturating_mul(T::MaxProxies::get().into())) + ] + fn remove_proxies(origin) { + let who = ensure_signed(origin)?; + let (_, old_deposit) = Proxies::::take(&who); + T::Currency::unreserve(&who, old_deposit); + } + + /// Spawn a fresh new account that is guaranteed to be otherwise inaccessible, and + /// initialize it with a proxy of `proxy_type` for `origin` sender. + /// + /// Requires a `Signed` origin. + /// + /// - `proxy_type`: The type of the proxy that the sender will be registered as over the + /// new account. This will almost always be the most permissive `ProxyType` possible to + /// allow for maximum flexibility. + /// - `index`: A disambiguation index, in case this is called multiple times in the same + /// transaction (e.g. with `utility::batch`). Unless you're using `batch` you probably just + /// want to use `0`. + /// + /// Fails with `Duplicate` if this has already been called in this transaction, from the + /// same sender, with the same parameters. + /// + /// Fails if there are insufficient funds to pay for deposit. + /// + /// # + /// P is the number of proxies the user has + /// - Base weight: 36.48 + .039 * P µs + /// - DB weight: 1 storage read and write. + /// # + #[weight = T::DbWeight::get().reads_writes(1, 1) + .saturating_add(36 * WEIGHT_PER_MICROS) + .saturating_add((40 * WEIGHT_PER_NANOS).saturating_mul(T::MaxProxies::get().into())) + ] + fn anonymous(origin, proxy_type: T::ProxyType, index: u16) { + let who = ensure_signed(origin)?; + + let anonymous = Self::anonymous_account(&who, &proxy_type, index, None); + ensure!(!Proxies::::contains_key(&anonymous), Error::::Duplicate); + let deposit = T::ProxyDepositBase::get() + T::ProxyDepositFactor::get(); + T::Currency::reserve(&who, deposit)?; + Proxies::::insert(&anonymous, (vec![(who.clone(), proxy_type.clone())], deposit)); + Self::deposit_event(RawEvent::AnonymousCreated(anonymous, who, proxy_type, index)); + } + + /// Removes a previously spawned anonymous proxy. + /// + /// WARNING: **All access to this account will be lost.** Any funds held in it will be + /// inaccessible. + /// + /// Requires a `Signed` origin, and the sender account must have been created by a call to + /// `anonymous` with corresponding parameters. + /// + /// - `spawner`: The account that originally called `anonymous` to create this account. + /// - `index`: The disambiguation index originally passed to `anonymous`. Probably `0`. + /// - `proxy_type`: The proxy type originally passed to `anonymous`. + /// - `height`: The height of the chain when the call to `anonymous` was processed. + /// - `ext_index`: The extrinsic index in which the call to `anonymous` was processed. + /// + /// Fails with `NoPermission` in case the caller is not a previously created anonymous + /// account whose `anonymous` call has corresponding parameters. + /// + /// # + /// P is the number of proxies the user has + /// - Base weight: 15.65 + .137 * P µs + /// - DB weight: 1 storage read and write. + /// # + #[weight = T::DbWeight::get().reads_writes(1, 1) + .saturating_add(15 * WEIGHT_PER_MICROS) + .saturating_add((140 * WEIGHT_PER_NANOS).saturating_mul(T::MaxProxies::get().into())) + ] + fn kill_anonymous(origin, + spawner: T::AccountId, + proxy_type: T::ProxyType, + index: u16, + #[compact] height: T::BlockNumber, + #[compact] ext_index: u32, + ) { + let who = ensure_signed(origin)?; + + let when = (height, ext_index); + let proxy = Self::anonymous_account(&spawner, &proxy_type, index, Some(when)); + ensure!(proxy == who, Error::::NoPermission); + + let (_, deposit) = Proxies::::take(&who); + T::Currency::unreserve(&spawner, deposit); + } + } +} + +impl Module { + pub fn anonymous_account( + who: &T::AccountId, + proxy_type: &T::ProxyType, + index: u16, + maybe_when: Option<(T::BlockNumber, u32)>, + ) -> T::AccountId { + let (height, ext_index) = maybe_when.unwrap_or_else(|| ( + system::Module::::block_number(), + system::Module::::extrinsic_index().unwrap_or_default() + )); + let entropy = (b"modlpy/proxy____", who, height, ext_index, proxy_type, index) + .using_encoded(blake2_256); + T::AccountId::decode(&mut &entropy[..]).unwrap_or_default() + } +} diff --git a/frame/proxy/src/tests.rs b/frame/proxy/src/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..63d5c9e575d9cf81c9189237f52a6da1db60e15a --- /dev/null +++ b/frame/proxy/src/tests.rs @@ -0,0 +1,353 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for Proxy Pallet + +#![cfg(test)] + +use super::*; + +use frame_support::{ + assert_ok, assert_noop, impl_outer_origin, parameter_types, impl_outer_dispatch, + weights::Weight, impl_outer_event, RuntimeDebug, dispatch::DispatchError, traits::Filter, +}; +use codec::{Encode, Decode}; +use sp_core::H256; +use sp_runtime::{Perbill, traits::{BlakeTwo256, IdentityLookup}, testing::Header}; +use crate as proxy; + +impl_outer_origin! { + pub enum Origin for Test where system = frame_system {} +} +impl_outer_event! { + pub enum TestEvent for Test { + system, + pallet_balances, + proxy, + pallet_utility, + } +} +impl_outer_dispatch! { + pub enum Call for Test where origin: Origin { + frame_system::System, + pallet_balances::Balances, + proxy::Proxy, + pallet_utility::Utility, + } +} + +// For testing the pallet, we construct most of a mock runtime. This means +// first constructing a configuration type (`Test`) which `impl`s each of the +// configuration traits of pallets we want to use. +#[derive(Clone, Eq, PartialEq)] +pub struct Test; +parameter_types! { + pub const BlockHashCount: u64 = 250; + pub const MaximumBlockWeight: Weight = 1024; + pub const MaximumBlockLength: u32 = 2 * 1024; + pub const AvailableBlockRatio: Perbill = Perbill::one(); +} +impl frame_system::Trait for Test { + type BaseCallFilter = BaseFilter; + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Call = Call; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = TestEvent; + type BlockHashCount = BlockHashCount; + type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; + type MaximumBlockLength = MaximumBlockLength; + type AvailableBlockRatio = AvailableBlockRatio; + type Version = (); + type ModuleToIndex = (); + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); +} +parameter_types! { + pub const ExistentialDeposit: u64 = 1; +} +impl pallet_balances::Trait for Test { + type Balance = u64; + type Event = TestEvent; + type DustRemoval = (); + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; +} +impl pallet_utility::Trait for Test { + type Event = TestEvent; + type Call = Call; +} +parameter_types! { + pub const ProxyDepositBase: u64 = 1; + pub const ProxyDepositFactor: u64 = 1; + pub const MaxProxies: u16 = 4; +} +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Encode, Decode, RuntimeDebug)] +pub enum ProxyType { + Any, + JustTransfer, + JustUtility, +} +impl Default for ProxyType { fn default() -> Self { Self::Any } } +impl InstanceFilter for ProxyType { + fn filter(&self, c: &Call) -> bool { + match self { + ProxyType::Any => true, + ProxyType::JustTransfer => matches!(c, Call::Balances(pallet_balances::Call::transfer(..))), + ProxyType::JustUtility => matches!(c, Call::Utility(..)), + } + } + fn is_superset(&self, o: &Self) -> bool { + self == &ProxyType::Any || self == o + } +} +pub struct BaseFilter; +impl Filter for BaseFilter { + fn filter(c: &Call) -> bool { + match *c { + // Remark is used as a no-op call in the benchmarking + Call::System(SystemCall::remark(_)) => true, + Call::System(_) => false, + _ => true, + } + } +} +impl Trait for Test { + type Event = TestEvent; + type Call = Call; + type Currency = Balances; + type ProxyType = ProxyType; + type ProxyDepositBase = ProxyDepositBase; + type ProxyDepositFactor = ProxyDepositFactor; + type MaxProxies = MaxProxies; +} + +type System = frame_system::Module; +type Balances = pallet_balances::Module; +type Utility = pallet_utility::Module; +type Proxy = Module; + +use frame_system::Call as SystemCall; +use pallet_balances::Call as BalancesCall; +use pallet_balances::Error as BalancesError; +use pallet_balances::Event as BalancesEvent; +use pallet_utility::Call as UtilityCall; +use pallet_utility::Event as UtilityEvent; +use super::Call as ProxyCall; + +pub fn new_test_ext() -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + pallet_balances::GenesisConfig:: { + balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 2)], + }.assimilate_storage(&mut t).unwrap(); + let mut ext = sp_io::TestExternalities::new(t); + ext.execute_with(|| System::set_block_number(1)); + ext +} + +fn last_event() -> TestEvent { + system::Module::::events().pop().expect("Event expected").event +} + +fn expect_event>(e: E) { + assert_eq!(last_event(), e.into()); +} + +fn last_events(n: usize) -> Vec { + system::Module::::events().into_iter().rev().take(n).rev().map(|e| e.event).collect() +} + +fn expect_events(e: Vec) { + assert_eq!(last_events(e.len()), e); +} + +#[test] +fn filtering_works() { + new_test_ext().execute_with(|| { + Balances::mutate_account(&1, |a| a.free = 1000); + assert_ok!(Proxy::add_proxy(Origin::signed(1), 2, ProxyType::Any)); + assert_ok!(Proxy::add_proxy(Origin::signed(1), 3, ProxyType::JustTransfer)); + assert_ok!(Proxy::add_proxy(Origin::signed(1), 4, ProxyType::JustUtility)); + + let call = Box::new(Call::Balances(BalancesCall::transfer(6, 1))); + assert_ok!(Proxy::proxy(Origin::signed(2), 1, None, call.clone())); + expect_event(RawEvent::ProxyExecuted(Ok(()))); + assert_ok!(Proxy::proxy(Origin::signed(3), 1, None, call.clone())); + expect_event(RawEvent::ProxyExecuted(Ok(()))); + assert_ok!(Proxy::proxy(Origin::signed(4), 1, None, call.clone())); + expect_event(RawEvent::ProxyExecuted(Err(DispatchError::BadOrigin))); + + let derivative_id = Utility::derivative_account_id(1, 0); + Balances::mutate_account(&derivative_id, |a| a.free = 1000); + let inner = Box::new(Call::Balances(BalancesCall::transfer(6, 1))); + + let call = Box::new(Call::Utility(UtilityCall::as_derivative(0, inner.clone()))); + assert_ok!(Proxy::proxy(Origin::signed(2), 1, None, call.clone())); + expect_event(RawEvent::ProxyExecuted(Ok(()))); + assert_ok!(Proxy::proxy(Origin::signed(3), 1, None, call.clone())); + expect_event(RawEvent::ProxyExecuted(Err(DispatchError::BadOrigin))); + assert_ok!(Proxy::proxy(Origin::signed(4), 1, None, call.clone())); + expect_event(RawEvent::ProxyExecuted(Err(DispatchError::BadOrigin))); + + let call = Box::new(Call::Utility(UtilityCall::batch(vec![*inner]))); + assert_ok!(Proxy::proxy(Origin::signed(2), 1, None, call.clone())); + expect_events(vec![UtilityEvent::BatchCompleted.into(), RawEvent::ProxyExecuted(Ok(())).into()]); + assert_ok!(Proxy::proxy(Origin::signed(3), 1, None, call.clone())); + expect_event(RawEvent::ProxyExecuted(Err(DispatchError::BadOrigin))); + assert_ok!(Proxy::proxy(Origin::signed(4), 1, None, call.clone())); + expect_events(vec![ + UtilityEvent::BatchInterrupted(0, DispatchError::BadOrigin).into(), + RawEvent::ProxyExecuted(Ok(())).into(), + ]); + + let inner = Box::new(Call::Proxy(ProxyCall::add_proxy(5, ProxyType::Any))); + let call = Box::new(Call::Utility(UtilityCall::batch(vec![*inner]))); + assert_ok!(Proxy::proxy(Origin::signed(2), 1, None, call.clone())); + expect_events(vec![UtilityEvent::BatchCompleted.into(), RawEvent::ProxyExecuted(Ok(())).into()]); + assert_ok!(Proxy::proxy(Origin::signed(3), 1, None, call.clone())); + expect_event(RawEvent::ProxyExecuted(Err(DispatchError::BadOrigin))); + assert_ok!(Proxy::proxy(Origin::signed(4), 1, None, call.clone())); + expect_events(vec![ + UtilityEvent::BatchInterrupted(0, DispatchError::BadOrigin).into(), + RawEvent::ProxyExecuted(Ok(())).into(), + ]); + + let call = Box::new(Call::Proxy(ProxyCall::remove_proxies())); + assert_ok!(Proxy::proxy(Origin::signed(3), 1, None, call.clone())); + expect_event(RawEvent::ProxyExecuted(Err(DispatchError::BadOrigin))); + assert_ok!(Proxy::proxy(Origin::signed(4), 1, None, call.clone())); + expect_event(RawEvent::ProxyExecuted(Err(DispatchError::BadOrigin))); + assert_ok!(Proxy::proxy(Origin::signed(2), 1, None, call.clone())); + expect_events(vec![BalancesEvent::::Unreserved(1, 5).into(), RawEvent::ProxyExecuted(Ok(())).into()]); + }); +} + +#[test] +fn add_remove_proxies_works() { + new_test_ext().execute_with(|| { + assert_ok!(Proxy::add_proxy(Origin::signed(1), 2, ProxyType::Any)); + assert_noop!(Proxy::add_proxy(Origin::signed(1), 2, ProxyType::Any), Error::::Duplicate); + assert_eq!(Balances::reserved_balance(1), 2); + assert_ok!(Proxy::add_proxy(Origin::signed(1), 2, ProxyType::JustTransfer)); + assert_eq!(Balances::reserved_balance(1), 3); + assert_ok!(Proxy::add_proxy(Origin::signed(1), 3, ProxyType::Any)); + assert_eq!(Balances::reserved_balance(1), 4); + assert_ok!(Proxy::add_proxy(Origin::signed(1), 4, ProxyType::JustUtility)); + assert_eq!(Balances::reserved_balance(1), 5); + assert_noop!(Proxy::add_proxy(Origin::signed(1), 4, ProxyType::Any), Error::::TooMany); + assert_noop!(Proxy::remove_proxy(Origin::signed(1), 3, ProxyType::JustTransfer), Error::::NotFound); + assert_ok!(Proxy::remove_proxy(Origin::signed(1), 4, ProxyType::JustUtility)); + assert_eq!(Balances::reserved_balance(1), 4); + assert_ok!(Proxy::remove_proxy(Origin::signed(1), 3, ProxyType::Any)); + assert_eq!(Balances::reserved_balance(1), 3); + assert_ok!(Proxy::remove_proxy(Origin::signed(1), 2, ProxyType::Any)); + assert_eq!(Balances::reserved_balance(1), 2); + assert_ok!(Proxy::remove_proxy(Origin::signed(1), 2, ProxyType::JustTransfer)); + assert_eq!(Balances::reserved_balance(1), 0); + }); +} + +#[test] +fn cannot_add_proxy_without_balance() { + new_test_ext().execute_with(|| { + assert_ok!(Proxy::add_proxy(Origin::signed(5), 3, ProxyType::Any)); + assert_eq!(Balances::reserved_balance(5), 2); + assert_noop!( + Proxy::add_proxy(Origin::signed(5), 4, ProxyType::Any), + BalancesError::::InsufficientBalance + ); + }); +} + +#[test] +fn proxying_works() { + new_test_ext().execute_with(|| { + assert_ok!(Proxy::add_proxy(Origin::signed(1), 2, ProxyType::JustTransfer)); + assert_ok!(Proxy::add_proxy(Origin::signed(1), 3, ProxyType::Any)); + + let call = Box::new(Call::Balances(BalancesCall::transfer(6, 1))); + assert_noop!(Proxy::proxy(Origin::signed(4), 1, None, call.clone()), Error::::NotProxy); + assert_noop!( + Proxy::proxy(Origin::signed(2), 1, Some(ProxyType::Any), call.clone()), + Error::::NotProxy + ); + assert_ok!(Proxy::proxy(Origin::signed(2), 1, None, call.clone())); + expect_event(RawEvent::ProxyExecuted(Ok(()))); + assert_eq!(Balances::free_balance(6), 1); + + let call = Box::new(Call::System(SystemCall::set_code(vec![]))); + assert_ok!(Proxy::proxy(Origin::signed(3), 1, None, call.clone())); + expect_event(RawEvent::ProxyExecuted(Err(DispatchError::BadOrigin))); + + let call = Box::new(Call::Balances(BalancesCall::transfer_keep_alive(6, 1))); + assert_ok!(Call::Proxy(super::Call::proxy(1, None, call.clone())).dispatch(Origin::signed(2))); + expect_event(RawEvent::ProxyExecuted(Err(DispatchError::BadOrigin))); + assert_ok!(Proxy::proxy(Origin::signed(3), 1, None, call.clone())); + expect_event(RawEvent::ProxyExecuted(Ok(()))); + assert_eq!(Balances::free_balance(6), 2); + }); +} + +#[test] +fn anonymous_works() { + new_test_ext().execute_with(|| { + assert_ok!(Proxy::anonymous(Origin::signed(1), ProxyType::Any, 0)); + let anon = Proxy::anonymous_account(&1, &ProxyType::Any, 0, None); + expect_event(RawEvent::AnonymousCreated(anon.clone(), 1, ProxyType::Any, 0)); + + // other calls to anonymous allowed as long as they're not exactly the same. + assert_ok!(Proxy::anonymous(Origin::signed(1), ProxyType::JustTransfer, 0)); + assert_ok!(Proxy::anonymous(Origin::signed(1), ProxyType::Any, 1)); + let anon2 = Proxy::anonymous_account(&2, &ProxyType::Any, 0, None); + assert_ok!(Proxy::anonymous(Origin::signed(2), ProxyType::Any, 0)); + assert_noop!(Proxy::anonymous(Origin::signed(1), ProxyType::Any, 0), Error::::Duplicate); + System::set_extrinsic_index(1); + assert_ok!(Proxy::anonymous(Origin::signed(1), ProxyType::Any, 0)); + System::set_extrinsic_index(0); + System::set_block_number(2); + assert_ok!(Proxy::anonymous(Origin::signed(1), ProxyType::Any, 0)); + + let call = Box::new(Call::Balances(BalancesCall::transfer(6, 1))); + assert_ok!(Balances::transfer(Origin::signed(3), anon, 5)); + assert_ok!(Proxy::proxy(Origin::signed(1), anon, None, call)); + expect_event(RawEvent::ProxyExecuted(Ok(()))); + assert_eq!(Balances::free_balance(6), 1); + + let call = Box::new(Call::Proxy(ProxyCall::kill_anonymous(1, ProxyType::Any, 0, 1, 0))); + assert_ok!(Proxy::proxy(Origin::signed(2), anon2, None, call.clone())); + let de = DispatchError::from(Error::::NoPermission).stripped(); + expect_event(RawEvent::ProxyExecuted(Err(de))); + assert_noop!( + Proxy::kill_anonymous(Origin::signed(1), 1, ProxyType::Any, 0, 1, 0), + Error::::NoPermission + ); + assert_eq!(Balances::free_balance(1), 0); + assert_ok!(Proxy::proxy(Origin::signed(1), anon, None, call.clone())); + assert_eq!(Balances::free_balance(1), 2); + assert_noop!(Proxy::proxy(Origin::signed(1), anon, None, call.clone()), Error::::NotProxy); + }); +} diff --git a/frame/randomness-collective-flip/Cargo.toml b/frame/randomness-collective-flip/Cargo.toml index efd41d96b05355105586f1a469f8efaf4df0720b..64324bc8c59b6b908a4a9575865ba3ee395c45df 100644 --- a/frame/randomness-collective-flip/Cargo.toml +++ b/frame/randomness-collective-flip/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-randomness-collective-flip" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,15 +13,15 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] safe-mix = { version = "1.0", default-features = false } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-io = { version = "2.0.0-alpha.8", path = "../../primitives/io" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-io = { version = "2.0.0-rc4", path = "../../primitives/io" } [features] default = ["std"] diff --git a/frame/randomness-collective-flip/src/lib.rs b/frame/randomness-collective-flip/src/lib.rs index 16f54fbc4470ed43f93f16574bc45c731ff1a691..0cf44de679c33e92cd79f9e546fd1e1165d7abc0 100644 --- a/frame/randomness-collective-flip/src/lib.rs +++ b/frame/randomness-collective-flip/src/lib.rs @@ -158,6 +158,7 @@ mod tests { } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -173,6 +174,7 @@ mod tests { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); diff --git a/frame/recovery/Cargo.toml b/frame/recovery/Cargo.toml index 5d4ea80fca45ef696dbe5020e2eba12369b65ab3..63f4d4dcdd14672a7b96a8b0f920a058f1dad5a9 100644 --- a/frame/recovery/Cargo.toml +++ b/frame/recovery/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-recovery" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,17 +13,17 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } enumflags2 = { version = "0.6.2" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } [features] default = ["std"] diff --git a/frame/recovery/src/lib.rs b/frame/recovery/src/lib.rs index cd3ba76b37012cd819d8849de691766e4228246f..9c7503666a7f47970e4119a40157c70d6e515cb0 100644 --- a/frame/recovery/src/lib.rs +++ b/frame/recovery/src/lib.rs @@ -160,7 +160,7 @@ use codec::{Encode, Decode}; use frame_support::{ decl_module, decl_event, decl_storage, decl_error, ensure, - Parameter, RuntimeDebug, weights::{GetDispatchInfo, FunctionOf, Pays}, + Parameter, RuntimeDebug, weights::GetDispatchInfo, traits::{Currency, ReservableCurrency, Get, BalanceStatus}, dispatch::PostDispatchInfo, }; @@ -320,6 +320,18 @@ decl_module! { pub struct Module for enum Call where origin: T::Origin { type Error = Error; + /// The base amount of currency needed to reserve for creating a recovery configuration. + const ConfigDepositBase: BalanceOf = T::ConfigDepositBase::get(); + + /// The amount of currency needed per additional user when creating a recovery configuration. + const FriendDepositFactor: BalanceOf = T::FriendDepositFactor::get(); + + /// The maximum amount of friends allowed in a recovery configuration. + const MaxFriends: u16 = T::MaxFriends::get(); + + /// The base amount of currency needed to reserve for starting a recovery. + const RecoveryDeposit: BalanceOf = T::RecoveryDeposit::get(); + /// Deposit one of this module's events by using the default implementation. fn deposit_event() = default; @@ -336,11 +348,7 @@ decl_module! { /// - The weight of the `call` + 10,000. /// - One storage lookup to check account is recovered by `who`. O(1) /// # - #[weight = FunctionOf( - |args: (&T::AccountId, &Box<::Call>)| args.1.get_dispatch_info().weight + 10_000, - |args: (&T::AccountId, &Box<::Call>)| args.1.get_dispatch_info().class, - Pays::Yes, - )] + #[weight = (call.get_dispatch_info().weight + 10_000, call.get_dispatch_info().class)] fn as_recovered(origin, account: T::AccountId, call: Box<::Call> diff --git a/frame/recovery/src/mock.rs b/frame/recovery/src/mock.rs index 6345eac5a09896297c43b672ba3b5bbab4933e6a..101778f3ea21b23ab76a1c30f77498e7cc0c09fe 100644 --- a/frame/recovery/src/mock.rs +++ b/frame/recovery/src/mock.rs @@ -64,6 +64,7 @@ parameter_types! { } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Call = Call; type Index = u64; @@ -79,6 +80,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/recovery/src/tests.rs b/frame/recovery/src/tests.rs index 5192bdaca85e20c493613a71f0419eef688f6014..8e9484f0fb089588cfe1ef9c347fcbe6d517ae8d 100644 --- a/frame/recovery/src/tests.rs +++ b/frame/recovery/src/tests.rs @@ -46,7 +46,7 @@ fn set_recovered_works() { // Not accessible by a normal user assert_noop!(Recovery::set_recovered(Origin::signed(1), 5, 1), BadOrigin); // Root can set a recovered account though - assert_ok!(Recovery::set_recovered(Origin::ROOT, 5, 1)); + assert_ok!(Recovery::set_recovered(Origin::root(), 5, 1)); // Account 1 should now be able to make a call through account 5 let call = Box::new(Call::Balances(BalancesCall::transfer(1, 100))); assert_ok!(Recovery::as_recovered(Origin::signed(1), 5, call)); diff --git a/frame/scheduler/Cargo.toml b/frame/scheduler/Cargo.toml index 6fdf61cf93e636a9559db57d58487112ffb8d034..43507bd364f0f626254293902e465f8aab77405d 100644 --- a/frame/scheduler/Cargo.toml +++ b/frame/scheduler/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-scheduler" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Unlicense" @@ -11,16 +11,16 @@ description = "FRAME example pallet" [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../benchmarking", optional = true } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core", default-features = false } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core", default-features = false } [features] default = ["std"] diff --git a/frame/scheduler/src/benchmarking.rs b/frame/scheduler/src/benchmarking.rs index 975c10e3b6c8f1a2ddc7e4c1e1b4b6a6e824a768..5c580b5525c367152f8e664db7793a6c3a485009 100644 --- a/frame/scheduler/src/benchmarking.rs +++ b/frame/scheduler/src/benchmarking.rs @@ -29,6 +29,7 @@ use crate::Module as Scheduler; use frame_system::Module as System; const MAX_SCHEDULED: u32 = 50; +const BLOCK_NUMBER: u32 = 2; // Add `n` named items to the schedule fn fill_schedule (when: T::BlockNumber, n: u32) -> Result<(), &'static str> { @@ -55,7 +56,7 @@ benchmarks! { schedule { let s in 0 .. MAX_SCHEDULED; - let when = T::BlockNumber::one(); + let when = BLOCK_NUMBER.into(); let periodic = Some((T::BlockNumber::one(), 100)); let priority = 0; // Essentially a no-op call. @@ -72,7 +73,7 @@ benchmarks! { cancel { let s in 1 .. MAX_SCHEDULED; - let when: T::BlockNumber = 2.into(); + let when = BLOCK_NUMBER.into(); fill_schedule::(when, s)?; assert_eq!(Agenda::::get(when).len(), s as usize); @@ -92,7 +93,7 @@ benchmarks! { schedule_named { let s in 0 .. MAX_SCHEDULED; let id = s.encode(); - let when = T::BlockNumber::one(); + let when = BLOCK_NUMBER.into(); let periodic = Some((T::BlockNumber::one(), 100)); let priority = 0; // Essentially a no-op call. @@ -109,7 +110,7 @@ benchmarks! { cancel_named { let s in 1 .. MAX_SCHEDULED; - let when = T::BlockNumber::one(); + let when = BLOCK_NUMBER.into(); fill_schedule::(when, s)?; }: _(RawOrigin::Root, 0.encode()) @@ -127,9 +128,9 @@ benchmarks! { on_initialize { let s in 0 .. MAX_SCHEDULED; - let when = T::BlockNumber::one(); + let when = BLOCK_NUMBER.into(); fill_schedule::(when, s)?; - }: { Scheduler::::on_initialize(T::BlockNumber::one()); } + }: { Scheduler::::on_initialize(BLOCK_NUMBER.into()); } verify { assert_eq!(System::::event_count(), s); // Next block should have all the schedules again diff --git a/frame/scheduler/src/lib.rs b/frame/scheduler/src/lib.rs index 08f53cc5924a8a6085165c55a0bde4dd400f0aa7..6b47e6258708eebe88672bbdc0910bb60335bfc6 100644 --- a/frame/scheduler/src/lib.rs +++ b/frame/scheduler/src/lib.rs @@ -16,31 +16,29 @@ // limitations under the License. //! # Scheduler +//! A module for scheduling dispatches. //! -//! \# Scheduler +//! - [`scheduler::Trait`](./trait.Trait.html) +//! - [`Call`](./enum.Call.html) +//! - [`Module`](./struct.Module.html) //! -//! - \[`scheduler::Trait`](./trait.Trait.html) -//! - \[`Call`](./enum.Call.html) -//! - \[`Module`](./struct.Module.html) +//! ## Overview //! -//! \## Overview +//! This module exposes capabilities for scheduling dispatches to occur at a +//! specified block number or at a specified period. These scheduled dispatches +//! may be named or anonymous and may be canceled. //! -//! // Short description of pallet's purpose. -//! // Links to Traits that should be implemented. -//! // What this pallet is for. -//! // What functionality the pallet provides. -//! // When to use the pallet (use case examples). -//! // How it is used. -//! // Inputs it uses and the source of each input. -//! // Outputs it produces. +//! ## Interface //! -//! \## Terminology +//! ### Dispatchable Functions //! -//! \## Goals -//! -//! \## Interface -//! -//! \### Dispatchable Functions +//! * `schedule` - schedule a dispatch, which may be periodic, to occur at a +//! specified block and with a specified priority. +//! * `cancel` - cancel a scheduled dispatch, specified by block number and +//! index. +//! * `schedule_named` - augments the `schedule` interface with an additional +//! `Vec` parameter that can be used for identification. +//! * `cancel_named` - the named complement to the cancel function. // Ensure we're `no_std` when compiling for Wasm. #![cfg_attr(not(feature = "std"), no_std)] @@ -121,6 +119,8 @@ decl_error! { FailedToSchedule, /// Failed to cancel a scheduled call FailedToCancel, + /// Given target block number is in the past. + TargetBlockNumberInPast, } } @@ -147,7 +147,7 @@ decl_module! { call: Box<::Call>, ) { ensure_root(origin)?; - let _ = Self::do_schedule(when, maybe_periodic, priority, *call); + Self::do_schedule(when, maybe_periodic, priority, *call)?; } /// Cancel an anonymously scheduled task. @@ -296,7 +296,11 @@ impl Module { maybe_periodic: Option>, priority: schedule::Priority, call: ::Call - ) -> TaskAddress { + ) -> Result, DispatchError> { + if when <= frame_system::Module::::block_number() { + return Err(Error::::TargetBlockNumberInPast.into()) + } + // sanitize maybe_periodic let maybe_periodic = maybe_periodic .filter(|p| p.1 > 1 && !p.0.is_zero()) @@ -306,7 +310,8 @@ impl Module { Agenda::::append(when, s); let index = Agenda::::decode_len(when).unwrap_or(1) as u32 - 1; Self::deposit_event(RawEvent::Scheduled(when, index)); - (when, index) + + Ok((when, index)) } fn do_cancel((when, index): TaskAddress) -> Result<(), DispatchError> { @@ -333,6 +338,10 @@ impl Module { return Err(Error::::FailedToSchedule)? } + if when <= frame_system::Module::::block_number() { + return Err(Error::::TargetBlockNumberInPast.into()) + } + // sanitize maybe_periodic let maybe_periodic = maybe_periodic .filter(|p| p.1 > 1 && !p.0.is_zero()) @@ -345,6 +354,7 @@ impl Module { let address = (when, index); Lookup::::insert(&id, &address); Self::deposit_event(RawEvent::Scheduled(when, index)); + Ok(address) } @@ -368,7 +378,7 @@ impl schedule::Anon::Call> for Module maybe_periodic: Option>, priority: schedule::Priority, call: ::Call - ) -> Self::Address { + ) -> Result { Self::do_schedule(when, maybe_periodic, priority, call) } @@ -401,8 +411,7 @@ mod tests { use frame_support::{ impl_outer_event, impl_outer_origin, impl_outer_dispatch, parameter_types, assert_ok, - traits::{OnInitialize, OnFinalize}, - weights::{DispatchClass, FunctionOf, Pays, constants::RocksDbWeight}, + assert_err, traits::{OnInitialize, OnFinalize, Filter}, weights::constants::RocksDbWeight, }; use sp_core::H256; // The testing primitives are very useful for avoiding having to work with signatures @@ -441,11 +450,7 @@ mod tests { pub struct Module for enum Call where origin: ::Origin { fn deposit_event() = default; - #[weight = FunctionOf( - |args: (&u32, &Weight)| *args.1, - |_: (&u32, &Weight)| DispatchClass::Normal, - Pays::Yes, - )] + #[weight = *weight] fn log(origin, i: u32, weight: Weight) { ensure_root(origin)?; Self::deposit_event(Event::Logged(i, weight)); @@ -475,6 +480,15 @@ mod tests { scheduler, } } + + // Scheduler must dispatch with root and no filter, this tests base filter is indeed not used. + pub struct BaseFilter; + impl Filter for BaseFilter { + fn filter(call: &Call) -> bool { + !matches!(call, Call::Logger(_)) + } + } + // For testing the pallet, we construct most of a mock runtime. This means // first constructing a configuration type (`Test`) which `impl`s each of the // configuration traits of pallets we want to use. @@ -487,8 +501,9 @@ mod tests { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl system::Trait for Test { + type BaseCallFilter = BaseFilter; type Origin = Origin; - type Call = (); + type Call = Call; type Index = u64; type BlockNumber = u64; type Hash = H256; @@ -502,6 +517,7 @@ mod tests { type DbWeight = RocksDbWeight; type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -514,7 +530,7 @@ mod tests { type Event = (); } parameter_types! { - pub const MaximumSchedulerWeight: Weight = Perbill::from_percent(80) * MaximumBlockWeight::get(); + pub MaximumSchedulerWeight: Weight = Perbill::from_percent(80) * MaximumBlockWeight::get(); } impl Trait for Test { type Event = (); @@ -544,7 +560,9 @@ mod tests { #[test] fn basic_scheduling_works() { new_test_ext().execute_with(|| { - Scheduler::do_schedule(4, None, 127, Call::Logger(logger::Call::log(42, 1000))); + let call = Call::Logger(logger::Call::log(42, 1000)); + assert!(!::BaseCallFilter::filter(&call)); + let _ = Scheduler::do_schedule(4, None, 127, call); run_to_block(3); assert!(logger::log().is_empty()); run_to_block(4); @@ -558,7 +576,7 @@ mod tests { fn periodic_scheduling_works() { new_test_ext().execute_with(|| { // at #4, every 3 blocks, 3 times. - Scheduler::do_schedule(4, Some((3, 3)), 127, Call::Logger(logger::Call::log(42, 1000))); + let _ = Scheduler::do_schedule(4, Some((3, 3)), 127, Call::Logger(logger::Call::log(42, 1000))); run_to_block(3); assert!(logger::log().is_empty()); run_to_block(4); @@ -581,7 +599,7 @@ mod tests { new_test_ext().execute_with(|| { // at #4. Scheduler::do_schedule_named(1u32.encode(), 4, None, 127, Call::Logger(logger::Call::log(69, 1000))).unwrap(); - let i = Scheduler::do_schedule(4, None, 127, Call::Logger(logger::Call::log(42, 1000))); + let i = Scheduler::do_schedule(4, None, 127, Call::Logger(logger::Call::log(42, 1000))).unwrap(); run_to_block(3); assert!(logger::log().is_empty()); assert_ok!(Scheduler::do_cancel_named(1u32.encode())); @@ -614,8 +632,8 @@ mod tests { #[test] fn scheduler_respects_weight_limits() { new_test_ext().execute_with(|| { - Scheduler::do_schedule(4, None, 127, Call::Logger(logger::Call::log(42, MaximumSchedulerWeight::get() / 2))); - Scheduler::do_schedule(4, None, 127, Call::Logger(logger::Call::log(69, MaximumSchedulerWeight::get() / 2))); + let _ = Scheduler::do_schedule(4, None, 127, Call::Logger(logger::Call::log(42, MaximumSchedulerWeight::get() / 2))); + let _ = Scheduler::do_schedule(4, None, 127, Call::Logger(logger::Call::log(69, MaximumSchedulerWeight::get() / 2))); // 69 and 42 do not fit together run_to_block(4); assert_eq!(logger::log(), vec![42u32]); @@ -627,8 +645,8 @@ mod tests { #[test] fn scheduler_respects_hard_deadlines_more() { new_test_ext().execute_with(|| { - Scheduler::do_schedule(4, None, 0, Call::Logger(logger::Call::log(42, MaximumSchedulerWeight::get() / 2))); - Scheduler::do_schedule(4, None, 0, Call::Logger(logger::Call::log(69, MaximumSchedulerWeight::get() / 2))); + let _ = Scheduler::do_schedule(4, None, 0, Call::Logger(logger::Call::log(42, MaximumSchedulerWeight::get() / 2))); + let _ = Scheduler::do_schedule(4, None, 0, Call::Logger(logger::Call::log(69, MaximumSchedulerWeight::get() / 2))); // With base weights, 69 and 42 should not fit together, but do because of hard deadlines run_to_block(4); assert_eq!(logger::log(), vec![42u32, 69u32]); @@ -638,8 +656,8 @@ mod tests { #[test] fn scheduler_respects_priority_ordering() { new_test_ext().execute_with(|| { - Scheduler::do_schedule(4, None, 1, Call::Logger(logger::Call::log(42, MaximumSchedulerWeight::get() / 2))); - Scheduler::do_schedule(4, None, 0, Call::Logger(logger::Call::log(69, MaximumSchedulerWeight::get() / 2))); + let _ = Scheduler::do_schedule(4, None, 1, Call::Logger(logger::Call::log(42, MaximumSchedulerWeight::get() / 2))); + let _ = Scheduler::do_schedule(4, None, 0, Call::Logger(logger::Call::log(69, MaximumSchedulerWeight::get() / 2))); run_to_block(4); assert_eq!(logger::log(), vec![69u32, 42u32]); }); @@ -648,9 +666,24 @@ mod tests { #[test] fn scheduler_respects_priority_ordering_with_soft_deadlines() { new_test_ext().execute_with(|| { - Scheduler::do_schedule(4, None, 255, Call::Logger(logger::Call::log(42, MaximumSchedulerWeight::get() / 3))); - Scheduler::do_schedule(4, None, 127, Call::Logger(logger::Call::log(69, MaximumSchedulerWeight::get() / 2))); - Scheduler::do_schedule(4, None, 126, Call::Logger(logger::Call::log(2600, MaximumSchedulerWeight::get() / 2))); + let _ = Scheduler::do_schedule( + 4, + None, + 255, + Call::Logger(logger::Call::log(42, MaximumSchedulerWeight::get() / 3)), + ); + let _ = Scheduler::do_schedule( + 4, + None, + 127, + Call::Logger(logger::Call::log(69, MaximumSchedulerWeight::get() / 2)), + ); + let _ = Scheduler::do_schedule( + 4, + None, + 126, + Call::Logger(logger::Call::log(2600, MaximumSchedulerWeight::get() / 2)), + ); // 2600 does not fit with 69 or 42, but has higher priority, so will go through run_to_block(4); @@ -672,11 +705,27 @@ mod tests { // Named assert_ok!(Scheduler::do_schedule_named(1u32.encode(), 1, None, 255, Call::Logger(logger::Call::log(3, MaximumSchedulerWeight::get() / 3)))); // Anon Periodic - Scheduler::do_schedule(1, Some((1000, 3)), 128, Call::Logger(logger::Call::log(42, MaximumSchedulerWeight::get() / 3))); + let _ = Scheduler::do_schedule( + 1, + Some((1000, 3)), + 128, + Call::Logger(logger::Call::log(42, MaximumSchedulerWeight::get() / 3)), + ); // Anon - Scheduler::do_schedule(1, None, 127, Call::Logger(logger::Call::log(69, MaximumSchedulerWeight::get() / 2))); + let _ = Scheduler::do_schedule( + 1, + None, + 127, + Call::Logger(logger::Call::log(69, MaximumSchedulerWeight::get() / 2)), + ); // Named Periodic - assert_ok!(Scheduler::do_schedule_named(2u32.encode(), 1, Some((1000, 3)), 126, Call::Logger(logger::Call::log(2600, MaximumSchedulerWeight::get() / 2)))); + assert_ok!(Scheduler::do_schedule_named( + 2u32.encode(), + 1, + Some((1000, 3)), + 126, + Call::Logger(logger::Call::log(2600, MaximumSchedulerWeight::get() / 2)), + )); // Will include the named periodic only let actual_weight = Scheduler::on_initialize(1); @@ -707,17 +756,42 @@ mod tests { new_test_ext().execute_with(|| { let call = Box::new(Call::Logger(logger::Call::log(69, 1000))); let call2 = Box::new(Call::Logger(logger::Call::log(42, 1000))); - assert_ok!(Scheduler::schedule_named(Origin::ROOT, 1u32.encode(), 4, None, 127, call)); - assert_ok!(Scheduler::schedule(Origin::ROOT, 4, None, 127, call2)); + assert_ok!(Scheduler::schedule_named(Origin::root(), 1u32.encode(), 4, None, 127, call)); + assert_ok!(Scheduler::schedule(Origin::root(), 4, None, 127, call2)); run_to_block(3); // Scheduled calls are in the agenda. assert_eq!(Agenda::::get(4).len(), 2); assert!(logger::log().is_empty()); - assert_ok!(Scheduler::cancel_named(Origin::ROOT, 1u32.encode())); - assert_ok!(Scheduler::cancel(Origin::ROOT, 4, 1)); + assert_ok!(Scheduler::cancel_named(Origin::root(), 1u32.encode())); + assert_ok!(Scheduler::cancel(Origin::root(), 4, 1)); // Scheduled calls are made NONE, so should not effect state run_to_block(100); assert!(logger::log().is_empty()); }); } + + #[test] + fn fails_to_schedule_task_in_the_past() { + new_test_ext().execute_with(|| { + run_to_block(3); + + let call = Box::new(Call::Logger(logger::Call::log(69, 1000))); + let call2 = Box::new(Call::Logger(logger::Call::log(42, 1000))); + + assert_err!( + Scheduler::schedule_named(Origin::root(), 1u32.encode(), 2, None, 127, call), + Error::::TargetBlockNumberInPast, + ); + + assert_err!( + Scheduler::schedule(Origin::root(), 2, None, 127, call2.clone()), + Error::::TargetBlockNumberInPast, + ); + + assert_err!( + Scheduler::schedule(Origin::root(), 3, None, 127, call2), + Error::::TargetBlockNumberInPast, + ); + }); + } } diff --git a/frame/scored-pool/Cargo.toml b/frame/scored-pool/Cargo.toml index e24505355d08bfedc3558bfb49c1f4bdfe645a65..05fc56fc65a28d7030277c339567336d6c9ae25b 100644 --- a/frame/scored-pool/Cargo.toml +++ b/frame/scored-pool/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-scored-pool" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,17 +12,17 @@ description = "FRAME pallet for scored pools" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } [dev-dependencies] -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } [features] default = ["std"] diff --git a/frame/scored-pool/src/lib.rs b/frame/scored-pool/src/lib.rs index ba56298493a99b4148472e4db976393ff20bbf0a..81ee92aeb4641be5493f9f991ded0a10768c1700 100644 --- a/frame/scored-pool/src/lib.rs +++ b/frame/scored-pool/src/lib.rs @@ -308,7 +308,7 @@ decl_module! { /// Kick a member `who` from the set. /// - /// May only be called from `KickOrigin` or root. + /// May only be called from `T::KickOrigin`. /// /// The `index` parameter of this function must be set to /// the index of `dest` in the `Pool`. @@ -318,9 +318,7 @@ decl_module! { dest: ::Source, index: u32 ) { - T::KickOrigin::try_origin(origin) - .map(|_| ()) - .or_else(ensure_root)?; + T::KickOrigin::ensure_origin(origin)?; let who = T::Lookup::lookup(dest)?; @@ -333,7 +331,7 @@ decl_module! { /// Score a member `who` with `score`. /// - /// May only be called from `ScoreOrigin` or root. + /// May only be called from `T::ScoreOrigin`. /// /// The `index` parameter of this function must be set to /// the index of the `dest` in the `Pool`. @@ -344,9 +342,7 @@ decl_module! { index: u32, score: T::Score ) { - T::ScoreOrigin::try_origin(origin) - .map(|_| ()) - .or_else(ensure_root)?; + T::ScoreOrigin::ensure_origin(origin)?; let who = T::Lookup::lookup(dest)?; diff --git a/frame/scored-pool/src/mock.rs b/frame/scored-pool/src/mock.rs index aae86973a9a9d39228b678992f834518193acc9d..87a56ca27dba01ff20d9d9f9d055e48b79c812ec 100644 --- a/frame/scored-pool/src/mock.rs +++ b/frame/scored-pool/src/mock.rs @@ -55,6 +55,7 @@ ord_parameter_types! { } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -70,6 +71,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/session/Cargo.toml b/frame/session/Cargo.toml index 61fa57521e08903f42f108d298f7635c11b443c5..c882df7115f3878b848dfdb02818ac7e7dad85c2 100644 --- a/frame/session/Cargo.toml +++ b/frame/session/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-session" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,21 +13,21 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-session = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/session" } -sp-staking = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/staking" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -pallet-timestamp = { version = "2.0.0-alpha.8", default-features = false, path = "../timestamp" } -sp-trie = { optional = true, path = "../../primitives/trie", default-features = false, version = "2.0.0-alpha.8"} -sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.8"} +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-session = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/session" } +sp-staking = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/staking" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +pallet-timestamp = { version = "2.0.0-rc4", default-features = false, path = "../timestamp" } +sp-trie = { version = "2.0.0-rc4", optional = true, default-features = false, path = "../../primitives/trie" } impl-trait-for-tuples = "0.1.3" [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-application-crypto = { version = "2.0.0-alpha.8", path = "../../primitives/application-crypto" } +sp-application-crypto = { version = "2.0.0-rc4", path = "../../primitives/application-crypto" } lazy_static = "1.4.0" [features] @@ -37,11 +37,12 @@ std = [ "serde", "codec/std", "sp-std/std", + "sp-io/std", "frame-support/std", + "sp-core/std", "sp-runtime/std", "sp-session/std", "sp-staking/std", "pallet-timestamp/std", "sp-trie/std", - "sp-io/std", ] diff --git a/frame/session/benchmarking/Cargo.toml b/frame/session/benchmarking/Cargo.toml index e06377a0b4556b689dc78c81a55b18f2ba4bf02f..391b80237ef737567f5e62ce87a0a87cde0aa1b8 100644 --- a/frame/session/benchmarking/Cargo.toml +++ b/frame/session/benchmarking/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-session-benchmarking" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,22 +12,22 @@ description = "FRAME sessions pallet benchmarking" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/runtime" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../../system" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../../benchmarking" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../../support" } -pallet-staking = { version = "2.0.0-alpha.8", default-features = false, features = ["runtime-benchmarks"], path = "../../staking" } -pallet-session = { version = "2.0.0-alpha.8", default-features = false, path = "../../session" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/runtime" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../../system" } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../../benchmarking" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../../support" } +pallet-staking = { version = "2.0.0-rc4", default-features = false, features = ["runtime-benchmarks"], path = "../../staking" } +pallet-session = { version = "2.0.0-rc4", default-features = false, path = "../../session" } [dev-dependencies] serde = { version = "1.0.101" } -codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -pallet-staking-reward-curve = { version = "2.0.0-alpha.8", path = "../../staking/reward-curve" } -sp-io ={ path = "../../../primitives/io", version = "2.0.0-alpha.8"} -pallet-timestamp = { version = "2.0.0-alpha.8", path = "../../timestamp" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../../balances" } +codec = { package = "parity-scale-codec", version = "1.3.1", features = ["derive"] } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +pallet-staking-reward-curve = { version = "2.0.0-rc4", path = "../../staking/reward-curve" } +sp-io ={ version = "2.0.0-rc4", path = "../../../primitives/io" } +pallet-timestamp = { version = "2.0.0-rc4", path = "../../timestamp" } +pallet-balances = { version = "2.0.0-rc4", path = "../../balances" } [features] default = ["std"] diff --git a/frame/session/benchmarking/src/lib.rs b/frame/session/benchmarking/src/lib.rs index 04b7d556026da81efb0573ad37fc482a736964e5..0df4dcfbd9bad083da763bbe32066b053db328ba 100644 --- a/frame/session/benchmarking/src/lib.rs +++ b/frame/session/benchmarking/src/lib.rs @@ -45,7 +45,7 @@ benchmarks! { set_keys { let n in 1 .. MAX_NOMINATIONS as u32; - let v_stash = create_validator_with_nominators::(n, MAX_NOMINATIONS as u32)?; + let v_stash = create_validator_with_nominators::(n, MAX_NOMINATIONS as u32, false)?; let v_controller = pallet_staking::Module::::bonded(&v_stash).ok_or("not stash")?; let keys = T::Keys::default(); let proof: Vec = vec![0,1,2,3]; @@ -53,7 +53,7 @@ benchmarks! { purge_keys { let n in 1 .. MAX_NOMINATIONS as u32; - let v_stash = create_validator_with_nominators::(n, MAX_NOMINATIONS as u32)?; + let v_stash = create_validator_with_nominators::(n, MAX_NOMINATIONS as u32, false)?; let v_controller = pallet_staking::Module::::bonded(&v_stash).ok_or("not stash")?; let keys = T::Keys::default(); let proof: Vec = vec![0,1,2,3]; diff --git a/frame/session/benchmarking/src/mock.rs b/frame/session/benchmarking/src/mock.rs index 9ec017cf22083d5e5d1cd2ac58daa0e806dc74f4..ee04f1a04645d22d9508a6a4466fc0c5d65ba391 100644 --- a/frame/session/benchmarking/src/mock.rs +++ b/frame/session/benchmarking/src/mock.rs @@ -58,6 +58,7 @@ impl Convert for CurrencyToVoteHandler { pub struct Test; impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = AccountIndex; type BlockNumber = BlockNumber; @@ -73,6 +74,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = (); type AvailableBlockRatio = (); type MaximumBlockLength = (); type Version = (); @@ -182,6 +184,7 @@ impl pallet_staking::Trait for Test { type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; type UnsignedPriority = UnsignedPriority; type MaxIterations = (); + type MinSolutionScoreBump = (); } impl crate::Trait for Test {} diff --git a/frame/session/src/historical.rs b/frame/session/src/historical/mod.rs similarity index 97% rename from frame/session/src/historical.rs rename to frame/session/src/historical/mod.rs index a1c286eb39245548fa4c460dc7d2113f0412b262..20c3d57464c89c9a3f6b4877e64210e489a79513 100644 --- a/frame/session/src/historical.rs +++ b/frame/session/src/historical/mod.rs @@ -37,6 +37,10 @@ use sp_trie::{MemoryDB, Trie, TrieMut, Recorder, EMPTY_PREFIX}; use sp_trie::trie_types::{TrieDBMut, TrieDB}; use super::{SessionIndex, Module as SessionModule}; +mod shared; +pub mod offchain; +pub mod onchain; + /// Trait necessary for the historical module. pub trait Trait: super::Trait { /// Full identification of the validator. @@ -116,6 +120,7 @@ impl crate::SessionManager for NoteHistoricalRoot { fn new_session(new_index: SessionIndex) -> Option> { + StoredRange::mutate(|range| { range.get_or_insert_with(|| (new_index, new_index)).1 = new_index + 1; }); @@ -143,10 +148,13 @@ impl crate::SessionManager for NoteHistoricalRoot>::start_session(start_index) } + fn end_session(end_index: SessionIndex) { + onchain::store_session_validator_set_to_offchain::(end_index); >::end_session(end_index) } } @@ -154,7 +162,7 @@ impl crate::SessionManager for NoteHistoricalRoot = (::ValidatorId, ::FullIdentification); -/// a trie instance for checking and generating proofs. +/// A trie instance for checking and generating proofs. pub struct ProvingTrie { db: MemoryDB, root: T::Hash, @@ -250,7 +258,6 @@ impl ProvingTrie { .ok()? .and_then(|raw| >::decode(&mut &*raw).ok()) } - } impl> frame_support::traits::KeyOwnerProofSystem<(KeyTypeId, D)> @@ -311,9 +318,9 @@ impl> frame_support::traits::KeyOwnerProofSystem<(KeyTy } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; - use sp_core::crypto::key_types::DUMMY; + use sp_runtime::key_types::DUMMY; use sp_runtime::testing::UintAuthorityId; use crate::mock::{ NEXT_VALIDATORS, force_new_session, @@ -323,7 +330,7 @@ mod tests { type Historical = Module; - fn new_test_ext() -> sp_io::TestExternalities { + pub(crate) fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); crate::GenesisConfig:: { keys: NEXT_VALIDATORS.with(|l| diff --git a/frame/session/src/historical/offchain.rs b/frame/session/src/historical/offchain.rs new file mode 100644 index 0000000000000000000000000000000000000000..97655d1a18b3280b85e7d2074e8071152611dfea --- /dev/null +++ b/frame/session/src/historical/offchain.rs @@ -0,0 +1,263 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Off-chain logic for creating a proof based data provided by on-chain logic. +//! +//! Validator-set extracting an iterator from an off-chain worker stored list containing +//! historical validator-sets. +//! Based on the logic of historical slashing, but the validation is done off-chain. +//! Use [`fn store_current_session_validator_set_to_offchain()`](super::onchain) to store the +//! required data to the offchain validator set. +//! This is used in conjunction with [`ProvingTrie`](super::ProvingTrie) and +//! the off-chain indexing API. + +use sp_runtime::{offchain::storage::StorageValueRef, KeyTypeId}; +use sp_session::MembershipProof; + +use super::super::{Module as SessionModule, SessionIndex}; +use super::{IdentificationTuple, ProvingTrie, Trait}; + +use super::shared; +use sp_std::prelude::*; + + +/// A set of validators, which was used for a fixed session index. +struct ValidatorSet { + validator_set: Vec>, +} + +impl ValidatorSet { + /// Load the set of validators for a particular session index from the off-chain storage. + /// + /// If none is found or decodable given `prefix` and `session`, it will return `None`. + /// Empty validator sets should only ever exist for genesis blocks. + pub fn load_from_offchain_db(session_index: SessionIndex) -> Option { + let derived_key = shared::derive_key(shared::PREFIX, session_index); + StorageValueRef::persistent(derived_key.as_ref()) + .get::>() + .flatten() + .map(|validator_set| Self { validator_set }) + } + + #[inline] + fn len(&self) -> usize { + self.validator_set.len() + } +} + +/// Implement conversion into iterator for usage +/// with [ProvingTrie](super::ProvingTrie::generate_for). +impl sp_std::iter::IntoIterator for ValidatorSet { + type Item = (T::ValidatorId, T::FullIdentification); + type IntoIter = sp_std::vec::IntoIter; + fn into_iter(self) -> Self::IntoIter { + self.validator_set.into_iter() + } +} + +/// Create a proof based on the data available in the off-chain database. +/// +/// Based on the yielded `MembershipProof` the implementer may decide what +/// to do, i.e. in case of a failed proof, enqueue a transaction back on +/// chain reflecting that, with all its consequences such as i.e. slashing. +pub fn prove_session_membership>( + session_index: SessionIndex, + session_key: (KeyTypeId, D), +) -> Option { + let validators = ValidatorSet::::load_from_offchain_db(session_index)?; + let count = validators.len() as u32; + let trie = ProvingTrie::::generate_for(validators.into_iter()).ok()?; + + let (id, data) = session_key; + trie.prove(id, data.as_ref()) + .map(|trie_nodes| MembershipProof { + session: session_index, + trie_nodes, + validator_count: count, + }) +} + + +/// Attempt to prune anything that is older than `first_to_keep` session index. +/// +/// Due to re-organisation it could be that the `first_to_keep` might be less +/// than the stored one, in which case the conservative choice is made to keep records +/// up to the one that is the lesser. +pub fn prune_older_than(first_to_keep: SessionIndex) { + let derived_key = shared::LAST_PRUNE.to_vec(); + let entry = StorageValueRef::persistent(derived_key.as_ref()); + match entry.mutate(|current: Option>| -> Result<_, ()> { + match current { + Some(Some(current)) if current < first_to_keep => Ok(first_to_keep), + // do not move the cursor, if the new one would be behind ours + Some(Some(current)) => Ok(current), + None => Ok(first_to_keep), + // if the storage contains undecodable data, overwrite with current anyways + // which might leak some entries being never purged, but that is acceptable + // in this context + Some(None) => Ok(first_to_keep), + } + }) { + Ok(Ok(new_value)) => { + // on a re-org this is not necessarily true, with the above they might be equal + if new_value < first_to_keep { + for session_index in new_value..first_to_keep { + let derived_key = shared::derive_key(shared::PREFIX, session_index); + let _ = StorageValueRef::persistent(derived_key.as_ref()).clear(); + } + } + } + Ok(Err(_)) => {} // failed to store the value calculated with the given closure + Err(_) => {} // failed to calculate the value to store with the given closure + } +} + +/// Keep the newest `n` items, and prune all items older than that. +pub fn keep_newest(n_to_keep: usize) { + let session_index = >::current_index(); + let n_to_keep = n_to_keep as SessionIndex; + if n_to_keep < session_index { + prune_older_than::(session_index - n_to_keep) + } +} + +#[cfg(test)] +mod tests { + use super::super::{onchain, Module}; + use super::*; + use crate::mock::{ + force_new_session, set_next_validators, Session, System, Test, NEXT_VALIDATORS, + }; + use codec::Encode; + use frame_support::traits::{KeyOwnerProofSystem, OnInitialize}; + use sp_core::crypto::key_types::DUMMY; + use sp_core::offchain::{ + testing::TestOffchainExt, + OffchainExt, + StorageKind, + }; + + use sp_runtime::testing::UintAuthorityId; + + type Historical = Module; + + pub fn new_test_ext() -> sp_io::TestExternalities { + let mut ext = frame_system::GenesisConfig::default() + .build_storage::() + .expect("Failed to create test externalities."); + + crate::GenesisConfig:: { + keys: NEXT_VALIDATORS.with(|l| { + l.borrow() + .iter() + .cloned() + .map(|i| (i, i, UintAuthorityId(i).into())) + .collect() + }), + } + .assimilate_storage(&mut ext) + .unwrap(); + + + let mut ext = sp_io::TestExternalities::new(ext); + + let (offchain, offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db()); + + const ITERATIONS: u32 = 5u32; + let mut seed = [0u8; 32]; + seed[0..4].copy_from_slice(&ITERATIONS.to_le_bytes()); + offchain_state.write().seed = seed; + + ext.register_extension(OffchainExt::new(offchain)); + ext + } + + #[test] + fn encode_decode_roundtrip() { + use codec::{Decode, Encode}; + use super::super::super::Trait as SessionTrait; + use super::super::Trait as HistoricalTrait; + + let sample = ( + 22u32 as ::ValidatorId, + 7_777_777 as ::FullIdentification); + + let encoded = sample.encode(); + let decoded = Decode::decode(&mut encoded.as_slice()).expect("Must decode"); + assert_eq!(sample, decoded); + } + + #[test] + fn onchain_to_offchain() { + let mut ext = new_test_ext(); + + const DATA: &[u8] = &[7,8,9,10,11]; + ext.execute_with(|| { + b"alphaomega"[..].using_encoded(|key| sp_io::offchain_index::set(key, DATA)); + }); + + ext.persist_offchain_overlay(); + + ext.execute_with(|| { + let data = + b"alphaomega"[..].using_encoded(|key| { + sp_io::offchain::local_storage_get(StorageKind::PERSISTENT, key) + }); + assert_eq!(data, Some(DATA.to_vec())); + }); + } + + + #[test] + fn historical_proof_offchain() { + let mut ext = new_test_ext(); + let encoded_key_1 = UintAuthorityId(1).encode(); + + ext.execute_with(|| { + set_next_validators(vec![1, 2]); + force_new_session(); + + System::set_block_number(1); + Session::on_initialize(1); + + // "on-chain" + onchain::store_current_session_validator_set_to_offchain::(); + assert_eq!(>::current_index(), 1); + + set_next_validators(vec![7, 8]); + + force_new_session(); + }); + + ext.persist_offchain_overlay(); + + ext.execute_with(|| { + + + System::set_block_number(2); + Session::on_initialize(2); + assert_eq!(>::current_index(), 2); + + // "off-chain" + let proof = prove_session_membership::(1, (DUMMY, &encoded_key_1)); + assert!(proof.is_some()); + let proof = proof.expect("Must be Some(Proof)"); + + assert!(Historical::check_proof((DUMMY, &encoded_key_1[..]), proof.clone()).is_some()); + }); + } +} diff --git a/frame/session/src/historical/onchain.rs b/frame/session/src/historical/onchain.rs new file mode 100644 index 0000000000000000000000000000000000000000..745603a49829be556cddd7f1fbfe244a1ea0a93d --- /dev/null +++ b/frame/session/src/historical/onchain.rs @@ -0,0 +1,62 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! On-chain logic to store a validator-set for deferred validation using an off-chain worker. + +use codec::Encode; +use sp_runtime::traits::Convert; + +use super::super::Trait as SessionTrait; +use super::super::{Module as SessionModule, SessionIndex}; +use super::Trait as HistoricalTrait; + +use super::shared; +use sp_std::prelude::*; + +/// Store the validator-set associated to the `session_index` to the off-chain database. +/// +/// Further processing is then done [`off-chain side`](super::offchain). +/// +/// **Must** be called from on-chain, i.e. a call that originates from +/// `on_initialize(..)` or `on_finalization(..)`. +/// **Must** be called during the session, which validator-set is to be stored for further +/// off-chain processing. Otherwise the `FullIdentification` might not be available. +pub fn store_session_validator_set_to_offchain( + session_index: SessionIndex, +) { + let encoded_validator_list = >::validators() + .into_iter() + .filter_map(|validator_id: ::ValidatorId| { + let full_identification = + <::FullIdentificationOf>::convert(validator_id.clone()); + full_identification.map(|full_identification| (validator_id, full_identification)) + }) + .collect::>(); + + encoded_validator_list.using_encoded(|encoded_validator_list| { + let derived_key = shared::derive_key(shared::PREFIX, session_index); + sp_io::offchain_index::set(derived_key.as_slice(), encoded_validator_list); + }); +} + +/// Store the validator set associated to the _current_ session index to the off-chain database. +/// +/// See [`fn store_session_validator_set_...(..)`](Self::store_session_validator_set_to_offchain) +/// for further information and restrictions. +pub fn store_current_session_validator_set_to_offchain() { + store_session_validator_set_to_offchain::(>::current_index()); +} diff --git a/frame/session/src/historical/shared.rs b/frame/session/src/historical/shared.rs new file mode 100644 index 0000000000000000000000000000000000000000..fda0361b05959ab388a34d31d7fce5e1efc8ea4b --- /dev/null +++ b/frame/session/src/historical/shared.rs @@ -0,0 +1,39 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Shared logic between on-chain and off-chain components used for slashing using an off-chain +//! worker. + + +use super::SessionIndex; +use sp_std::prelude::*; +use codec::Encode; + +pub(super) const PREFIX: &[u8] = b"session_historical"; +pub(super) const LAST_PRUNE: &[u8] = b"session_historical_last_prune"; + +/// Derive the key used to store the list of validators +pub(super) fn derive_key>(prefix: P, session_index: SessionIndex) -> Vec { + let prefix: &[u8] = prefix.as_ref(); + session_index.using_encoded(|encoded_session_index| { + prefix.into_iter() + .chain(b"/".into_iter()) + .chain(encoded_session_index.into_iter()) + .copied() + .collect::>() + }) +} \ No newline at end of file diff --git a/frame/session/src/lib.rs b/frame/session/src/lib.rs index 47517702cc57a6ea5ac8b39e6a6add7ea6562ccd..6f5630adf9ecf5836d610d763cbe2892290cab1e 100644 --- a/frame/session/src/lib.rs +++ b/frame/session/src/lib.rs @@ -219,7 +219,7 @@ pub trait SessionHandler { /// All the key type ids this session handler can process. /// /// The order must be the same as it expects them in - /// [`on_new_session`](Self::on_new_session) and [`on_genesis_session`](Self::on_genesis_session). + /// [`on_new_session`](Self::on_new_session) and [`on_genesis_session`](Self::on_genesis_session). const KEY_TYPE_IDS: &'static [KeyTypeId]; /// The given validator set will be used for the genesis session. diff --git a/frame/session/src/mock.rs b/frame/session/src/mock.rs index 30fb35b2823187c82df32299bcbb37d3345b4d33..51ca3bc790aad2adbbd29c32adf1457a31314f3d 100644 --- a/frame/session/src/mock.rs +++ b/frame/session/src/mock.rs @@ -173,6 +173,7 @@ parameter_types! { } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -188,6 +189,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); diff --git a/frame/society/Cargo.toml b/frame/society/Cargo.toml index 00f5aa651b999265c061ba890871d36ea129c4d8..67c4c3296639a8683e0ff3ceaedea72aba144731 100644 --- a/frame/society/Cargo.toml +++ b/frame/society/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-society" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,24 +13,23 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.8"} -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } rand_chacha = { version = "0.2", default-features = false } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-io ={ version = "2.0.0-rc4", path = "../../primitives/io" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } [features] default = ["std"] std = [ "codec/std", "serde", - "sp-io/std", "sp-runtime/std", "rand_chacha/std", "sp-std/std", diff --git a/frame/society/src/lib.rs b/frame/society/src/lib.rs index 122ed06b2911917d2347ce03717052ca3857d290..684fe5043745a4bdaaad32cc798b80dee0316471 100644 --- a/frame/society/src/lib.rs +++ b/frame/society/src/lib.rs @@ -532,7 +532,7 @@ decl_module! { /// /// Total Complexity: O(M + B + C + logM + logB + X) /// # - #[weight = 50_000_000] + #[weight = T::MaximumBlockWeight::get() / 10] pub fn bid(origin, value: BalanceOf) -> DispatchResult { let who = ensure_signed(origin)?; ensure!(!>::contains_key(&who), Error::::Suspended); @@ -571,7 +571,7 @@ decl_module! { /// /// Total Complexity: O(B + X) /// # - #[weight = 20_000_000] + #[weight = T::MaximumBlockWeight::get() / 10] pub fn unbid(origin, pos: u32) -> DispatchResult { let who = ensure_signed(origin)?; @@ -641,7 +641,7 @@ decl_module! { /// /// Total Complexity: O(M + B + C + logM + logB + X) /// # - #[weight = 50_000_000] + #[weight = T::MaximumBlockWeight::get() / 10] pub fn vouch(origin, who: T::AccountId, value: BalanceOf, tip: BalanceOf) -> DispatchResult { let voucher = ensure_signed(origin)?; // Check user is not suspended. @@ -682,7 +682,7 @@ decl_module! { /// /// Total Complexity: O(B) /// # - #[weight = 20_000_000] + #[weight = T::MaximumBlockWeight::get() / 10] pub fn unvouch(origin, pos: u32) -> DispatchResult { let voucher = ensure_signed(origin)?; ensure!(Self::vouching(&voucher) == Some(VouchingStatus::Vouching), Error::::NotVouching); @@ -720,7 +720,7 @@ decl_module! { /// /// Total Complexity: O(M + logM + C) /// # - #[weight = 30_000_000] + #[weight = T::MaximumBlockWeight::get() / 10] pub fn vote(origin, candidate: ::Source, approve: bool) { let voter = ensure_signed(origin)?; let candidate = T::Lookup::lookup(candidate)?; @@ -751,7 +751,7 @@ decl_module! { /// /// Total Complexity: O(M + logM) /// # - #[weight = 20_000_000] + #[weight = T::MaximumBlockWeight::get() / 10] pub fn defender_vote(origin, approve: bool) { let voter = ensure_signed(origin)?; let members = >::get(); @@ -783,7 +783,7 @@ decl_module! { /// /// Total Complexity: O(M + logM + P + X) /// # - #[weight = 30_000_000] + #[weight = T::MaximumBlockWeight::get() / 10] pub fn payout(origin) { let who = ensure_signed(origin)?; @@ -825,7 +825,7 @@ decl_module! { /// /// Total Complexity: O(1) /// # - #[weight = 0] + #[weight = T::MaximumBlockWeight::get() / 10] fn found(origin, founder: T::AccountId, max_members: u32, rules: Vec) { T::FounderSetOrigin::ensure_origin(origin)?; ensure!(!>::exists(), Error::::AlreadyFounded); @@ -852,7 +852,7 @@ decl_module! { /// /// Total Complexity: O(1) /// # - #[weight = 20_000_000] + #[weight = T::MaximumBlockWeight::get() / 10] fn unfound(origin) { let founder = ensure_signed(origin)?; ensure!(Founder::::get() == Some(founder.clone()), Error::::NotFounder); @@ -894,7 +894,7 @@ decl_module! { /// /// Total Complexity: O(M + logM + B) /// # - #[weight = 30_000_000] + #[weight = T::MaximumBlockWeight::get() / 10] fn judge_suspended_member(origin, who: T::AccountId, forgive: bool) { T::SuspensionJudgementOrigin::ensure_origin(origin)?; ensure!(>::contains_key(&who), Error::::NotSuspended); @@ -965,7 +965,7 @@ decl_module! { /// /// Total Complexity: O(M + logM + B + X) /// # - #[weight = 50_000_000] + #[weight = T::MaximumBlockWeight::get() / 10] fn judge_suspended_candidate(origin, who: T::AccountId, judgement: Judgement) { T::SuspensionJudgementOrigin::ensure_origin(origin)?; if let Some((value, kind)) = >::get(&who) { @@ -1025,7 +1025,7 @@ decl_module! { /// /// Total Complexity: O(1) /// # - #[weight = 0] + #[weight = T::MaximumBlockWeight::get() / 10] fn set_max_members(origin, max: u32) { ensure_root(origin)?; ensure!(max > 1, Error::::MaxMembers); @@ -1036,10 +1036,14 @@ decl_module! { fn on_initialize(n: T::BlockNumber) -> Weight { let mut members = vec![]; + let mut weight = 0; + // Run a candidate/membership rotation if (n % T::RotationPeriod::get()).is_zero() { members = >::get(); Self::rotate_period(&mut members); + + weight += T::MaximumBlockWeight::get() / 20; } // Run a challenge rotation @@ -1049,9 +1053,11 @@ decl_module! { members = >::get(); } Self::rotate_challenge(&mut members); + + weight += T::MaximumBlockWeight::get() / 20; } - 0 + weight } } } diff --git a/frame/society/src/mock.rs b/frame/society/src/mock.rs index 3b5cb550a412253d85afc522f1b6d4ed06ecfc03..89a0691b93fed3c03097c699a098574aecc498f2 100644 --- a/frame/society/src/mock.rs +++ b/frame/society/src/mock.rs @@ -65,6 +65,7 @@ ord_parameter_types! { } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -80,6 +81,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/society/src/tests.rs b/frame/society/src/tests.rs index 8f18ecba469b61ab3831f6fbd22714de953b7933..0374c7bcd7a6079d1b7960a5f87fef0e89d17547 100644 --- a/frame/society/src/tests.rs +++ b/frame/society/src/tests.rs @@ -813,7 +813,7 @@ fn max_limits_work() { // No candidates because full assert_eq!(Society::candidates().len(), 0); // Increase member limit - assert_ok!(Society::set_max_members(Origin::ROOT, 200)); + assert_ok!(Society::set_max_members(Origin::root(), 200)); // Rotate period run_to_block(16); // Candidates are back! diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 38fb18b56a27e6a3f58fea8ebb636d55bb75b8fe..144095cfa97744ac22ec36be1d7be7d796e20a4e 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-staking" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,55 +12,44 @@ description = "FRAME pallet staking" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-phragmen = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/phragmen" } -sp-io ={ version = "2.0.0-alpha.8", path = "../../primitives/io", default-features = false } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-staking = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/staking" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -pallet-session = { version = "2.0.0-alpha.8", features = ["historical"], path = "../session", default-features = false } -pallet-authorship = { version = "2.0.0-alpha.8", default-features = false, path = "../authorship" } -sp-application-crypto = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/application-crypto" } static_assertions = "1.1.0" - -# Optional imports for tesing-utils feature -pallet-indices = { version = "2.0.0-alpha.8", optional = true, path = "../indices", default-features = false } -sp-core = { version = "2.0.0-alpha.8", optional = true, path = "../../primitives/core", default-features = false } -rand = { version = "0.7.3", optional = true, default-features = false } +serde = { version = "1.0.101", optional = true } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-npos-elections = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/npos-elections" } +sp-io ={ version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/staking" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +pallet-session = { version = "2.0.0-rc4", default-features = false, features = ["historical"], path = "../session" } +pallet-authorship = { version = "2.0.0-rc4", default-features = false, path = "../authorship" } +sp-application-crypto = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/application-crypto" } # Optional imports for benchmarking -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../benchmarking", optional = true } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } rand_chacha = { version = "0.2", default-features = false, optional = true } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-storage = { version = "2.0.0-alpha.8", path = "../../primitives/storage" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } -pallet-timestamp = { version = "2.0.0-alpha.8", path = "../timestamp" } -pallet-staking-reward-curve = { version = "2.0.0-alpha.8", path = "../staking/reward-curve" } -substrate-test-utils = { version = "2.0.0-alpha.8", path = "../../test-utils" } -frame-benchmarking = { version = "2.0.0-alpha.8", path = "../benchmarking" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-storage = { version = "2.0.0-rc4", path = "../../primitives/storage" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } +pallet-timestamp = { version = "2.0.0-rc4", path = "../timestamp" } +pallet-staking-reward-curve = { version = "2.0.0-rc4", path = "../staking/reward-curve" } +substrate-test-utils = { version = "2.0.0-rc4", path = "../../test-utils" } +frame-benchmarking = { version = "2.0.0-rc4", path = "../benchmarking" } rand_chacha = { version = "0.2" } parking_lot = "0.10.2" env_logger = "0.7.1" hex = "0.4" [features] -testing-utils = [ - "std", - "pallet-indices/std", - "sp-core/std", - "rand/std", -] default = ["std"] std = [ "serde", "codec/std", "sp-std/std", - "sp-phragmen/std", + "sp-npos-elections/std", "sp-io/std", "frame-support/std", "sp-runtime/std", @@ -69,9 +58,8 @@ std = [ "frame-system/std", "pallet-authorship/std", "sp-application-crypto/std", - "sp-core/std", ] runtime-benchmarks = [ - "rand_chacha", "frame-benchmarking", + "rand_chacha", ] diff --git a/frame/staking/fuzzer/Cargo.lock b/frame/staking/fuzzer/Cargo.lock index a45f33fdce2492109446030c1a082e7cc32f2f19..e451e12d101310d0ec36d08f0105b8fbd496cc64 100644 --- a/frame/staking/fuzzer/Cargo.lock +++ b/frame/staking/fuzzer/Cargo.lock @@ -1003,7 +1003,7 @@ dependencies = [ "sp-application-crypto", "sp-core", "sp-io", - "sp-phragmen", + "sp-npos-elections", "sp-runtime", "sp-staking", "sp-std", @@ -1027,7 +1027,7 @@ dependencies = [ "rand 0.7.3", "sp-core", "sp-io", - "sp-phragmen", + "sp-npos-elections", "sp-runtime", "sp-std", ] @@ -1751,19 +1751,19 @@ dependencies = [ ] [[package]] -name = "sp-phragmen" +name = "sp-npos-elections" version = "2.0.0-alpha.5" dependencies = [ "parity-scale-codec", "serde", - "sp-phragmen-compact", + "sp-npos-elections-compact", "sp-runtime", "sp-std", ] [[package]] -name = "sp-phragmen-compact" -version = "2.0.0-dev" +name = "sp-npos-elections-compact" +version = "2.0.0-rc3" dependencies = [ "proc-macro-crate", "proc-macro2", diff --git a/frame/staking/fuzzer/Cargo.toml b/frame/staking/fuzzer/Cargo.toml index 9fa74d695d17c74c9a1efc93e635f367db26baca..5cd0ae1180a70e3d28d8bbad2b16d761e5311cbf 100644 --- a/frame/staking/fuzzer/Cargo.toml +++ b/frame/staking/fuzzer/Cargo.toml @@ -14,20 +14,20 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] honggfuzz = "0.5" -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -pallet-staking = { version = "2.0.0-alpha.8", path = "..", features = ["testing-utils"] } -pallet-staking-reward-curve = { version = "2.0.0-alpha.8", path = "../reward-curve" } -pallet-session = { version = "2.0.0-alpha.8", path = "../../session" } -pallet-indices = { version = "2.0.0-alpha.8", path = "../../indices" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../../balances" } -pallet-timestamp = { version = "2.0.0-alpha.8", path = "../../timestamp" } -frame-system = { version = "2.0.0-alpha.8", path = "../../system" } -frame-support = { version = "2.0.0-alpha.8", path = "../../support" } -sp-std = { version = "2.0.0-alpha.8", path = "../../../primitives/std" } -sp-io ={ version = "2.0.0-alpha.8", path = "../../../primitives/io" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-phragmen = { version = "2.0.0-alpha.8", path = "../../../primitives/phragmen" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +pallet-staking = { version = "2.0.0-rc4", path = "..", features = ["runtime-benchmarks"] } +pallet-staking-reward-curve = { version = "2.0.0-rc4", path = "../reward-curve" } +pallet-session = { version = "2.0.0-rc4", path = "../../session" } +pallet-indices = { version = "2.0.0-rc4", path = "../../indices" } +pallet-balances = { version = "2.0.0-rc4", path = "../../balances" } +pallet-timestamp = { version = "2.0.0-rc4", path = "../../timestamp" } +frame-system = { version = "2.0.0-rc4", path = "../../system" } +frame-support = { version = "2.0.0-rc4", path = "../../support" } +sp-std = { version = "2.0.0-rc4", path = "../../../primitives/std" } +sp-io ={ version = "2.0.0-rc4", path = "../../../primitives/io" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-npos-elections = { version = "2.0.0-rc4", path = "../../../primitives/npos-elections" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } [[bin]] name = "submit_solution" diff --git a/frame/staking/fuzzer/src/mock.rs b/frame/staking/fuzzer/src/mock.rs index b1e7ed273c550b3193e03c088994da6292a2ee50..d1e471fadb7f746b2ee4679d4c72d0f90c62553e 100644 --- a/frame/staking/fuzzer/src/mock.rs +++ b/frame/staking/fuzzer/src/mock.rs @@ -57,10 +57,12 @@ impl Convert for CurrencyToVoteHandler { pub struct Test; impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = (); type Index = AccountIndex; type BlockNumber = BlockNumber; type Call = Call; @@ -184,6 +186,7 @@ impl pallet_staking::Trait for Test { type ElectionLookahead = (); type Call = Call; type MaxIterations = MaxIterations; + type MinSolutionScoreBump = (); type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; type UnsignedPriority = (); } diff --git a/frame/staking/fuzzer/src/submit_solution.rs b/frame/staking/fuzzer/src/submit_solution.rs index 067ab0c631968cd07f5d295aa72940ad4beb2774..7293cf23890a5869e70e102e0c613d52c13829c0 100644 --- a/frame/staking/fuzzer/src/submit_solution.rs +++ b/frame/staking/fuzzer/src/submit_solution.rs @@ -16,17 +16,18 @@ // limitations under the License. //! Fuzzing for staking pallet. +//! +//! HFUZZ_RUN_ARGS="-n 8" cargo hfuzz run submit_solution use honggfuzz::fuzz; use mock::Test; -use pallet_staking::testing_utils::{ - USER, get_seq_phragmen_solution, get_weak_solution, setup_chain_stakers, - set_validator_count, signed_account, -}; -use frame_support::{assert_ok, storage::StorageValue}; -use sp_runtime::{traits::Dispatchable, DispatchError}; +use pallet_staking::testing_utils::*; +use frame_support::{assert_ok, storage::StorageValue, traits::UnfilteredDispatchable}; +use frame_system::RawOrigin; +use sp_runtime::DispatchError; use sp_core::offchain::{testing::TestOffchainExt, OffchainExt}; +use pallet_staking::{EraElectionStatus, ElectionStatus, Module as Staking, Call as StakingCall}; mod mock; @@ -43,7 +44,9 @@ enum Mode { } pub fn new_test_ext(iterations: u32) -> sp_io::TestExternalities { - let mut ext: sp_io::TestExternalities = frame_system::GenesisConfig::default().build_storage::().map(Into::into) + let mut ext: sp_io::TestExternalities = frame_system::GenesisConfig::default() + .build_storage::() + .map(Into::into) .expect("Failed to create test externalities."); let (offchain, offchain_state) = TestOffchainExt::new(); @@ -69,96 +72,109 @@ fn main() { loop { fuzz!(|data: (u32, u32, u32, u32, u32)| { let (mut num_validators, mut num_nominators, mut edge_per_voter, mut to_elect, mode_u32) = data; + // always run with 5 iterations. let mut ext = new_test_ext(5); let mode: Mode = unsafe { std::mem::transmute(mode_u32) }; num_validators = to_range(num_validators, 50, 1000); num_nominators = to_range(num_nominators, 50, 2000); edge_per_voter = to_range(edge_per_voter, 1, 16); to_elect = to_range(to_elect, 20, num_validators); + let do_reduce = true; - println!("+++ instance with params {} / {} / {} / {:?}({}) / {}", + println!("+++ instance with params {} / {} / {} / {} / {:?}({})", num_nominators, num_validators, edge_per_voter, + to_elect, mode, mode_u32, - to_elect, ); ext.execute_with(|| { // initial setup - set_validator_count::(to_elect); - pallet_staking::testing_utils::init_active_era(); - setup_chain_stakers::( + init_active_era(); + + assert_ok!(create_validators_with_nominators_for_era::( num_validators, num_nominators, - edge_per_voter, - ); - >::put(pallet_staking::ElectionStatus::Open(1)); + edge_per_voter as usize, + true, + None, + )); + + >::put(ElectionStatus::Open(1)); + assert!(>::create_stakers_snapshot().0); - println!("++ Chain setup done."); + let origin = RawOrigin::Signed(create_funded_user::("fuzzer", 0, 100)); // stuff to submit - let (winners, compact, score) = match mode { + let (winners, compact, score, size) = match mode { Mode::InitialSubmission => { /* No need to setup anything */ get_seq_phragmen_solution::(do_reduce) }, Mode::StrongerSubmission => { - let (winners, compact, score) = get_weak_solution::(false); + let (winners, compact, score, size) = get_weak_solution::(false); println!("Weak on chain score = {:?}", score); assert_ok!( - >::submit_election_solution( - signed_account::(USER), + >::submit_election_solution( + origin.clone().into(), winners, compact, score, - pallet_staking::testing_utils::active_era::(), + current_era::(), + size, ) ); get_seq_phragmen_solution::(do_reduce) }, Mode::WeakerSubmission => { - let (winners, compact, score) = get_seq_phragmen_solution::(do_reduce); + let (winners, compact, score, size) = get_seq_phragmen_solution::(do_reduce); println!("Strong on chain score = {:?}", score); assert_ok!( - >::submit_election_solution( - signed_account::(USER), + >::submit_election_solution( + origin.clone().into(), winners, compact, score, - pallet_staking::testing_utils::active_era::(), + current_era::(), + size, ) ); get_weak_solution::(false) } }; - println!("++ Submission ready. Score = {:?}", score); - // must have chosen correct number of winners. - assert_eq!(winners.len() as u32, >::validator_count()); + assert_eq!(winners.len() as u32, >::validator_count()); // final call and origin - let call = pallet_staking::Call::::submit_election_solution( + let call = StakingCall::::submit_election_solution( winners, compact, score, - pallet_staking::testing_utils::active_era::(), + current_era::(), + size, ); - let caller = signed_account::(USER); // actually submit match mode { Mode::WeakerSubmission => { assert_eq!( - call.dispatch(caller.into()).unwrap_err().error, - DispatchError::Module { index: 0, error: 16, message: Some("PhragmenWeakSubmission") }, + call.dispatch_bypass_filter(origin.clone().into()).unwrap_err().error, + DispatchError::Module { + index: 0, + error: 16, + message: Some("PhragmenWeakSubmission"), + }, ); }, - // NOTE: so exhaustive pattern doesn't work here.. maybe some rust issue? or due to `#[repr(u32)]`? - Mode::InitialSubmission | Mode::StrongerSubmission => assert!(call.dispatch(caller.into()).is_ok()), + // NOTE: so exhaustive pattern doesn't work here.. maybe some rust issue? + // or due to `#[repr(u32)]`? + Mode::InitialSubmission | Mode::StrongerSubmission => { + assert_ok!(call.dispatch_bypass_filter(origin.into())); + } }; }) }); diff --git a/frame/staking/reward-curve/Cargo.toml b/frame/staking/reward-curve/Cargo.toml index 52ba05e40914755ffbb51ce702ef06e94343f609..3d677c7456d6f51a7012cd97a85ce06df72be13d 100644 --- a/frame/staking/reward-curve/Cargo.toml +++ b/frame/staking/reward-curve/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-staking-reward-curve" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -21,4 +21,4 @@ proc-macro2 = "1.0.6" proc-macro-crate = "0.1.4" [dev-dependencies] -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index cd6d0badd0b8fd75870ab3844776c3962dd53ec6..b2035c22b675c258ecd7bce220f154db00628d54 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -18,53 +18,17 @@ //! Staking pallet benchmarking. use super::*; - -use rand_chacha::{rand_core::{RngCore, SeedableRng}, ChaChaRng}; - -use sp_runtime::traits::{Dispatchable, One}; -use sp_io::hashing::blake2_256; - -use frame_system::RawOrigin; -use frame_benchmarking::{benchmarks, account}; - use crate::Module as Staking; +use testing_utils::*; +use sp_runtime::traits::One; +use frame_system::RawOrigin; +pub use frame_benchmarking::{benchmarks, account}; const SEED: u32 = 0; const MAX_SPANS: u32 = 100; const MAX_VALIDATORS: u32 = 1000; const MAX_SLASHES: u32 = 1000; -fn create_funded_user(string: &'static str, n: u32) -> T::AccountId { - let user = account(string, n, SEED); - let balance = T::Currency::minimum_balance() * 100.into(); - T::Currency::make_free_balance_be(&user, balance); - user -} - -pub fn create_stash_controller(n: u32) -> Result<(T::AccountId, T::AccountId), &'static str> { - let stash = create_funded_user::("stash", n); - let controller = create_funded_user::("controller", n); - let controller_lookup: ::Source = T::Lookup::unlookup(controller.clone()); - let reward_destination = RewardDestination::Staked; - let amount = T::Currency::minimum_balance() * 10.into(); - Staking::::bond(RawOrigin::Signed(stash.clone()).into(), controller_lookup, amount, reward_destination)?; - return Ok((stash, controller)) -} - -fn create_validators(max: u32) -> Result::Source>, &'static str> { - let mut validators: Vec<::Source> = Vec::with_capacity(max as usize); - for i in 0 .. max { - let (stash, controller) = create_stash_controller::(i)?; - let validator_prefs = ValidatorPrefs { - commission: Perbill::from_percent(50), - }; - Staking::::validate(RawOrigin::Signed(controller).into(), validator_prefs)?; - let stash_lookup: ::Source = T::Lookup::unlookup(stash); - validators.push(stash_lookup); - } - Ok(validators) -} - // Add slashing spans to a user account. Not relevant for actual use, only to benchmark // read and write operations. fn add_slashing_spans(who: &T::AccountId, spans: u32) { @@ -81,51 +45,19 @@ fn add_slashing_spans(who: &T::AccountId, spans: u32) { SlashingSpans::::insert(who, slashing_spans); } -// This function generates v validators and n nominators who are randomly nominating up to MAX_NOMINATIONS. -pub fn create_validators_with_nominators_for_era(v: u32, n: u32) -> Result<(), &'static str> { - let mut validators: Vec<::Source> = Vec::with_capacity(v as usize); - let mut rng = ChaChaRng::from_seed(SEED.using_encoded(blake2_256)); - - // Create v validators - for i in 0 .. v { - let (v_stash, v_controller) = create_stash_controller::(i)?; - let validator_prefs = ValidatorPrefs { - commission: Perbill::from_percent(50), - }; - Staking::::validate(RawOrigin::Signed(v_controller.clone()).into(), validator_prefs)?; - let stash_lookup: ::Source = T::Lookup::unlookup(v_stash.clone()); - validators.push(stash_lookup.clone()); - } - - // Create n nominators - for j in 0 .. n { - let (_n_stash, n_controller) = create_stash_controller::(u32::max_value() - j)?; - - // Have them randomly validate - let mut available_validators = validators.clone(); - let mut selected_validators: Vec<::Source> = Vec::with_capacity(MAX_NOMINATIONS); - for _ in 0 .. v.min(MAX_NOMINATIONS as u32) { - let selected = rng.next_u32() as usize % available_validators.len(); - let validator = available_validators.remove(selected); - selected_validators.push(validator); - } - Staking::::nominate(RawOrigin::Signed(n_controller.clone()).into(), selected_validators)?; - } - - ValidatorCount::put(v); - - Ok(()) -} - -// This function generates one validator being nominated by n nominators, and returns -//the validator stash account. It also starts an era and creates pending payouts. -pub fn create_validator_with_nominators(n: u32, upper_bound: u32) -> Result { +// This function generates one validator being nominated by n nominators, and returns the validator +// stash account. It also starts an era and creates pending payouts. +pub fn create_validator_with_nominators( + n: u32, + upper_bound: u32, + dead: bool, +) -> Result { let mut points_total = 0; let mut points_individual = Vec::new(); MinimumValidatorCount::put(0); - let (v_stash, v_controller) = create_stash_controller::(0)?; + let (v_stash, v_controller) = create_stash_controller::(0, 100)?; let validator_prefs = ValidatorPrefs { commission: Perbill::from_percent(50), }; @@ -137,7 +69,11 @@ pub fn create_validator_with_nominators(n: u32, upper_bound: u32) -> R // Give the validator n nominators, but keep total users in the system the same. for i in 0 .. upper_bound { - let (_n_stash, n_controller) = create_stash_controller::(u32::max_value() - i)?; + let (_n_stash, n_controller) = if !dead { + create_stash_controller::(u32::max_value() - i, 100)? + } else { + create_stash_and_dead_controller::(u32::max_value() - i, 100)? + }; if i < n { Staking::::nominate(RawOrigin::Signed(n_controller.clone()).into(), vec![stash_lookup.clone()])?; } @@ -174,8 +110,8 @@ benchmarks! { bond { let u in ...; - let stash = create_funded_user::("stash",u); - let controller = create_funded_user::("controller", u); + let stash = create_funded_user::("stash", u, 100); + let controller = create_funded_user::("controller", u, 100); let controller_lookup: ::Source = T::Lookup::unlookup(controller.clone()); let reward_destination = RewardDestination::Staked; let amount = T::Currency::minimum_balance() * 10.into(); @@ -187,7 +123,7 @@ benchmarks! { bond_extra { let u in ...; - let (stash, controller) = create_stash_controller::(u)?; + let (stash, controller) = create_stash_controller::(u, 100)?; let max_additional = T::Currency::minimum_balance() * 10.into(); let ledger = Ledger::::get(&controller).ok_or("ledger not created before")?; let original_bonded: BalanceOf = ledger.active; @@ -200,7 +136,7 @@ benchmarks! { unbond { let u in ...; - let (_, controller) = create_stash_controller::(u)?; + let (_, controller) = create_stash_controller::(u, 100)?; let amount = T::Currency::minimum_balance() * 10.into(); let ledger = Ledger::::get(&controller).ok_or("ledger not created before")?; let original_bonded: BalanceOf = ledger.active; @@ -215,7 +151,7 @@ benchmarks! { withdraw_unbonded_update { // Slashing Spans let s in 0 .. MAX_SPANS; - let (stash, controller) = create_stash_controller::(0)?; + let (stash, controller) = create_stash_controller::(0, 100)?; add_slashing_spans::(&stash, s); let amount = T::Currency::minimum_balance() * 5.into(); // Half of total Staking::::unbond(RawOrigin::Signed(controller.clone()).into(), amount)?; @@ -233,7 +169,7 @@ benchmarks! { withdraw_unbonded_kill { // Slashing Spans let s in 0 .. MAX_SPANS; - let (stash, controller) = create_stash_controller::(0)?; + let (stash, controller) = create_stash_controller::(0, 100)?; add_slashing_spans::(&stash, s); let amount = T::Currency::minimum_balance() * 10.into(); Staking::::unbond(RawOrigin::Signed(controller.clone()).into(), amount)?; @@ -247,7 +183,7 @@ benchmarks! { validate { let u in ...; - let (stash, controller) = create_stash_controller::(u)?; + let (stash, controller) = create_stash_controller::(u, 100)?; let prefs = ValidatorPrefs::default(); }: _(RawOrigin::Signed(controller), prefs) verify { @@ -257,8 +193,8 @@ benchmarks! { // Worst case scenario, MAX_NOMINATIONS nominate { let n in 1 .. MAX_NOMINATIONS as u32; - let (stash, controller) = create_stash_controller::(n + 1)?; - let validators = create_validators::(n)?; + let (stash, controller) = create_stash_controller::(n + 1, 100)?; + let validators = create_validators::(n, 100)?; }: _(RawOrigin::Signed(controller), validators) verify { assert!(Nominators::::contains_key(stash)); @@ -266,12 +202,12 @@ benchmarks! { chill { let u in ...; - let (_, controller) = create_stash_controller::(u)?; + let (_, controller) = create_stash_controller::(u, 100)?; }: _(RawOrigin::Signed(controller)) set_payee { let u in ...; - let (stash, controller) = create_stash_controller::(u)?; + let (stash, controller) = create_stash_controller::(u, 100)?; assert_eq!(Payee::::get(&stash), RewardDestination::Staked); }: _(RawOrigin::Signed(controller), RewardDestination::Controller) verify { @@ -280,8 +216,8 @@ benchmarks! { set_controller { let u in ...; - let (stash, _) = create_stash_controller::(u)?; - let new_controller = create_funded_user::("new_controller", u); + let (stash, _) = create_stash_controller::(u, 100)?; + let new_controller = create_funded_user::("new_controller", u, 100); let new_controller_lookup = T::Lookup::unlookup(new_controller.clone()); }: _(RawOrigin::Signed(stash), new_controller_lookup) verify { @@ -319,7 +255,7 @@ benchmarks! { force_unstake { // Slashing Spans let s in 0 .. MAX_SPANS; - let (stash, controller) = create_stash_controller::(0)?; + let (stash, controller) = create_stash_controller::(0, 100)?; add_slashing_spans::(&stash, s); }: _(RawOrigin::Root, stash, s) verify { @@ -343,7 +279,8 @@ benchmarks! { payout_stakers { let n in 1 .. T::MaxNominatorRewardedPerValidator::get() as u32; - let validator = create_validator_with_nominators::(n, T::MaxNominatorRewardedPerValidator::get() as u32)?; + let validator = create_validator_with_nominators::(n, T::MaxNominatorRewardedPerValidator::get() as u32, true)?; + let current_era = CurrentEra::get().unwrap(); let caller = account("caller", 0, SEED); let balance_before = T::Currency::free_balance(&validator); @@ -354,9 +291,23 @@ benchmarks! { assert!(balance_before < balance_after); } + payout_stakers_alive_controller { + let n in 1 .. T::MaxNominatorRewardedPerValidator::get() as u32; + let validator = create_validator_with_nominators::(n, T::MaxNominatorRewardedPerValidator::get() as u32, false)?; + + let current_era = CurrentEra::get().unwrap(); + let caller = account("caller", 0, SEED); + let balance_before = T::Currency::free_balance(&validator); + }: payout_stakers(RawOrigin::Signed(caller), validator.clone(), current_era) + verify { + // Validator has been paid! + let balance_after = T::Currency::free_balance(&validator); + assert!(balance_before < balance_after); + } + rebond { let l in 1 .. MAX_UNLOCKING_CHUNKS as u32; - let (_, controller) = create_stash_controller::(u)?; + let (_, controller) = create_stash_controller::(u, 100)?; let mut staking_ledger = Ledger::::get(controller.clone()).unwrap(); let unlock_chunk = UnlockChunk::> { value: 1.into(), @@ -394,7 +345,7 @@ benchmarks! { reap_stash { let s in 1 .. MAX_SPANS; - let (stash, controller) = create_stash_controller::(0)?; + let (stash, controller) = create_stash_controller::(0, 100)?; add_slashing_spans::(&stash, s); T::Currency::make_free_balance_be(&stash, 0.into()); }: _(RawOrigin::Signed(controller), stash.clone(), s) @@ -406,7 +357,7 @@ benchmarks! { let v in 1 .. 10; let n in 1 .. 100; MinimumValidatorCount::put(0); - create_validators_with_nominators_for_era::(v, n)?; + create_validators_with_nominators_for_era::(v, n, MAX_NOMINATIONS, false, None)?; let session_index = SessionIndex::one(); }: { let validators = Staking::::new_era(session_index).ok_or("`new_era` failed")?; @@ -415,7 +366,7 @@ benchmarks! { do_slash { let l in 1 .. MAX_UNLOCKING_CHUNKS as u32; - let (stash, controller) = create_stash_controller::(0)?; + let (stash, controller) = create_stash_controller::(0, 100)?; let mut staking_ledger = Ledger::::get(controller.clone()).unwrap(); let unlock_chunk = UnlockChunk::> { value: 1.into(), @@ -443,7 +394,7 @@ benchmarks! { let v in 1 .. 10; let n in 1 .. 100; MinimumValidatorCount::put(0); - create_validators_with_nominators_for_era::(v, n)?; + create_validators_with_nominators_for_era::(v, n, MAX_NOMINATIONS, false, None)?; // Start a new Era let new_validators = Staking::::new_era(SessionIndex::one()).unwrap(); assert!(new_validators.len() == v as usize); @@ -451,12 +402,12 @@ benchmarks! { let current_era = CurrentEra::get().unwrap(); let mut points_total = 0; let mut points_individual = Vec::new(); - let mut payout_calls = Vec::new(); + let mut payout_calls_arg = Vec::new(); for validator in new_validators.iter() { points_total += 10; points_individual.push((validator.clone(), 10)); - payout_calls.push(Call::::payout_stakers(validator.clone(), current_era)) + payout_calls_arg.push((validator.clone(), current_era)); } // Give Era Points @@ -473,9 +424,201 @@ benchmarks! { let caller: T::AccountId = account("caller", 0, SEED); }: { - for call in payout_calls { - call.dispatch(RawOrigin::Signed(caller.clone()).into())?; + for arg in payout_calls_arg { + >::payout_stakers(RawOrigin::Signed(caller.clone()).into(), arg.0, arg.1)?; + } + } + + // This benchmark create `v` validators intent, `n` nominators intent, each nominator nominate + // MAX_NOMINATIONS in the set of the first `w` validators. + // It builds a solution with `w` winners composed of nominated validators randomly nominated, + // `a` assignment with MAX_NOMINATIONS. + submit_solution_initial { + // number of validator intent + let v in 1000 .. 2000; + // number of nominator intent + let n in 1000 .. 2000; + // number of assignments. Basically, number of active nominators. + let a in 200 .. 500; + // number of winners, also ValidatorCount + let w in 16 .. 100; + + ensure!(w as usize >= MAX_NOMINATIONS, "doesn't support lower value"); + + let winners = create_validators_with_nominators_for_era::( + v, + n, + MAX_NOMINATIONS, + false, + Some(w), + )?; + + // needed for the solution to be generates. + assert!(>::create_stakers_snapshot().0); + + // set number of winners + ValidatorCount::put(w); + + // create a assignments in total for the w winners. + let (winners, assignments) = create_assignments_for_offchain::(a, winners)?; + + let ( + winners, + compact, + score, + size + ) = offchain_election::prepare_submission::(assignments, winners, false).unwrap(); + + // needed for the solution to be accepted + >::put(ElectionStatus::Open(T::BlockNumber::from(1u32))); + + let era = >::current_era().unwrap_or(0); + let caller: T::AccountId = account("caller", n, SEED); + }: { + let result = >::submit_election_solution( + RawOrigin::Signed(caller.clone()).into(), + winners, + compact, + score.clone(), + era, + size, + ); + assert!(result.is_ok()); + } + verify { + // new solution has been accepted. + assert_eq!(>::queued_score().unwrap(), score); + } + + // same as submit_solution_initial but we place a very weak solution on chian first. + submit_solution_better { + // number of validator intent + let v in 1000 .. 2000; + // number of nominator intent + let n in 1000 .. 2000; + // number of assignments. Basically, number of active nominators. + let a in 200 .. 500; + // number of winners, also ValidatorCount + let w in 16 .. 100; + + ensure!(w as usize >= MAX_NOMINATIONS, "doesn't support lower value"); + + let winners = create_validators_with_nominators_for_era::( + v, + n, + MAX_NOMINATIONS, + false, + Some(w), + )?; + + // needed for the solution to be generates. + assert!(>::create_stakers_snapshot().0); + + // set number of winners + ValidatorCount::put(w); + + // create a assignments in total for the w winners. + let (winners, assignments) = create_assignments_for_offchain::(a, winners)?; + + let single_winner = winners[0].0.clone(); + + let ( + winners, + compact, + score, + size + ) = offchain_election::prepare_submission::(assignments, winners, false).unwrap(); + + // needed for the solution to be accepted + >::put(ElectionStatus::Open(T::BlockNumber::from(1u32))); + + let era = >::current_era().unwrap_or(0); + let caller: T::AccountId = account("caller", n, SEED); + + // submit a very bad solution on-chain + { + // this is needed to fool the chain to accept this solution. + ValidatorCount::put(1); + let (winners, compact, score, size) = get_single_winner_solution::(single_winner)?; + assert!( + >::submit_election_solution( + RawOrigin::Signed(caller.clone()).into(), + winners, + compact, + score.clone(), + era, + size, + ).is_ok()); + + // new solution has been accepted. + assert_eq!(>::queued_score().unwrap(), score); + ValidatorCount::put(w); } + }: { + let result = >::submit_election_solution( + RawOrigin::Signed(caller.clone()).into(), + winners, + compact, + score.clone(), + era, + size, + ); + assert!(result.is_ok()); + } + verify { + // new solution has been accepted. + assert_eq!(>::queued_score().unwrap(), score); + } + + // This will be early rejected based on the score. + submit_solution_weaker { + // number of validator intent + let v in 1000 .. 2000; + // number of nominator intent + let n in 1000 .. 2000; + + MinimumValidatorCount::put(0); + create_validators_with_nominators_for_era::(v, n, MAX_NOMINATIONS, false, None)?; + + // needed for the solution to be generates. + assert!(>::create_stakers_snapshot().0); + + // needed for the solution to be accepted + >::put(ElectionStatus::Open(T::BlockNumber::from(1u32))); + let caller: T::AccountId = account("caller", n, SEED); + let era = >::current_era().unwrap_or(0); + + // submit a seq-phragmen with all the good stuff on chain. + { + let (winners, compact, score, size) = get_seq_phragmen_solution::(true); + assert!( + >::submit_election_solution( + RawOrigin::Signed(caller.clone()).into(), + winners, + compact, + score.clone(), + era, + size, + ).is_ok() + ); + + // new solution has been accepted. + assert_eq!(>::queued_score().unwrap(), score); + } + + // prepare a bad solution. This will be very early rejected. + let (winners, compact, score, size) = get_weak_solution::(true); + }: { + assert!( + >::submit_election_solution( + RawOrigin::Signed(caller.clone()).into(), + winners, + compact, + score.clone(), + era, + size, + ).is_err() + ); } } @@ -491,7 +634,8 @@ mod tests { let v = 10; let n = 100; - create_validators_with_nominators_for_era::(v,n).unwrap(); + create_validators_with_nominators_for_era::(v, n, MAX_NOMINATIONS, false, None) + .unwrap(); let count_validators = Validators::::iter().count(); let count_nominators = Nominators::::iter().count(); @@ -509,6 +653,7 @@ mod tests { let validator_stash = create_validator_with_nominators::( n, ::MaxNominatorRewardedPerValidator::get() as u32, + false, ).unwrap(); let current_era = CurrentEra::get().unwrap(); @@ -529,6 +674,7 @@ mod tests { let validator_stash = create_validator_with_nominators::( n, ::MaxNominatorRewardedPerValidator::get() as u32, + false, ).unwrap(); // Add 20 slashing spans @@ -589,12 +735,25 @@ mod tests { assert_ok!(test_benchmark_force_unstake::()); assert_ok!(test_benchmark_cancel_deferred_slash::()); assert_ok!(test_benchmark_payout_stakers::()); + assert_ok!(test_benchmark_payout_stakers_alive_controller::()); assert_ok!(test_benchmark_rebond::()); assert_ok!(test_benchmark_set_history_depth::()); assert_ok!(test_benchmark_reap_stash::()); assert_ok!(test_benchmark_new_era::()); assert_ok!(test_benchmark_do_slash::()); assert_ok!(test_benchmark_payout_all::()); + // only run one of them to same time on the CI. ignore the other two. + assert_ok!(test_benchmark_submit_solution_initial::()); }); } + + #[test] + #[ignore] + fn test_benchmarks_offchain() { + ExtBuilder::default().has_stakers(false).build().execute_with(|| { + assert_ok!(test_benchmark_submit_solution_better::()); + assert_ok!(test_benchmark_submit_solution_weaker::()); + }); + } + } diff --git a/frame/staking/src/inflation.rs b/frame/staking/src/inflation.rs index 04bfc98357a7f4958a8a77da6538a77af9cbee6f..2161fe20af8299a5d163f7b17f5ae1a0e7e059d1 100644 --- a/frame/staking/src/inflation.rs +++ b/frame/staking/src/inflation.rs @@ -20,7 +20,7 @@ //! The staking rate in NPoS is the total amount of tokens staked by nominators and validators, //! divided by the total token supply. -use sp_runtime::{Perbill, traits::AtLeast32Bit, curve::PiecewiseLinear}; +use sp_runtime::{Perbill, traits::AtLeast32BitUnsigned, curve::PiecewiseLinear}; /// The total payout to all validators (and their nominators) per era and maximum payout. /// @@ -34,7 +34,7 @@ pub fn compute_total_payout( npos_token_staked: N, total_tokens: N, era_duration: u64 -) -> (N, N) where N: AtLeast32Bit + Clone { +) -> (N, N) where N: AtLeast32BitUnsigned + Clone { // Milliseconds per year for the Julian year (365.25 days). const MILLISECONDS_PER_YEAR: u64 = 1000 * 3600 * 24 * 36525 / 100; diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 20e34bf43f8542261e5ad550b3bf55175f3a0342..e71ded362c7b1606b87104a573f0148f5c98cc99 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -25,25 +25,25 @@ //! //! ## Overview //! -//! The Staking module is the means by which a set of network maintainers (known as _authorities_ -//! in some contexts and _validators_ in others) are chosen based upon those who voluntarily place -//! funds under deposit. Under deposit, those funds are rewarded under normal operation but are -//! held at pain of _slash_ (expropriation) should the staked maintainer be found not to be -//! discharging its duties properly. +//! The Staking module is the means by which a set of network maintainers (known as _authorities_ in +//! some contexts and _validators_ in others) are chosen based upon those who voluntarily place +//! funds under deposit. Under deposit, those funds are rewarded under normal operation but are held +//! at pain of _slash_ (expropriation) should the staked maintainer be found not to be discharging +//! its duties properly. //! //! ### Terminology //! //! //! - Staking: The process of locking up funds for some time, placing them at risk of slashing -//! (loss) in order to become a rewarded maintainer of the network. +//! (loss) in order to become a rewarded maintainer of the network. //! - Validating: The process of running a node to actively maintain the network, either by -//! producing blocks or guaranteeing finality of the chain. +//! producing blocks or guaranteeing finality of the chain. //! - Nominating: The process of placing staked funds behind one or more validators in order to -//! share in any reward, and punishment, they take. +//! share in any reward, and punishment, they take. //! - Stash account: The account holding an owner's funds used for staking. //! - Controller account: The account that controls an owner's funds for staking. //! - Era: A (whole) number of sessions, which is the period that the validator set (and each -//! validator's active nominator set) is recalculated and where rewards are paid out. +//! validator's active nominator set) is recalculated and where rewards are paid out. //! - Slash: The punishment of a staker by reducing its funds. //! //! ### Goals @@ -106,10 +106,10 @@ //! valid behavior_ while _punishing any misbehavior or lack of availability_. //! //! Rewards must be claimed for each era before it gets too old by `$HISTORY_DEPTH` using the -//! `payout_stakers` call. Any account can call `payout_stakers`, which pays the reward to -//! the validator as well as its nominators. -//! Only the [`T::MaxNominatorRewardedPerValidator`] biggest stakers can claim their reward. This -//! is to limit the i/o cost to mutate storage for each nominator's account. +//! `payout_stakers` call. Any account can call `payout_stakers`, which pays the reward to the +//! validator as well as its nominators. Only the [`Trait::MaxNominatorRewardedPerValidator`] +//! biggest stakers can claim their reward. This is to limit the i/o cost to mutate storage for each +//! nominator's account. //! //! Slashing can occur at any point in time, once misbehavior is reported. Once slashing is //! determined, a value is deducted from the balance of the validator and all the nominators who @@ -118,8 +118,8 @@ //! Slashing logic is further described in the documentation of the `slashing` module. //! //! Similar to slashing, rewards are also shared among a validator and its associated nominators. -//! Yet, the reward funds are not always transferred to the stash account and can be configured. -//! See [Reward Calculation](#reward-calculation) for more details. +//! Yet, the reward funds are not always transferred to the stash account and can be configured. See +//! [Reward Calculation](#reward-calculation) for more details. //! //! #### Chilling //! @@ -157,15 +157,15 @@ //! pub trait Trait: staking::Trait {} //! //! decl_module! { -//! pub struct Module for enum Call where origin: T::Origin { -//! /// Reward a validator. -//! #[weight = 0] -//! pub fn reward_myself(origin) -> dispatch::DispatchResult { -//! let reported = ensure_signed(origin)?; -//! >::reward_by_ids(vec![(reported, 10)]); -//! Ok(()) -//! } -//! } +//! pub struct Module for enum Call where origin: T::Origin { +//! /// Reward a validator. +//! #[weight = 0] +//! pub fn reward_myself(origin) -> dispatch::DispatchResult { +//! let reported = ensure_signed(origin)?; +//! >::reward_by_ids(vec![(reported, 10)]); +//! Ok(()) +//! } +//! } //! } //! # fn main() { } //! ``` @@ -208,8 +208,8 @@ //! The validator and its nominator split their reward as following: //! //! The validator can declare an amount, named -//! [`commission`](./struct.ValidatorPrefs.html#structfield.commission), that does not -//! get shared with the nominators at each reward payout through its +//! [`commission`](./struct.ValidatorPrefs.html#structfield.commission), that does not get shared +//! with the nominators at each reward payout through its //! [`ValidatorPrefs`](./struct.ValidatorPrefs.html). This value gets deducted from the total reward //! that is paid to the validator and its nominators. The remaining portion is split among the //! validator and all of the nominators that nominated the validator, proportional to the value @@ -218,8 +218,8 @@ //! [`others`](./struct.Exposure.html#structfield.others) by //! [`total`](./struct.Exposure.html#structfield.total) in [`Exposure`](./struct.Exposure.html)). //! -//! All entities who receive a reward have the option to choose their reward destination -//! through the [`Payee`](./struct.Payee.html) storage item (see +//! All entities who receive a reward have the option to choose their reward destination through the +//! [`Payee`](./struct.Payee.html) storage item (see //! [`set_payee`](enum.Call.html#variant.set_payee)), to be one of the following: //! //! - Controller account, (obviously) not increasing the staked value. @@ -244,9 +244,8 @@ //! //! ### Election Algorithm //! -//! The current election algorithm is implemented based on Phragmén. -//! The reference implementation can be found -//! [here](https://github.com/w3f/consensus/tree/master/NPoS). +//! The current election algorithm is implemented based on Phragmén. The reference implementation +//! can be found [here](https://github.com/w3f/consensus/tree/master/NPoS). //! //! The election algorithm, aside from electing the validators with the most stake value and votes, //! tries to divide the nominator votes among candidates in an equal manner. To further assure this, @@ -256,8 +255,8 @@ //! //! ## GenesisConfig //! -//! The Staking module depends on the [`GenesisConfig`](./struct.GenesisConfig.html). -//! The `GenesisConfig` is optional and allow to set some initial stakers. +//! The Staking module depends on the [`GenesisConfig`](./struct.GenesisConfig.html). The +//! `GenesisConfig` is optional and allow to set some initial stakers. //! //! ## Related Modules //! @@ -272,7 +271,7 @@ mod mock; #[cfg(test)] mod tests; -#[cfg(feature = "testing-utils")] +#[cfg(any(feature = "runtime-benchmarks", test))] pub mod testing_utils; #[cfg(any(feature = "runtime-benchmarks", test))] pub mod benchmarking; @@ -290,10 +289,13 @@ use sp_std::{ }; use codec::{HasCompact, Encode, Decode}; use frame_support::{ - decl_module, decl_event, decl_storage, ensure, decl_error, debug, + decl_module, decl_event, decl_storage, ensure, decl_error, weights::{Weight, constants::{WEIGHT_PER_MICROS, WEIGHT_PER_NANOS}}, storage::IterableStorageMap, - dispatch::{IsSubType, DispatchResult, DispatchResultWithPostInfo}, + dispatch::{ + IsSubType, DispatchResult, DispatchResultWithPostInfo, DispatchErrorWithPostInfo, + WithPostDispatchInfo, + }, traits::{ Currency, LockIdentifier, LockableCurrency, WithdrawReasons, OnUnbalanced, Imbalance, Get, UnixTime, EstimateNextNewSession, EnsureOrigin, @@ -301,11 +303,11 @@ use frame_support::{ }; use pallet_session::historical; use sp_runtime::{ - Perbill, PerU16, PerThing, RuntimeDebug, + Percent, Perbill, PerU16, PerThing, RuntimeDebug, DispatchError, curve::PiecewiseLinear, traits::{ - Convert, Zero, StaticLookup, CheckedSub, Saturating, SaturatedConversion, AtLeast32Bit, - Dispatchable, + Convert, Zero, StaticLookup, CheckedSub, Saturating, SaturatedConversion, + AtLeast32BitUnsigned, Dispatchable, }, transaction_validity::{ TransactionValidityError, TransactionValidity, ValidTransaction, InvalidTransaction, @@ -322,9 +324,10 @@ use frame_system::{ self as system, ensure_signed, ensure_root, ensure_none, offchain::SendTransactionTypes, }; -use sp_phragmen::{ - ExtendedBalance, Assignment, PhragmenScore, PhragmenResult, build_support_map, evaluate_support, - elect, generate_compact_solution_type, is_score_better, VotingLimit, SupportMap, VoteWeight, +use sp_npos_elections::{ + ExtendedBalance, Assignment, ElectionScore, ElectionResult as PrimitiveElectionResult, + build_support_map, evaluate_support, seq_phragmen, generate_compact_solution_type, + is_score_better, VotingLimit, SupportMap, VoteWeight, }; const DEFAULT_MINIMUM_VALIDATOR_COUNT: u32 = 4; @@ -332,13 +335,14 @@ const STAKING_ID: LockIdentifier = *b"staking "; pub const MAX_UNLOCKING_CHUNKS: usize = 32; pub const MAX_NOMINATIONS: usize = ::LIMIT; -// syntactic sugar for logging -#[cfg(feature = "std")] -const LOG_TARGET: &'static str = "staking"; +pub(crate) const LOG_TARGET: &'static str = "staking"; + +// syntactic sugar for logging. +#[macro_export] macro_rules! log { ($level:tt, $patter:expr $(, $values:expr)* $(,)?) => { - debug::native::$level!( - target: LOG_TARGET, + frame_support::debug::$level!( + target: crate::LOG_TARGET, $patter $(, $values)* ) }; @@ -372,17 +376,17 @@ generate_compact_solution_type!(pub GenericCompactAssignments, 16); pub struct ActiveEraInfo { /// Index of era. pub index: EraIndex, - /// Moment of start expresed as millisecond from `$UNIX_EPOCH`. + /// Moment of start expressed as millisecond from `$UNIX_EPOCH`. /// /// Start can be none if start hasn't been set for the era yet, /// Start is set on the first on_finalize of the era to guarantee usage of `Time`. start: Option, } -/// Accuracy used for on-chain phragmen. +/// Accuracy used for on-chain election. pub type ChainAccuracy = Perbill; -/// Accuracy used for off-chain phragmen. This better be small. +/// Accuracy used for off-chain election. This better be small. pub type OffchainAccuracy = PerU16; /// The balance type of this module. @@ -489,7 +493,7 @@ pub struct StakingLedger { impl< AccountId, - Balance: HasCompact + Copy + Saturating + AtLeast32Bit, + Balance: HasCompact + Copy + Saturating + AtLeast32BitUnsigned, > StakingLedger { /// Remove entries from `unlocking` that are sufficiently old and reduce the /// total by the sum of their balances. @@ -540,7 +544,7 @@ impl< } impl StakingLedger where - Balance: AtLeast32Bit + Saturating + Copy, + Balance: AtLeast32BitUnsigned + Saturating + Copy, { /// Slash the validator for a given amount of balance. This can grow the value /// of the slash in the case that the validator has less than `minimum_balance` @@ -680,6 +684,22 @@ pub enum ElectionStatus { Open(BlockNumber), } +/// Some indications about the size of the election. This must be submitted with the solution. +/// +/// Note that these values must reflect the __total__ number, not only those that are present in the +/// solution. In short, these should be the same size as the size of the values dumped in +/// `SnapshotValidators` and `SnapshotNominators`. +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, Default)] +pub struct ElectionSize { + /// Number of validators in the snapshot of the current election round. + #[codec(compact)] + pub validators: ValidatorIndex, + /// Number of nominators in the snapshot of the current election round. + #[codec(compact)] + pub nominators: NominatorIndex, +} + + impl ElectionStatus { fn is_open_at(&self, n: BlockNumber) -> bool { *self == Self::Open(n) @@ -743,6 +763,72 @@ impl SessionInterface<::AccountId> for T whe } } +pub mod weight { + use super::*; + + /// All weight notes are pertaining to the case of a better solution, in which we execute + /// the longest code path. + /// Weight: 0 + (0.63 μs * v) + (0.36 μs * n) + (96.53 μs * a ) + (8 μs * w ) with: + /// * v validators in snapshot validators, + /// * n nominators in snapshot nominators, + /// * a assignment in the submitted solution + /// * w winners in the submitted solution + /// + /// State reads: + /// - Initial checks: + /// - ElectionState, CurrentEra, QueuedScore + /// - SnapshotValidators.len() + SnapShotNominators.len() + /// - ValidatorCount + /// - SnapshotValidators + /// - SnapshotNominators + /// - Iterate over nominators: + /// - compact.len() * Nominators(who) + /// - (non_self_vote_edges) * SlashingSpans + /// - For `assignment_ratio_to_staked`: Basically read the staked value of each stash. + /// - (winners.len() + compact.len()) * (Ledger + Bonded) + /// - TotalIssuance (read a gzillion times potentially, but well it is cached.) + /// - State writes: + /// - QueuedElected, QueuedScore + pub fn weight_for_submit_solution( + winners: &Vec, + compact: &CompactAssignments, + size: &ElectionSize, + ) -> Weight { + (630 * WEIGHT_PER_NANOS).saturating_mul(size.validators as Weight) + .saturating_add((360 * WEIGHT_PER_NANOS).saturating_mul(size.nominators as Weight)) + .saturating_add((96 * WEIGHT_PER_MICROS).saturating_mul(compact.len() as Weight)) + .saturating_add((8 * WEIGHT_PER_MICROS).saturating_mul(winners.len() as Weight)) + // Initial checks + .saturating_add(T::DbWeight::get().reads(8)) + // Nominators + .saturating_add(T::DbWeight::get().reads(compact.len() as Weight)) + // SlashingSpans (upper bound for invalid solution) + .saturating_add(T::DbWeight::get().reads(compact.edge_count() as Weight)) + // `assignment_ratio_to_staked` + .saturating_add(T::DbWeight::get().reads(2 * ((winners.len() + compact.len()) as Weight))) + .saturating_add(T::DbWeight::get().reads(1)) + // write queued score and elected + .saturating_add(T::DbWeight::get().writes(2)) + } + + /// Weight of `submit_solution` in case of a correct submission. + /// + /// refund: we charged compact.len() * read(1) for SlashingSpans. A valid solution only reads + /// winners.len(). + pub fn weight_for_correct_submit_solution( + winners: &Vec, + compact: &CompactAssignments, + size: &ElectionSize, + ) -> Weight { + // NOTE: for consistency, we re-compute the original weight to maintain their relation and + // prevent any foot-guns. + let original_weight = weight_for_submit_solution::(winners, compact, size); + original_weight + .saturating_sub(T::DbWeight::get().reads(compact.edge_count() as Weight)) + .saturating_add(T::DbWeight::get().reads(winners.len() as Weight)) + } +} + pub trait Trait: frame_system::Trait + SendTransactionTypes> { /// The staking balance. type Currency: LockableCurrency; @@ -755,8 +841,9 @@ pub trait Trait: frame_system::Trait + SendTransactionTypes> { /// Convert a balance into a number used for election calculation. This must fit into a `u64` /// but is allowed to be sensibly lossy. The `u64` is used to communicate with the - /// [`sp_phragmen`] crate which accepts u64 numbers and does operations in 128. Consequently, - /// the backward convert is used convert the u128s from phragmen back to a [`BalanceOf`]. + /// [`sp_npos_elections`] crate which accepts u64 numbers and does operations in 128. + /// Consequently, the backward convert is used convert the u128s from sp-elections back to a + /// [`BalanceOf`]. type CurrencyToVote: Convert, VoteWeight> + Convert>; /// Tokens have been minted and are unused for validator-reward. @@ -778,9 +865,10 @@ pub trait Trait: frame_system::Trait + SendTransactionTypes> { /// Number of eras that staked funds must remain bonded for. type BondingDuration: Get; - /// Number of eras that slashes are deferred by, after computation. This - /// should be less than the bonding duration. Set to 0 if slashes should be - /// applied immediately, without opportunity for intervention. + /// Number of eras that slashes are deferred by, after computation. + /// + /// This should be less than the bonding duration. Set to 0 if slashes + /// should be applied immediately, without opportunity for intervention. type SlashDeferDuration: Get; /// The origin which can cancel a deferred slash. Root can always do this. @@ -796,19 +884,27 @@ pub trait Trait: frame_system::Trait + SendTransactionTypes> { /// Something that can estimate the next session change, accurately or as a best effort guess. type NextNewSession: EstimateNextNewSession; - /// How many blocks ahead of the era, within the last do we try to run the phragmen offchain? + /// The number of blocks before the end of the era from which election submissions are allowed. + /// /// Setting this to zero will disable the offchain compute and only on-chain seq-phragmen will /// be used. + /// + /// This is bounded by being within the last session. Hence, setting it to a value more than the + /// length of a session will be pointless. type ElectionLookahead: Get; /// The overarching call type. type Call: Dispatchable + From> + IsSubType> + Clone; - /// Maximum number of equalise iterations to run in the offchain submission. If set to 0, - /// equalize will not be executed at all. + /// Maximum number of balancing iterations to run in the offchain submission. + /// + /// If set to 0, balance_solution will not be executed at all. type MaxIterations: Get; - /// The maximum number of nominator rewarded for each validator. + /// The threshold of improvement that should be provided for a new solution to be accepted. + type MinSolutionScoreBump: Get; + + /// The maximum number of nominators rewarded for each validator. /// /// For each validator only the `$MaxNominatorRewardedPerValidator` biggest stakers can claim /// their reward. This used to limit the i/o cost for the nominator payout. @@ -839,19 +935,20 @@ impl Default for Forcing { fn default() -> Self { Forcing::NotForcing } } -// A value placed in storage that represents the current version of the Staking storage. -// This value is used by the `on_runtime_upgrade` logic to determine whether we run -// storage migration logic. This should match directly with the semantic versions of the Rust crate. +// A value placed in storage that represents the current version of the Staking storage. This value +// is used by the `on_runtime_upgrade` logic to determine whether we run storage migration logic. +// This should match directly with the semantic versions of the Rust crate. #[derive(Encode, Decode, Clone, Copy, PartialEq, Eq, RuntimeDebug)] enum Releases { V1_0_0Ancient, V2_0_0, V3_0_0, + V4_0_0, } impl Default for Releases { fn default() -> Self { - Releases::V3_0_0 + Releases::V4_0_0 } } @@ -861,9 +958,9 @@ decl_storage! { /// /// Information is kept for eras in `[current_era - history_depth; current_era]`. /// - /// Must be more than the number of eras delayed by session otherwise. - /// I.e. active era must always be in history. - /// I.e. `active_era > current_era - history_depth` must be guaranteed. + /// Must be more than the number of eras delayed by session otherwise. I.e. active era must + /// always be in history. I.e. `active_era > current_era - history_depth` must be + /// guaranteed. HistoryDepth get(fn history_depth) config(): u32 = 84; /// The ideal number of staking participants. @@ -1023,7 +1120,7 @@ decl_storage! { pub QueuedElected get(fn queued_elected): Option>>; /// The score of the current [`QueuedElected`]. - pub QueuedScore get(fn queued_score): Option; + pub QueuedScore get(fn queued_score): Option; /// Flag to control the execution of the offchain election. When `Open(_)`, we accept /// solutions to be submitted. @@ -1037,10 +1134,7 @@ decl_storage! { /// Storage version of the pallet. /// /// This is set to v3.0.0 for new networks. - StorageVersion build(|_: &GenesisConfig| Releases::V3_0_0): Releases; - - /// The era where we migrated from Lazy Payouts to Simple Payouts - MigrateEra: Option; + StorageVersion build(|_: &GenesisConfig| Releases::V4_0_0): Releases; } add_extra_genesis { config(stakers): @@ -1090,6 +1184,8 @@ decl_event!( OldSlashingReportDiscarded(SessionIndex), /// A new set of stakers was elected with the given computation method. StakingElection(ElectionCompute), + /// A new solution for the upcoming election has been stored. + SolutionStored(ElectionCompute), /// An account has bonded this amount. /// /// NOTE: This event is only emitted when funds are bonded via a dispatchable. Notably, @@ -1163,6 +1259,8 @@ decl_error! { PhragmenBogusEdge, /// The claimed score does not match with the one computed from the data. PhragmenBogusScore, + /// The election size is invalid. + PhragmenBogusElectionSize, /// The call is not allowed at the given time due to restrictions of election period. CallNotAllowed, /// Incorrect previous history depth input provided. @@ -1180,6 +1278,36 @@ decl_module! { /// Number of eras that staked funds must remain bonded for. const BondingDuration: EraIndex = T::BondingDuration::get(); + /// Number of eras that slashes are deferred by, after computation. + /// + /// This should be less than the bonding duration. + /// Set to 0 if slashes should be applied immediately, without opportunity for + /// intervention. + const SlashDeferDuration: EraIndex = T::SlashDeferDuration::get(); + + /// The number of blocks before the end of the era from which election submissions are allowed. + /// + /// Setting this to zero will disable the offchain compute and only on-chain seq-phragmen will + /// be used. + /// + /// This is bounded by being within the last session. Hence, setting it to a value more than the + /// length of a session will be pointless. + const ElectionLookahead: T::BlockNumber = T::ElectionLookahead::get(); + + /// Maximum number of balancing iterations to run in the offchain submission. + /// + /// If set to 0, balance_solution will not be executed at all. + const MaxIterations: u32 = T::MaxIterations::get(); + + /// The threshold of improvement that should be provided for a new solution to be accepted. + const MinSolutionScoreBump: Perbill = T::MinSolutionScoreBump::get(); + + /// The maximum number of nominators rewarded for each validator. + /// + /// For each validator only the `$MaxNominatorRewardedPerValidator` biggest stakers can claim + /// their reward. This used to limit the i/o cost for the nominator payout. + const MaxNominatorRewardedPerValidator: u32 = T::MaxNominatorRewardedPerValidator::get(); + type Error = Error; fn deposit_event() = default; @@ -1201,7 +1329,7 @@ decl_module! { // either current session final based on the plan, or we're forcing. (Self::is_current_session_final() || Self::will_era_be_forced()) { - if let Some(next_session_change) = T::NextNewSession::estimate_next_new_session(now){ + if let Some(next_session_change) = T::NextNewSession::estimate_next_new_session(now) { if let Some(remaining) = next_session_change.checked_sub(&now) { if remaining <= T::ElectionLookahead::get() && !remaining.is_zero() { // create snapshot. @@ -1243,7 +1371,7 @@ decl_module! { log!(debug, "skipping offchain worker in open election window due to [{}]", why); } else { if let Err(e) = compute_offchain_election::() { - log!(warn, "💸 Error in phragmen offchain worker: {:?}", e); + log!(error, "💸 Error in election offchain worker: {:?}", e); } else { log!(debug, "Executed offchain worker thread without errors."); } @@ -1677,6 +1805,34 @@ decl_module! { ValidatorCount::put(new); } + /// Increments the ideal number of validators. + /// + /// The dispatch origin must be Root. + /// + /// # + /// Base Weight: 1.717 µs + /// Read/Write: Validator Count + /// # + #[weight = 2 * WEIGHT_PER_MICROS + T::DbWeight::get().reads_writes(1, 1)] + fn increase_validator_count(origin, #[compact] additional: u32) { + ensure_root(origin)?; + ValidatorCount::mutate(|n| *n += additional); + } + + /// Scale up the ideal number of validators by a factor. + /// + /// The dispatch origin must be Root. + /// + /// # + /// Base Weight: 1.717 µs + /// Read/Write: Validator Count + /// # + #[weight = 2 * WEIGHT_PER_MICROS + T::DbWeight::get().reads_writes(1, 1)] + fn scale_validator_count(origin, factor: Percent) { + ensure_root(origin)?; + ValidatorCount::mutate(|n| *n += factor * *n); + } + /// Force there to be no new eras indefinitely. /// /// The dispatch origin must be Root. @@ -1772,7 +1928,7 @@ decl_module! { /// Cancel enactment of a deferred slash. /// - /// Can be called by either the root origin or the `T::SlashCancelOrigin`. + /// Can be called by the `T::SlashCancelOrigin`. /// /// Parameters: era and indices of the slashes for that era to kill. /// @@ -1789,9 +1945,7 @@ decl_module! { .saturating_add((35 * WEIGHT_PER_MICROS).saturating_mul(slash_indices.len() as Weight)) ] fn cancel_deferred_slash(origin, era: EraIndex, slash_indices: Vec) { - T::SlashCancelOrigin::try_origin(origin) - .map(|_| ()) - .or_else(ensure_root)?; + T::SlashCancelOrigin::ensure_origin(origin)?; ensure!(!slash_indices.is_empty(), Error::::EmptyTargets); ensure!(is_sorted_and_unique(&slash_indices), Error::::NotSortedAndUnique); @@ -1808,69 +1962,6 @@ decl_module! { ::UnappliedSlashes::insert(&era, &unapplied); } - /// **This extrinsic will be removed after `MigrationEra + HistoryDepth` has passed, giving - /// opportunity for users to claim all rewards before moving to Simple Payouts. After this - /// time, you should use `payout_stakers` instead.** - /// - /// Make one nominator's payout for one era. - /// - /// - `who` is the controller account of the nominator to pay out. - /// - `era` may not be lower than one following the most recently paid era. If it is higher, - /// then it indicates an instruction to skip the payout of all previous eras. - /// - `validators` is the list of all validators that `who` had exposure to during `era`, - /// alongside the index of `who` in the clipped exposure of the validator. - /// I.e. each element is a tuple of - /// `(validator, index of `who` in clipped exposure of validator)`. - /// If it is incomplete, then less than the full reward will be paid out. - /// It must not exceed `MAX_NOMINATIONS`. - /// - /// WARNING: once an era is payed for a validator such validator can't claim the payout of - /// previous era. - /// - /// WARNING: Incorrect arguments here can result in loss of payout. Be very careful. - /// - /// # - /// - Number of storage read of `O(validators)`; `validators` is the argument of the call, - /// and is bounded by `MAX_NOMINATIONS`. - /// - Each storage read is `O(N)` size and decode complexity; `N` is the maximum - /// nominations that can be given to a single validator. - /// - Computation complexity: `O(MAX_NOMINATIONS * logN)`; `MAX_NOMINATIONS` is the - /// maximum number of validators that may be nominated by a single nominator, it is - /// bounded only economically (all nominators are required to place a minimum stake). - /// # - #[weight = 500_000_000] - fn payout_nominator(origin, era: EraIndex, validators: Vec<(T::AccountId, u32)>) - -> DispatchResult - { - let ctrl = ensure_signed(origin)?; - Self::do_payout_nominator(ctrl, era, validators) - } - - /// **This extrinsic will be removed after `MigrationEra + HistoryDepth` has passed, giving - /// opportunity for users to claim all rewards before moving to Simple Payouts. After this - /// time, you should use `payout_stakers` instead.** - /// - /// Make one validator's payout for one era. - /// - /// - `who` is the controller account of the validator to pay out. - /// - `era` may not be lower than one following the most recently paid era. If it is higher, - /// then it indicates an instruction to skip the payout of all previous eras. - /// - /// WARNING: once an era is payed for a validator such validator can't claim the payout of - /// previous era. - /// - /// WARNING: Incorrect arguments here can result in loss of payout. Be very careful. - /// - /// # - /// - Time complexity: O(1). - /// - Contains a limited number of reads and writes. - /// # - #[weight = 500_000_000] - fn payout_validator(origin, era: EraIndex) -> DispatchResult { - let ctrl = ensure_signed(origin)?; - Self::do_payout_validator(ctrl, era) - } - /// Pay out all the stakers behind a single validator for a single era. /// /// - `validator_stash` is the stash account of the validator. Their nominators, up to @@ -1887,18 +1978,19 @@ decl_module! { /// - Contains a limited number of reads and writes. /// ----------- /// N is the Number of payouts for the validator (including the validator) - /// Base Weight: 110 + 54.2 * N µs (Median Slopes) + /// Base Weight: + /// - Reward Destination Staked: 110 + 54.2 * N µs (Median Slopes) + /// - Reward Destination Controller (Creating): 120 + 41.95 * N µs (Median Slopes) /// DB Weight: - /// - Read: EraElectionStatus, CurrentEra, HistoryDepth, MigrateEra, ErasValidatorReward, + /// - Read: EraElectionStatus, CurrentEra, HistoryDepth, ErasValidatorReward, /// ErasStakersClipped, ErasRewardPoints, ErasValidatorPrefs (8 items) /// - Read Each: Bonded, Ledger, Payee, Locks, System Account (5 items) /// - Write Each: System Account, Locks, Ledger (3 items) - // TODO: Remove read on Migrate Era /// # #[weight = - 110 * WEIGHT_PER_MICROS + 120 * WEIGHT_PER_MICROS + 54 * WEIGHT_PER_MICROS * Weight::from(T::MaxNominatorRewardedPerValidator::get()) - + T::DbWeight::get().reads(8) + + T::DbWeight::get().reads(7) + T::DbWeight::get().reads(5) * Weight::from(T::MaxNominatorRewardedPerValidator::get() + 1) + T::DbWeight::get().writes(3) * Weight::from(T::MaxNominatorRewardedPerValidator::get() + 1) ] @@ -2018,7 +2110,7 @@ decl_module! { T::Currency::remove_lock(STAKING_ID, &stash); } - /// Submit a phragmen result to the chain. If the solution: + /// Submit an election result to the chain. If the solution: /// /// 1. is valid. /// 2. has a better score than a potentially existing solution on chain. @@ -2031,7 +2123,7 @@ decl_module! { /// 2. `assignments`: the compact version of an assignment vector that encodes the edge /// weights. /// - /// Both of which may be computed using [`phragmen`], or any other algorithm. + /// Both of which may be computed using _phragmen_, or any other algorithm. /// /// Additionally, the submitter must provide: /// @@ -2063,51 +2155,26 @@ decl_module! { /// minimized (to ensure less variance) /// /// # - /// E: number of edges. m: size of winner committee. n: number of nominators. d: edge degree - /// (16 for now) v: number of on-chain validator candidates. - /// - /// NOTE: given a solution which is reduced, we can enable a new check the ensure `|E| < n + - /// m`. We don't do this _yet_, but our offchain worker code executes it nonetheless. - /// - /// major steps (all done in `check_and_replace_solution`): - /// - /// - Storage: O(1) read `ElectionStatus`. - /// - Storage: O(1) read `PhragmenScore`. - /// - Storage: O(1) read `ValidatorCount`. - /// - Storage: O(1) length read from `SnapshotValidators`. - /// - /// - Storage: O(v) reads of `AccountId` to fetch `snapshot_validators`. - /// - Memory: O(m) iterations to map winner index to validator id. - /// - Storage: O(n) reads `AccountId` to fetch `snapshot_nominators`. - /// - Memory: O(n + m) reads to map index to `AccountId` for un-compact. - /// - /// - Storage: O(e) accountid reads from `Nomination` to read correct nominations. - /// - Storage: O(e) calls into `slashable_balance_of_vote_weight` to convert ratio to staked. - /// - /// - Memory: build_support_map. O(e). - /// - Memory: evaluate_support: O(E). - /// - /// - Storage: O(e) writes to `QueuedElected`. - /// - Storage: O(1) write to `QueuedScore` - /// - /// The weight of this call is 1/10th of the blocks total weight. + /// See `crate::weight` module. /// # - #[weight = 100_000_000_000] + #[weight = weight::weight_for_submit_solution::(winners, compact, size)] pub fn submit_election_solution( origin, winners: Vec, - compact_assignments: CompactAssignments, - score: PhragmenScore, + compact: CompactAssignments, + score: ElectionScore, era: EraIndex, - ) { + size: ElectionSize, + ) -> DispatchResultWithPostInfo { let _who = ensure_signed(origin)?; Self::check_and_replace_solution( winners, - compact_assignments, + compact, ElectionCompute::Signed, score, era, - )? + size, + ) } /// Unsigned version of `submit_election_solution`. @@ -2115,26 +2182,34 @@ decl_module! { /// Note that this must pass the [`ValidateUnsigned`] check which only allows transactions /// from the local node to be included. In other words, only the block author can include a /// transaction in the block. - #[weight = 100_000_000_000] + /// + /// # + /// See `crate::weight` module. + /// # + #[weight = weight::weight_for_submit_solution::(winners, compact, size)] pub fn submit_election_solution_unsigned( origin, winners: Vec, - compact_assignments: CompactAssignments, - score: PhragmenScore, + compact: CompactAssignments, + score: ElectionScore, era: EraIndex, - ) { + size: ElectionSize, + ) -> DispatchResultWithPostInfo { ensure_none(origin)?; - Self::check_and_replace_solution( + let adjustments = Self::check_and_replace_solution( winners, - compact_assignments, + compact, ElectionCompute::Unsigned, score, era, - )? - // TODO: instead of returning an error, panic. This makes the entire produced block - // invalid. - // This ensures that block authors will not ever try and submit a solution which is not - // an improvement, since they will lose their authoring points/rewards. + size, + ).expect( + "An unsigned solution can only be submitted by validators; A validator should \ + always produce correct solutions, else this block should not be imported, thus \ + effectively depriving the validators from their authoring reward. Hence, this panic + is expected." + ); + Ok(adjustments) } } } @@ -2142,6 +2217,7 @@ decl_module! { impl Module { /// The total balance that can be slashed from a stash account as of right now. pub fn slashable_balance_of(stash: &T::AccountId) -> BalanceOf { + // Weight note: consider making the stake accessible through stash. Self::bonded(stash).and_then(Self::ledger).map(|l| l.active).unwrap_or_default() } @@ -2156,7 +2232,7 @@ impl Module { /// /// This data is used to efficiently evaluate election results. returns `true` if the operation /// is successful. - fn create_stakers_snapshot() -> (bool, Weight) { + pub fn create_stakers_snapshot() -> (bool, Weight) { let mut consumed_weight = 0; let mut add_db_reads_writes = |reads, writes| { consumed_weight += T::DbWeight::get().reads_writes(reads, writes); @@ -2198,143 +2274,6 @@ impl Module { >::kill(); } - fn do_payout_nominator(ctrl: T::AccountId, era: EraIndex, validators: Vec<(T::AccountId, u32)>) - -> DispatchResult - { - // validators len must not exceed `MAX_NOMINATIONS` to avoid querying more validator - // exposure than necessary. - if validators.len() > MAX_NOMINATIONS { - return Err(Error::::InvalidNumberOfNominations.into()); - } - // If migrate_era is not populated, then you should use `payout_stakers` - let migrate_era = MigrateEra::get().ok_or(Error::::InvalidEraToReward)?; - // This payout mechanism will only work for eras before the migration. - // Subsequent payouts should use `payout_stakers`. - ensure!(era < migrate_era, Error::::InvalidEraToReward); - let current_era = CurrentEra::get().ok_or(Error::::InvalidEraToReward)?; - ensure!(era <= current_era, Error::::InvalidEraToReward); - let history_depth = Self::history_depth(); - ensure!(era >= current_era.saturating_sub(history_depth), Error::::InvalidEraToReward); - - // Note: if era has no reward to be claimed, era may be future. better not to update - // `nominator_ledger.last_reward` in this case. - let era_payout = >::get(&era) - .ok_or_else(|| Error::::InvalidEraToReward)?; - - let mut nominator_ledger = >::get(&ctrl).ok_or_else(|| Error::::NotController)?; - - ensure!( - Self::era_election_status().is_closed() || Self::payee(&nominator_ledger.stash) != RewardDestination::Staked, - Error::::CallNotAllowed, - ); - - nominator_ledger.claimed_rewards.retain(|&x| x >= current_era.saturating_sub(history_depth)); - match nominator_ledger.claimed_rewards.binary_search(&era) { - Ok(_) => Err(Error::::AlreadyClaimed)?, - Err(pos) => nominator_ledger.claimed_rewards.insert(pos, era), - } - - >::insert(&ctrl, &nominator_ledger); - - let mut reward = Perbill::zero(); - let era_reward_points = >::get(&era); - - for (validator, nominator_index) in validators.into_iter() { - let commission = Self::eras_validator_prefs(&era, &validator).commission; - let validator_exposure = >::get(&era, &validator); - - if let Some(nominator_exposure) = validator_exposure.others - .get(nominator_index as usize) - { - if nominator_exposure.who != nominator_ledger.stash { - continue; - } - - let nominator_exposure_part = Perbill::from_rational_approximation( - nominator_exposure.value, - validator_exposure.total, - ); - let validator_point = era_reward_points.individual.get(&validator) - .map(|points| *points) - .unwrap_or_else(|| Zero::zero()); - let validator_point_part = Perbill::from_rational_approximation( - validator_point, - era_reward_points.total, - ); - reward = reward.saturating_add( - validator_point_part - .saturating_mul(Perbill::one().saturating_sub(commission)) - .saturating_mul(nominator_exposure_part) - ); - } - } - - if let Some(imbalance) = Self::make_payout(&nominator_ledger.stash, reward * era_payout) { - Self::deposit_event(RawEvent::Reward(ctrl, imbalance.peek())); - } - - Ok(()) - } - - fn do_payout_validator(ctrl: T::AccountId, era: EraIndex) -> DispatchResult { - // If migrate_era is not populated, then you should use `payout_stakers` - let migrate_era = MigrateEra::get().ok_or(Error::::InvalidEraToReward)?; - // This payout mechanism will only work for eras before the migration. - // Subsequent payouts should use `payout_stakers`. - ensure!(era < migrate_era, Error::::InvalidEraToReward); - let current_era = CurrentEra::get().ok_or(Error::::InvalidEraToReward)?; - ensure!(era <= current_era, Error::::InvalidEraToReward); - let history_depth = Self::history_depth(); - ensure!(era >= current_era.saturating_sub(history_depth), Error::::InvalidEraToReward); - - // Note: if era has no reward to be claimed, era may be future. better not to update - // `ledger.last_reward` in this case. - let era_payout = >::get(&era) - .ok_or_else(|| Error::::InvalidEraToReward)?; - - let mut ledger = >::get(&ctrl).ok_or_else(|| Error::::NotController)?; - - ensure!( - Self::era_election_status().is_closed() || Self::payee(&ledger.stash) != RewardDestination::Staked, - Error::::CallNotAllowed, - ); - - ledger.claimed_rewards.retain(|&x| x >= current_era.saturating_sub(history_depth)); - match ledger.claimed_rewards.binary_search(&era) { - Ok(_) => Err(Error::::AlreadyClaimed)?, - Err(pos) => ledger.claimed_rewards.insert(pos, era), - } - - >::insert(&ctrl, &ledger); - - let era_reward_points = >::get(&era); - let commission = Self::eras_validator_prefs(&era, &ledger.stash).commission; - let exposure = >::get(&era, &ledger.stash); - - let exposure_part = Perbill::from_rational_approximation( - exposure.own, - exposure.total, - ); - let validator_point = era_reward_points.individual.get(&ledger.stash) - .map(|points| *points) - .unwrap_or_else(|| Zero::zero()); - let validator_point_part = Perbill::from_rational_approximation( - validator_point, - era_reward_points.total, - ); - let reward = validator_point_part.saturating_mul( - commission.saturating_add( - Perbill::one().saturating_sub(commission).saturating_mul(exposure_part) - ) - ); - - if let Some(imbalance) = Self::make_payout(&ledger.stash, reward * era_payout) { - Self::deposit_event(RawEvent::Reward(ctrl, imbalance.peek())); - } - - Ok(()) - } - fn do_payout_stakers( validator_stash: T::AccountId, era: EraIndex, @@ -2345,13 +2284,6 @@ impl Module { let history_depth = Self::history_depth(); ensure!(era >= current_era.saturating_sub(history_depth), Error::::InvalidEraToReward); - // If there was no migration, then this function is always valid. - if let Some(migrate_era) = MigrateEra::get() { - // This payout mechanism will only work for eras on and after the migration. - // Payouts before then should use `payout_nominator`/`payout_validator`. - ensure!(migrate_era <= era, Error::::InvalidEraToReward); - } - // Note: if era has no reward to be claimed, era may be future. better not to update // `ledger.claimed_rewards` in this case. let era_payout = >::get(&era) @@ -2465,7 +2397,7 @@ impl Module { match dest { RewardDestination::Controller => Self::bonded(stash) .and_then(|controller| - T::Currency::deposit_into_existing(&controller, amount).ok() + Some(T::Currency::deposit_creating(&controller, amount)) ), RewardDestination::Stash => T::Currency::deposit_into_existing(stash, amount).ok(), @@ -2500,16 +2432,19 @@ impl Module { Forcing::ForceAlways => (), Forcing::NotForcing if era_length >= T::SessionsPerEra::get() => (), _ => { - // not forcing, not a new era either. If final, set the flag. - if era_length + 1 >= T::SessionsPerEra::get() { + // Either `ForceNone`, or `NotForcing && era_length < T::SessionsPerEra::get()`. + if era_length + 1 == T::SessionsPerEra::get() { IsCurrentSessionFinal::put(true); + } else if era_length >= T::SessionsPerEra::get() { + // Should only happen when we are ready to trigger an era but we have ForceNone, + // otherwise previous arm would short circuit. + Self::close_election_window(); } return None }, } // new era. - IsCurrentSessionFinal::put(false); Self::new_era(session_index) } else { // Set initial era @@ -2518,31 +2453,36 @@ impl Module { } /// Basic and cheap checks that we perform in validate unsigned, and in the execution. - pub fn pre_dispatch_checks(score: PhragmenScore, era: EraIndex) -> Result<(), Error> { + /// + /// State reads: ElectionState, CurrentEr, QueuedScore. + /// + /// This function does weight refund in case of errors, which is based upon the fact that it is + /// called at the very beginning of the call site's function. + pub fn pre_dispatch_checks(score: ElectionScore, era: EraIndex) -> DispatchResultWithPostInfo { // discard solutions that are not in-time // check window open ensure!( Self::era_election_status().is_open(), - Error::::PhragmenEarlySubmission, + Error::::PhragmenEarlySubmission.with_weight(T::DbWeight::get().reads(1)), ); // check current era. if let Some(current_era) = Self::current_era() { ensure!( current_era == era, - Error::::PhragmenEarlySubmission, + Error::::PhragmenEarlySubmission.with_weight(T::DbWeight::get().reads(2)), ) } // assume the given score is valid. Is it better than what we have on-chain, if we have any? if let Some(queued_score) = Self::queued_score() { ensure!( - is_score_better(queued_score, score), - Error::::PhragmenWeakSubmission, + is_score_better(score, queued_score, T::MinSolutionScoreBump::get()), + Error::::PhragmenWeakSubmission.with_weight(T::DbWeight::get().reads(3)), ) } - Ok(()) + Ok(None.into()) } /// Checks a given solution and if correct and improved, writes it on chain as the queued result @@ -2551,23 +2491,48 @@ impl Module { winners: Vec, compact_assignments: CompactAssignments, compute: ElectionCompute, - claimed_score: PhragmenScore, + claimed_score: ElectionScore, era: EraIndex, - ) -> Result<(), Error> { + election_size: ElectionSize, + ) -> DispatchResultWithPostInfo { // Do the basic checks. era, claimed score and window open. Self::pre_dispatch_checks(claimed_score, era)?; + // the weight that we will refund in case of a correct submission. We compute this now + // because the data needed for it will be consumed further down. + let adjusted_weight = weight::weight_for_correct_submit_solution::( + &winners, + &compact_assignments, + &election_size, + ); // Check that the number of presented winners is sane. Most often we have more candidates - // that we need. Then it should be Self::validator_count(). Else it should be all the + // than we need. Then it should be `Self::validator_count()`. Else it should be all the // candidates. - let snapshot_length = >::decode_len() + let snapshot_validators_length = >::decode_len() + .map(|l| l as u32) .ok_or_else(|| Error::::SnapshotUnavailable)?; + // size of the solution must be correct. + ensure!( + snapshot_validators_length == u32::from(election_size.validators), + Error::::PhragmenBogusElectionSize, + ); + // check the winner length only here and when we know the length of the snapshot validators // length. - let desired_winners = Self::validator_count().min(snapshot_length as u32); + let desired_winners = Self::validator_count().min(snapshot_validators_length); ensure!(winners.len() as u32 == desired_winners, Error::::PhragmenBogusWinnerCount); + let snapshot_nominators_len = >::decode_len() + .map(|l| l as u32) + .ok_or_else(|| Error::::SnapshotUnavailable)?; + + // rest of the size of the solution must be correct. + ensure!( + snapshot_nominators_len == election_size.nominators, + Error::::PhragmenBogusElectionSize, + ); + // decode snapshot validators. let snapshot_validators = Self::snapshot_validators() .ok_or(Error::::SnapshotUnavailable)?; @@ -2581,7 +2546,7 @@ impl Module { }).collect::, Error>>()?; // decode the rest of the snapshot. - let snapshot_nominators = >::snapshot_nominators() + let snapshot_nominators = Self::snapshot_nominators() .ok_or(Error::::SnapshotUnavailable)?; // helpers @@ -2615,7 +2580,7 @@ impl Module { // have bigger problems. log!(error, "💸 detected an error in the staking locking and snapshot."); // abort. - return Err(Error::::PhragmenBogusNominator); + return Err(Error::::PhragmenBogusNominator.into()); } if !is_validator { @@ -2632,14 +2597,14 @@ impl Module { // each target in the provided distribution must be actually nominated by the // nominator after the last non-zero slash. if nomination.targets.iter().find(|&tt| tt == t).is_none() { - return Err(Error::::PhragmenBogusNomination); + return Err(Error::::PhragmenBogusNomination.into()); } if ::SlashingSpans::get(&t).map_or( false, |spans| nomination.submitted_in < spans.last_nonzero_slash(), ) { - return Err(Error::::PhragmenSlashedNomination); + return Err(Error::::PhragmenSlashedNomination.into()); } } } else { @@ -2656,7 +2621,7 @@ impl Module { } // convert into staked assignments. - let staked_assignments = sp_phragmen::assignment_ratio_to_staked( + let staked_assignments = sp_npos_elections::assignment_ratio_to_staked( assignments, Self::slashable_balance_of_vote_weight, ); @@ -2679,8 +2644,9 @@ impl Module { let exposures = Self::collect_exposure(supports); log!( info, - "💸 A better solution (with compute {:?}) has been validated and stored on chain.", + "💸 A better solution (with compute {:?} and score {:?}) has been validated and stored on chain.", compute, + submitted_score, ); // write new results. @@ -2691,8 +2657,10 @@ impl Module { }); QueuedScore::put(submitted_score); - Ok(()) + // emit event. + Self::deposit_event(RawEvent::SolutionStored(compute)); + Ok(Some(adjusted_weight).into()) } /// Start a session potentially starting an era. @@ -2810,6 +2778,17 @@ impl Module { maybe_new_validators } + + /// Remove all the storage items associated with the election. + fn close_election_window() { + // Close window. + >::put(ElectionStatus::Closed); + // Kill snapshots. + Self::kill_stakers_snapshot(); + // Don't track final session. + IsCurrentSessionFinal::put(false); + } + /// Select the new validator set at the end of the era. /// /// Runs [`try_do_phragmen`] and updates the following storage items: @@ -2830,12 +2809,9 @@ impl Module { elected_stashes, exposures, compute, - }) = Self::try_do_phragmen() { - // We have chosen the new validator set. Submission is no longer allowed. - >::put(ElectionStatus::Closed); - - // kill the snapshots. - Self::kill_stakers_snapshot(); + }) = Self::try_do_election() { + // Totally close the election round and data. + Self::close_election_window(); // Populate Stakers and write slot stake. let mut total_stake: BalanceOf = Zero::zero(); @@ -2879,12 +2855,12 @@ impl Module { } /// Select a new validator set from the assembled stakers and their role preferences. It tries - /// first to peek into [`QueuedElected`]. Otherwise, it runs a new phragmen. + /// first to peek into [`QueuedElected`]. Otherwise, it runs a new on-chain phragmen election. /// /// If [`QueuedElected`] and [`QueuedScore`] exists, they are both removed. No further storage /// is updated. - fn try_do_phragmen() -> Option>> { - // a phragmen result from either a stored submission or locally executed one. + fn try_do_election() -> Option>> { + // an election result from either a stored submission or locally executed one. let next_result = >::take().or_else(|| Self::do_phragmen_with_post_processing::(ElectionCompute::OnChain) ); @@ -2896,11 +2872,11 @@ impl Module { next_result } - /// Execute phragmen and return the new results. The edge weights are processed into support + /// Execute election and return the new results. The edge weights are processed into support /// values. /// - /// This is basically a wrapper around [`do_phragmen`] which translates `PhragmenResult` into - /// `ElectionResult`. + /// This is basically a wrapper around [`do_phragmen`] which translates + /// `PrimitiveElectionResult` into `ElectionResult`. /// /// No storage item is updated. fn do_phragmen_with_post_processing(compute: ElectionCompute) @@ -2915,7 +2891,7 @@ impl Module { .collect::>(); let assignments = phragmen_result.assignments; - let staked_assignments = sp_phragmen::assignment_ratio_to_staked( + let staked_assignments = sp_npos_elections::assignment_ratio_to_staked( assignments, Self::slashable_balance_of_vote_weight, ); @@ -2946,13 +2922,13 @@ impl Module { } } - /// Execute phragmen and return the new results. No post-processing is applied and the raw edge - /// weights are returned. + /// Execute phragmen election and return the new results. No post-processing is applied and the + /// raw edge weights are returned. /// /// Self votes are added and nominations before the most recent slashing span are reaped. /// /// No storage item is updated. - fn do_phragmen() -> Option> { + fn do_phragmen() -> Option> { let mut all_nominators: Vec<(T::AccountId, VoteWeight, Vec)> = Vec::new(); let mut all_validators = Vec::new(); for (validator, _) in >::iter() { @@ -2981,7 +2957,7 @@ impl Module { (n, s, ns) })); - elect::<_, Accuracy>( + seq_phragmen::<_, Accuracy>( Self::validator_count() as usize, Self::minimum_validator_count().max(1) as usize, all_validators, @@ -2989,14 +2965,14 @@ impl Module { ) } - /// Consume a set of [`Supports`] from [`sp_phragmen`] and collect them into a [`Exposure`] + /// Consume a set of [`Supports`] from [`sp_npos_elections`] and collect them into a [`Exposure`] fn collect_exposure(supports: SupportMap) -> Vec<(T::AccountId, Exposure>)> { let to_balance = |e: ExtendedBalance| >>::convert(e); supports.into_iter().map(|(validator, support)| { // build `struct exposure` from `support` - let mut others = Vec::new(); + let mut others = Vec::with_capacity(support.voters.len()); let mut own: BalanceOf = Zero::zero(); let mut total: BalanceOf = Zero::zero(); support.voters @@ -3029,11 +3005,11 @@ impl Module { /// - after a `withdraw_unbond()` call that frees all of a stash's bonded balance. /// - through `reap_stash()` if the balance has fallen to zero (through slashing). fn kill_stash(stash: &T::AccountId, num_slashing_spans: u32) -> DispatchResult { - let controller = Bonded::::get(stash).ok_or(Error::::NotStash)?; + let controller = >::get(stash).ok_or(Error::::NotStash)?; slashing::clear_stash_metadata::(stash, num_slashing_spans)?; - Bonded::::remove(stash); + >::remove(stash); >::remove(&controller); >::remove(stash); @@ -3218,7 +3194,9 @@ impl Convert> } /// This is intended to be used with `FilterHistoricalOffences`. -impl OnOffenceHandler> for Module where +impl + OnOffenceHandler, Weight> +for Module where T: pallet_session::Trait::AccountId>, T: pallet_session::historical::Trait< FullIdentification = Exposure<::AccountId, BalanceOf>, @@ -3226,24 +3204,32 @@ impl OnOffenceHandler, T::SessionHandler: pallet_session::SessionHandler<::AccountId>, T::SessionManager: pallet_session::SessionManager<::AccountId>, - T::ValidatorIdOf: Convert<::AccountId, Option<::AccountId>> + T::ValidatorIdOf: Convert< + ::AccountId, + Option<::AccountId>, + >, { fn on_offence( offenders: &[OffenceDetails>], slash_fraction: &[Perbill], slash_session: SessionIndex, - ) -> Result<(), ()> { + ) -> Result { if !Self::can_report() { return Err(()) } let reward_proportion = SlashRewardFraction::get(); + let mut consumed_weight: Weight = 0; + let mut add_db_reads_writes = |reads, writes| { + consumed_weight += T::DbWeight::get().reads_writes(reads, writes); + }; let active_era = { let active_era = Self::active_era(); + add_db_reads_writes(1, 0); if active_era.is_none() { // this offence need not be re-submitted. - return Ok(()) + return Ok(consumed_weight) } active_era.expect("value checked not to be `None`; qed").index }; @@ -3252,6 +3238,7 @@ impl OnOffenceHandler OnOffenceHandler return Ok(()), // before bonding period. defensive - should be filtered out. Some(&(ref slash_era, _)) => *slash_era, + // before bonding period. defensive - should be filtered out. + None => return Ok(consumed_weight), } }; @@ -3274,14 +3263,18 @@ impl OnOffenceHandler OnOffenceHandler(unapplied); + { + let slash_cost = (6, 5); + let reward_cost = (2, 2); + add_db_reads_writes( + (1 + nominators_len) * slash_cost.0 + reward_cost.0 * reporters_len, + (1 + nominators_len) * slash_cost.1 + reward_cost.1 * reporters_len + ); + } } else { // defer to end of some `slash_defer_duration` from now. ::UnappliedSlashes::mutate( active_era, move |for_later| for_later.push(unapplied), ); + add_db_reads_writes(1, 1); } + } else { + add_db_reads_writes(4 /* fetch_spans */, 5 /* kick_out_if_recent */) } } - Ok(()) + Ok(consumed_weight) } fn can_report() -> bool { @@ -3345,12 +3357,6 @@ impl ReportOffence } } -impl From> for InvalidTransaction { - fn from(e: Error) -> Self { - InvalidTransaction::Custom(e.as_u8()) - } -} - #[allow(deprecated)] impl frame_support::unsigned::ValidateUnsigned for Module { type Call = Call; @@ -3360,6 +3366,7 @@ impl frame_support::unsigned::ValidateUnsigned for Module { _, score, era, + _, ) = call { use offchain_election::DEFAULT_LONGEVITY; @@ -3372,9 +3379,14 @@ impl frame_support::unsigned::ValidateUnsigned for Module { } } - if let Err(e) = Self::pre_dispatch_checks(*score, *era) { - log!(debug, "validate unsigned pre dispatch checks failed due to {:?}.", e); - return InvalidTransaction::from(e).into(); + if let Err(error_with_post_info) = Self::pre_dispatch_checks(*score, *era) { + let invalid = to_invalid(error_with_post_info); + log!( + debug, + "validate unsigned pre dispatch checks failed due to error #{:?}.", + invalid, + ); + return invalid .into(); } log!(debug, "validateUnsigned succeeded for a solution at era {}.", era); @@ -3402,13 +3414,28 @@ impl frame_support::unsigned::ValidateUnsigned for Module { } } - fn pre_dispatch(_: &Self::Call) -> Result<(), TransactionValidityError> { - // IMPORTANT NOTE: By default, a sane `pre-dispatch` should always do the same checks as - // `validate_unsigned` and overriding this should be done with care. this module has only - // one unsigned entry point, in which we call into `>::pre_dispatch_checks()` - // which is all the important checks that we do in `validate_unsigned`. Hence, we can safely - // override this to save some time. - Ok(()) + fn pre_dispatch(call: &Self::Call) -> Result<(), TransactionValidityError> { + if let Call::submit_election_solution_unsigned( + _, + _, + score, + era, + _, + ) = call { + // IMPORTANT NOTE: These checks are performed in the dispatch call itself, yet we need + // to duplicate them here to prevent a block producer from putting a previously + // validated, yet no longer valid solution on chain. + // OPTIMISATION NOTE: we could skip this in the `submit_election_solution_unsigned` + // since we already do it here. The signed version needs it though. Yer for now we keep + // this duplicate check here so both signed and unsigned can use a singular + // `check_and_replace_solution`. + Self::pre_dispatch_checks(*score, *era) + .map(|_| ()) + .map_err(to_invalid) + .map_err(Into::into) + } else { + Err(InvalidTransaction::Call.into()) + } } } @@ -3416,3 +3443,14 @@ impl frame_support::unsigned::ValidateUnsigned for Module { fn is_sorted_and_unique(list: &[u32]) -> bool { list.windows(2).all(|w| w[0] < w[1]) } + +/// convert a DispatchErrorWithPostInfo to a custom InvalidTransaction with the inner code being the +/// error number. +fn to_invalid(error_with_post_info: DispatchErrorWithPostInfo) -> InvalidTransaction { + let error = error_with_post_info.error; + let error_number = match error { + DispatchError::Module { error, ..} => error, + _ => 0, + }; + InvalidTransaction::Custom(error_number) +} diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index a2d53f689577ca56081d31fd7b3f3ef2493e4b9d..3860dba90f3507ee799820b98a9634c4acf20cf8 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -17,7 +17,7 @@ //! Test utilities -use std::{collections::{HashSet, HashMap}, cell::RefCell}; +use std::{collections::HashSet, cell::RefCell}; use sp_runtime::Perbill; use sp_runtime::curve::PiecewiseLinear; use sp_runtime::traits::{IdentityLookup, Convert, SaturatedConversion, Zero}; @@ -31,8 +31,8 @@ use frame_support::{ weights::{Weight, constants::RocksDbWeight}, }; use sp_io; -use sp_phragmen::{ - build_support_map, evaluate_support, reduce, ExtendedBalance, StakedAssignment, PhragmenScore, +use sp_npos_elections::{ + build_support_map, evaluate_support, reduce, ExtendedBalance, StakedAssignment, ElectionScore, VoteWeight, }; use crate::*; @@ -200,6 +200,7 @@ parameter_types! { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = AccountIndex; type BlockNumber = BlockNumber; @@ -215,6 +216,7 @@ impl frame_system::Trait for Test { type DbWeight = RocksDbWeight; type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); @@ -285,6 +287,7 @@ parameter_types! { pub const RewardCurve: &'static PiecewiseLinear<'static> = &I_NPOS; pub const MaxNominatorRewardedPerValidator: u32 = 64; pub const UnsignedPriority: u64 = 1 << 20; + pub const MinSolutionScoreBump: Perbill = Perbill::zero(); } thread_local! { @@ -320,6 +323,7 @@ impl Trait for Test { type ElectionLookahead = ElectionLookahead; type Call = Call; type MaxIterations = MaxIterations; + type MinSolutionScoreBump = MinSolutionScoreBump; type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; type UnsignedPriority = UnsignedPriority; } @@ -780,7 +784,7 @@ pub(crate) fn add_slash(who: &AccountId) { // distributed evenly. pub(crate) fn horrible_phragmen_with_post_processing( do_reduce: bool, -) -> (CompactAssignments, Vec, PhragmenScore) { +) -> (CompactAssignments, Vec, ElectionScore) { let mut backing_stake_of: BTreeMap = BTreeMap::new(); // self stake @@ -852,7 +856,11 @@ pub(crate) fn horrible_phragmen_with_post_processing( let support = build_support_map::(&winners, &staked_assignment).0; let score = evaluate_support(&support); - assert!(sp_phragmen::is_score_better(score, better_score)); + assert!(sp_npos_elections::is_score_better::( + better_score, + score, + MinSolutionScoreBump::get(), + )); score }; @@ -872,7 +880,7 @@ pub(crate) fn horrible_phragmen_with_post_processing( // convert back to ratio assignment. This takes less space. let assignments_reduced = - sp_phragmen::assignment_staked_to_ratio::(staked_assignment); + sp_npos_elections::assignment_staked_to_ratio::(staked_assignment); let compact = CompactAssignments::from_assignment(assignments_reduced, nominator_index, validator_index) @@ -890,13 +898,13 @@ pub(crate) fn prepare_submission_with( do_reduce: bool, iterations: usize, tweak: impl FnOnce(&mut Vec>), -) -> (CompactAssignments, Vec, PhragmenScore) { - // run phragmen on the default stuff. - let sp_phragmen::PhragmenResult { +) -> (CompactAssignments, Vec, ElectionScore) { + // run election on the default stuff. + let sp_npos_elections::ElectionResult { winners, assignments, } = Staking::do_phragmen::().unwrap(); - let winners = sp_phragmen::to_without_backing(winners); + let winners = sp_npos_elections::to_without_backing(winners); let stake_of = |who: &AccountId| -> VoteWeight { >::convert( @@ -904,11 +912,11 @@ pub(crate) fn prepare_submission_with( ) }; - let mut staked = sp_phragmen::assignment_ratio_to_staked(assignments, stake_of); + let mut staked = sp_npos_elections::assignment_ratio_to_staked(assignments, stake_of); let (mut support_map, _) = build_support_map::(&winners, &staked); if iterations > 0 { - sp_phragmen::equalize( + sp_npos_elections::balance_solution( &mut staked, &mut support_map, Zero::zero(), @@ -945,11 +953,11 @@ pub(crate) fn prepare_submission_with( ) }; - let assignments_reduced = sp_phragmen::assignment_staked_to_ratio(staked); + let assignments_reduced = sp_npos_elections::assignment_staked_to_ratio(staked); // re-compute score by converting, yet again, into staked type let score = { - let staked = sp_phragmen::assignment_ratio_to_staked( + let staked = sp_npos_elections::assignment_ratio_to_staked( assignments_reduced.clone(), Staking::slashable_balance_of_vote_weight, ); @@ -973,38 +981,6 @@ pub(crate) fn prepare_submission_with( (compact, winners, score) } -/// Make all validator and nominator request their payment -pub(crate) fn make_all_reward_payment_before_migration(era: EraIndex) { - let validators_with_reward = ErasRewardPoints::::get(era).individual.keys() - .cloned() - .collect::>(); - - // reward nominators - let mut nominator_controllers = HashMap::new(); - for validator in Staking::eras_reward_points(era).individual.keys() { - let validator_exposure = Staking::eras_stakers_clipped(era, validator); - for (nom_index, nom) in validator_exposure.others.iter().enumerate() { - if let Some(nom_ctrl) = Staking::bonded(nom.who) { - nominator_controllers.entry(nom_ctrl) - .or_insert(vec![]) - .push((validator.clone(), nom_index as u32)); - } - } - } - for (nominator_controller, validators_with_nom_index) in nominator_controllers { - assert_ok!(Staking::payout_nominator( - Origin::signed(nominator_controller), - era, - validators_with_nom_index, - )); - } - - // reward validators - for validator_controller in validators_with_reward.iter().filter_map(Staking::bonded) { - assert_ok!(Staking::payout_validator(Origin::signed(validator_controller), era)); - } -} - /// Make all validator and nominator request their payment pub(crate) fn make_all_reward_payment(era: EraIndex) { let validators_with_reward = ErasRewardPoints::::get(era).individual.keys() diff --git a/frame/staking/src/offchain_election.rs b/frame/staking/src/offchain_election.rs index 4e32d75f21d76ac463baf921a75e83d4661e928f..79f3a5c2d94fec73f2297e6a18de804c8d0f7d2b 100644 --- a/frame/staking/src/offchain_election.rs +++ b/frame/staking/src/offchain_election.rs @@ -20,36 +20,37 @@ use codec::Decode; use crate::{ Call, CompactAssignments, Module, NominatorIndex, OffchainAccuracy, Trait, ValidatorIndex, + ElectionSize, }; use frame_system::offchain::SubmitTransaction; -use sp_phragmen::{ - build_support_map, evaluate_support, reduce, Assignment, ExtendedBalance, PhragmenResult, - PhragmenScore, equalize, +use sp_npos_elections::{ + build_support_map, evaluate_support, reduce, Assignment, ExtendedBalance, ElectionResult, + ElectionScore, balance_solution, }; use sp_runtime::offchain::storage::StorageValueRef; use sp_runtime::{PerThing, RuntimeDebug, traits::{TrailingZeroInput, Zero}}; -use frame_support::{debug, traits::Get}; +use frame_support::traits::Get; use sp_std::{convert::TryInto, prelude::*}; /// Error types related to the offchain election machinery. #[derive(RuntimeDebug)] pub enum OffchainElectionError { - /// Phragmen election returned None. This means less candidate that minimum number of needed + /// election returned None. This means less candidate that minimum number of needed /// validators were present. The chain is in trouble and not much that we can do about it. ElectionFailed, /// Submission to the transaction pool failed. PoolSubmissionFailed, /// The snapshot data is not available. SnapshotUnavailable, - /// Error from phragmen crate. This usually relates to compact operation. - PhragmenError(sp_phragmen::Error), + /// Error from npos-election crate. This usually relates to compact operation. + InternalElectionError(sp_npos_elections::Error), /// One of the computed winners is invalid. InvalidWinner, } -impl From for OffchainElectionError { - fn from(e: sp_phragmen::Error) -> Self { - Self::PhragmenError(e) +impl From for OffchainElectionError { + fn from(e: sp_npos_elections::Error) -> Self { + Self::InternalElectionError(e) } } @@ -106,14 +107,14 @@ pub(crate) fn set_check_offchain_execution_status( /// unsigned transaction, without any signature. pub(crate) fn compute_offchain_election() -> Result<(), OffchainElectionError> { // compute raw solution. Note that we use `OffchainAccuracy`. - let PhragmenResult { + let ElectionResult { winners, assignments, } = >::do_phragmen::() .ok_or(OffchainElectionError::ElectionFailed)?; // process and prepare it for submission. - let (winners, compact, score) = prepare_submission::(assignments, winners, true)?; + let (winners, compact, score, size) = prepare_submission::(assignments, winners, true)?; // defensive-only: current era can never be none except genesis. let current_era = >::current_era().unwrap_or_default(); @@ -124,20 +125,27 @@ pub(crate) fn compute_offchain_election() -> Result<(), OffchainElecti compact, score, current_era, + size, ).into(); SubmitTransaction::>::submit_unsigned_transaction(call) .map_err(|_| OffchainElectionError::PoolSubmissionFailed) } -/// Takes a phragmen result and spits out some data that can be submitted to the chain. + +/// Takes an election result and spits out some data that can be submitted to the chain. /// /// This does a lot of stuff; read the inline comments. pub fn prepare_submission( assignments: Vec>, winners: Vec<(T::AccountId, ExtendedBalance)>, do_reduce: bool, -) -> Result<(Vec, CompactAssignments, PhragmenScore), OffchainElectionError> where +) -> Result<( + Vec, + CompactAssignments, + ElectionScore, + ElectionSize, +), OffchainElectionError> where ExtendedBalance: From<::Inner>, { // make sure that the snapshot is available. @@ -161,26 +169,26 @@ pub fn prepare_submission( }; // Clean winners. - let winners = sp_phragmen::to_without_backing(winners); + let winners = sp_npos_elections::to_without_backing(winners); // convert into absolute value and to obtain the reduced version. - let mut staked = sp_phragmen::assignment_ratio_to_staked( + let mut staked = sp_npos_elections::assignment_ratio_to_staked( assignments, >::slashable_balance_of_vote_weight, ); let (mut support_map, _) = build_support_map::(&winners, &staked); - // equalize a random number of times. + // balance a random number of times. let iterations_executed = match T::MaxIterations::get() { 0 => { - // Don't run equalize at all + // Don't run balance_solution at all 0 } iterations @ _ => { let seed = sp_io::offchain::random_seed(); let iterations = ::decode(&mut TrailingZeroInput::new(seed.as_ref())) .expect("input is padded with zeroes; qed") % iterations.saturating_add(1); - equalize( + balance_solution( &mut staked, &mut support_map, Zero::zero(), @@ -195,7 +203,8 @@ pub fn prepare_submission( } // Convert back to ratio assignment. This takes less space. - let low_accuracy_assignment = sp_phragmen::assignment_staked_to_ratio(staked); + let low_accuracy_assignment = sp_npos_elections::assignment_staked_to_ratio_normalized(staked) + .map_err(|e| OffchainElectionError::from(e))?; // convert back to staked to compute the score in the receiver's accuracy. This can be done // nicer, for now we do it as such since this code is not time-critical. This ensure that the @@ -206,7 +215,7 @@ pub fn prepare_submission( // assignment set is also all multiples of this value. After reduce, this no longer holds. Hence // converting to ratio thereafter is not trivially reversible. let score = { - let staked = sp_phragmen::assignment_ratio_to_staked( + let staked = sp_npos_elections::assignment_ratio_to_staked( low_accuracy_assignment.clone(), >::slashable_balance_of_vote_weight, ); @@ -235,12 +244,18 @@ pub fn prepare_submission( } } - debug::native::debug!( - target: "staking", + // both conversions are safe; snapshots are not created if they exceed. + let size = ElectionSize { + validators: snapshot_validators.len() as ValidatorIndex, + nominators: snapshot_nominators.len() as NominatorIndex, + }; + + crate::log!( + info, "prepared solution after {} equalization iterations with score {:?}", iterations_executed, score, ); - Ok((winners_indexed, compact, score)) + Ok((winners_indexed, compact, score, size)) } diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index 6a95838a096f51497c759d059a9b16a5f7d46f03..27a2575eb0df33d7871552ecee95cf351241aa5d 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -15,142 +15,145 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Testing utils for staking. Needs the `testing-utils` feature to be enabled. -//! -//! Note that these helpers should NOT be used with the actual crate tests, but are rather designed -//! for when the module is being externally tested (i.e. fuzzing, benchmarking, e2e tests). Enabling -//! this feature in the current crate's Cargo.toml will leak all of this into a normal release -//! build. Just don't do it. +//! Testing utils for staking. Provides some common functions to setup staking state, such as +//! bonding validators, nominators, and generating different types of solutions. use crate::*; -use codec::{Decode, Encode}; -use frame_support::assert_ok; +use crate::Module as Staking; +use frame_benchmarking::{account}; use frame_system::RawOrigin; -use pallet_indices::address::Address; -use rand::Rng; -use sp_core::hashing::blake2_256; -use sp_phragmen::{ - build_support_map, evaluate_support, reduce, Assignment, PhragmenScore, StakedAssignment, -}; - -const CTRL_PREFIX: u32 = 1000; -const NOMINATOR_PREFIX: u32 = 1_000_000; - -/// A dummy suer. -pub const USER: u32 = 999_999_999; - -/// Address type of the `T` -pub type AddressOf = Address<::AccountId, u32>; - -/// Random number in the range `[a, b]`. -pub fn random(a: u32, b: u32) -> u32 { - rand::thread_rng().gen_range(a, b) +use sp_io::hashing::blake2_256; +use rand_chacha::{rand_core::{RngCore, SeedableRng}, ChaChaRng}; +use sp_npos_elections::*; + +const SEED: u32 = 0; + +/// Grab a funded user. +pub fn create_funded_user(string: &'static str, n: u32, balance_factor: u32) -> T::AccountId { + let user = account(string, n, SEED); + let balance = T::Currency::minimum_balance() * balance_factor.into(); + T::Currency::make_free_balance_be(&user, balance); + // ensure T::CurrencyToVote will work correctly. + T::Currency::issue(balance); + user } -/// Set the desired validator count, with related storage items. -pub fn set_validator_count(to_elect: u32) { - ValidatorCount::put(to_elect); - MinimumValidatorCount::put(to_elect / 2); - >::put(ElectionStatus::Closed); +/// Create a stash and controller pair. +pub fn create_stash_controller(n: u32, balance_factor: u32) + -> Result<(T::AccountId, T::AccountId), &'static str> +{ + let stash = create_funded_user::("stash", n, balance_factor); + let controller = create_funded_user::("controller", n, balance_factor); + let controller_lookup: ::Source = T::Lookup::unlookup(controller.clone()); + let reward_destination = RewardDestination::Staked; + let amount = T::Currency::minimum_balance() * (balance_factor / 10).max(1).into(); + Staking::::bond(RawOrigin::Signed(stash.clone()).into(), controller_lookup, amount, reward_destination)?; + return Ok((stash, controller)) } -/// Build an account with the given index. -pub fn account(index: u32) -> T::AccountId { - let entropy = (b"benchmark/staking", index).using_encoded(blake2_256); - T::AccountId::decode(&mut &entropy[..]).unwrap_or_default() +/// Create a stash and controller pair, where the controller is dead, and payouts go to controller. +/// This is used to test worst case payout scenarios. +pub fn create_stash_and_dead_controller(n: u32, balance_factor: u32) + -> Result<(T::AccountId, T::AccountId), &'static str> +{ + let stash = create_funded_user::("stash", n, balance_factor); + // controller has no funds + let controller = create_funded_user::("controller", n, 0); + let controller_lookup: ::Source = T::Lookup::unlookup(controller.clone()); + let reward_destination = RewardDestination::Controller; + let amount = T::Currency::minimum_balance() * (balance_factor / 10).max(1).into(); + Staking::::bond(RawOrigin::Signed(stash.clone()).into(), controller_lookup, amount, reward_destination)?; + return Ok((stash, controller)) } -/// Build an address given Index -pub fn address(index: u32) -> AddressOf { - pallet_indices::address::Address::Id(account::(index)) +/// create `max` validators. +pub fn create_validators( + max: u32, + balance_factor: u32, +) -> Result::Source>, &'static str> { + let mut validators: Vec<::Source> = Vec::with_capacity(max as usize); + for i in 0 .. max { + let (stash, controller) = create_stash_controller::(i, balance_factor)?; + let validator_prefs = ValidatorPrefs { + commission: Perbill::from_percent(50), + }; + Staking::::validate(RawOrigin::Signed(controller).into(), validator_prefs)?; + let stash_lookup: ::Source = T::Lookup::unlookup(stash); + validators.push(stash_lookup); + } + Ok(validators) } -/// Generate signed origin from `who`. -pub fn signed(who: T::AccountId) -> T::Origin { - RawOrigin::Signed(who).into() -} +/// This function generates validators and nominators who are randomly nominating +/// `edge_per_nominator` random validators (until `to_nominate` if provided). +/// +/// Parameters: +/// - `validators`: number of bonded validators +/// - `nominators`: number of bonded nominators. +/// - `edge_per_nominator`: number of edge (vote) per nominator. +/// - `randomize_stake`: whether to randomize the stakes. +/// - `to_nominate`: if `Some(n)`, only the first `n` bonded validator are voted upon. +/// Else, all of them are considered and `edge_per_nominator` random validators are voted for. +/// +/// Return the validators choosen to be nominated. +pub fn create_validators_with_nominators_for_era( + validators: u32, + nominators: u32, + edge_per_nominator: usize, + randomize_stake: bool, + to_nominate: Option, +) -> Result::Source>, &'static str> { + let mut validators_stash: Vec<::Source> + = Vec::with_capacity(validators as usize); + let mut rng = ChaChaRng::from_seed(SEED.using_encoded(blake2_256)); + + // Create validators + for i in 0 .. validators { + let balance_factor = if randomize_stake { rng.next_u32() % 255 + 10 } else { 100u32 }; + let (v_stash, v_controller) = create_stash_controller::(i, balance_factor)?; + let validator_prefs = ValidatorPrefs { + commission: Perbill::from_percent(50), + }; + Staking::::validate(RawOrigin::Signed(v_controller.clone()).into(), validator_prefs)?; + let stash_lookup: ::Source = T::Lookup::unlookup(v_stash.clone()); + validators_stash.push(stash_lookup.clone()); + } -/// Generate signed origin from `index`. -pub fn signed_account(index: u32) -> T::Origin { - signed::(account::(index)) -} + let to_nominate = to_nominate.unwrap_or(validators_stash.len() as u32) as usize; + let validator_choosen = validators_stash[0..to_nominate].to_vec(); + + // Create nominators + for j in 0 .. nominators { + let balance_factor = if randomize_stake { rng.next_u32() % 255 + 10 } else { 100u32 }; + let (_n_stash, n_controller) = create_stash_controller::( + u32::max_value() - j, + balance_factor, + )?; + + // Have them randomly validate + let mut available_validators = validator_choosen.clone(); + let mut selected_validators: Vec<::Source> = + Vec::with_capacity(edge_per_nominator); + + for _ in 0 .. validators.min(edge_per_nominator as u32) { + let selected = rng.next_u32() as usize % available_validators.len(); + let validator = available_validators.remove(selected); + selected_validators.push(validator); + } + Staking::::nominate(RawOrigin::Signed(n_controller.clone()).into(), selected_validators)?; + } -/// Bond a validator. -pub fn bond_validator(stash: T::AccountId, ctrl: u32, val: BalanceOf) -where - T::Lookup: StaticLookup>, -{ - let _ = T::Currency::make_free_balance_be(&stash, val); - assert_ok!(>::bond( - signed::(stash), - address::(ctrl), - val, - RewardDestination::Controller - )); - assert_ok!(>::validate( - signed_account::(ctrl), - ValidatorPrefs::default() - )); -} + ValidatorCount::put(validators); -pub fn bond_nominator( - stash: T::AccountId, - ctrl: u32, - val: BalanceOf, - target: Vec>, -) where - T::Lookup: StaticLookup>, -{ - let _ = T::Currency::make_free_balance_be(&stash, val); - assert_ok!(>::bond( - signed::(stash), - address::(ctrl), - val, - RewardDestination::Controller - )); - assert_ok!(>::nominate(signed_account::(ctrl), target)); + Ok(validator_choosen) } -/// Bond `nun_validators` validators and `num_nominator` nominators with `edge_per_voter` random -/// votes per nominator. -pub fn setup_chain_stakers(num_validators: u32, num_voters: u32, edge_per_voter: u32) -where - T::Lookup: StaticLookup>, -{ - (0..num_validators).for_each(|i| { - bond_validator::( - account::(i), - i + CTRL_PREFIX, - >::from(random(1, 1000)) * T::Currency::minimum_balance(), - ); - }); - - (0..num_voters).for_each(|i| { - let mut targets: Vec> = Vec::with_capacity(edge_per_voter as usize); - let mut all_targets = (0..num_validators) - .map(|t| address::(t)) - .collect::>(); - assert!(num_validators >= edge_per_voter); - (0..edge_per_voter).for_each(|_| { - let target = all_targets.remove(random(0, all_targets.len() as u32 - 1) as usize); - targets.push(target); - }); - bond_nominator::( - account::(i + NOMINATOR_PREFIX), - i + NOMINATOR_PREFIX + CTRL_PREFIX, - >::from(random(1, 1000)) * T::Currency::minimum_balance(), - targets, - ); - }); - - >::create_stakers_snapshot(); -} /// Build a _really bad_ but acceptable solution for election. This should always yield a solution /// which has a less score than the seq-phragmen. pub fn get_weak_solution( do_reduce: bool, -) -> (Vec, CompactAssignments, PhragmenScore) { +) -> (Vec, CompactAssignments, ElectionScore, ElectionSize) { let mut backing_stake_of: BTreeMap> = BTreeMap::new(); // self stake @@ -159,68 +162,19 @@ pub fn get_weak_solution( >::slashable_balance_of(&who) }); - // add nominator stuff - >::iter().for_each(|(who, nomination)| { - nomination.targets.into_iter().for_each(|v| { - *backing_stake_of.entry(v).or_insert(Zero::zero()) += - >::slashable_balance_of(&who) - }) - }); - - // elect winners + // elect winners. We chose the.. least backed ones. let mut sorted: Vec = backing_stake_of.keys().cloned().collect(); sorted.sort_by_key(|x| backing_stake_of.get(x).unwrap()); let winners: Vec = sorted .iter() + .rev() .cloned() .take(>::validator_count() as usize) .collect(); let mut staked_assignments: Vec> = Vec::new(); - >::iter().for_each(|(who, nomination)| { - let mut dist: Vec<(T::AccountId, ExtendedBalance)> = Vec::new(); - nomination.targets.into_iter().for_each(|v| { - if winners.iter().find(|&w| *w == v).is_some() { - dist.push((v, ExtendedBalance::zero())); - } - }); - - if dist.len() == 0 { - return; - } - - // assign real stakes. just split the stake. - let stake = , u64>>::convert( - >::slashable_balance_of(&who), - ) as ExtendedBalance; - - let mut sum: ExtendedBalance = Zero::zero(); - let dist_len = dist.len() as ExtendedBalance; - - // assign main portion - // only take the first half into account. This should highly imbalance stuff, which is good. - dist.iter_mut() - .take(if dist_len > 1 { - (dist_len as usize) / 2 - } else { - 1 - }) - .for_each(|(_, w)| { - let partial = stake / dist_len; - *w = partial; - sum += partial; - }); - - // assign the leftover to last. - let leftover = stake - sum; - let last = dist.last_mut().unwrap(); - last.1 += leftover; - - staked_assignments.push(StakedAssignment { - who, - distribution: dist, - }); - }); + // you could at this point start adding some of the nominator's stake, but for now we don't. + // This solution must be bad. // add self support to winners. winners.iter().for_each(|w| { @@ -255,28 +209,22 @@ pub fn get_weak_solution( .position(|x| x == a) .and_then(|i| >::try_into(i).ok()) }; - let stake_of = |who: &T::AccountId| -> ExtendedBalance { + let stake_of = |who: &T::AccountId| -> VoteWeight { , u64>>::convert( >::slashable_balance_of(who), - ) as ExtendedBalance + ) }; // convert back to ratio assignment. This takes less space. - let low_accuracy_assignment: Vec> = - staked_assignments - .into_iter() - .map(|sa| sa.into_assignment(true)) - .collect(); + let low_accuracy_assignment = assignment_staked_to_ratio_normalized(staked_assignments) + .expect("Failed to normalize"); // re-calculate score based on what the chain will decode. let score = { - let staked: Vec> = low_accuracy_assignment - .iter() - .map(|a| { - let stake = stake_of(&a.who); - a.clone().into_staked(stake, true) - }) - .collect(); + let staked = assignment_ratio_to_staked::<_, OffchainAccuracy, _>( + low_accuracy_assignment.clone(), + stake_of + ); let (support_map, _) = build_support_map::(winners.as_slice(), staked.as_slice()); @@ -304,15 +252,20 @@ pub fn get_weak_solution( }) .collect::>(); - (winners, compact, score) + let size = ElectionSize { + validators: snapshot_validators.len() as ValidatorIndex, + nominators: snapshot_nominators.len() as NominatorIndex, + }; + + (winners, compact, score, size) } /// Create a solution for seq-phragmen. This uses the same internal function as used by the offchain /// worker code. pub fn get_seq_phragmen_solution( do_reduce: bool, -) -> (Vec, CompactAssignments, PhragmenScore) { - let sp_phragmen::PhragmenResult { +) -> (Vec, CompactAssignments, ElectionScore, ElectionSize) { + let sp_npos_elections::ElectionResult { winners, assignments, } = >::do_phragmen::().unwrap(); @@ -320,29 +273,40 @@ pub fn get_seq_phragmen_solution( offchain_election::prepare_submission::(assignments, winners, do_reduce).unwrap() } -/// Remove all validator, nominators, votes and exposures. -pub fn clean(era: EraIndex) - where - ::AccountId: codec::EncodeLike, - u32: codec::EncodeLike, -{ - >::iter().for_each(|(k, _)| { - let ctrl = >::bonded(&k).unwrap(); - >::remove(&k); - >::remove(&k); - >::remove(&ctrl); - >::remove(k, era); - }); - >::iter().for_each(|(k, _)| >::remove(k)); - >::remove_all(); - >::remove_all(); - >::kill(); - QueuedScore::kill(); +/// Returns a solution in which only one winner is elected with just a self vote. +pub fn get_single_winner_solution( + winner: T::AccountId +) -> Result<(Vec, CompactAssignments, ElectionScore, ElectionSize), &'static str> { + let snapshot_validators = >::snapshot_validators().unwrap(); + let snapshot_nominators = >::snapshot_nominators().unwrap(); + + let val_index = snapshot_validators.iter().position(|x| *x == winner).ok_or("not a validator")?; + let nom_index = snapshot_nominators.iter().position(|x| *x == winner).ok_or("not a nominator")?; + + let stake = >::slashable_balance_of(&winner); + let stake = , VoteWeight>>::convert(stake) + as ExtendedBalance; + + let val_index = val_index as ValidatorIndex; + let nom_index = nom_index as NominatorIndex; + + let winners = vec![val_index]; + let compact = CompactAssignments { + votes1: vec![(nom_index, val_index)], + ..Default::default() + }; + let score = [stake, stake, stake * stake]; + let size = ElectionSize { + validators: snapshot_validators.len() as ValidatorIndex, + nominators: snapshot_nominators.len() as NominatorIndex, + }; + + Ok((winners, compact, score, size)) } /// get the active era. -pub fn active_era() -> EraIndex { - >::active_era().unwrap().index +pub fn current_era() -> EraIndex { + >::current_era().unwrap_or(0) } /// initialize the first era. @@ -352,3 +316,33 @@ pub fn init_active_era() { start: None, }) } + +/// Create random assignments for the given list of winners. Each assignment will have +/// MAX_NOMINATIONS edges. +pub fn create_assignments_for_offchain( + num_assignments: u32, + winners: Vec<::Source>, +) -> Result< + ( + Vec<(T::AccountId, ExtendedBalance)>, + Vec>, + ), + &'static str +> { + let ratio = OffchainAccuracy::from_rational_approximation(1, MAX_NOMINATIONS); + let assignments: Vec> = >::iter() + .take(num_assignments as usize) + .map(|(n, t)| Assignment { + who: n, + distribution: t.targets.iter().map(|v| (v.clone(), ratio)).collect(), + }) + .collect(); + + ensure!(assignments.len() == num_assignments as usize, "must bench for `a` assignments"); + + let winners = winners.into_iter().map(|v| { + (::lookup(v).unwrap(), 0) + }).collect(); + + Ok((winners, assignments)) +} diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 80ffc4b7bf78ad052dc5c9158b15d130fa772030..a3cfed9e2f260e105948a0245dadc9bed3e175a5 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -47,7 +47,7 @@ fn force_unstake_works() { // Force unstake needs correct number of slashing spans (for weight calculation) assert_noop!(Staking::force_unstake(Origin::signed(11), 11, 0), BadOrigin); // We now force them to unstake - assert_ok!(Staking::force_unstake(Origin::ROOT, 11, 2)); + assert_ok!(Staking::force_unstake(Origin::root(), 11, 2)); // No longer bonded. assert_eq!(Staking::bonded(&11), None); // Transfer works. @@ -337,7 +337,7 @@ fn staking_should_work() { claimed_rewards: vec![0], }) ); - // e.g. it cannot spend more than 500 that it has free from the total 2000 + // e.g. it cannot reserve more than 500 that it has free from the total 2000 assert_noop!( Balances::reserve(&3, 501), BalancesError::::LiquidityRestrictions @@ -406,41 +406,6 @@ fn no_candidate_emergency_condition() { #[test] fn nominating_and_rewards_should_work() { - // PHRAGMEN OUTPUT: running this test with the reference impl gives: - // - // Sequential Phragmén gives - // 10 is elected with stake 2200.0 and score 0.0003333333333333333 - // 20 is elected with stake 1800.0 and score 0.0005555555555555556 - - // 10 has load 0.0003333333333333333 and supported - // 10 with stake 1000.0 - // 20 has load 0.0005555555555555556 and supported - // 20 with stake 1000.0 - // 30 has load 0 and supported - // 30 with stake 0 - // 40 has load 0 and supported - // 40 with stake 0 - // 2 has load 0.0005555555555555556 and supported - // 10 with stake 600.0 20 with stake 400.0 30 with stake 0.0 - // 4 has load 0.0005555555555555556 and supported - // 10 with stake 600.0 20 with stake 400.0 40 with stake 0.0 - - // Sequential Phragmén with post processing gives - // 10 is elected with stake 2000.0 and score 0.0003333333333333333 - // 20 is elected with stake 2000.0 and score 0.0005555555555555556 - - // 10 has load 0.0003333333333333333 and supported - // 10 with stake 1000.0 - // 20 has load 0.0005555555555555556 and supported - // 20 with stake 1000.0 - // 30 has load 0 and supported - // 30 with stake 0 - // 40 has load 0 and supported - // 40 with stake 0 - // 2 has load 0.0005555555555555556 and supported - // 10 with stake 400.0 20 with stake 600.0 30 with stake 0 - // 4 has load 0.0005555555555555556 and supported - // 10 with stake 600.0 20 with stake 400.0 40 with stake 0.0 ExtBuilder::default() .nominate(false) .validator_pool(true) @@ -477,7 +442,7 @@ fn nominating_and_rewards_should_work() { mock::start_era(1); - // 10 and 20 have more votes, they will be chosen by phragmen. + // 10 and 20 have more votes, they will be chosen. assert_eq_uvec!(validator_controllers(), vec![20, 10]); // OLD validators must have already received some rewards. @@ -818,10 +783,10 @@ fn cannot_reserve_staked_balance() { assert_eq!(Balances::free_balance(11), 1000); // Confirm account 11 (via controller 10) is totally staked assert_eq!(Staking::eras_stakers(Staking::active_era().unwrap().index, 11).own, 1000); - // Confirm account 11 cannot transfer as a result + // Confirm account 11 cannot reserve as a result assert_noop!( Balances::reserve(&11, 1), - BalancesError::::LiquidityRestrictions + BalancesError::::LiquidityRestrictions, ); // Give account 11 extra free balance @@ -1512,7 +1477,7 @@ fn on_free_balance_zero_stash_removes_validator() { assert_eq!(Balances::total_balance(&11), 0); // Reap the stash - assert_ok!(Staking::reap_stash(Origin::NONE, 11, 0)); + assert_ok!(Staking::reap_stash(Origin::none(), 11, 0)); // Check storage items do not exist assert!(!>::contains_key(&10)); @@ -1568,7 +1533,7 @@ fn on_free_balance_zero_stash_removes_nominator() { assert_eq!(Balances::total_balance(&11), 0); // Reap the stash - assert_ok!(Staking::reap_stash(Origin::NONE, 11, 0)); + assert_ok!(Staking::reap_stash(Origin::none(), 11, 0)); // Check storage items do not exist assert!(!>::contains_key(&10)); @@ -1963,7 +1928,7 @@ fn offence_forces_new_era() { #[test] fn offence_ensures_new_era_without_clobbering() { ExtBuilder::default().build_and_execute(|| { - assert_ok!(Staking::force_new_era_always(Origin::ROOT)); + assert_ok!(Staking::force_new_era_always(Origin::root())); assert_eq!(Staking::force_era(), Forcing::ForceAlways); on_offence_now( @@ -2337,8 +2302,8 @@ fn garbage_collection_after_slashing() { assert_eq!(slashing_spans.iter().count(), 2); // reap_stash respects num_slashing_spans so that weight is accurate - assert_noop!(Staking::reap_stash(Origin::NONE, 11, 0), Error::::IncorrectSlashingSpans); - assert_ok!(Staking::reap_stash(Origin::NONE, 11, 2)); + assert_noop!(Staking::reap_stash(Origin::none(), 11, 0), Error::::IncorrectSlashingSpans); + assert_ok!(Staking::reap_stash(Origin::none(), 11, 2)); assert!(::SlashingSpans::get(&11).is_none()); assert_eq!(::SpanSlash::get(&(11, 0)).amount_slashed(), &0); @@ -2626,11 +2591,11 @@ fn remove_deferred() { // fails if empty assert_noop!( - Staking::cancel_deferred_slash(Origin::ROOT, 1, vec![]), + Staking::cancel_deferred_slash(Origin::root(), 1, vec![]), Error::::EmptyTargets ); - assert_ok!(Staking::cancel_deferred_slash(Origin::ROOT, 1, vec![0])); + assert_ok!(Staking::cancel_deferred_slash(Origin::root(), 1, vec![0])); assert_eq!(Balances::free_balance(11), 1000); assert_eq!(Balances::free_balance(101), 2000); @@ -2727,21 +2692,21 @@ fn remove_multi_deferred() { // fails if list is not sorted assert_noop!( - Staking::cancel_deferred_slash(Origin::ROOT, 1, vec![2, 0, 4]), + Staking::cancel_deferred_slash(Origin::root(), 1, vec![2, 0, 4]), Error::::NotSortedAndUnique ); // fails if list is not unique assert_noop!( - Staking::cancel_deferred_slash(Origin::ROOT, 1, vec![0, 2, 2]), + Staking::cancel_deferred_slash(Origin::root(), 1, vec![0, 2, 2]), Error::::NotSortedAndUnique ); // fails if bad index assert_noop!( - Staking::cancel_deferred_slash(Origin::ROOT, 1, vec![1, 2, 3, 4, 5]), + Staking::cancel_deferred_slash(Origin::root(), 1, vec![1, 2, 3, 4, 5]), Error::::InvalidSlashIndex ); - assert_ok!(Staking::cancel_deferred_slash(Origin::ROOT, 1, vec![0, 2, 4])); + assert_ok!(Staking::cancel_deferred_slash(Origin::root(), 1, vec![0, 2, 4])); let slashes = ::UnappliedSlashes::get(&1); assert_eq!(slashes.len(), 2); @@ -2753,7 +2718,10 @@ fn remove_multi_deferred() { mod offchain_phragmen { use crate::*; use codec::Encode; - use frame_support::{assert_noop, assert_ok}; + use frame_support::{ + assert_noop, assert_ok, assert_err_with_weight, + dispatch::DispatchResultWithPostInfo, + }; use sp_runtime::transaction_validity::TransactionSource; use mock::*; use parking_lot::RwLock; @@ -2762,7 +2730,7 @@ mod offchain_phragmen { OffchainExt, TransactionPoolExt, }; use sp_io::TestExternalities; - use sp_phragmen::StakedAssignment; + use sp_npos_elections::StakedAssignment; use frame_support::traits::OffchainWorker; use std::sync::Arc; use substrate_test_utils::assert_eq_uvec; @@ -2808,6 +2776,29 @@ mod offchain_phragmen { pool_state } + fn election_size() -> ElectionSize { + ElectionSize { + validators: Staking::snapshot_validators().unwrap().len() as ValidatorIndex, + nominators: Staking::snapshot_nominators().unwrap().len() as NominatorIndex, + } + } + + fn submit_solution( + origin: Origin, + winners: Vec, + compact: CompactAssignments, + score: ElectionScore, + ) -> DispatchResultWithPostInfo { + Staking::submit_election_solution( + origin, + winners, + compact, + score, + current_era(), + election_size(), + ) + } + #[test] fn is_current_session_final_works() { ExtBuilder::default() @@ -2833,7 +2824,7 @@ mod offchain_phragmen { } #[test] - fn offchain_election_flag_is_triggered() { + fn offchain_window_is_triggered() { ExtBuilder::default() .session_per_era(5) .session_length(10) @@ -2893,16 +2884,13 @@ mod offchain_phragmen { } #[test] - fn offchain_election_flag_is_triggered_when_forcing() { + fn offchain_window_is_triggered_when_forcing() { ExtBuilder::default() .session_per_era(5) .session_length(10) .election_lookahead(3) .build() .execute_with(|| { - run_to_block(7); - assert_session_era!(0, 0); - run_to_block(12); ForceEra::put(Forcing::ForceNew); run_to_block(13); @@ -2910,11 +2898,90 @@ mod offchain_phragmen { run_to_block(17); // instead of 47 assert_eq!(Staking::era_election_status(), ElectionStatus::Open(17)); + + run_to_block(20); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + }) + } + + #[test] + fn offchain_window_is_triggered_when_force_always() { + ExtBuilder::default() + .session_per_era(5) + .session_length(10) + .election_lookahead(3) + .build() + .execute_with(|| { + + ForceEra::put(Forcing::ForceAlways); + run_to_block(16); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + + run_to_block(17); // instead of 37 + assert_eq!(Staking::era_election_status(), ElectionStatus::Open(17)); + + run_to_block(20); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + + run_to_block(26); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + + run_to_block(27); // next one again + assert_eq!(Staking::era_election_status(), ElectionStatus::Open(27)); + }) + } + + #[test] + fn offchain_window_closes_when_forcenone() { + ExtBuilder::default() + .session_per_era(5) + .session_length(10) + .election_lookahead(3) + .build() + .execute_with(|| { + ForceEra::put(Forcing::ForceNone); + + run_to_block(36); + assert_session_era!(3, 0); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + + // opens + run_to_block(37); + assert_eq!(Staking::era_election_status(), ElectionStatus::Open(37)); + assert!(Staking::is_current_session_final()); + assert!(Staking::snapshot_validators().is_some()); + + // closes normally + run_to_block(40); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + assert!(!Staking::is_current_session_final()); + assert!(Staking::snapshot_validators().is_none()); + assert_session_era!(4, 0); + + run_to_block(47); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + assert_session_era!(4, 0); + + run_to_block(57); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + assert_session_era!(5, 0); + + run_to_block(67); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + + // Will not open again as scheduled + run_to_block(87); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + assert_session_era!(8, 0); + + run_to_block(90); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + assert_session_era!(9, 0); }) } #[test] - fn election_on_chain_fallback_works() { + fn offchain_window_on_chain_fallback_works() { ExtBuilder::default().build_and_execute(|| { start_session(1); start_session(2); @@ -2996,16 +3063,30 @@ mod offchain_phragmen { assert!(Staking::snapshot_validators().is_some()); let (compact, winners, score) = prepare_submission_with(true, 2, |_| {}); - assert_ok!(Staking::submit_election_solution( + assert_ok!(submit_solution( Origin::signed(10), winners, compact, score, - current_era(), )); let queued_result = Staking::queued_elected().unwrap(); assert_eq!(queued_result.compute, ElectionCompute::Signed); + assert_eq!( + System::events() + .into_iter() + .map(|r| r.event) + .filter_map(|e| { + if let MetaEvent::staking(inner) = e { + Some(inner) + } else { + None + } + }) + .last() + .unwrap(), + RawEvent::SolutionStored(ElectionCompute::Signed), + ); run_to_block(15); assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); @@ -3039,13 +3120,7 @@ mod offchain_phragmen { assert_eq!(Staking::era_election_status(), ElectionStatus::Open(12)); let (compact, winners, score) = prepare_submission_with(true, 2, |_| {}); - assert_ok!(Staking::submit_election_solution( - Origin::signed(10), - winners, - compact, - score, - current_era(), - )); + assert_ok!(submit_solution(Origin::signed(10), winners, compact, score)); let queued_result = Staking::queued_elected().unwrap(); assert_eq!(queued_result.compute, ElectionCompute::Signed); @@ -3088,15 +3163,17 @@ mod offchain_phragmen { let (compact, winners, score) = prepare_submission_with(true, 2, |_| {}); Staking::kill_stakers_snapshot(); - assert_noop!( + assert_err_with_weight!( Staking::submit_election_solution( Origin::signed(10), - winners, - compact, + winners.clone(), + compact.clone(), score, current_era(), + ElectionSize::default(), ), Error::::PhragmenEarlySubmission, + Some(::DbWeight::get().reads(1)), ); }) } @@ -3115,25 +3192,24 @@ mod offchain_phragmen { // a good solution let (compact, winners, score) = prepare_submission_with(true, 2, |_| {}); - assert_ok!(Staking::submit_election_solution( + assert_ok!(submit_solution( Origin::signed(10), winners, compact, score, - current_era(), )); // a bad solution let (compact, winners, score) = horrible_phragmen_with_post_processing(false); - assert_noop!( - Staking::submit_election_solution( + assert_err_with_weight!( + submit_solution( Origin::signed(10), - winners, - compact, + winners.clone(), + compact.clone(), score, - current_era(), ), Error::::PhragmenWeakSubmission, + Some(::DbWeight::get().reads(3)) ); }) } @@ -3152,22 +3228,20 @@ mod offchain_phragmen { // a meeeeh solution let (compact, winners, score) = horrible_phragmen_with_post_processing(false); - assert_ok!(Staking::submit_election_solution( + assert_ok!(submit_solution( Origin::signed(10), winners, compact, score, - current_era(), )); // a better solution let (compact, winners, score) = prepare_submission_with(true, 2, |_| {}); - assert_ok!(Staking::submit_election_solution( + assert_ok!(submit_solution( Origin::signed(10), winners, compact, score, - current_era(), )); }) } @@ -3246,7 +3320,7 @@ mod offchain_phragmen { &inner, ), TransactionValidity::Ok(ValidTransaction { - // the proposed slot stake, with equalize. + // the proposed slot stake, with balance_solution. priority: UnsignedPriority::get() + 1250, requires: vec![], provides: vec![("StakingOffchain", active_era()).encode()], @@ -3268,12 +3342,11 @@ mod offchain_phragmen { run_to_block(12); // put a good solution on-chain let (compact, winners, score) = prepare_submission_with(true, 2, |_| {}); - assert_ok!(Staking::submit_election_solution( + assert_ok!(submit_solution( Origin::signed(10), winners, compact, score, - current_era(), ),); // now run the offchain worker in the same chain state. @@ -3318,6 +3391,28 @@ mod offchain_phragmen { assert_eq!(winners.len(), 3); + assert_noop!( + submit_solution( + Origin::signed(10), + winners, + compact, + score, + ), + Error::::PhragmenBogusWinnerCount, + ); + }) + } + + #[test] + fn invalid_phragmen_result_solution_size() { + ExtBuilder::default() + .offchain_phragmen_ext() + .build() + .execute_with(|| { + run_to_block(12); + + let (compact, winners, score) = prepare_submission_with(true, 2, |_| {}); + assert_noop!( Staking::submit_election_solution( Origin::signed(10), @@ -3325,8 +3420,9 @@ mod offchain_phragmen { compact, score, current_era(), + ElectionSize::default(), ), - Error::::PhragmenBogusWinnerCount, + Error::::PhragmenBogusElectionSize, ); }) } @@ -3350,12 +3446,11 @@ mod offchain_phragmen { assert_eq!(winners.len(), 3); assert_noop!( - Staking::submit_election_solution( + submit_solution( Origin::signed(10), winners, compact, score, - current_era(), ), Error::::PhragmenBogusWinnerCount, ); @@ -3379,12 +3474,11 @@ mod offchain_phragmen { assert_eq!(winners.len(), 4); // all good. We chose 4 and it works. - assert_ok!(Staking::submit_election_solution( + assert_ok!(submit_solution( Origin::signed(10), winners, compact, score, - current_era(), ),); }) } @@ -3410,12 +3504,11 @@ mod offchain_phragmen { // The error type sadly cannot be more specific now. assert_noop!( - Staking::submit_election_solution( + submit_solution( Origin::signed(10), winners, compact, score, - current_era(), ), Error::::PhragmenBogusCompact, ); @@ -3443,12 +3536,11 @@ mod offchain_phragmen { // The error type sadly cannot be more specific now. assert_noop!( - Staking::submit_election_solution( + submit_solution( Origin::signed(10), winners, compact, score, - current_era(), ), Error::::PhragmenBogusCompact, ); @@ -3475,12 +3567,11 @@ mod offchain_phragmen { let winners = vec![0, 1, 2, 4]; assert_noop!( - Staking::submit_election_solution( + submit_solution( Origin::signed(10), winners, compact, score, - current_era(), ), Error::::PhragmenBogusWinner, ); @@ -3511,12 +3602,11 @@ mod offchain_phragmen { }); assert_noop!( - Staking::submit_election_solution( + submit_solution( Origin::signed(10), winners, compact, score, - current_era(), ), Error::::PhragmenBogusEdge, ); @@ -3547,12 +3637,11 @@ mod offchain_phragmen { }); assert_noop!( - Staking::submit_election_solution( + submit_solution( Origin::signed(10), winners, compact, score, - current_era(), ), Error::::PhragmenBogusSelfVote, ); @@ -3583,12 +3672,11 @@ mod offchain_phragmen { // This raises score issue. assert_noop!( - Staking::submit_election_solution( + submit_solution( Origin::signed(10), winners, compact, score, - current_era(), ), Error::::PhragmenBogusSelfVote, ); @@ -3618,12 +3706,11 @@ mod offchain_phragmen { } assert_noop!( - Staking::submit_election_solution( + submit_solution( Origin::signed(10), winners, compact, score, - current_era(), ), Error::::PhragmenBogusCompact, ); @@ -3660,12 +3747,11 @@ mod offchain_phragmen { }); assert_noop!( - Staking::submit_election_solution( + submit_solution( Origin::signed(10), winners, compact, score, - current_era(), ), Error::::PhragmenBogusNomination, ); @@ -3723,12 +3809,11 @@ mod offchain_phragmen { }); // can be submitted. - assert_ok!(Staking::submit_election_solution( + assert_ok!(submit_solution( Origin::signed(10), winners, compact, score, - current_era(), )); // a wrong solution. @@ -3742,12 +3827,11 @@ mod offchain_phragmen { // is rejected. assert_noop!( - Staking::submit_election_solution( + submit_solution( Origin::signed(10), winners, compact, score, - current_era(), ), Error::::PhragmenSlashedNomination, ); @@ -3770,12 +3854,11 @@ mod offchain_phragmen { score[0] += 1; assert_noop!( - Staking::submit_election_solution( + submit_solution( Origin::signed(10), winners, compact, score, - current_era(), ), Error::::PhragmenBogusScore, ); @@ -4160,16 +4243,16 @@ fn test_max_nominator_rewarded_per_validator_and_cant_steal_someone_else_reward( fn set_history_depth_works() { ExtBuilder::default().build_and_execute(|| { mock::start_era(10); - Staking::set_history_depth(Origin::ROOT, 20, 0).unwrap(); + Staking::set_history_depth(Origin::root(), 20, 0).unwrap(); assert!(::ErasTotalStake::contains_key(10 - 4)); assert!(::ErasTotalStake::contains_key(10 - 5)); - Staking::set_history_depth(Origin::ROOT, 4, 0).unwrap(); + Staking::set_history_depth(Origin::root(), 4, 0).unwrap(); assert!(::ErasTotalStake::contains_key(10 - 4)); assert!(!::ErasTotalStake::contains_key(10 - 5)); - Staking::set_history_depth(Origin::ROOT, 3, 0).unwrap(); + Staking::set_history_depth(Origin::root(), 3, 0).unwrap(); assert!(!::ErasTotalStake::contains_key(10 - 4)); assert!(!::ErasTotalStake::contains_key(10 - 5)); - Staking::set_history_depth(Origin::ROOT, 8, 0).unwrap(); + Staking::set_history_depth(Origin::root(), 8, 0).unwrap(); assert!(!::ErasTotalStake::contains_key(10 - 4)); assert!(!::ErasTotalStake::contains_key(10 - 5)); }); @@ -4343,327 +4426,46 @@ fn bond_during_era_correctly_populates_claimed_rewards() { }); } -/* These migration tests below can be removed once migration code is removed */ - -#[test] -fn rewards_should_work_before_migration() { - // should check that before migration: - // * rewards get recorded per session - // * rewards get paid per Era - // * Check that nominators are also rewarded - ExtBuilder::default().nominate(true).build_and_execute(|| { - MigrateEra::put(10); - let init_balance_10 = Balances::total_balance(&10); - let init_balance_11 = Balances::total_balance(&11); - let init_balance_20 = Balances::total_balance(&20); - let init_balance_21 = Balances::total_balance(&21); - let init_balance_100 = Balances::total_balance(&100); - let init_balance_101 = Balances::total_balance(&101); - - // Check state - Payee::::insert(11, RewardDestination::Controller); - Payee::::insert(21, RewardDestination::Controller); - Payee::::insert(101, RewardDestination::Controller); - - >::reward_by_ids(vec![(11, 50)]); - >::reward_by_ids(vec![(11, 50)]); - // This is the second validator of the current elected set. - >::reward_by_ids(vec![(21, 50)]); - - // Compute total payout now for whole duration as other parameter won't change - let total_payout_0 = current_total_payout_for_duration(3 * 1000); - assert!(total_payout_0 > 10); // Test is meaningful if reward something - - start_session(1); - - assert_eq!(Balances::total_balance(&10), init_balance_10); - assert_eq!(Balances::total_balance(&11), init_balance_11); - assert_eq!(Balances::total_balance(&20), init_balance_20); - assert_eq!(Balances::total_balance(&21), init_balance_21); - assert_eq!(Balances::total_balance(&100), init_balance_100); - assert_eq!(Balances::total_balance(&101), init_balance_101); - assert_eq_uvec!(Session::validators(), vec![11, 21]); - assert_eq!(Staking::eras_reward_points(Staking::active_era().unwrap().index), EraRewardPoints { - total: 50*3, - individual: vec![(11, 100), (21, 50)].into_iter().collect(), - }); - let part_for_10 = Perbill::from_rational_approximation::(1000, 1125); - let part_for_20 = Perbill::from_rational_approximation::(1000, 1375); - let part_for_100_from_10 = Perbill::from_rational_approximation::(125, 1125); - let part_for_100_from_20 = Perbill::from_rational_approximation::(375, 1375); - - start_session(2); - start_session(3); - - assert_eq!(Staking::active_era().unwrap().index, 1); - mock::make_all_reward_payment_before_migration(0); - - assert_eq_error_rate!(Balances::total_balance(&10), init_balance_10 + part_for_10 * total_payout_0*2/3, 2); - assert_eq_error_rate!(Balances::total_balance(&11), init_balance_11, 2); - assert_eq_error_rate!(Balances::total_balance(&20), init_balance_20 + part_for_20 * total_payout_0*1/3, 2); - assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); - assert_eq_error_rate!( - Balances::total_balance(&100), - init_balance_100 - + part_for_100_from_10 * total_payout_0 * 2/3 - + part_for_100_from_20 * total_payout_0 * 1/3, - 2 - ); - assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); - - assert_eq_uvec!(Session::validators(), vec![11, 21]); - >::reward_by_ids(vec![(11, 1)]); - - // Compute total payout now for whole duration as other parameter won't change - let total_payout_1 = current_total_payout_for_duration(3 * 1000); - assert!(total_payout_1 > 10); // Test is meaningful if reward something - - mock::start_era(2); - mock::make_all_reward_payment_before_migration(1); - - assert_eq_error_rate!(Balances::total_balance(&10), init_balance_10 + part_for_10 * (total_payout_0 * 2/3 + total_payout_1), 2); - assert_eq_error_rate!(Balances::total_balance(&11), init_balance_11, 2); - assert_eq_error_rate!(Balances::total_balance(&20), init_balance_20 + part_for_20 * total_payout_0 * 1/3, 2); - assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); - assert_eq_error_rate!( - Balances::total_balance(&100), - init_balance_100 - + part_for_100_from_10 * (total_payout_0 * 2/3 + total_payout_1) - + part_for_100_from_20 * total_payout_0 * 1/3, - 2 - ); - assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); - }); -} - -#[test] -fn migrate_era_should_work() { - // should check that before and after migration: - // * rewards get recorded per session - // * rewards get paid per Era - // * Check that nominators are also rewarded - ExtBuilder::default().nominate(true).build_and_execute(|| { - MigrateEra::put(1); - let init_balance_10 = Balances::total_balance(&10); - let init_balance_11 = Balances::total_balance(&11); - let init_balance_20 = Balances::total_balance(&20); - let init_balance_21 = Balances::total_balance(&21); - let init_balance_100 = Balances::total_balance(&100); - let init_balance_101 = Balances::total_balance(&101); - - // Check state - Payee::::insert(11, RewardDestination::Controller); - Payee::::insert(21, RewardDestination::Controller); - Payee::::insert(101, RewardDestination::Controller); - - >::reward_by_ids(vec![(11, 50)]); - >::reward_by_ids(vec![(11, 50)]); - // This is the second validator of the current elected set. - >::reward_by_ids(vec![(21, 50)]); - - // Compute total payout now for whole duration as other parameter won't change - let total_payout_0 = current_total_payout_for_duration(3 * 1000); - assert!(total_payout_0 > 10); // Test is meaningful if reward something - - start_session(1); - - assert_eq!(Balances::total_balance(&10), init_balance_10); - assert_eq!(Balances::total_balance(&11), init_balance_11); - assert_eq!(Balances::total_balance(&20), init_balance_20); - assert_eq!(Balances::total_balance(&21), init_balance_21); - assert_eq!(Balances::total_balance(&100), init_balance_100); - assert_eq!(Balances::total_balance(&101), init_balance_101); - assert_eq_uvec!(Session::validators(), vec![11, 21]); - assert_eq!(Staking::eras_reward_points(Staking::active_era().unwrap().index), EraRewardPoints { - total: 50*3, - individual: vec![(11, 100), (21, 50)].into_iter().collect(), - }); - let part_for_10 = Perbill::from_rational_approximation::(1000, 1125); - let part_for_20 = Perbill::from_rational_approximation::(1000, 1375); - let part_for_100_from_10 = Perbill::from_rational_approximation::(125, 1125); - let part_for_100_from_20 = Perbill::from_rational_approximation::(375, 1375); - - start_session(2); - start_session(3); - - assert_eq!(Staking::active_era().unwrap().index, 1); - mock::make_all_reward_payment_before_migration(0); - - assert_eq_error_rate!(Balances::total_balance(&10), init_balance_10 + part_for_10 * total_payout_0*2/3, 2); - assert_eq_error_rate!(Balances::total_balance(&11), init_balance_11, 2); - assert_eq_error_rate!(Balances::total_balance(&20), init_balance_20 + part_for_20 * total_payout_0*1/3, 2); - assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); - assert_eq_error_rate!( - Balances::total_balance(&100), - init_balance_100 - + part_for_100_from_10 * total_payout_0 * 2/3 - + part_for_100_from_20 * total_payout_0 * 1/3, - 2 - ); - assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); - - assert_eq_uvec!(Session::validators(), vec![11, 21]); - >::reward_by_ids(vec![(11, 1)]); - - // Compute total payout now for whole duration as other parameter won't change - let total_payout_1 = current_total_payout_for_duration(3 * 1000); - assert!(total_payout_1 > 10); // Test is meaningful if reward something - - mock::start_era(2); - mock::make_all_reward_payment(1); - - assert_eq_error_rate!(Balances::total_balance(&10), init_balance_10 + part_for_10 * (total_payout_0 * 2/3 + total_payout_1), 2); - assert_eq_error_rate!(Balances::total_balance(&11), init_balance_11, 2); - assert_eq_error_rate!(Balances::total_balance(&20), init_balance_20 + part_for_20 * total_payout_0 * 1/3, 2); - assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); - assert_eq_error_rate!( - Balances::total_balance(&100), - init_balance_100 - + part_for_100_from_10 * (total_payout_0 * 2/3 + total_payout_1) - + part_for_100_from_20 * total_payout_0 * 1/3, - 2 - ); - assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); - }); -} - -#[test] -#[should_panic] -fn migrate_era_should_handle_error() { - ExtBuilder::default().nominate(true).build_and_execute(|| { - MigrateEra::put(1); - let init_balance_10 = Balances::total_balance(&10); - let init_balance_11 = Balances::total_balance(&11); - let init_balance_20 = Balances::total_balance(&20); - let init_balance_21 = Balances::total_balance(&21); - let init_balance_100 = Balances::total_balance(&100); - let init_balance_101 = Balances::total_balance(&101); - - // Check state - Payee::::insert(11, RewardDestination::Controller); - Payee::::insert(21, RewardDestination::Controller); - Payee::::insert(101, RewardDestination::Controller); - - >::reward_by_ids(vec![(11, 50)]); - >::reward_by_ids(vec![(11, 50)]); - // This is the second validator of the current elected set. - >::reward_by_ids(vec![(21, 50)]); - - // Compute total payout now for whole duration as other parameter won't change - let total_payout_0 = current_total_payout_for_duration(3 * 1000); - assert!(total_payout_0 > 10); // Test is meaningful if reward something - - start_session(1); - - assert_eq!(Balances::total_balance(&10), init_balance_10); - assert_eq!(Balances::total_balance(&11), init_balance_11); - assert_eq!(Balances::total_balance(&20), init_balance_20); - assert_eq!(Balances::total_balance(&21), init_balance_21); - assert_eq!(Balances::total_balance(&100), init_balance_100); - assert_eq!(Balances::total_balance(&101), init_balance_101); - assert_eq_uvec!(Session::validators(), vec![11, 21]); - assert_eq!(Staking::eras_reward_points(Staking::active_era().unwrap().index), EraRewardPoints { - total: 50*3, - individual: vec![(11, 100), (21, 50)].into_iter().collect(), - }); - - start_session(2); - start_session(3); - - assert_eq!(Staking::active_era().unwrap().index, 1); - mock::make_all_reward_payment(0); - }); -} - #[test] -#[should_panic] -fn migrate_era_should_handle_errors_2() { - // should check that before and after migration: - // * rewards get recorded per session - // * rewards get paid per Era - // * Check that nominators are also rewarded +fn offences_weight_calculated_correctly() { ExtBuilder::default().nominate(true).build_and_execute(|| { - MigrateEra::put(1); - let init_balance_10 = Balances::total_balance(&10); - let init_balance_11 = Balances::total_balance(&11); - let init_balance_20 = Balances::total_balance(&20); - let init_balance_21 = Balances::total_balance(&21); - let init_balance_100 = Balances::total_balance(&100); - let init_balance_101 = Balances::total_balance(&101); - - // Check state - Payee::::insert(11, RewardDestination::Controller); - Payee::::insert(21, RewardDestination::Controller); - Payee::::insert(101, RewardDestination::Controller); - - >::reward_by_ids(vec![(11, 50)]); - >::reward_by_ids(vec![(11, 50)]); - // This is the second validator of the current elected set. - >::reward_by_ids(vec![(21, 50)]); - - // Compute total payout now for whole duration as other parameter won't change - let total_payout_0 = current_total_payout_for_duration(3 * 1000); - assert!(total_payout_0 > 10); // Test is meaningful if reward something - - start_session(1); - - assert_eq!(Balances::total_balance(&10), init_balance_10); - assert_eq!(Balances::total_balance(&11), init_balance_11); - assert_eq!(Balances::total_balance(&20), init_balance_20); - assert_eq!(Balances::total_balance(&21), init_balance_21); - assert_eq!(Balances::total_balance(&100), init_balance_100); - assert_eq!(Balances::total_balance(&101), init_balance_101); - assert_eq_uvec!(Session::validators(), vec![11, 21]); - assert_eq!(Staking::eras_reward_points(Staking::active_era().unwrap().index), EraRewardPoints { - total: 50*3, - individual: vec![(11, 100), (21, 50)].into_iter().collect(), - }); - let part_for_10 = Perbill::from_rational_approximation::(1000, 1125); - let part_for_20 = Perbill::from_rational_approximation::(1000, 1375); - let part_for_100_from_10 = Perbill::from_rational_approximation::(125, 1125); - let part_for_100_from_20 = Perbill::from_rational_approximation::(375, 1375); - - start_session(2); - start_session(3); - - assert_eq!(Staking::active_era().unwrap().index, 1); - mock::make_all_reward_payment_before_migration(0); - - assert_eq_error_rate!(Balances::total_balance(&10), init_balance_10 + part_for_10 * total_payout_0*2/3, 2); - assert_eq_error_rate!(Balances::total_balance(&11), init_balance_11, 2); - assert_eq_error_rate!(Balances::total_balance(&20), init_balance_20 + part_for_20 * total_payout_0*1/3, 2); - assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); - assert_eq_error_rate!( - Balances::total_balance(&100), - init_balance_100 - + part_for_100_from_10 * total_payout_0 * 2/3 - + part_for_100_from_20 * total_payout_0 * 1/3, - 2 - ); - assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); + // On offence with zero offenders: 4 Reads, 1 Write + let zero_offence_weight = ::DbWeight::get().reads_writes(4, 1); + assert_eq!(Staking::on_offence(&[], &[Perbill::from_percent(50)], 0), Ok(zero_offence_weight)); - assert_eq_uvec!(Session::validators(), vec![11, 21]); - >::reward_by_ids(vec![(11, 1)]); + // On Offence with N offenders, Unapplied: 4 Reads, 1 Write + 4 Reads, 5 Writes + let n_offence_unapplied_weight = ::DbWeight::get().reads_writes(4, 1) + + ::DbWeight::get().reads_writes(4, 5); - // Compute total payout now for whole duration as other parameter won't change - let total_payout_1 = current_total_payout_for_duration(3 * 1000); - assert!(total_payout_1 > 10); // Test is meaningful if reward something + let offenders: Vec::AccountId, pallet_session::historical::IdentificationTuple>> + = (1..10).map(|i| + OffenceDetails { + offender: (i, Staking::eras_stakers(Staking::active_era().unwrap().index, i)), + reporters: vec![], + } + ).collect(); + assert_eq!(Staking::on_offence(&offenders, &[Perbill::from_percent(50)], 0), Ok(n_offence_unapplied_weight)); - mock::start_era(2); - mock::make_all_reward_payment_before_migration(1); + // On Offence with one offenders, Applied + let one_offender = [ + OffenceDetails { + offender: (11, Staking::eras_stakers(Staking::active_era().unwrap().index, 11)), + reporters: vec![1], + }, + ]; - assert_eq_error_rate!(Balances::total_balance(&10), init_balance_10 + part_for_10 * (total_payout_0 * 2/3 + total_payout_1), 2); - assert_eq_error_rate!(Balances::total_balance(&11), init_balance_11, 2); - assert_eq_error_rate!(Balances::total_balance(&20), init_balance_20 + part_for_20 * total_payout_0 * 1/3, 2); - assert_eq_error_rate!(Balances::total_balance(&21), init_balance_21, 2); - assert_eq_error_rate!( - Balances::total_balance(&100), - init_balance_100 - + part_for_100_from_10 * (total_payout_0 * 2/3 + total_payout_1) - + part_for_100_from_20 * total_payout_0 * 1/3, - 2 - ); - assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); + let n = 1; // Number of offenders + let rw = 3 + 3 * n; // rw reads and writes + let one_offence_unapplied_weight = ::DbWeight::get().reads_writes(4, 1) + + ::DbWeight::get().reads_writes(rw, rw) + // One `slash_cost` + + ::DbWeight::get().reads_writes(6, 5) + // `slash_cost` * nominators (1) + + ::DbWeight::get().reads_writes(6, 5) + // `reward_cost` * reporters (1) + + ::DbWeight::get().reads_writes(2, 2); + + assert_eq!(Staking::on_offence(&one_offender, &[Perbill::from_percent(50)], 0), Ok(one_offence_unapplied_weight)); }); } @@ -4699,3 +4501,33 @@ fn on_initialize_weight_is_correct() { assert_eq!(final_weight, Staking::on_initialize(System::block_number())); }); } + + +#[test] +fn payout_creates_controller() { + // Here we will test validator can set `max_nominators_payout` and it works. + // We also test that `payout_extra_nominators` works. + ExtBuilder::default().has_stakers(false).build_and_execute(|| { + let balance = 1000; + // Create three validators: + bond_validator(11, 10, balance); // Default(64) + + // Create a stash/controller pair + bond_nominator(1234, 1337, 100, vec![11]); + + // kill controller + assert_ok!(Balances::transfer(Origin::signed(1337), 1234, 100)); + assert_eq!(Balances::free_balance(1337), 0); + + mock::start_era(1); + Staking::reward_by_ids(vec![(11, 1)]); + // Compute total payout now for whole duration as other parameter won't change + let total_payout_0 = current_total_payout_for_duration(3 * 1000); + assert!(total_payout_0 > 100); // Test is meaningful if reward something + mock::start_era(2); + assert_ok!(Staking::payout_stakers(Origin::signed(1337), 11, 1)); + + // Controller is created + assert!(Balances::free_balance(1337) > 0); + }) +} diff --git a/frame/sudo/Cargo.toml b/frame/sudo/Cargo.toml index 1eb8de2cd003148eb13836486e8373f7d702e2e3..8bb549977069fe7bc25ae6925bf150ab8add749a 100644 --- a/frame/sudo/Cargo.toml +++ b/frame/sudo/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-sudo" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,15 +13,15 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } [features] default = ["std"] diff --git a/frame/sudo/src/lib.rs b/frame/sudo/src/lib.rs index 3d5d1b25821271868fb7c005c07357433df69f43..233e75e869006feddce80cae29311a52271c5109 100644 --- a/frame/sudo/src/lib.rs +++ b/frame/sudo/src/lib.rs @@ -88,12 +88,12 @@ #![cfg_attr(not(feature = "std"), no_std)] use sp_std::prelude::*; -use sp_runtime::{DispatchResult, traits::{StaticLookup, Dispatchable}}; +use sp_runtime::{DispatchResult, traits::StaticLookup}; use frame_support::{ Parameter, decl_module, decl_event, decl_storage, decl_error, ensure, }; -use frame_support::weights::{Weight, GetDispatchInfo, FunctionOf, Pays}; +use frame_support::{weights::{Weight, GetDispatchInfo}, traits::UnfilteredDispatchable}; use frame_system::{self as system, ensure_signed}; #[cfg(test)] @@ -106,7 +106,7 @@ pub trait Trait: frame_system::Trait { type Event: From> + Into<::Event>; /// A sudo-able call. - type Call: Parameter + Dispatchable + GetDispatchInfo; + type Call: Parameter + UnfilteredDispatchable + GetDispatchInfo; } decl_module! { @@ -126,17 +126,13 @@ decl_module! { /// - One DB write (event). /// - Weight of derivative `call` execution + 10,000. /// # - #[weight = FunctionOf( - |args: (&Box<::Call>,)| args.0.get_dispatch_info().weight + 10_000, - |args: (&Box<::Call>,)| args.0.get_dispatch_info().class, - Pays::Yes, - )] + #[weight = (call.get_dispatch_info().weight + 10_000, call.get_dispatch_info().class)] fn sudo(origin, call: Box<::Call>) { // This is a public call, so we ensure that the origin is some signed account. let sender = ensure_signed(origin)?; ensure!(sender == Self::key(), Error::::RequireSudo); - let res = call.dispatch(frame_system::RawOrigin::Root.into()); + let res = call.dispatch_bypass_filter(frame_system::RawOrigin::Root.into()); Self::deposit_event(RawEvent::Sudid(res.map(|_| ()).map_err(|e| e.error))); } @@ -150,17 +146,13 @@ decl_module! { /// - O(1). /// - The weight of this call is defined by the caller. /// # - #[weight = FunctionOf( - |(_, &weight): (&Box<::Call>,&Weight,)| weight, - |(call, _): (&Box<::Call>,&Weight,)| call.get_dispatch_info().class, - Pays::Yes, - )] + #[weight = (*_weight, call.get_dispatch_info().class)] fn sudo_unchecked_weight(origin, call: Box<::Call>, _weight: Weight) { // This is a public call, so we ensure that the origin is some signed account. let sender = ensure_signed(origin)?; ensure!(sender == Self::key(), Error::::RequireSudo); - let res = call.dispatch(frame_system::RawOrigin::Root.into()); + let res = call.dispatch_bypass_filter(frame_system::RawOrigin::Root.into()); Self::deposit_event(RawEvent::Sudid(res.map(|_| ()).map_err(|e| e.error))); } @@ -195,15 +187,7 @@ decl_module! { /// - One DB write (event). /// - Weight of derivative `call` execution + 10,000. /// # - #[weight = FunctionOf( - |args: (&::Source, &Box<::Call>,)| { - args.1.get_dispatch_info().weight + 10_000 - }, - |args: (&::Source, &Box<::Call>,)| { - args.1.get_dispatch_info().class - }, - Pays::Yes, - )] + #[weight = (call.get_dispatch_info().weight + 10_000, call.get_dispatch_info().class)] fn sudo_as(origin, who: ::Source, call: Box<::Call>) { // This is a public call, so we ensure that the origin is some signed account. let sender = ensure_signed(origin)?; @@ -211,7 +195,7 @@ decl_module! { let who = T::Lookup::lookup(who)?; - let res = match call.dispatch(frame_system::RawOrigin::Signed(who).into()) { + let res = match call.dispatch_bypass_filter(frame_system::RawOrigin::Signed(who).into()) { Ok(_) => true, Err(e) => { sp_runtime::print(e); diff --git a/frame/sudo/src/mock.rs b/frame/sudo/src/mock.rs index e853a29f3937061cfa64f9e139aaa5c5134e4177..3bf67f581b6e1e35f852d6167b270c6059351561 100644 --- a/frame/sudo/src/mock.rs +++ b/frame/sudo/src/mock.rs @@ -20,14 +20,15 @@ use super::*; use frame_support::{ impl_outer_origin, impl_outer_dispatch, impl_outer_event, parameter_types, - weights::{Weight, DispatchClass} + weights::Weight, }; use sp_core::H256; // The testing primitives are very useful for avoiding having to work with signatures -// or public keys. +// or public keys. use sp_runtime::{Perbill, traits::{BlakeTwo256, IdentityLookup}, testing::Header}; use sp_io; use crate as sudo; +use frame_support::traits::Filter; // Logger module to track execution. pub mod logger { @@ -56,25 +57,17 @@ pub mod logger { pub struct Module for enum Call where origin: ::Origin { fn deposit_event() = default; - #[weight = FunctionOf( - |args: (&i32, &Weight)| *args.1, - DispatchClass::Normal, - Pays::Yes, - )] + #[weight = *weight] fn privileged_i32_log(origin, i: i32, weight: Weight){ - // Ensure that the `origin` is `Root`. + // Ensure that the `origin` is `Root`. ensure_root(origin)?; ::append(i); Self::deposit_event(RawEvent::AppendI32(i, weight)); } - #[weight = FunctionOf( - |args: (&i32, &Weight)| *args.1, - DispatchClass::Normal, - Pays::Yes, - )] + #[weight = *weight] fn non_privileged_log(origin, i: i32, weight: Weight){ - // Ensure that the `origin` is some signed account. + // Ensure that the `origin` is some signed account. let sender = ensure_signed(origin)?; ::append(i); >::append(sender.clone()); @@ -120,7 +113,15 @@ parameter_types! { pub const AvailableBlockRatio: Perbill = Perbill::one(); } +pub struct BlockEverything; +impl Filter for BlockEverything { + fn filter(_: &Call) -> bool { + false + } +} + impl frame_system::Trait for Test { + type BaseCallFilter = BlockEverything; type Origin = Origin; type Call = Call; type Index = u64; @@ -128,7 +129,7 @@ impl frame_system::Trait for Test { type Hash = H256; type Hashing = BlakeTwo256; type AccountId = u64; - type Lookup = IdentityLookup; + type Lookup = IdentityLookup; type Header = Header; type Event = TestEvent; type BlockHashCount = BlockHashCount; @@ -136,6 +137,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/support/Cargo.toml b/frame/support/Cargo.toml index e2063bf69ec875b17362bc4099fb81383886457a..596faf263992b56e1820cd345a4fe9972c74e338 100644 --- a/frame/support/Cargo.toml +++ b/frame/support/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-support" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -14,25 +14,26 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] log = "0.4" serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -frame-metadata = { version = "11.0.0-alpha.8", default-features = false, path = "../metadata" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-tracing = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/tracing" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -sp-arithmetic = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/arithmetic" } -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/inherents" } -frame-support-procedural = { version = "2.0.0-alpha.8", path = "./procedural" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +frame-metadata = { version = "11.0.0-rc4", default-features = false, path = "../metadata" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-tracing = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/tracing" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-arithmetic = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/arithmetic" } +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/inherents" } +frame-support-procedural = { version = "2.0.0-rc4", path = "./procedural" } paste = "0.1.6" once_cell = { version = "1", default-features = false, optional = true } -sp-state-machine = { version = "0.8.0-alpha.8", optional = true, path = "../../primitives/state-machine" } +sp-state-machine = { version = "0.8.0-rc4", optional = true, path = "../../primitives/state-machine" } bitmask = { version = "0.5.0", default-features = false } impl-trait-for-tuples = "0.1.3" +smallvec = "1.4.0" [dev-dependencies] pretty_assertions = "0.6.1" -frame-system = { version = "2.0.0-alpha.8", path = "../system" } +frame-system = { version = "2.0.0-rc4", path = "../system" } [features] default = ["std"] diff --git a/frame/support/procedural/Cargo.toml b/frame/support/procedural/Cargo.toml index 6ef28cb75fe7d0fb2af66091283ed4b413c87e6c..593b2a16351b6e18ea5c89821eb45c761d6ed573 100644 --- a/frame/support/procedural/Cargo.toml +++ b/frame/support/procedural/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-support-procedural" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro = true [dependencies] -frame-support-procedural-tools = { version = "2.0.0-alpha.8", path = "./tools" } +frame-support-procedural-tools = { version = "2.0.0-rc4", path = "./tools" } proc-macro2 = "1.0.6" quote = "1.0.3" syn = { version = "1.0.7", features = ["full"] } diff --git a/frame/support/procedural/src/construct_runtime/mod.rs b/frame/support/procedural/src/construct_runtime/mod.rs index 677c4894e9913ea6ef8a7567b92536811b776b53..569413cbbc3f07dd42da6aff95fe769c9aba3bd4 100644 --- a/frame/support/procedural/src/construct_runtime/mod.rs +++ b/frame/support/procedural/src/construct_runtime/mod.rs @@ -94,6 +94,7 @@ fn construct_runtime_parsed(definition: RuntimeDefinition) -> Result Result( .find(|decl| decl.name == SYSTEM_MODULE_NAME) .map(|decl| &decl.module) } + +fn decl_integrity_test(scrate: &TokenStream2) -> TokenStream2 { + quote!( + #[cfg(test)] + mod __construct_runtime_integrity_test { + use super::*; + + #[test] + pub fn runtime_integrity_tests() { + ::integrity_test(); + } + } + ) +} diff --git a/frame/support/procedural/src/storage/genesis_config/builder_def.rs b/frame/support/procedural/src/storage/genesis_config/builder_def.rs index d134108af90bb08cf1f89c27c11a88ef622374ff..a045794529c958d908dfbb34ff6b847c9af52235 100644 --- a/frame/support/procedural/src/storage/genesis_config/builder_def.rs +++ b/frame/support/procedural/src/storage/genesis_config/builder_def.rs @@ -55,10 +55,16 @@ impl BuilderDef { data = Some(match &line.storage_type { StorageLineTypeDef::Simple(_) if line.is_option => quote_spanned!(builder.span() => - let data = (#builder)(self); + // NOTE: the type of `data` is specified when used later in the code + let builder: fn(&Self) -> _ = #builder; + let data = builder(self); let data = Option::as_ref(&data); ), - _ => quote_spanned!(builder.span() => let data = &(#builder)(self); ), + _ => quote_spanned!(builder.span() => + // NOTE: the type of `data` is specified when used later in the code + let builder: fn(&Self) -> _ = #builder; + let data = &builder(self); + ), }); } else if let Some(config) = &line.config { is_generic |= line.is_generic; diff --git a/frame/support/procedural/src/storage/storage_struct.rs b/frame/support/procedural/src/storage/storage_struct.rs index 6052dbfd4afea6d56b7ade0a708fb63a1d29308f..4cacb35c49d9abc0324a5009ee3ead917851e9a6 100644 --- a/frame/support/procedural/src/storage/storage_struct.rs +++ b/frame/support/procedural/src/storage/storage_struct.rs @@ -86,7 +86,10 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre Ident::new(INHERENT_INSTANCE_NAME, Span::call_site()) }; - let storage_name_str = syn::LitStr::new(&line.name.to_string(), line.name.span()); + let storage_name_bstr = syn::LitByteStr::new( + line.name.to_string().as_ref(), + line.name.span() + ); let storage_generator_trait = &line.storage_generator_trait; let storage_struct = &line.storage_struct; @@ -107,7 +110,7 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre } fn storage_prefix() -> &'static [u8] { - #storage_name_str.as_bytes() + #storage_name_bstr } fn from_optional_value_to_query(v: Option<#value_type>) -> Self::Query { @@ -131,7 +134,7 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre } fn storage_prefix() -> &'static [u8] { - #storage_name_str.as_bytes() + #storage_name_bstr } } @@ -146,7 +149,7 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre } fn storage_prefix() -> &'static [u8] { - #storage_name_str.as_bytes() + #storage_name_bstr } fn from_optional_value_to_query(v: Option<#value_type>) -> Self::Query { @@ -171,7 +174,7 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre } fn storage_prefix() -> &'static [u8] { - #storage_name_str.as_bytes() + #storage_name_bstr } } @@ -189,7 +192,7 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre } fn storage_prefix() -> &'static [u8] { - #storage_name_str.as_bytes() + #storage_name_bstr } fn from_optional_value_to_query(v: Option<#value_type>) -> Self::Query { diff --git a/frame/support/procedural/tools/Cargo.toml b/frame/support/procedural/tools/Cargo.toml index d49a90895fa33f6ba0a22bb58ed2c20edcf11448..a00dd97a66a0bb3de7c3c08d305b99e291181c90 100644 --- a/frame/support/procedural/tools/Cargo.toml +++ b/frame/support/procedural/tools/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-support-procedural-tools" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,7 +12,7 @@ description = "Proc macro helpers for procedural macros" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -frame-support-procedural-tools-derive = { version = "2.0.0-alpha.8", path = "./derive" } +frame-support-procedural-tools-derive = { version = "2.0.0-rc4", path = "./derive" } proc-macro2 = "1.0.6" quote = "1.0.3" syn = { version = "1.0.7", features = ["full", "visit"] } diff --git a/frame/support/procedural/tools/derive/Cargo.toml b/frame/support/procedural/tools/derive/Cargo.toml index f5f9c83b55718e667324618744c2866c69c7e560..3da66cf6928fa71b4cffd8bb1104e47ddce44127 100644 --- a/frame/support/procedural/tools/derive/Cargo.toml +++ b/frame/support/procedural/tools/derive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-support-procedural-tools-derive" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" diff --git a/frame/support/src/dispatch.rs b/frame/support/src/dispatch.rs index efade6d4a561fbc803f955652ef46bccbcdb1f84..760c94502d919edc81810c8e592bf0b0f5e56661 100644 --- a/frame/support/src/dispatch.rs +++ b/frame/support/src/dispatch.rs @@ -29,7 +29,7 @@ pub use crate::weights::{ PaysFee, PostDispatchInfo, WithPostDispatchInfo, }; pub use sp_runtime::{traits::Dispatchable, DispatchError}; -pub use crate::traits::{CallMetadata, GetCallMetadata, GetCallName}; +pub use crate::traits::{CallMetadata, GetCallMetadata, GetCallName, UnfilteredDispatchable}; /// The return typ of a `Dispatchable` in frame. When returned explicitly from /// a dispatchable function it allows overriding the default `PostDispatchInfo` @@ -47,10 +47,9 @@ pub type DispatchResult = Result<(), sp_runtime::DispatchError>; pub type DispatchErrorWithPostInfo = sp_runtime::DispatchErrorWithPostInfo; -/// Serializable version of Dispatchable. -/// This value can be used as a "function" in an extrinsic. -pub trait Callable { - type Call: Dispatchable + Codec + Clone + PartialEq + Eq; +/// Serializable version of pallet dispatchable. +pub trait Callable { + type Call: UnfilteredDispatchable + Codec + Clone + PartialEq + Eq; } // dirty hack to work around serde_derive issue @@ -270,8 +269,11 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// * `fn on_finalize() -> frame_support::weights::Weight` /// /// * `offchain_worker`: Executes at the beginning of a block and produces extrinsics for a future block -/// upon completion. Using this function will implement the -/// [`OffchainWorker`](./traits/trait.OffchainWorker.html) trait. +/// upon completion. Using this function will implement the +/// [`OffchainWorker`](./traits/trait.OffchainWorker.html) trait. +/// * `integrity_test`: Executes in a test generated by `construct_runtime`, note it doesn't +/// execute in an externalities-provided environment. Implement +/// [`IntegrityTest`](./trait.IntegrityTest.html) trait. #[macro_export] macro_rules! decl_module { // Entry point #1. @@ -281,7 +283,7 @@ macro_rules! decl_module { $trait_instance:ident: $trait_name:ident $( , I: $instantiable:path $( = $module_default_instance:path )? )? > - for enum $call_type:ident where origin: $origin_type:ty $(, $where_ty:ty: $where_bound:path )* { + for enum $call_type:ident where origin: $origin_type:ty $(, $where_ty:ty: $where_bound:path )* $(,)? { $( $t:tt )* } ) => { @@ -299,6 +301,7 @@ macro_rules! decl_module { {} {} {} + {} [] $($t)* ); @@ -314,6 +317,7 @@ macro_rules! decl_module { origin: $origin_type:ty, system = $system:ident $(, $where_ty:ty: $where_bound:path )* + $(,)? { $($t:tt)* } @@ -332,6 +336,7 @@ macro_rules! decl_module { {} {} {} + {} [] $($t)* ); @@ -350,6 +355,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* $vis:vis fn deposit_event() = default; @@ -367,6 +373,7 @@ macro_rules! decl_module { { $( $offchain )* } { $( $constants )* } { $( $error_type )* } + { $( $integrity_test)* } [ $( $dispatchables )* ] $($rest)* ); @@ -383,6 +390,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* $vis:vis fn deposit_event @@ -392,6 +400,29 @@ macro_rules! decl_module { "`deposit_event` function is reserved and must follow the syntax: `$vis:vis fn deposit_event() = default;`" ); }; + // Compile error on `deposit_event` being added a second time. + (@normalize + $(#[$attr:meta])* + pub struct $mod_type:ident< + $trait_instance:ident: $trait_name:ident$(, I: $instantiable:path $(= $module_default_instance:path)?)? + > + for enum $call_type:ident where origin: $origin_type:ty, system = $system:ident + { $( $other_where_bounds:tt )* } + { $( $deposit_event:tt )+ } + { $( $on_initialize:tt )* } + { $( $on_runtime_upgrade:tt )* } + { $( $on_finalize:tt )* } + { $( $offchain:tt )* } + { $( $constants:tt )* } + { $( $error_type:tt )* } + { $( $integrity_test:tt )* } + [ $( $dispatchables:tt )* ] + $(#[doc = $doc_attr:tt])* + $vis:vis fn deposit_event() = default; + $($rest:tt)* + ) => { + compile_error!("`deposit_event` can only be passed once as input."); + }; // Add on_finalize (@normalize $(#[$attr:meta])* @@ -405,6 +436,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* fn on_finalize( $( $param_name:ident : $param:ty ),* $(,)? ) { $( $impl:tt )* } @@ -424,6 +456,7 @@ macro_rules! decl_module { { $( $offchain )* } { $( $constants )* } { $( $error_type )* } + { $( $integrity_test)* } [ $( $dispatchables )* ] $($rest)* ); @@ -441,6 +474,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* #[weight = $weight:expr] @@ -452,6 +486,30 @@ macro_rules! decl_module { `on_initialize` or `on_runtime_upgrade` instead" ); }; + // Compile error on `on_finalize` being added a second time. + (@normalize + $(#[$attr:meta])* + pub struct $mod_type:ident< + $trait_instance:ident: $trait_name:ident$(, I: $instantiable:path $(= $module_default_instance:path)?)? + > + for enum $call_type:ident where origin: $origin_type:ty, system = $system:ident + { $( $other_where_bounds:tt )* } + { $( $deposit_event:tt )* } + { $( $on_initialize:tt )* } + { $( $on_runtime_upgrade:tt )* } + { $( $on_finalize:tt )+ } + { $( $offchain:tt )* } + { $( $constants:tt )* } + { $( $error_type:tt )* } + { $( $integrity_test:tt )* } + [ $( $dispatchables:tt )* ] + $(#[doc = $doc_attr:tt])* + #[weight = $weight:expr] + fn on_finalize( $( $param_name:ident : $param:ty ),* $(,)? ) { $( $impl:tt )* } + $($rest:tt)* + ) => { + compile_error!("`on_finalize` can only be passed once as input."); + }; // compile_error on_runtime_upgrade, without a given weight removed syntax. (@normalize $(#[$attr:meta])* @@ -467,6 +525,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* fn on_runtime_upgrade( $( $param_name:ident : $param:ty ),* $(,)? ) { $( $impl:tt )* } @@ -491,6 +550,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* #[weight = $weight:expr] @@ -517,6 +577,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* fn on_runtime_upgrade( $( $param_name:ident : $param:ty ),* $(,)? ) -> $return:ty { $( $impl:tt )* } @@ -536,10 +597,98 @@ macro_rules! decl_module { { $( $offchain )* } { $( $constants )* } { $( $error_type )* } + { $( $integrity_test)* } [ $( $dispatchables )* ] $($rest)* ); }; + // Compile error on `on_runtime_upgrade` being added a second time. + (@normalize + $(#[$attr:meta])* + pub struct $mod_type:ident< + $trait_instance:ident: $trait_name:ident$(, I: $instantiable:path $(= $module_default_instance:path)?)? + > + for enum $call_type:ident where origin: $origin_type:ty, system = $system:ident + { $( $other_where_bounds:tt )* } + { $( $deposit_event:tt )* } + { $( $on_initialize:tt )* } + { $( $on_runtime_upgrade:tt )+ } + { $( $on_finalize:tt )* } + { $( $offchain:tt )* } + { $( $constants:tt )* } + { $( $error_type:tt )* } + { $( $integrity_test:tt )* } + [ $( $dispatchables:tt )* ] + $(#[doc = $doc_attr:tt])* + fn on_runtime_upgrade( $( $param_name:ident : $param:ty ),* $(,)? ) -> $return:ty { $( $impl:tt )* } + $($rest:tt)* + ) => { + compile_error!("`on_runtime_upgrade` can only be passed once as input."); + }; + // Add integrity_test + (@normalize + $(#[$attr:meta])* + pub struct $mod_type:ident< + $trait_instance:ident: $trait_name:ident$(, I: $instantiable:path $(= $module_default_instance:path)?)? + > + for enum $call_type:ident where origin: $origin_type:ty, system = $system:ident + { $( $other_where_bounds:tt )* } + { $( $deposit_event:tt )* } + { $( $on_initialize:tt )* } + { $( $on_runtime_upgrade:tt )* } + { $( $on_finalize:tt )* } + { $( $offchain:tt )* } + { $( $constants:tt )* } + { $( $error_type:tt )* } + {} + [ $( $dispatchables:tt )* ] + $(#[doc = $doc_attr:tt])* + fn integrity_test() { $( $impl:tt )* } + $($rest:tt)* + ) => { + $crate::decl_module!(@normalize + $(#[$attr])* + pub struct $mod_type<$trait_instance: $trait_name$(, I: $instantiable $(= $module_default_instance)?)?> + for enum $call_type where origin: $origin_type, system = $system + { $( $other_where_bounds )* } + { $( $deposit_event )* } + { $( $on_initialize )* } + { $( $on_runtime_upgrade )* } + { $( $on_finalize )* } + { $( $offchain )* } + { $( $constants )* } + { $( $error_type )* } + { + $(#[doc = $doc_attr])* + fn integrity_test() { $( $impl)* } + } + [ $( $dispatchables )* ] + $($rest)* + ); + }; + // Compile error on `integrity_test` being added a second time. + (@normalize + $(#[$attr:meta])* + pub struct $mod_type:ident< + $trait_instance:ident: $trait_name:ident$(, I: $instantiable:path $(= $module_default_instance:path)?)? + > + for enum $call_type:ident where origin: $origin_type:ty, system = $system:ident + { $( $other_where_bounds:tt )* } + { $( $deposit_event:tt )* } + { $( $on_initialize:tt )* } + { $( $on_runtime_upgrade:tt )* } + { $( $on_finalize:tt )* } + { $( $offchain:tt )* } + { $( $constants:tt )* } + { $( $error_type:tt )* } + { $( $integrity_test:tt )+ } + [ $( $dispatchables:tt )* ] + $(#[doc = $doc_attr:tt])* + fn integrity_test() { $( $impl:tt )* } + $($rest:tt)* + ) => { + compile_error!("`integrity_test` can only be passed once as input."); + }; // compile_error on_initialize, without a given weight removed syntax. (@normalize $(#[$attr:meta])* @@ -555,6 +704,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* fn on_initialize( $( $param_name:ident : $param:ty ),* $(,)? ) { $( $impl:tt )* } @@ -579,6 +729,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* #[weight = $weight:expr] @@ -605,6 +756,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* fn on_initialize( $( $param_name:ident : $param:ty ),* $(,)? ) -> $return:ty { $( $impl:tt )* } @@ -624,10 +776,34 @@ macro_rules! decl_module { { $( $offchain )* } { $( $constants )* } { $( $error_type )* } + { $( $integrity_test)* } [ $( $dispatchables )* ] $($rest)* ); }; + // Compile error on trying to add a second `on_initialize`. + (@normalize + $(#[$attr:meta])* + pub struct $mod_type:ident< + $trait_instance:ident: $trait_name:ident$(, I: $instantiable:path $(= $module_default_instance:path)?)? + > + for enum $call_type:ident where origin: $origin_type:ty, system = $system:ident + { $( $other_where_bounds:tt )* } + { $( $deposit_event:tt )* } + { $( $on_initialize:tt )+ } + { $( $on_runtime_upgrade:tt )* } + { $( $on_finalize:tt )* } + { $( $offchain:tt )* } + { $( $constants:tt )* } + { $( $error_type:tt )* } + { $( $integrity_test:tt )* } + [ $( $dispatchables:tt )* ] + $(#[doc = $doc_attr:tt])* + fn on_initialize( $( $param_name:ident : $param:ty ),* $(,)? ) -> $return:ty { $( $impl:tt )* } + $($rest:tt)* + ) => { + compile_error!("`on_initialize` can only be passed once as input."); + }; (@normalize $(#[$attr:meta])* pub struct $mod_type:ident< @@ -643,6 +819,7 @@ macro_rules! decl_module { { } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* fn offchain_worker( $( $param_name:ident : $param:ty ),* $(,)? ) { $( $impl:tt )* } @@ -662,11 +839,34 @@ macro_rules! decl_module { { fn offchain_worker( $( $param_name : $param ),* ) { $( $impl )* } } { $( $constants )* } { $( $error_type )* } + { $( $integrity_test)* } [ $( $dispatchables )* ] $($rest)* ); }; - + // Compile error on trying to add a second `offchain_worker`. + (@normalize + $(#[$attr:meta])* + pub struct $mod_type:ident< + $trait_instance:ident: $trait_name:ident$(, I: $instantiable:path $(= $module_default_instance:path)?)? + > + for enum $call_type:ident where origin: $origin_type:ty, system = $system:ident + { $( $other_where_bounds:tt )* } + { $( $deposit_event:tt )* } + { $( $on_initialize:tt )* } + { $( $on_runtime_upgrade:tt )* } + { $( $on_finalize:tt )* } + { $( $offchain:tt )+ } + { $( $constants:tt )* } + { $( $error_type:tt )* } + { $( $integrity_test:tt )* } + [ $( $dispatchables:tt )* ] + $(#[doc = $doc_attr:tt])* + fn offchain_worker( $( $param_name:ident : $param:ty ),* $(,)? ) -> $return:ty { $( $impl:tt )* } + $($rest:tt)* + ) => { + compile_error!("`offchain_worker` can only be passed once as input."); + }; // This puts a constant in the parsed constants list. (@normalize $(#[$attr:meta])* @@ -683,6 +883,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $( #[doc = $doc_attr:tt] )* const $name:ident: $ty:ty = $value:expr; @@ -707,6 +908,7 @@ macro_rules! decl_module { $name: $ty = $value; } { $( $error_type )* } + { $( $integrity_test)* } [ $( $dispatchables )* ] $($rest)* ); @@ -728,6 +930,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* type Error = $error_type:ty; @@ -747,6 +950,7 @@ macro_rules! decl_module { { $( $offchain )* } { $( $constants )* } { $error_type } + { $( $integrity_test)* } [ $( $dispatchables )* ] $($rest)* ); @@ -767,6 +971,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { } + { $( $integrity_test:tt )* } [ $($t:tt)* ] $($rest:tt)* ) => { @@ -784,6 +989,7 @@ macro_rules! decl_module { { $( $offchain )* } { $( $constants )* } { &'static str } + { $( $integrity_test)* } [ $($t)* ] $($rest)* ); @@ -805,6 +1011,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $error_type:ty } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* #[weight = $weight:expr] @@ -827,6 +1034,7 @@ macro_rules! decl_module { { $( $offchain )* } { $( $constants )* } { $error_type } + { $( $integrity_test)* } [ $( $dispatchables )* $(#[doc = $doc_attr])* @@ -855,6 +1063,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* $fn_vis:vis fn $fn_name:ident( @@ -881,6 +1090,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* $(#[weight = $weight:expr])? @@ -907,6 +1117,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* $(#[weight = $weight:expr])? @@ -933,6 +1144,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] $(#[doc = $doc_attr:tt])* $(#[weight = $weight:expr])? @@ -960,6 +1172,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $( $error_type:tt )* } + { $( $integrity_test:tt )* } [ $( $dispatchables:tt )* ] ) => { $crate::decl_module!(@imp @@ -976,6 +1189,7 @@ macro_rules! decl_module { { $( $offchain )* } { $( $constants )* } { $( $error_type )* } + { $( $integrity_test)* } ); }; @@ -1005,6 +1219,7 @@ macro_rules! decl_module { impl<$trait_instance: $trait_name$(, $instance: $instantiable)?> $module<$trait_instance $(, $instance)?> where $( $other_where_bounds )* { + /// Deposits an event using `frame_system::Module::deposit_event`. $vis fn deposit_event( event: impl Into<< $trait_instance as $trait_name $(<$instance>)? >::Event> ) { @@ -1081,6 +1296,32 @@ macro_rules! decl_module { {} }; + (@impl_integrity_test + $module:ident<$trait_instance:ident: $trait_name:ident$(, $instance:ident: $instantiable:path)?>; + { $( $other_where_bounds:tt )* } + $(#[doc = $doc_attr:tt])* + fn integrity_test() { $( $impl:tt )* } + ) => { + impl<$trait_instance: $trait_name$(, $instance: $instantiable)?> + $crate::traits::IntegrityTest + for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* + { + $(#[doc = $doc_attr])* + fn integrity_test() { + $( $impl )* + } + } + }; + + (@impl_integrity_test + $module:ident<$trait_instance:ident: $trait_name:ident$(, $instance:ident: $instantiable:path)?>; + { $( $other_where_bounds:tt )* } + ) => { + impl<$trait_instance: $trait_name$(, $instance: $instantiable)?> + $crate::traits::IntegrityTest + for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* + {} + }; (@impl_on_finalize $module:ident<$trait_instance:ident: $trait_name:ident$(, $instance:ident: $instantiable:path)?>; @@ -1340,6 +1581,7 @@ macro_rules! decl_module { { $( $offchain:tt )* } { $( $constants:tt )* } { $error_type:ty } + { $( $integrity_test:tt )* } ) => { $crate::__check_reserved_fn_name! { $( $fn_name )* } @@ -1366,7 +1608,6 @@ macro_rules! decl_module { $( $on_runtime_upgrade )* } - $crate::decl_module! { @impl_on_finalize $mod_type<$trait_instance: $trait_name $(, $instance: $instantiable)?>; @@ -1388,6 +1629,13 @@ macro_rules! decl_module { $( $deposit_event )* } + $crate::decl_module! { + @impl_integrity_test + $mod_type<$trait_instance: $trait_name $(, $instance: $instantiable)?>; + { $( $other_where_bounds )* } + $( $integrity_test )* + } + /// Can also be called using [`Call`]. /// /// [`Call`]: enum.Call.html @@ -1402,6 +1650,8 @@ macro_rules! decl_module { $error_type; $from; $(#[doc = $doc_attr])* + /// + /// NOTE: Calling this function will bypass origin filters. $fn_vis fn $fn_name ( $from $(, $param_name : $param )* ) $( -> $result )* { $( $impl )* } @@ -1546,14 +1796,11 @@ macro_rules! decl_module { } } - impl<$trait_instance: $trait_name $(, $instance: $instantiable)?> $crate::dispatch::Dispatchable + impl<$trait_instance: $trait_name $(, $instance: $instantiable)?> $crate::traits::UnfilteredDispatchable for $call_type<$trait_instance $(, $instance)?> where $( $other_where_bounds )* { - type Trait = $trait_instance; type Origin = $origin_type; - type Info = $crate::weights::DispatchInfo; - type PostInfo = $crate::weights::PostDispatchInfo; - fn dispatch(self, _origin: Self::Origin) -> $crate::dispatch::DispatchResultWithPostInfo { + fn dispatch_bypass_filter(self, _origin: Self::Origin) -> $crate::dispatch::DispatchResultWithPostInfo { match self { $( $call_type::$fn_name( $( $param_name ),* ) => { @@ -1574,17 +1821,6 @@ macro_rules! decl_module { type Call = $call_type<$trait_instance $(, $instance)?>; } - impl<$trait_instance: $trait_name $(, $instance: $instantiable)?> $mod_type<$trait_instance $(, $instance)?> - where $( $other_where_bounds )* - { - #[doc(hidden)] - pub fn dispatch>( - d: D, - origin: D::Origin - ) -> $crate::dispatch::DispatchResultWithPostInfo { - d.dispatch(origin) - } - } $crate::__dispatch_impl_metadata! { $mod_type<$trait_instance: $trait_name $(, $instance: $instantiable)?> { $( $other_where_bounds )* } @@ -1684,6 +1920,20 @@ macro_rules! impl_outer_dispatch { fn dispatch( self, origin: $origin, + ) -> $crate::dispatch::DispatchResultWithPostInfo { + if !::filter_call(&origin, &self) { + return $crate::sp_std::result::Result::Err($crate::dispatch::DispatchError::BadOrigin.into()) + } + + $crate::traits::UnfilteredDispatchable::dispatch_bypass_filter(self, origin) + } + } + + impl $crate::traits::UnfilteredDispatchable for $call_type { + type Origin = $origin; + fn dispatch_bypass_filter( + self, + origin: $origin, ) -> $crate::dispatch::DispatchResultWithPostInfo { $crate::impl_outer_dispatch! { @DISPATCH_MATCH @@ -1696,6 +1946,7 @@ macro_rules! impl_outer_dispatch { } } } + $( impl $crate::dispatch::IsSubType<$crate::dispatch::CallableCallFor<$camelcase, $runtime>> for $call_type { #[allow(unreachable_patterns)] @@ -1731,7 +1982,8 @@ macro_rules! impl_outer_dispatch { $origin { $( $generated )* - $call_type::$name(call) => call.dispatch($origin), + $call_type::$name(call) => + $crate::traits::UnfilteredDispatchable::dispatch_bypass_filter(call, $origin), } $index + 1; $( $rest ),* @@ -2011,6 +2263,9 @@ macro_rules! __check_reserved_fn_name { (offchain_worker $( $rest:ident )*) => { $crate::__check_reserved_fn_name!(@compile_error offchain_worker); }; + (integrity_test $( $rest:ident )*) => { + $crate::__check_reserved_fn_name!(@compile_error integrity_test); + }; ($t:ident $( $rest:ident )*) => { $crate::__check_reserved_fn_name!($( $rest )*); }; @@ -2046,25 +2301,39 @@ mod tests { use super::*; use crate::weights::{DispatchInfo, DispatchClass, Pays}; use crate::traits::{ - CallMetadata, GetCallMetadata, GetCallName, OnInitialize, OnFinalize, OnRuntimeUpgrade + CallMetadata, GetCallMetadata, GetCallName, OnInitialize, OnFinalize, OnRuntimeUpgrade, + IntegrityTest, }; pub trait Trait: system::Trait + Sized where Self::AccountId: From { - type Origin; type BlockNumber: Into; - type Call: From>; } pub mod system { - use super::*; - pub trait Trait { type AccountId; + type Call; + type BaseCallFilter; + type Origin: crate::traits::OriginTrait; } - pub fn ensure_root(_: R) -> DispatchResult { - Ok(()) + #[derive(Clone, PartialEq, Eq, Debug)] + pub enum RawOrigin { + Root, + Signed(AccountId), + None, } + + impl From> for RawOrigin { + fn from(s: Option) -> RawOrigin { + match s { + Some(who) => RawOrigin::Signed(who), + None => RawOrigin::None, + } + } + } + + pub type Origin = RawOrigin<::AccountId>; } decl_module! { @@ -2095,6 +2364,8 @@ mod tests { fn on_finalize(n: T::BlockNumber,) { if n.into() == 42 { panic!("on_finalize") } } fn on_runtime_upgrade() -> Weight { 10 } fn offchain_worker() {} + /// Some doc + fn integrity_test() { panic!("integrity_test") } } } @@ -2169,21 +2440,26 @@ mod tests { pub struct TraitImpl {} impl Trait for TraitImpl { - type Origin = u32; type BlockNumber = u32; - type Call = OuterCall; } type Test = Module; + impl_outer_origin!{ + pub enum OuterOrigin for TraitImpl where system = system {} + } + impl_outer_dispatch! { - pub enum OuterCall for TraitImpl where origin: u32 { + pub enum OuterCall for TraitImpl where origin: OuterOrigin { self::Test, } } impl system::Trait for TraitImpl { + type Origin = OuterOrigin; type AccountId = u32; + type Call = OuterCall; + type BaseCallFilter = (); } #[test] @@ -2281,4 +2557,10 @@ mod tests { let module_names = OuterCall::get_module_names(); assert_eq!(["Test"], module_names); } + + #[test] + #[should_panic(expected = "integrity_test")] + fn integrity_test_should_work() { + as IntegrityTest>::integrity_test(); + } } diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 11109fb177536f60cdc611f6dc50c72710d0bbc7..196bddbdf5bce15bbd42b1011f34add1ece834ef 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -44,21 +44,21 @@ pub use paste; #[doc(hidden)] pub use sp_state_machine::BasicExternalities; #[doc(hidden)] -pub use sp_io::storage::root as storage_root; +pub use sp_io::{storage::root as storage_root, self}; #[doc(hidden)] pub use sp_runtime::RuntimeDebug; #[macro_use] pub mod debug; #[macro_use] +mod origin; +#[macro_use] pub mod dispatch; pub mod storage; mod hash; #[macro_use] pub mod event; #[macro_use] -mod origin; -#[macro_use] pub mod metadata; #[macro_use] pub mod inherent; @@ -84,21 +84,66 @@ pub use sp_runtime::{self, ConsensusEngineId, print, traits::Printable}; #[derive(Debug)] pub enum Never {} -/// Macro for easily creating a new implementation of the `Get` trait. Use similarly to -/// how you would declare a `const`: +/// Create new implementations of the [`Get`](crate::traits::Get) trait. +/// +/// The so-called parameter type can be created in three different ways: +/// +/// - Using `const` to create a parameter type that provides a `const` getter. +/// It is required that the `value` is const. +/// +/// - Declare the parameter type without `const` to have more freedom when creating the value. +/// +/// - Using `storage` to create a storage parameter type. This type is special as it tries to +/// load the value from the storage under a fixed key. If the value could not be found in the +/// storage, the given default value will be returned. It is required that the value implements +/// [`Encode`](codec::Encode) and [`Decode`](codec::Decode). The key for looking up the value +/// in the storage is built using the following formular: +/// +/// `twox_128(":" ++ NAME ++ ":")` where `NAME` is the name that is passed as type name. /// -/// ```no_compile +/// # Examples +/// +/// ``` +/// # use frame_support::traits::Get; +/// # use frame_support::parameter_types; +/// // This function cannot be used in a const context. +/// fn non_const_expression() -> u64 { 99 } +/// +/// const FIXED_VALUE: u64 = 10; /// parameter_types! { -/// pub const Argument: u64 = 42; +/// pub const Argument: u64 = 42 + FIXED_VALUE; +/// /// Visibility of the type is optional +/// OtherArgument: u64 = non_const_expression(); +/// pub storage StorageArgument: u64 = 5; /// } +/// /// trait Config { -/// type Parameter: Get; +/// type Parameter: Get; +/// type OtherParameter: Get; +/// type StorageParameter: Get; /// } +/// /// struct Runtime; /// impl Config for Runtime { -/// type Parameter = Argument; +/// type Parameter = Argument; +/// type OtherParameter = OtherArgument; +/// type StorageParameter = StorageArgument; /// } /// ``` +/// +/// # Invalid example: +/// +/// ```compile_fail +/// # use frame_support::traits::Get; +/// # use frame_support::parameter_types; +/// // This function cannot be used in a const context. +/// fn non_const_expression() -> u64 { 99 } +/// +/// parameter_types! { +/// pub const Argument: u64 = non_const_expression(); +/// } +/// ``` + #[macro_export] macro_rules! parameter_types { ( @@ -108,21 +153,89 @@ macro_rules! parameter_types { ) => ( $( #[ $attr ] )* $vis struct $name; - $crate::parameter_types!{IMPL $name , $type , $value} - $crate::parameter_types!{ $( $rest )* } + $crate::parameter_types!(IMPL_CONST $name , $type , $value); + $crate::parameter_types!( $( $rest )* ); + ); + ( + $( #[ $attr:meta ] )* + $vis:vis $name:ident: $type:ty = $value:expr; + $( $rest:tt )* + ) => ( + $( #[ $attr ] )* + $vis struct $name; + $crate::parameter_types!(IMPL $name, $type, $value); + $crate::parameter_types!( $( $rest )* ); + ); + ( + $( #[ $attr:meta ] )* + $vis:vis storage $name:ident: $type:ty = $value:expr; + $( $rest:tt )* + ) => ( + $( #[ $attr ] )* + $vis struct $name; + $crate::parameter_types!(IMPL_STORAGE $name, $type, $value); + $crate::parameter_types!( $( $rest )* ); ); () => (); - (IMPL $name:ident , $type:ty , $value:expr) => { + (IMPL_CONST $name:ident, $type:ty, $value:expr) => { + impl $name { + /// Returns the value of this parameter type. + pub const fn get() -> $type { + $value + } + } + + impl> $crate::traits::Get for $name { + fn get() -> I { + I::from($value) + } + } + }; + (IMPL $name:ident, $type:ty, $value:expr) => { impl $name { + /// Returns the value of this parameter type. pub fn get() -> $type { $value } } + impl> $crate::traits::Get for $name { fn get() -> I { I::from($value) } } + }; + (IMPL_STORAGE $name:ident, $type:ty, $value:expr) => { + impl $name { + /// Returns the key for this parameter type. + pub fn key() -> [u8; 16] { + $crate::sp_io::hashing::twox_128( + concat!(":", stringify!($name), ":").as_bytes() + ) + } + + /// Set the value of this parameter type in the storage. + /// + /// This needs to be executed in an externalities provided + /// environment. + pub fn set(value: &$type) { + $crate::storage::unhashed::put(&Self::key(), value); + } + + /// Returns the value of this parameter type. + /// + /// This needs to be executed in an externalities provided + /// environment. + pub fn get() -> $type { + $crate::storage::unhashed::get(&Self::key()).unwrap_or_else(|| $value) + } + } + + impl> $crate::traits::Get for $name { + fn get() -> I { + I::from(Self::get()) + } + } } } @@ -218,6 +331,7 @@ macro_rules! assert_err_ignore_postinfo { } } +/// Assert an expression returns error with the given weight. #[macro_export] #[cfg(feature = "std")] macro_rules! assert_err_with_weight { @@ -268,6 +382,7 @@ mod tests { StorageEntryModifier, DefaultByteGetter, StorageHasher, }; use sp_std::marker::PhantomData; + use sp_io::TestExternalities; pub trait Trait { type BlockNumber: Codec + EncodeLike + Default; @@ -313,7 +428,7 @@ mod tests { type Origin = u32; } - fn new_test_ext() -> sp_io::TestExternalities { + fn new_test_ext() -> TestExternalities { GenesisConfig::default().build_storage().unwrap().into() } @@ -648,4 +763,20 @@ mod tests { let metadata = Module::::storage_metadata(); pretty_assertions::assert_eq!(EXPECTED_METADATA, metadata); } + + parameter_types! { + storage StorageParameter: u64 = 10; + } + + #[test] + fn check_storage_parameter_type_works() { + TestExternalities::default().execute_with(|| { + assert_eq!(sp_io::hashing::twox_128(b":StorageParameter:"), StorageParameter::key()); + + assert_eq!(10, StorageParameter::get()); + + StorageParameter::set(&300); + assert_eq!(300, StorageParameter::get()); + }) + } } diff --git a/frame/support/src/metadata.rs b/frame/support/src/metadata.rs index 7248d6bc4df5c088e3a7c7be2f7ff718fc013356..d6ec9a7373952348e2104c11fca08ac0d4246755 100644 --- a/frame/support/src/metadata.rs +++ b/frame/support/src/metadata.rs @@ -290,6 +290,7 @@ mod tests { use super::*; pub trait Trait: 'static { + type BaseCallFilter; const ASSOCIATED_CONST: u64 = 500; type Origin: Into, Self::Origin>> + From>; @@ -297,6 +298,7 @@ mod tests { type BlockNumber: From + Encode; type SomeValue: Get; type ModuleToIndex: crate::traits::ModuleToIndex; + type Call; } decl_module! { @@ -436,11 +438,13 @@ mod tests { } impl system::Trait for TestRuntime { + type BaseCallFilter = (); type Origin = Origin; type AccountId = u32; type BlockNumber = u32; type SomeValue = SystemValue; type ModuleToIndex = (); + type Call = Call; } impl_runtime_metadata!( diff --git a/frame/support/src/origin.rs b/frame/support/src/origin.rs index f96ec07af0a9c648f34c5990b2caccfc3cac9e72..77fe86cc5575bf9ac2f1b157e3571eede59e4b4e 100644 --- a/frame/support/src/origin.rs +++ b/frame/support/src/origin.rs @@ -45,19 +45,23 @@ macro_rules! impl_outer_origin { $( $rest_with_system:tt )* } ) => { - $crate::impl_outer_origin!( - $( #[$attr] )*; - $name; - $runtime; - $system; - Modules { $( $rest_with_system )* }; - ); + $crate::paste::item! { + $crate::impl_outer_origin!( + $( #[$attr] )*; + $name; + [< $name Caller >]; + $runtime; + $system; + Modules { $( $rest_with_system )* }; + ); + } }; // Generic + Instance ( $(#[$attr:meta])*; $name:ident; + $caller_name:ident; $runtime:ident; $system:ident; Modules { @@ -69,6 +73,7 @@ macro_rules! impl_outer_origin { $crate::impl_outer_origin!( $( #[$attr] )*; $name; + $caller_name; $runtime; $system; Modules { $( $( $rest_module )* )? }; @@ -80,6 +85,7 @@ macro_rules! impl_outer_origin { ( $(#[$attr:meta])*; $name:ident; + $caller_name:ident; $runtime:ident; $system:ident; Modules { @@ -91,6 +97,7 @@ macro_rules! impl_outer_origin { $crate::impl_outer_origin!( $( #[$attr] )*; $name; + $caller_name; $runtime; $system; Modules { $( $rest_module )* }; @@ -102,6 +109,7 @@ macro_rules! impl_outer_origin { ( $(#[$attr:meta])*; $name:ident; + $caller_name:ident; $runtime:ident; $system:ident; Modules { @@ -113,6 +121,7 @@ macro_rules! impl_outer_origin { $crate::impl_outer_origin!( $( #[$attr] )*; $name; + $caller_name; $runtime; $system; Modules { $( $( $rest_module )* )? }; @@ -124,6 +133,7 @@ macro_rules! impl_outer_origin { ( $(#[$attr:meta])*; $name:ident; + $caller_name:ident; $runtime:ident; $system:ident; Modules { @@ -135,6 +145,7 @@ macro_rules! impl_outer_origin { $crate::impl_outer_origin!( $( #[$attr] )*; $name; + $caller_name; $runtime; $system; Modules { $( $( $rest_module )* )? }; @@ -146,16 +157,78 @@ macro_rules! impl_outer_origin { ( $(#[$attr:meta])*; $name:ident; + $caller_name:ident; $runtime:ident; $system:ident; Modules { }; $( $module:ident $( < $generic:ident > )? $( { $generic_instance:ident } )? ,)* ) => { + // WARNING: All instance must hold the filter `frame_system::Trait::BaseCallFilter`, except + // when caller is system Root. One can use `OriginTrait::reset_filter` to do so. + #[derive(Clone)] + pub struct $name { + caller: $caller_name, + filter: $crate::sp_std::rc::Rc::Call) -> bool>>, + } + + #[cfg(not(feature = "std"))] + impl $crate::sp_std::fmt::Debug for $name { + fn fmt( + &self, + fmt: &mut $crate::sp_std::fmt::Formatter + ) -> $crate::sp_std::result::Result<(), $crate::sp_std::fmt::Error> { + fmt.write_str("") + } + } + + #[cfg(feature = "std")] + impl $crate::sp_std::fmt::Debug for $name { + fn fmt( + &self, + fmt: &mut $crate::sp_std::fmt::Formatter + ) -> $crate::sp_std::result::Result<(), $crate::sp_std::fmt::Error> { + fmt.debug_struct(stringify!($name)) + .field("caller", &self.caller) + .field("filter", &"[function ptr]") + .finish() + } + } + + impl $crate::traits::OriginTrait for $name { + type Call = <$runtime as $system::Trait>::Call; + type PalletsOrigin = $caller_name; + + fn add_filter(&mut self, filter: impl Fn(&Self::Call) -> bool + 'static) { + let f = self.filter.clone(); + + self.filter = $crate::sp_std::rc::Rc::new(Box::new(move |call| { + f(call) && filter(call) + })); + } + + fn reset_filter(&mut self) { + let filter = < + <$runtime as $system::Trait>::BaseCallFilter + as $crate::traits::Filter<<$runtime as $system::Trait>::Call> + >::filter; + + self.filter = $crate::sp_std::rc::Rc::new(Box::new(filter)); + } + + fn set_caller_from(&mut self, other: impl Into) { + self.caller = other.into().caller + } + + fn filter_call(&self, call: &Self::Call) -> bool { + (self.filter)(call) + } + } + $crate::paste::item! { #[derive(Clone, PartialEq, Eq, $crate::RuntimeDebug)] $(#[$attr])* #[allow(non_camel_case_types)] - pub enum $name { + pub enum $caller_name { system($system::Origin<$runtime>), $( [< $module $( _ $generic_instance )? >] @@ -168,20 +241,42 @@ macro_rules! impl_outer_origin { #[allow(dead_code)] impl $name { - pub const NONE: Self = $name::system($system::RawOrigin::None); - pub const ROOT: Self = $name::system($system::RawOrigin::Root); + /// Create with system none origin and `frame-system::Trait::BaseCallFilter`. + pub fn none() -> Self { + $system::RawOrigin::None.into() + } + /// Create with system root origin and no filter. + pub fn root() -> Self { + $system::RawOrigin::Root.into() + } + /// Create with system signed origin and `frame-system::Trait::BaseCallFilter`. pub fn signed(by: <$runtime as $system::Trait>::AccountId) -> Self { - $name::system($system::RawOrigin::Signed(by)) + $system::RawOrigin::Signed(by).into() } } + impl From<$system::Origin<$runtime>> for $name { + /// Convert to runtime origin: + /// * root origin is built with no filter + /// * others use `frame-system::Trait::BaseCallFilter` fn from(x: $system::Origin<$runtime>) -> Self { - $name::system(x) + let mut o = $name { + caller: $caller_name::system(x), + filter: $crate::sp_std::rc::Rc::new(Box::new(|_| true)), + }; + + // Root has no filter + if !matches!(o.caller, $caller_name::system($system::Origin::<$runtime>::Root)) { + $crate::traits::OriginTrait::reset_filter(&mut o); + } + + o } } impl Into<$crate::sp_std::result::Result<$system::Origin<$runtime>, $name>> for $name { + /// NOTE: converting to pallet origin loses the origin filter information. fn into(self) -> $crate::sp_std::result::Result<$system::Origin<$runtime>, Self> { - if let $name::system(l) = self { + if let $caller_name::system(l) = self.caller { Ok(l) } else { Err(self) @@ -189,6 +284,8 @@ macro_rules! impl_outer_origin { } } impl From::AccountId>> for $name { + /// Convert to runtime origin with caller being system signed or none and use filter + /// `frame-system::Trait::BaseCallFilter`. fn from(x: Option<<$runtime as $system::Trait>::AccountId>) -> Self { <$system::Origin<$runtime>>::from(x).into() } @@ -196,8 +293,14 @@ macro_rules! impl_outer_origin { $( $crate::paste::item! { impl From<$module::Origin < $( $generic )? $(, $module::$generic_instance )? > > for $name { + /// Convert to runtime origin using `frame-system::Trait::BaseCallFilter`. fn from(x: $module::Origin < $( $generic )? $(, $module::$generic_instance )? >) -> Self { - $name::[< $module $( _ $generic_instance )? >](x) + let mut o = $name { + caller: $caller_name::[< $module $( _ $generic_instance )? >](x), + filter: $crate::sp_std::rc::Rc::new(Box::new(|_| true)), + }; + $crate::traits::OriginTrait::reset_filter(&mut o); + o } } impl Into< @@ -206,11 +309,12 @@ macro_rules! impl_outer_origin { $name, >> for $name { + /// NOTE: converting to pallet origin loses the origin filter information. fn into(self) -> $crate::sp_std::result::Result< $module::Origin < $( $generic )? $(, $module::$generic_instance )? >, Self, > { - if let $name::[< $module $( _ $generic_instance )? >](l) = self { + if let $caller_name::[< $module $( _ $generic_instance )? >](l) = self.caller { Ok(l) } else { Err(self) @@ -224,9 +328,12 @@ macro_rules! impl_outer_origin { #[cfg(test)] mod tests { + use crate::traits::{Filter, OriginTrait}; mod system { pub trait Trait { type AccountId; + type Call; + type BaseCallFilter; } #[derive(Clone, PartialEq, Eq, Debug)] @@ -263,8 +370,17 @@ mod tests { #[derive(Clone, PartialEq, Eq, Debug)] pub struct TestRuntime; + pub struct BaseCallFilter; + impl Filter for BaseCallFilter { + fn filter(c: &u32) -> bool { + *c % 2 == 0 + } + } + impl system::Trait for TestRuntime { type AccountId = u32; + type Call = u32; + type BaseCallFilter = BaseCallFilter; } impl_outer_origin!( @@ -298,4 +414,35 @@ mod tests { impl_outer_origin!( pub enum OriginEmpty for TestRuntime where system = system {} ); + + #[test] + fn test_default_filter() { + assert_eq!(OriginWithSystem::root().filter_call(&0), true); + assert_eq!(OriginWithSystem::root().filter_call(&1), true); + assert_eq!(OriginWithSystem::none().filter_call(&0), true); + assert_eq!(OriginWithSystem::none().filter_call(&1), false); + assert_eq!(OriginWithSystem::signed(0).filter_call(&0), true); + assert_eq!(OriginWithSystem::signed(0).filter_call(&1), false); + assert_eq!(OriginWithSystem::from(Some(0)).filter_call(&0), true); + assert_eq!(OriginWithSystem::from(Some(0)).filter_call(&1), false); + assert_eq!(OriginWithSystem::from(None).filter_call(&0), true); + assert_eq!(OriginWithSystem::from(None).filter_call(&1), false); + assert_eq!(OriginWithSystem::from(origin_without_generic::Origin).filter_call(&0), true); + assert_eq!(OriginWithSystem::from(origin_without_generic::Origin).filter_call(&1), false); + + let mut origin = OriginWithSystem::from(Some(0)); + + origin.add_filter(|c| *c % 2 == 1); + assert_eq!(origin.filter_call(&0), false); + assert_eq!(origin.filter_call(&1), false); + + origin.set_caller_from(OriginWithSystem::root()); + assert!(matches!(origin.caller, OriginWithSystemCaller::system(system::RawOrigin::Root))); + assert_eq!(origin.filter_call(&0), false); + assert_eq!(origin.filter_call(&1), false); + + origin.reset_filter(); + assert_eq!(origin.filter_call(&0), true); + assert_eq!(origin.filter_call(&1), false); + } } diff --git a/frame/support/src/storage/mod.rs b/frame/support/src/storage/mod.rs index 6d0ef91ce1e17e4f81c69a624827c9a252d1fa7b..c2d7ceef0fee6315deab20aa0794ff73e604d7c4 100644 --- a/frame/support/src/storage/mod.rs +++ b/frame/support/src/storage/mod.rs @@ -29,6 +29,33 @@ pub mod child; pub mod generator; pub mod migration; +/// Describes whether a storage transaction should be committed or rolled back. +pub enum TransactionOutcome { + /// Transaction should be committed. + Commit(T), + /// Transaction should be rolled back. + Rollback(T), +} + +/// Execute the supplied function in a new storage transaction. +/// +/// All changes to storage performed by the supplied function are discarded if the returned +/// outcome is `TransactionOutcome::Rollback`. +/// +/// Transactions can be nested to any depth. Commits happen to the parent transaction. +pub fn with_transaction(f: impl FnOnce() -> TransactionOutcome) -> R { + use sp_io::storage::{ + start_transaction, commit_transaction, rollback_transaction, + }; + use TransactionOutcome::*; + + start_transaction(); + match f() { + Commit(res) => { commit_transaction(); res }, + Rollback(res) => { rollback_transaction(); res }, + } +} + /// A trait for working with macro-generated storage values under the substrate storage API. /// /// Details on implementation can be found at diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs index 99a5abcbacb4423d017c880bbc5d2149e9e2ed77..f7e7710b32936c2f4afe94d6787d6e7fbe99d8b2 100644 --- a/frame/support/src/traits.rs +++ b/frame/support/src/traits.rs @@ -25,7 +25,7 @@ use sp_core::u32_trait::Value as U32; use sp_runtime::{ RuntimeDebug, ConsensusEngineId, DispatchResult, DispatchError, traits::{ MaybeSerializeDeserialize, AtLeast32Bit, Saturating, TrailingZeroInput, Bounded, Zero, - BadOrigin + BadOrigin, AtLeast32BitUnsigned }, }; use crate::dispatch::Parameter; @@ -33,6 +33,268 @@ use crate::storage::StorageMap; use crate::weights::Weight; use impl_trait_for_tuples::impl_for_tuples; +/// Re-expected for the macro. +#[doc(hidden)] +pub use sp_std::{mem::{swap, take}, cell::RefCell, vec::Vec, boxed::Box}; + +/// Simple trait for providing a filter over a reference to some type. +pub trait Filter { + /// Determine if a given value should be allowed through the filter (returns `true`) or not. + fn filter(_: &T) -> bool; +} + +impl Filter for () { + fn filter(_: &T) -> bool { true } +} + +/// Trait to add a constraint onto the filter. +pub trait FilterStack: Filter { + /// The type used to archive the stack. + type Stack; + + /// Add a new `constraint` onto the filter. + fn push(constraint: impl Fn(&T) -> bool + 'static); + + /// Removes the most recently pushed, and not-yet-popped, constraint from the filter. + fn pop(); + + /// Clear the filter, returning a value that may be used later to `restore` it. + fn take() -> Self::Stack; + + /// Restore the filter from a previous `take` operation. + fn restore(taken: Self::Stack); +} + +/// Guard type for pushing a constraint to a `FilterStack` and popping when dropped. +pub struct FilterStackGuard, T>(PhantomData<(F, T)>); + +/// Guard type for clearing all pushed constraints from a `FilterStack` and reinstating them when +/// dropped. +pub struct ClearFilterGuard, T>(Option, PhantomData); + +impl, T> FilterStackGuard { + /// Create a new instance, adding a new `constraint` onto the filter `T`, and popping it when + /// this instance is dropped. + pub fn new(constraint: impl Fn(&T) -> bool + 'static) -> Self { + F::push(constraint); + Self(PhantomData) + } +} + +impl, T> Drop for FilterStackGuard { + fn drop(&mut self) { + F::pop(); + } +} + +impl, T> ClearFilterGuard { + /// Create a new instance, adding a new `constraint` onto the filter `T`, and popping it when + /// this instance is dropped. + pub fn new() -> Self { + Self(Some(F::take()), PhantomData) + } +} + +impl, T> Drop for ClearFilterGuard { + fn drop(&mut self) { + if let Some(taken) = self.0.take() { + F::restore(taken); + } + } +} + +/// Simple trait for providing a filter over a reference to some type, given an instance of itself. +pub trait InstanceFilter: Sized + Send + Sync { + /// Determine if a given value should be allowed through the filter (returns `true`) or not. + fn filter(&self, _: &T) -> bool; + + /// Determines whether `self` matches at least everything that `_o` does. + fn is_superset(&self, _o: &Self) -> bool { false } +} + +impl InstanceFilter for () { + fn filter(&self, _: &T) -> bool { true } + fn is_superset(&self, _o: &Self) -> bool { true } +} + +#[macro_export] +macro_rules! impl_filter_stack { + ($target:ty, $base:ty, $call:ty, $module:ident) => { + #[cfg(feature = "std")] + mod $module { + #[allow(unused_imports)] + use super::*; + use $crate::traits::{swap, take, RefCell, Vec, Box, Filter, FilterStack}; + + thread_local! { + static FILTER: RefCell bool + 'static>>> = RefCell::new(Vec::new()); + } + + impl Filter<$call> for $target { + fn filter(call: &$call) -> bool { + <$base>::filter(call) && + FILTER.with(|filter| filter.borrow().iter().all(|f| f(call))) + } + } + + impl FilterStack<$call> for $target { + type Stack = Vec bool + 'static>>; + fn push(f: impl Fn(&$call) -> bool + 'static) { + FILTER.with(|filter| filter.borrow_mut().push(Box::new(f))); + } + fn pop() { + FILTER.with(|filter| filter.borrow_mut().pop()); + } + fn take() -> Self::Stack { + FILTER.with(|filter| take(filter.borrow_mut().as_mut())) + } + fn restore(mut s: Self::Stack) { + FILTER.with(|filter| swap(filter.borrow_mut().as_mut(), &mut s)); + } + } + } + + #[cfg(not(feature = "std"))] + mod $module { + #[allow(unused_imports)] + use super::*; + use $crate::traits::{swap, take, RefCell, Vec, Box, Filter, FilterStack}; + + struct ThisFilter(RefCell bool + 'static>>>); + // NOTE: Safe only in wasm (guarded above) because there's only one thread. + unsafe impl Send for ThisFilter {} + unsafe impl Sync for ThisFilter {} + + static FILTER: ThisFilter = ThisFilter(RefCell::new(Vec::new())); + + impl Filter<$call> for $target { + fn filter(call: &$call) -> bool { + <$base>::filter(call) && FILTER.0.borrow().iter().all(|f| f(call)) + } + } + + impl FilterStack<$call> for $target { + type Stack = Vec bool + 'static>>; + fn push(f: impl Fn(&$call) -> bool + 'static) { + FILTER.0.borrow_mut().push(Box::new(f)); + } + fn pop() { + FILTER.0.borrow_mut().pop(); + } + fn take() -> Self::Stack { + take(FILTER.0.borrow_mut().as_mut()) + } + fn restore(mut s: Self::Stack) { + swap(FILTER.0.borrow_mut().as_mut(), &mut s); + } + } + } + } +} + +/// Type that provide some integrity tests. +/// +/// This implemented for modules by `decl_module`. +#[impl_for_tuples(30)] +pub trait IntegrityTest { + /// Run integrity test. + /// + /// The test is not executed in a externalities provided environment. + fn integrity_test() {} +} + +#[cfg(test)] +mod test_impl_filter_stack { + use super::*; + + pub struct IsCallable; + pub struct BaseFilter; + impl Filter for BaseFilter { + fn filter(x: &u32) -> bool { x % 2 == 0 } + } + impl_filter_stack!( + crate::traits::test_impl_filter_stack::IsCallable, + crate::traits::test_impl_filter_stack::BaseFilter, + u32, + is_callable + ); + + #[test] + fn impl_filter_stack_should_work() { + assert!(IsCallable::filter(&36)); + assert!(IsCallable::filter(&40)); + assert!(IsCallable::filter(&42)); + assert!(!IsCallable::filter(&43)); + + IsCallable::push(|x| *x < 42); + assert!(IsCallable::filter(&36)); + assert!(IsCallable::filter(&40)); + assert!(!IsCallable::filter(&42)); + + IsCallable::push(|x| *x % 3 == 0); + assert!(IsCallable::filter(&36)); + assert!(!IsCallable::filter(&40)); + + IsCallable::pop(); + assert!(IsCallable::filter(&36)); + assert!(IsCallable::filter(&40)); + assert!(!IsCallable::filter(&42)); + + let saved = IsCallable::take(); + assert!(IsCallable::filter(&36)); + assert!(IsCallable::filter(&40)); + assert!(IsCallable::filter(&42)); + assert!(!IsCallable::filter(&43)); + + IsCallable::restore(saved); + assert!(IsCallable::filter(&36)); + assert!(IsCallable::filter(&40)); + assert!(!IsCallable::filter(&42)); + + IsCallable::pop(); + assert!(IsCallable::filter(&36)); + assert!(IsCallable::filter(&40)); + assert!(IsCallable::filter(&42)); + assert!(!IsCallable::filter(&43)); + } + + #[test] + fn guards_should_work() { + assert!(IsCallable::filter(&36)); + assert!(IsCallable::filter(&40)); + assert!(IsCallable::filter(&42)); + assert!(!IsCallable::filter(&43)); + { + let _guard_1 = FilterStackGuard::::new(|x| *x < 42); + assert!(IsCallable::filter(&36)); + assert!(IsCallable::filter(&40)); + assert!(!IsCallable::filter(&42)); + { + let _guard_2 = FilterStackGuard::::new(|x| *x % 3 == 0); + assert!(IsCallable::filter(&36)); + assert!(!IsCallable::filter(&40)); + } + assert!(IsCallable::filter(&36)); + assert!(IsCallable::filter(&40)); + assert!(!IsCallable::filter(&42)); + { + let _guard_2 = ClearFilterGuard::::new(); + assert!(IsCallable::filter(&36)); + assert!(IsCallable::filter(&40)); + assert!(IsCallable::filter(&42)); + assert!(!IsCallable::filter(&43)); + } + assert!(IsCallable::filter(&36)); + assert!(IsCallable::filter(&40)); + assert!(!IsCallable::filter(&42)); + } + assert!(IsCallable::filter(&36)); + assert!(IsCallable::filter(&40)); + assert!(IsCallable::filter(&42)); + assert!(!IsCallable::filter(&43)); + } +} + /// An abstraction of a value stored within storage, but possibly as part of a larger composite /// item. pub trait StoredMap { @@ -67,6 +329,10 @@ pub trait Happened { fn happened(t: &T); } +impl Happened for () { + fn happened(_: &T) {} +} + /// A shim for placing around a storage item in order to use it as a `StoredValue`. Ideally this /// wouldn't be needed as `StorageValue`s should blanket implement `StoredValue`s, however this /// would break the ability to have custom impls of `StoredValue`. The other workaround is to @@ -95,20 +361,23 @@ impl< fn get(k: &K) -> T { S::get(k) } fn is_explicit(k: &K) -> bool { S::contains_key(k) } fn insert(k: &K, t: T) { + let existed = S::contains_key(&k); S::insert(k, t); - if !S::contains_key(&k) { + if !existed { Created::happened(k); } } fn remove(k: &K) { - if S::contains_key(&k) { + let existed = S::contains_key(&k); + S::remove(k); + if existed { Removed::happened(&k); } - S::remove(k); } fn mutate(k: &K, f: impl FnOnce(&mut T) -> R) -> R { + let existed = S::contains_key(&k); let r = S::mutate(k, f); - if !S::contains_key(&k) { + if !existed { Created::happened(k); } r @@ -196,9 +465,11 @@ impl Len for T where ::IntoIter: Ex } } -/// A trait for querying a single fixed value from a type. +/// A trait for querying a single value from a type. +/// +/// It is not required that the value is constant. pub trait Get { - /// Return a constant value. + /// Return the current value. fn get() -> T; } @@ -521,7 +792,7 @@ pub enum SignedImbalance>{ impl< P: Imbalance, N: Imbalance, - B: AtLeast32Bit + FullCodec + Copy + MaybeSerializeDeserialize + Debug + Default, + B: AtLeast32BitUnsigned + FullCodec + Copy + MaybeSerializeDeserialize + Debug + Default, > SignedImbalance { pub fn zero() -> Self { SignedImbalance::Positive(P::zero()) @@ -584,7 +855,8 @@ impl< /// Abstraction over a fungible assets system. pub trait Currency { /// The balance of an account. - type Balance: AtLeast32Bit + FullCodec + Copy + MaybeSerializeDeserialize + Debug + Default; + type Balance: AtLeast32BitUnsigned + FullCodec + Copy + MaybeSerializeDeserialize + Debug + + Default; /// The opaque token type for an imbalance. This is returned by unbalanced operations /// and must be dealt with. It may be dropped but cannot be cloned. @@ -752,6 +1024,7 @@ pub trait Currency { } /// Status of funds. +#[derive(PartialEq, Eq, Clone, Copy, Encode, Decode, RuntimeDebug)] pub enum BalanceStatus { /// Funds are free, as corresponding to `free` item in Balances. Free, @@ -1158,11 +1431,19 @@ impl OnInitialize for Tuple { } } -/// The runtime upgrade trait. Implementing this lets you express what should happen -/// when the runtime upgrades, and changes may need to occur to your module. +/// The runtime upgrade trait. +/// +/// Implementing this lets you express what should happen when the runtime upgrades, +/// and changes may need to occur to your module. pub trait OnRuntimeUpgrade { /// Perform a module upgrade. /// + /// # Warning + /// + /// This function will be called before we initialized any runtime state, aka `on_initialize` + /// wasn't called yet. So, information like the block number and any other + /// block local data are not accessible. + /// /// Return the non-negotiable weight consumed for runtime upgrade. fn on_runtime_upgrade() -> crate::weights::Weight { 0 } } @@ -1211,12 +1492,12 @@ pub mod schedule { /// The highest priority. We invert the value so that normal sorting will place the highest /// priority at the beginning of the list. - pub const HIGHEST_PRORITY: Priority = 0; + pub const HIGHEST_PRIORITY: Priority = 0; /// Anything of this value or lower will definitely be scheduled on the block that they ask for, even /// if it breaches the `MaximumWeight` limitation. pub const HARD_DEADLINE: Priority = 63; /// The lowest priority. Most stuff should be around here. - pub const LOWEST_PRORITY: Priority = 255; + pub const LOWEST_PRIORITY: Priority = 255; /// A type that can be used as a scheduler. pub trait Anon { @@ -1233,7 +1514,7 @@ pub mod schedule { maybe_periodic: Option>, priority: Priority, call: Call - ) -> Self::Address; + ) -> Result; /// Cancel a scheduled task. If periodic, then it will cancel all further instances of that, /// also. @@ -1293,6 +1574,63 @@ pub trait EnsureOrigin { fn successful_origin() -> OuterOrigin; } +/// Type that can be dispatched with an origin but without checking the origin filter. +/// +/// Implemented for pallet dispatchable type by `decl_module` and for runtime dispatchable by +/// `construct_runtime` and `impl_outer_dispatch`. +pub trait UnfilteredDispatchable { + /// The origin type of the runtime, (i.e. `frame_system::Trait::Origin`). + type Origin; + + /// Dispatch this call but do not check the filter in origin. + fn dispatch_bypass_filter(self, origin: Self::Origin) -> crate::dispatch::DispatchResultWithPostInfo; +} + +/// Methods available on `frame_system::Trait::Origin`. +pub trait OriginTrait: Sized { + /// Runtime call type, as in `frame_system::Trait::Call` + type Call; + + /// The caller origin, overarching type of all pallets origins. + type PalletsOrigin; + + /// Add a filter to the origin. + fn add_filter(&mut self, filter: impl Fn(&Self::Call) -> bool + 'static); + + /// Reset origin filters to default one, i.e `frame_system::Trait::BaseCallFilter`. + fn reset_filter(&mut self); + + /// Replace the caller with caller from the other origin + fn set_caller_from(&mut self, other: impl Into); + + /// Filter the call, if false then call is filtered out. + fn filter_call(&self, call: &Self::Call) -> bool; +} + +/// Trait to be used when types are exactly same. +/// +/// This allow to convert back and forth from type, a reference and a mutable reference. +pub trait IsType: Into + From { + /// Cast reference. + fn from_ref(t: &T) -> &Self; + + /// Cast reference. + fn into_ref(&self) -> &T; + + /// Cast mutable reference. + fn from_mut(t: &mut T) -> &mut Self; + + /// Cast mutable reference. + fn into_mut(&mut self) -> &mut T; +} + +impl IsType for T { + fn from_ref(t: &T) -> &Self { t } + fn into_ref(&self) -> &T { self } + fn from_mut(t: &mut T) -> &mut Self { t } + fn into_mut(&mut self) -> &mut T { self } +} + #[cfg(test)] mod tests { use super::*; diff --git a/frame/support/src/weights.rs b/frame/support/src/weights.rs index 27f2aef586f526db87c577643f0c43643a6a0bfc..f614bc4706d87aee7f269a350037e37a87e42cf9 100644 --- a/frame/support/src/weights.rs +++ b/frame/support/src/weights.rs @@ -134,7 +134,10 @@ use sp_runtime::{ traits::SignedExtension, generic::{CheckedExtrinsic, UncheckedExtrinsic}, }; -use crate::dispatch::{DispatchErrorWithPostInfo, DispatchError}; +use crate::dispatch::{DispatchErrorWithPostInfo, DispatchResultWithPostInfo, DispatchError}; +use sp_runtime::traits::SaturatedConversion; +use sp_arithmetic::{Perbill, traits::{BaseArithmetic, Saturating, Unsigned}}; +use smallvec::{smallvec, SmallVec}; /// Re-export priority as type pub use sp_runtime::transaction_validity::TransactionPriority; @@ -269,18 +272,27 @@ pub struct PostDispatchInfo { impl PostDispatchInfo { /// Calculate how much (if any) weight was not used by the `Dispatchable`. pub fn calc_unspent(&self, info: &DispatchInfo) -> Weight { + info.weight - self.calc_actual_weight(info) + } + + /// Calculate how much weight was actually spent by the `Dispatchable`. + pub fn calc_actual_weight(&self, info: &DispatchInfo) -> Weight { if let Some(actual_weight) = self.actual_weight { - if actual_weight >= info.weight { - 0 - } else { - info.weight - actual_weight - } + actual_weight.min(info.weight) } else { - 0 + info.weight } } } +/// Extract the actual weight from a dispatch result if any or fall back to the default weight. +pub fn extract_actual_weight(result: &DispatchResultWithPostInfo, info: &DispatchInfo) -> Weight { + match result { + Ok(post_info) => &post_info, + Err(err) => &err.post_info, + }.calc_actual_weight(info) +} + impl From> for PostDispatchInfo { fn from(actual_weight: Option) -> Self { Self { @@ -413,9 +425,11 @@ impl PaysFee for (Weight, Pays) { /// with the same argument list as the dispatched, wrapped in a tuple. /// - `PF`: a `Pays` variant for whether this dispatch pays fee or not or a closure that /// returns a `Pays` variant with the same argument list as the dispatched, wrapped in a tuple. +#[deprecated = "Function arguments are available directly inside the annotation now."] pub struct FunctionOf(pub WD, pub CD, pub PF); // `WeighData` as a raw value +#[allow(deprecated)] impl WeighData for FunctionOf { fn weigh_data(&self, _: Args) -> Weight { self.0 @@ -423,6 +437,7 @@ impl WeighData for FunctionOf { } // `WeighData` as a closure +#[allow(deprecated)] impl WeighData for FunctionOf where WD : Fn(Args) -> Weight { @@ -432,6 +447,7 @@ impl WeighData for FunctionOf where } // `ClassifyDispatch` as a raw value +#[allow(deprecated)] impl ClassifyDispatch for FunctionOf { fn classify_dispatch(&self, _: Args) -> DispatchClass { self.1 @@ -439,6 +455,7 @@ impl ClassifyDispatch for FunctionOf } // `ClassifyDispatch` as a raw value +#[allow(deprecated)] impl ClassifyDispatch for FunctionOf where CD : Fn(Args) -> DispatchClass { @@ -448,6 +465,7 @@ impl ClassifyDispatch for FunctionOf where } // `PaysFee` as a raw value +#[allow(deprecated)] impl PaysFee for FunctionOf { fn pays_fee(&self, _: Args) -> Pays { self.2 @@ -455,6 +473,7 @@ impl PaysFee for FunctionOf { } // `PaysFee` as a closure +#[allow(deprecated)] impl PaysFee for FunctionOf where PF : Fn(Args) -> Pays { @@ -522,6 +541,90 @@ impl RuntimeDbWeight { } } +/// One coefficient and its position in the `WeightToFeePolynomial`. +/// +/// One term of polynomial is calculated as: +/// +/// ```ignore +/// coeff_integer * x^(degree) + coeff_frac * x^(degree) +/// ``` +/// +/// The `negative` value encodes whether the term is added or substracted from the +/// overall polynomial result. +#[derive(Clone, Encode, Decode)] +pub struct WeightToFeeCoefficient { + /// The integral part of the coefficient. + pub coeff_integer: Balance, + /// The fractional part of the coefficient. + pub coeff_frac: Perbill, + /// True iff the coefficient should be interpreted as negative. + pub negative: bool, + /// Degree/exponent of the term. + pub degree: u8, +} + +/// A list of coefficients that represent one polynomial. +pub type WeightToFeeCoefficients = SmallVec<[WeightToFeeCoefficient; 4]>; + +/// A trait that describes the weight to fee calculation as polynomial. +/// +/// An implementor should only implement the `polynomial` function. +pub trait WeightToFeePolynomial { + /// The type that is returned as result from polynomial evaluation. + type Balance: BaseArithmetic + From + Copy + Unsigned; + + /// Returns a polynomial that describes the weight to fee conversion. + /// + /// This is the only function that should be manually implemented. Please note + /// that all calculation is done in the probably unsigned `Balance` type. This means + /// that the order of coefficients is important as putting the negative coefficients + /// first will most likely saturate the result to zero mid evaluation. + fn polynomial() -> WeightToFeeCoefficients; + + /// Calculates the fee from the passed `weight` according to the `polynomial`. + /// + /// This should not be overriden in most circumstances. Calculation is done in the + /// `Balance` type and never overflows. All evaluation is saturating. + fn calc(weight: &Weight) -> Self::Balance { + Self::polynomial().iter().fold(Self::Balance::saturated_from(0u32), |mut acc, args| { + let w = Self::Balance::saturated_from(*weight).saturating_pow(args.degree.into()); + + // The sum could get negative. Therefore we only sum with the accumulator. + // The Perbill Mul implementation is non overflowing. + let frac = args.coeff_frac * w; + let integer = args.coeff_integer.saturating_mul(w); + + if args.negative { + acc = acc.saturating_sub(frac); + acc = acc.saturating_sub(integer); + } else { + acc = acc.saturating_add(frac); + acc = acc.saturating_add(integer); + } + + acc + }) + } +} + +/// Implementor of `WeightToFeePolynomial` that maps one unit of weight to one unit of fee. +pub struct IdentityFee(sp_std::marker::PhantomData); + +impl WeightToFeePolynomial for IdentityFee where + T: BaseArithmetic + From + Copy + Unsigned +{ + type Balance = T; + + fn polynomial() -> WeightToFeeCoefficients { + smallvec!(WeightToFeeCoefficient { + coeff_integer: 1u32.into(), + coeff_frac: Perbill::zero(), + negative: false, + degree: 1, + }) + } +} + #[cfg(test)] #[allow(dead_code)] mod tests { @@ -567,10 +670,10 @@ mod tests { fn f03(_origin) { unimplemented!(); } // weight = a x 10 + b - #[weight = FunctionOf(|args: (&u32, &u32)| (args.0 * 10 + args.1) as Weight, DispatchClass::Normal, Pays::Yes)] + #[weight = ((_a * 10 + _eb * 1) as Weight, DispatchClass::Normal, Pays::Yes)] fn f11(_origin, _a: u32, _eb: u32) { unimplemented!(); } - #[weight = FunctionOf(|_: (&u32, &u32)| 0, DispatchClass::Operational, Pays::Yes)] + #[weight = (0, DispatchClass::Operational, Pays::Yes)] fn f12(_origin, _a: u32, _eb: u32) { unimplemented!(); } #[weight = T::DbWeight::get().reads(3) + T::DbWeight::get().writes(2) + 10_000] @@ -616,4 +719,91 @@ mod tests { assert_eq!(Call::::f21().get_dispatch_info().weight, 45600); assert_eq!(Call::::f2().get_dispatch_info().class, DispatchClass::Normal); } + + #[test] + fn extract_actual_weight_works() { + let pre = DispatchInfo { + weight: 1000, + .. Default::default() + }; + assert_eq!(extract_actual_weight(&Ok(Some(7).into()), &pre), 7); + assert_eq!(extract_actual_weight(&Ok(Some(1000).into()), &pre), 1000); + assert_eq!( + extract_actual_weight(&Err(DispatchError::BadOrigin.with_weight(9)), &pre), + 9 + ); + } + + #[test] + fn extract_actual_weight_caps_at_pre_weight() { + let pre = DispatchInfo { + weight: 1000, + .. Default::default() + }; + assert_eq!(extract_actual_weight(&Ok(Some(1250).into()), &pre), 1000); + assert_eq!( + extract_actual_weight(&Err(DispatchError::BadOrigin.with_weight(1300)), &pre), + 1000 + ); + } + + type Balance = u64; + + // 0.5x^3 + 2.333x2 + 7x - 10_000 + struct Poly; + impl WeightToFeePolynomial for Poly { + type Balance = Balance; + + fn polynomial() -> WeightToFeeCoefficients { + smallvec![ + WeightToFeeCoefficient { + coeff_integer: 0, + coeff_frac: Perbill::from_fraction(0.5), + negative: false, + degree: 3 + }, + WeightToFeeCoefficient { + coeff_integer: 2, + coeff_frac: Perbill::from_rational_approximation(1u32, 3u32), + negative: false, + degree: 2 + }, + WeightToFeeCoefficient { + coeff_integer: 7, + coeff_frac: Perbill::zero(), + negative: false, + degree: 1 + }, + WeightToFeeCoefficient { + coeff_integer: 10_000, + coeff_frac: Perbill::zero(), + negative: true, + degree: 0 + }, + ] + } + } + + #[test] + fn polynomial_works() { + assert_eq!(Poly::calc(&100), 514033); + assert_eq!(Poly::calc(&10_123), 518917034928); + } + + #[test] + fn polynomial_does_not_underflow() { + assert_eq!(Poly::calc(&0), 0); + } + + #[test] + fn polynomial_does_not_overflow() { + assert_eq!(Poly::calc(&Weight::max_value()), Balance::max_value() - 10_000); + } + + #[test] + fn identity_fee_works() { + assert_eq!(IdentityFee::::calc(&0), 0); + assert_eq!(IdentityFee::::calc(&50), 50); + assert_eq!(IdentityFee::::calc(&Weight::max_value()), Balance::max_value()); + } } diff --git a/frame/support/test/Cargo.toml b/frame/support/test/Cargo.toml index 67e00f3d8d34d508a1e9f0e3d2413397acde3627..682001564bda22987412e6e2768ac15f383be105 100644 --- a/frame/support/test/Cargo.toml +++ b/frame/support/test/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-support-test" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,13 +13,14 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", default-features = false, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-io ={ path = "../../../primitives/io", default-features = false , version = "2.0.0-alpha.8"} -sp-state-machine = { version = "0.8.0-alpha.8", optional = true, path = "../../../primitives/state-machine" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../" } -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/inherents" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/runtime" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/core" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-io ={ version = "2.0.0-rc4", path = "../../../primitives/io", default-features = false } +sp-state-machine = { version = "0.8.0-rc4", optional = true, path = "../../../primitives/state-machine" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../" } +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/inherents" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/core" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/std" } trybuild = "1.0.17" pretty_assertions = "0.6.1" rustversion = "1.0.0" @@ -33,6 +34,7 @@ std = [ "frame-support/std", "sp-inherents/std", "sp-core/std", + "sp-std/std", "sp-runtime/std", "sp-state-machine", ] diff --git a/frame/support/test/tests/decl_error.rs b/frame/support/test/tests/construct_runtime.rs similarity index 86% rename from frame/support/test/tests/decl_error.rs rename to frame/support/test/tests/construct_runtime.rs index 9536d4e8195d4830bc00144985902961fca7ea59..10fc3319fb080fb34d70fd0dcddc3b991c4290ff 100644 --- a/frame/support/test/tests/decl_error.rs +++ b/frame/support/test/tests/construct_runtime.rs @@ -15,16 +15,24 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! General tests for construct_runtime macro, test for: +//! * error declareed with decl_error works +//! * integrity test is generated + #![recursion_limit="128"] use sp_runtime::{generic, traits::{BlakeTwo256, Block as _, Verify}, DispatchError}; use sp_core::{H256, sr25519}; - +use sp_std::cell::RefCell; mod system; pub trait Currency {} +thread_local! { + pub static INTEGRITY_TEST_EXEC: RefCell = RefCell::new(0); +} + mod module1 { use super::*; @@ -65,6 +73,10 @@ mod module2 { pub fn fail(_origin) -> frame_support::dispatch::DispatchResult { Err(Error::::Something.into()) } + + fn integrity_test() { + INTEGRITY_TEST_EXEC.with(|i| *i.borrow_mut() += 1); + } } } @@ -89,12 +101,14 @@ pub type BlockNumber = u64; pub type Index = u64; impl system::Trait for Runtime { + type BaseCallFilter = (); type Hash = H256; type Origin = Origin; type BlockNumber = BlockNumber; type AccountId = AccountId; type Event = Event; type ModuleToIndex = ModuleToIndex; + type Call = Call; } frame_support::construct_runtime!( @@ -137,3 +151,9 @@ fn check_module2_error_type() { Err(DispatchError::Module { index: 2, error: 0, message: Some("Something") }), ); } + +#[test] +fn integrity_test_works() { + __construct_runtime_integrity_test::runtime_integrity_tests(); + assert_eq!(INTEGRITY_TEST_EXEC.with(|i| *i.borrow()), 1); +} diff --git a/frame/support/test/tests/decl_module_ui.rs b/frame/support/test/tests/decl_module_ui.rs new file mode 100644 index 0000000000000000000000000000000000000000..90d105e7cfae31a795125484bc4ce1012556e401 --- /dev/null +++ b/frame/support/test/tests/decl_module_ui.rs @@ -0,0 +1,26 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//#[rustversion::attr(not(stable), ignore)] +#[test] +fn decl_module_ui() { + // As trybuild is using `cargo check`, we don't need the real WASM binaries. + std::env::set_var("BUILD_DUMMY_WASM_BINARY", "1"); + + let t = trybuild::TestCases::new(); + t.compile_fail("tests/decl_module_ui/*.rs"); +} diff --git a/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.rs b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.rs new file mode 100644 index 0000000000000000000000000000000000000000..4dbae05f07ff77869d3f7c6367079138d7cbf73f --- /dev/null +++ b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.rs @@ -0,0 +1,7 @@ +frame_support::decl_module! { + pub struct Module for enum Call where origin: T::Origin { + fn integrity_test() {} + + fn integrity_test() {} + } +} diff --git a/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.stderr b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.stderr new file mode 100644 index 0000000000000000000000000000000000000000..d6498961d31c8006ec5f37218f9ff16b19fc80e4 --- /dev/null +++ b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.stderr @@ -0,0 +1,25 @@ +error: `integrity_test` can only be passed once as input. + --> $DIR/reserved_keyword_two_times_integrity_test.rs:1:1 + | +1 | / frame_support::decl_module! { +2 | | pub struct Module for enum Call where origin: T::Origin { +3 | | fn integrity_test() {} +4 | | +5 | | fn integrity_test() {} +6 | | } +7 | | } + | |_^ + | + = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0601]: `main` function not found in crate `$CRATE` + --> $DIR/reserved_keyword_two_times_integrity_test.rs:1:1 + | +1 | / frame_support::decl_module! { +2 | | pub struct Module for enum Call where origin: T::Origin { +3 | | fn integrity_test() {} +4 | | +5 | | fn integrity_test() {} +6 | | } +7 | | } + | |_^ consider adding a `main` function to `$DIR/tests/decl_module_ui/reserved_keyword_two_times_integrity_test.rs` diff --git a/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_on_initialize.rs b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_on_initialize.rs new file mode 100644 index 0000000000000000000000000000000000000000..4f05134997e81e380010ff74a462c81f8833ac4d --- /dev/null +++ b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_on_initialize.rs @@ -0,0 +1,11 @@ +frame_support::decl_module! { + pub struct Module for enum Call where origin: T::Origin { + fn on_initialize() -> Weight { + 0 + } + + fn on_initialize() -> Weight { + 0 + } + } +} diff --git a/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_on_initialize.stderr b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_on_initialize.stderr new file mode 100644 index 0000000000000000000000000000000000000000..8a9f025046b7ee57323afd3c6faa0b2c6d5cda43 --- /dev/null +++ b/frame/support/test/tests/decl_module_ui/reserved_keyword_two_times_on_initialize.stderr @@ -0,0 +1,25 @@ +error: `on_initialize` can only be passed once as input. + --> $DIR/reserved_keyword_two_times_on_initialize.rs:1:1 + | +1 | / frame_support::decl_module! { +2 | | pub struct Module for enum Call where origin: T::Origin { +3 | | fn on_initialize() -> Weight { +4 | | 0 +... | +10 | | } +11 | | } + | |_^ + | + = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0601]: `main` function not found in crate `$CRATE` + --> $DIR/reserved_keyword_two_times_on_initialize.rs:1:1 + | +1 | / frame_support::decl_module! { +2 | | pub struct Module for enum Call where origin: T::Origin { +3 | | fn on_initialize() -> Weight { +4 | | 0 +... | +10 | | } +11 | | } + | |_^ consider adding a `main` function to `$DIR/tests/decl_module_ui/reserved_keyword_two_times_on_initialize.rs` diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index 45e280902a268b0d0a3c8d4d7e8e53ac846f9784..920554346f73ef3f2e4e2dfc4dc8991218412017 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -233,12 +233,14 @@ pub type BlockNumber = u64; pub type Index = u64; impl system::Trait for Runtime { + type BaseCallFilter= (); type Hash = H256; type Origin = Origin; type BlockNumber = BlockNumber; type AccountId = AccountId; type Event = Event; type ModuleToIndex = (); + type Call = Call; } frame_support::construct_runtime!( diff --git a/frame/support/test/tests/issue2219.rs b/frame/support/test/tests/issue2219.rs index cd357ba2667cb18c096a61be827311badb4b8c34..7166f202c7325d2aed5a4a27ce1daaf08ad24398 100644 --- a/frame/support/test/tests/issue2219.rs +++ b/frame/support/test/tests/issue2219.rs @@ -158,12 +158,14 @@ pub type Block = generic::Block; pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; impl system::Trait for Runtime { + type BaseCallFilter = (); type Hash = H256; type Origin = Origin; type BlockNumber = BlockNumber; type AccountId = AccountId; type Event = Event; type ModuleToIndex = (); + type Call = Call; } impl module::Trait for Runtime {} diff --git a/frame/support/test/tests/storage_transaction.rs b/frame/support/test/tests/storage_transaction.rs new file mode 100644 index 0000000000000000000000000000000000000000..bf6c70966b469ebc0d0e64ab2a231a8d6c319d32 --- /dev/null +++ b/frame/support/test/tests/storage_transaction.rs @@ -0,0 +1,159 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use codec::{Encode, Decode, EncodeLike}; +use frame_support::{ + StorageMap, StorageValue, storage::{with_transaction, TransactionOutcome::*}, +}; +use sp_io::TestExternalities; + +pub trait Trait { + type Origin; + type BlockNumber: Encode + Decode + EncodeLike + Default + Clone; +} + +frame_support::decl_module! { + pub struct Module for enum Call where origin: T::Origin {} +} + +frame_support::decl_storage!{ + trait Store for Module as StorageTransactions { + pub Value: u32; + pub Map: map hasher(twox_64_concat) String => u32; + } +} + + +#[test] +fn storage_transaction_basic_commit() { + TestExternalities::default().execute_with(|| { + + assert_eq!(Value::get(), 0); + assert!(!Map::contains_key("val0")); + + with_transaction(|| { + Value::set(99); + Map::insert("val0", 99); + assert_eq!(Value::get(), 99); + assert_eq!(Map::get("val0"), 99); + Commit(()) + }); + + assert_eq!(Value::get(), 99); + assert_eq!(Map::get("val0"), 99); + }); +} + +#[test] +fn storage_transaction_basic_rollback() { + TestExternalities::default().execute_with(|| { + + assert_eq!(Value::get(), 0); + assert_eq!(Map::get("val0"), 0); + + with_transaction(|| { + Value::set(99); + Map::insert("val0", 99); + assert_eq!(Value::get(), 99); + assert_eq!(Map::get("val0"), 99); + Rollback(()) + }); + + assert_eq!(Value::get(), 0); + assert_eq!(Map::get("val0"), 0); + }); +} + +#[test] +fn storage_transaction_rollback_then_commit() { + TestExternalities::default().execute_with(|| { + Value::set(1); + Map::insert("val1", 1); + + with_transaction(|| { + Value::set(2); + Map::insert("val1", 2); + Map::insert("val2", 2); + + with_transaction(|| { + Value::set(3); + Map::insert("val1", 3); + Map::insert("val2", 3); + Map::insert("val3", 3); + + assert_eq!(Value::get(), 3); + assert_eq!(Map::get("val1"), 3); + assert_eq!(Map::get("val2"), 3); + assert_eq!(Map::get("val3"), 3); + + Rollback(()) + }); + + assert_eq!(Value::get(), 2); + assert_eq!(Map::get("val1"), 2); + assert_eq!(Map::get("val2"), 2); + assert_eq!(Map::get("val3"), 0); + + Commit(()) + }); + + assert_eq!(Value::get(), 2); + assert_eq!(Map::get("val1"), 2); + assert_eq!(Map::get("val2"), 2); + assert_eq!(Map::get("val3"), 0); + }); +} + +#[test] +fn storage_transaction_commit_then_rollback() { + TestExternalities::default().execute_with(|| { + Value::set(1); + Map::insert("val1", 1); + + with_transaction(|| { + Value::set(2); + Map::insert("val1", 2); + Map::insert("val2", 2); + + with_transaction(|| { + Value::set(3); + Map::insert("val1", 3); + Map::insert("val2", 3); + Map::insert("val3", 3); + + assert_eq!(Value::get(), 3); + assert_eq!(Map::get("val1"), 3); + assert_eq!(Map::get("val2"), 3); + assert_eq!(Map::get("val3"), 3); + + Commit(()) + }); + + assert_eq!(Value::get(), 3); + assert_eq!(Map::get("val1"), 3); + assert_eq!(Map::get("val2"), 3); + assert_eq!(Map::get("val3"), 3); + + Rollback(()) + }); + + assert_eq!(Value::get(), 1); + assert_eq!(Map::get("val1"), 1); + assert_eq!(Map::get("val2"), 0); + assert_eq!(Map::get("val3"), 0); + }); +} diff --git a/frame/support/test/tests/system.rs b/frame/support/test/tests/system.rs index 821224d0a29cfd2ed7fa6c3f264d9f007832ede9..0d6a22fd1a3e199bb9c12f43eb99e01e9a4978ae 100644 --- a/frame/support/test/tests/system.rs +++ b/frame/support/test/tests/system.rs @@ -21,15 +21,17 @@ pub trait Trait: 'static + Eq + Clone { type Origin: Into, Self::Origin>> + From>; + type BaseCallFilter: frame_support::traits::Filter; type BlockNumber: Decode + Encode + EncodeLike + Clone + Default; type Hash; type AccountId: Encode + EncodeLike + Decode; + type Call; type Event: From>; type ModuleToIndex: frame_support::traits::ModuleToIndex; } frame_support::decl_module! { - pub struct Module for enum Call where origin: T::Origin {} + pub struct Module for enum Call where origin: T::Origin, {} } impl Module { diff --git a/frame/system/Cargo.toml b/frame/system/Cargo.toml index 9b4424d56a8e84319bf0cecb55b61bc4646580e6..2173ea8cee445719d9f53cb1f2c33b34c41e01bf 100644 --- a/frame/system/Cargo.toml +++ b/frame/system/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-system" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,19 +13,19 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { path = "../../primitives/io", default-features = false, version = "2.0.0-alpha.8"} -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-version = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/version" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", path = "../../primitives/io", default-features = false } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-version = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/version" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } impl-trait-for-tuples = "0.1.3" [dev-dependencies] criterion = "0.2.11" -sp-externalities = { version = "0.8.0-alpha.8", path = "../../primitives/externalities" } -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../test-utils/runtime/client" } +sp-externalities = { version = "0.8.0-rc4", path = "../../primitives/externalities" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../test-utils/runtime/client" } [features] default = ["std"] diff --git a/frame/system/benches/bench.rs b/frame/system/benches/bench.rs index 6c25e5d1c9775d81404e41f3e23ad905175108b0..56fd4b8c35200d2ff8cbb9721314bd0e78329af7 100644 --- a/frame/system/benches/bench.rs +++ b/frame/system/benches/bench.rs @@ -61,6 +61,7 @@ frame_support::parameter_types! { #[derive(Clone, Eq, PartialEq)] pub struct Runtime; impl system::Trait for Runtime { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -76,6 +77,7 @@ impl system::Trait for Runtime { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/system/benchmarking/Cargo.toml b/frame/system/benchmarking/Cargo.toml index 25a9e8cf86ab8059920f864bdef284326b57341d..c278bad150eae6a9c5a3c5c1aa3c8ce57285a798 100644 --- a/frame/system/benchmarking/Cargo.toml +++ b/frame/system/benchmarking/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-system-benchmarking" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,17 +12,17 @@ description = "FRAME System benchmarking" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/runtime" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../../benchmarking" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../../system" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../../support" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../../primitives/core" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/runtime" } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../../benchmarking" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../../system" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../../support" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../../primitives/core" } [dev-dependencies] serde = { version = "1.0.101" } -sp-io ={ path = "../../../primitives/io", version = "2.0.0-alpha.8"} +sp-io ={ version = "2.0.0-rc4", path = "../../../primitives/io" } [features] default = ["std"] diff --git a/frame/system/benchmarking/src/mock.rs b/frame/system/benchmarking/src/mock.rs index ed65f4df93471cc6a4226ac8b302710052b39c3f..9e41ff20164cddedde4552c56bdfe0cea32f5ee3 100644 --- a/frame/system/benchmarking/src/mock.rs +++ b/frame/system/benchmarking/src/mock.rs @@ -51,6 +51,7 @@ impl Dispatchable for Call { pub struct Test; impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = AccountIndex; type BlockNumber = BlockNumber; @@ -66,6 +67,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = (); type AvailableBlockRatio = (); type MaximumBlockLength = (); type Version = (); diff --git a/frame/system/rpc/runtime-api/Cargo.toml b/frame/system/rpc/runtime-api/Cargo.toml index 5d2811abcd0af3f7b56642fd8847dc7ad7ca3a74..8d340ad7defb81d23f3309b4872b915be65bf31b 100644 --- a/frame/system/rpc/runtime-api/Cargo.toml +++ b/frame/system/rpc/runtime-api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-system-rpc-runtime-api" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,8 +12,8 @@ description = "Runtime API definition required by System RPC extensions." targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-api = { version = "2.0.0-alpha.8", default-features = false, path = "../../../../primitives/api" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } +sp-api = { version = "2.0.0-rc4", default-features = false, path = "../../../../primitives/api" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } [features] default = ["std"] diff --git a/frame/system/src/accounts.scale b/frame/system/src/accounts.scale deleted file mode 100644 index bfd7e6277f20c53ceb1d664235bb6af18ec538ed..0000000000000000000000000000000000000000 Binary files a/frame/system/src/accounts.scale and /dev/null differ diff --git a/frame/system/src/extensions/check_genesis.rs b/frame/system/src/extensions/check_genesis.rs new file mode 100644 index 0000000000000000000000000000000000000000..d0a346519ca23b8661cd92285ec2d72e5753b7a9 --- /dev/null +++ b/frame/system/src/extensions/check_genesis.rs @@ -0,0 +1,58 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use codec::{Encode, Decode}; +use crate::{Trait, Module}; +use sp_runtime::{ + traits::{SignedExtension, Zero}, + transaction_validity::TransactionValidityError, +}; + +/// Genesis hash check to provide replay protection between different networks. +#[derive(Encode, Decode, Clone, Eq, PartialEq)] +pub struct CheckGenesis(sp_std::marker::PhantomData); + +impl sp_std::fmt::Debug for CheckGenesis { + #[cfg(feature = "std")] + fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + write!(f, "CheckGenesis") + } + + #[cfg(not(feature = "std"))] + fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + Ok(()) + } +} + +impl CheckGenesis { + /// Creates new `SignedExtension` to check genesis hash. + pub fn new() -> Self { + Self(sp_std::marker::PhantomData) + } +} + +impl SignedExtension for CheckGenesis { + type AccountId = T::AccountId; + type Call = ::Call; + type AdditionalSigned = T::Hash; + type Pre = (); + const IDENTIFIER: &'static str = "CheckGenesis"; + + fn additional_signed(&self) -> Result { + Ok(>::block_hash(T::BlockNumber::zero())) + } +} diff --git a/frame/system/src/extensions/check_mortality.rs b/frame/system/src/extensions/check_mortality.rs new file mode 100644 index 0000000000000000000000000000000000000000..cc7496df9a20ab32db2bd162d9142d245ba7dc73 --- /dev/null +++ b/frame/system/src/extensions/check_mortality.rs @@ -0,0 +1,124 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use codec::{Encode, Decode}; +use crate::{Trait, Module, BlockHash}; +use frame_support::StorageMap; +use sp_runtime::{ + generic::Era, + traits::{SignedExtension, DispatchInfoOf, SaturatedConversion}, + transaction_validity::{ + ValidTransaction, TransactionValidityError, InvalidTransaction, TransactionValidity, + }, +}; + +/// Check for transaction mortality. +#[derive(Encode, Decode, Clone, Eq, PartialEq)] +pub struct CheckMortality(Era, sp_std::marker::PhantomData); + +impl CheckMortality { + /// utility constructor. Used only in client/factory code. + pub fn from(era: Era) -> Self { + Self(era, sp_std::marker::PhantomData) + } +} + +impl sp_std::fmt::Debug for CheckMortality { + #[cfg(feature = "std")] + fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + write!(f, "CheckMortality({:?})", self.0) + } + + #[cfg(not(feature = "std"))] + fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + Ok(()) + } +} + +impl SignedExtension for CheckMortality { + type AccountId = T::AccountId; + type Call = T::Call; + type AdditionalSigned = T::Hash; + type Pre = (); + // TODO [#6483] rename to CheckMortality + const IDENTIFIER: &'static str = "CheckEra"; + + fn validate( + &self, + _who: &Self::AccountId, + _call: &Self::Call, + _info: &DispatchInfoOf, + _len: usize, + ) -> TransactionValidity { + let current_u64 = >::block_number().saturated_into::(); + let valid_till = self.0.death(current_u64); + Ok(ValidTransaction { + longevity: valid_till.saturating_sub(current_u64), + ..Default::default() + }) + } + + fn additional_signed(&self) -> Result { + let current_u64 = >::block_number().saturated_into::(); + let n = self.0.birth(current_u64).saturated_into::(); + if !>::contains_key(n) { + Err(InvalidTransaction::AncientBirthBlock.into()) + } else { + Ok(>::block_hash(n)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mock::{Test, new_test_ext, System, CALL}; + use frame_support::weights::{DispatchClass, DispatchInfo, Pays}; + use sp_core::H256; + + #[test] + fn signed_ext_check_era_should_work() { + new_test_ext().execute_with(|| { + // future + assert_eq!( + CheckMortality::::from(Era::mortal(4, 2)).additional_signed().err().unwrap(), + InvalidTransaction::AncientBirthBlock.into(), + ); + + // correct + System::set_block_number(13); + >::insert(12, H256::repeat_byte(1)); + assert!(CheckMortality::::from(Era::mortal(4, 12)).additional_signed().is_ok()); + }) + } + + #[test] + fn signed_ext_check_era_should_change_longevity() { + new_test_ext().execute_with(|| { + let normal = DispatchInfo { weight: 100, class: DispatchClass::Normal, pays_fee: Pays::Yes }; + let len = 0_usize; + let ext = ( + crate::CheckWeight::::default(), + CheckMortality::::from(Era::mortal(16, 256)), + ); + System::set_block_number(17); + >::insert(16, H256::repeat_byte(1)); + + assert_eq!(ext.validate(&1, CALL, &normal, len).unwrap().longevity, 15); + }) + } +} diff --git a/frame/system/src/extensions/check_nonce.rs b/frame/system/src/extensions/check_nonce.rs new file mode 100644 index 0000000000000000000000000000000000000000..1af3a1210aaf417c96019f03b43944184d435e22 --- /dev/null +++ b/frame/system/src/extensions/check_nonce.rs @@ -0,0 +1,145 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use codec::{Encode, Decode}; +use crate::Trait; +use frame_support::{ + weights::DispatchInfo, + StorageMap, +}; +use sp_runtime::{ + traits::{SignedExtension, DispatchInfoOf, Dispatchable, One}, + transaction_validity::{ + ValidTransaction, TransactionValidityError, InvalidTransaction, TransactionValidity, + TransactionLongevity, TransactionPriority, + }, +}; +use sp_std::vec; + +/// Nonce check and increment to give replay protection for transactions. +#[derive(Encode, Decode, Clone, Eq, PartialEq)] +pub struct CheckNonce(#[codec(compact)] T::Index); + +impl CheckNonce { + /// utility constructor. Used only in client/factory code. + pub fn from(nonce: T::Index) -> Self { + Self(nonce) + } +} + +impl sp_std::fmt::Debug for CheckNonce { + #[cfg(feature = "std")] + fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + write!(f, "CheckNonce({})", self.0) + } + + #[cfg(not(feature = "std"))] + fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + Ok(()) + } +} + +impl SignedExtension for CheckNonce where + T::Call: Dispatchable +{ + type AccountId = T::AccountId; + type Call = T::Call; + type AdditionalSigned = (); + type Pre = (); + const IDENTIFIER: &'static str = "CheckNonce"; + + fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } + + fn pre_dispatch( + self, + who: &Self::AccountId, + _call: &Self::Call, + _info: &DispatchInfoOf, + _len: usize, + ) -> Result<(), TransactionValidityError> { + let mut account = crate::Account::::get(who); + if self.0 != account.nonce { + return Err( + if self.0 < account.nonce { + InvalidTransaction::Stale + } else { + InvalidTransaction::Future + }.into() + ) + } + account.nonce += T::Index::one(); + crate::Account::::insert(who, account); + Ok(()) + } + + fn validate( + &self, + who: &Self::AccountId, + _call: &Self::Call, + info: &DispatchInfoOf, + _len: usize, + ) -> TransactionValidity { + // check index + let account = crate::Account::::get(who); + if self.0 < account.nonce { + return InvalidTransaction::Stale.into() + } + + let provides = vec![Encode::encode(&(who, self.0))]; + let requires = if account.nonce < self.0 { + vec![Encode::encode(&(who, self.0 - One::one()))] + } else { + vec![] + }; + + Ok(ValidTransaction { + priority: info.weight as TransactionPriority, + requires, + provides, + longevity: TransactionLongevity::max_value(), + propagate: true, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mock::{Test, new_test_ext, CALL}; + + #[test] + fn signed_ext_check_nonce_works() { + new_test_ext().execute_with(|| { + crate::Account::::insert(1, crate::AccountInfo { + nonce: 1, + refcount: 0, + data: 0, + }); + let info = DispatchInfo::default(); + let len = 0_usize; + // stale + assert!(CheckNonce::(0).validate(&1, CALL, &info, len).is_err()); + assert!(CheckNonce::(0).pre_dispatch(&1, CALL, &info, len).is_err()); + // correct + assert!(CheckNonce::(1).validate(&1, CALL, &info, len).is_ok()); + assert!(CheckNonce::(1).pre_dispatch(&1, CALL, &info, len).is_ok()); + // future + assert!(CheckNonce::(5).validate(&1, CALL, &info, len).is_ok()); + assert!(CheckNonce::(5).pre_dispatch(&1, CALL, &info, len).is_err()); + }) + } +} diff --git a/frame/system/src/extensions/check_spec_version.rs b/frame/system/src/extensions/check_spec_version.rs new file mode 100644 index 0000000000000000000000000000000000000000..8dc4d8d9ceddc52893795cda924366bdec44975f --- /dev/null +++ b/frame/system/src/extensions/check_spec_version.rs @@ -0,0 +1,58 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{Trait, Module}; +use codec::{Encode, Decode}; +use sp_runtime::{ + traits::SignedExtension, + transaction_validity::TransactionValidityError, +}; + +/// Ensure the runtime version registered in the transaction is the same as at present. +#[derive(Encode, Decode, Clone, Eq, PartialEq)] +pub struct CheckSpecVersion(sp_std::marker::PhantomData); + +impl sp_std::fmt::Debug for CheckSpecVersion { + #[cfg(feature = "std")] + fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + write!(f, "CheckSpecVersion") + } + + #[cfg(not(feature = "std"))] + fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + Ok(()) + } +} + +impl CheckSpecVersion { + /// Create new `SignedExtension` to check runtime version. + pub fn new() -> Self { + Self(sp_std::marker::PhantomData) + } +} + +impl SignedExtension for CheckSpecVersion { + type AccountId = T::AccountId; + type Call = ::Call; + type AdditionalSigned = u32; + type Pre = (); + const IDENTIFIER: &'static str = "CheckSpecVersion"; + + fn additional_signed(&self) -> Result { + Ok(>::runtime_version().spec_version) + } +} diff --git a/frame/system/src/extensions/check_tx_version.rs b/frame/system/src/extensions/check_tx_version.rs new file mode 100644 index 0000000000000000000000000000000000000000..ee6f3349365b9b9b21801eabd91ef32863a45007 --- /dev/null +++ b/frame/system/src/extensions/check_tx_version.rs @@ -0,0 +1,58 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{Trait, Module}; +use codec::{Encode, Decode}; +use sp_runtime::{ + traits::SignedExtension, + transaction_validity::TransactionValidityError, +}; + +/// Ensure the transaction version registered in the transaction is the same as at present. +#[derive(Encode, Decode, Clone, Eq, PartialEq)] +pub struct CheckTxVersion(sp_std::marker::PhantomData); + +impl sp_std::fmt::Debug for CheckTxVersion { + #[cfg(feature = "std")] + fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + write!(f, "CheckTxVersion") + } + + #[cfg(not(feature = "std"))] + fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + Ok(()) + } +} + +impl CheckTxVersion { + /// Create new `SignedExtension` to check transaction version. + pub fn new() -> Self { + Self(sp_std::marker::PhantomData) + } +} + +impl SignedExtension for CheckTxVersion { + type AccountId = T::AccountId; + type Call = ::Call; + type AdditionalSigned = u32; + type Pre = (); + const IDENTIFIER: &'static str = "CheckTxVersion"; + + fn additional_signed(&self) -> Result { + Ok(>::runtime_version().transaction_version) + } +} diff --git a/frame/system/src/extensions/check_weight.rs b/frame/system/src/extensions/check_weight.rs new file mode 100644 index 0000000000000000000000000000000000000000..d52138b1e3bbe20cc7bef9b6d1b70a04bfa70efd --- /dev/null +++ b/frame/system/src/extensions/check_weight.rs @@ -0,0 +1,644 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{Trait, Module}; +use codec::{Encode, Decode}; +use sp_runtime::{ + traits::{SignedExtension, DispatchInfoOf, Dispatchable, PostDispatchInfoOf, Printable}, + transaction_validity::{ + ValidTransaction, TransactionValidityError, InvalidTransaction, TransactionValidity, + TransactionPriority, + }, + Perbill, DispatchResult, +}; +use frame_support::{ + traits::{Get}, + weights::{PostDispatchInfo, DispatchInfo, DispatchClass}, + StorageValue, +}; + +/// Block resource (weight) limit check. +#[derive(Encode, Decode, Clone, Eq, PartialEq, Default)] +pub struct CheckWeight(sp_std::marker::PhantomData); + +impl CheckWeight where + T::Call: Dispatchable +{ + /// Get the quota ratio of each dispatch class type. This indicates that all operational and mandatory + /// dispatches can use the full capacity of any resource, while user-triggered ones can consume + /// a portion. + fn get_dispatch_limit_ratio(class: DispatchClass) -> Perbill { + match class { + DispatchClass::Operational | DispatchClass::Mandatory + => ::one(), + DispatchClass::Normal => T::AvailableBlockRatio::get(), + } + } + + /// Checks if the current extrinsic does not exceed `MaximumExtrinsicWeight` limit. + fn check_extrinsic_weight( + info: &DispatchInfoOf, + ) -> Result<(), TransactionValidityError> { + match info.class { + // Mandatory transactions are included in a block unconditionally, so + // we don't verify weight. + DispatchClass::Mandatory => Ok(()), + // Normal transactions must not exceed `MaximumExtrinsicWeight`. + DispatchClass::Normal => { + let maximum_weight = T::MaximumExtrinsicWeight::get(); + let extrinsic_weight = info.weight.saturating_add(T::ExtrinsicBaseWeight::get()); + if extrinsic_weight > maximum_weight { + Err(InvalidTransaction::ExhaustsResources.into()) + } else { + Ok(()) + } + }, + // For operational transactions we make sure it doesn't exceed + // the space alloted for `Operational` class. + DispatchClass::Operational => { + let maximum_weight = T::MaximumBlockWeight::get(); + let operational_limit = + Self::get_dispatch_limit_ratio(DispatchClass::Operational) * maximum_weight; + let operational_limit = + operational_limit.saturating_sub(T::BlockExecutionWeight::get()); + let extrinsic_weight = info.weight.saturating_add(T::ExtrinsicBaseWeight::get()); + if extrinsic_weight > operational_limit { + Err(InvalidTransaction::ExhaustsResources.into()) + } else { + Ok(()) + } + }, + } + } + + /// Checks if the current extrinsic can fit into the block with respect to block weight limits. + /// + /// Upon successes, it returns the new block weight as a `Result`. + fn check_block_weight( + info: &DispatchInfoOf, + ) -> Result { + let maximum_weight = T::MaximumBlockWeight::get(); + let mut all_weight = Module::::block_weight(); + match info.class { + // If we have a dispatch that must be included in the block, it ignores all the limits. + DispatchClass::Mandatory => { + let extrinsic_weight = info.weight.saturating_add(T::ExtrinsicBaseWeight::get()); + all_weight.add(extrinsic_weight, DispatchClass::Mandatory); + Ok(all_weight) + }, + // If we have a normal dispatch, we follow all the normal rules and limits. + DispatchClass::Normal => { + let normal_limit = Self::get_dispatch_limit_ratio(DispatchClass::Normal) * maximum_weight; + let extrinsic_weight = info.weight.checked_add(T::ExtrinsicBaseWeight::get()) + .ok_or(InvalidTransaction::ExhaustsResources)?; + all_weight.checked_add(extrinsic_weight, DispatchClass::Normal) + .map_err(|_| InvalidTransaction::ExhaustsResources)?; + if all_weight.get(DispatchClass::Normal) > normal_limit { + Err(InvalidTransaction::ExhaustsResources.into()) + } else { + Ok(all_weight) + } + }, + // If we have an operational dispatch, allow it if we have not used our full + // "operational space" (independent of existing fullness). + DispatchClass::Operational => { + let operational_limit = Self::get_dispatch_limit_ratio(DispatchClass::Operational) * maximum_weight; + let normal_limit = Self::get_dispatch_limit_ratio(DispatchClass::Normal) * maximum_weight; + let operational_space = operational_limit.saturating_sub(normal_limit); + + let extrinsic_weight = info.weight.checked_add(T::ExtrinsicBaseWeight::get()) + .ok_or(InvalidTransaction::ExhaustsResources)?; + all_weight.checked_add(extrinsic_weight, DispatchClass::Operational) + .map_err(|_| InvalidTransaction::ExhaustsResources)?; + + // If it would fit in normally, its okay + if all_weight.total() <= maximum_weight || + // If we have not used our operational space + all_weight.get(DispatchClass::Operational) <= operational_space { + Ok(all_weight) + } else { + Err(InvalidTransaction::ExhaustsResources.into()) + } + } + } + } + + /// Checks if the current extrinsic can fit into the block with respect to block length limits. + /// + /// Upon successes, it returns the new block length as a `Result`. + fn check_block_length( + info: &DispatchInfoOf, + len: usize, + ) -> Result { + let current_len = Module::::all_extrinsics_len(); + let maximum_len = T::MaximumBlockLength::get(); + let limit = Self::get_dispatch_limit_ratio(info.class) * maximum_len; + let added_len = len as u32; + let next_len = current_len.saturating_add(added_len); + if next_len > limit { + Err(InvalidTransaction::ExhaustsResources.into()) + } else { + Ok(next_len) + } + } + + /// get the priority of an extrinsic denoted by `info`. + fn get_priority(info: &DispatchInfoOf) -> TransactionPriority { + match info.class { + DispatchClass::Normal => info.weight.into(), + // Don't use up the whole priority space, to allow things like `tip` + // to be taken into account as well. + DispatchClass::Operational => TransactionPriority::max_value() / 2, + // Mandatory extrinsics are only for inherents; never transactions. + DispatchClass::Mandatory => TransactionPriority::min_value(), + } + } + + /// Creates new `SignedExtension` to check weight of the extrinsic. + pub fn new() -> Self { + Self(Default::default()) + } + + /// Do the pre-dispatch checks. This can be applied to both signed and unsigned. + /// + /// It checks and notes the new weight and length. + fn do_pre_dispatch( + info: &DispatchInfoOf, + len: usize, + ) -> Result<(), TransactionValidityError> { + let next_len = Self::check_block_length(info, len)?; + let next_weight = Self::check_block_weight(info)?; + Self::check_extrinsic_weight(info)?; + + crate::AllExtrinsicsLen::put(next_len); + crate::BlockWeight::put(next_weight); + Ok(()) + } + + /// Do the validate checks. This can be applied to both signed and unsigned. + /// + /// It only checks that the block weight and length limit will not exceed. + fn do_validate( + info: &DispatchInfoOf, + len: usize, + ) -> TransactionValidity { + // ignore the next length. If they return `Ok`, then it is below the limit. + let _ = Self::check_block_length(info, len)?; + // during validation we skip block limit check. Since the `validate_transaction` + // call runs on an empty block anyway, by this we prevent `on_initialize` weight + // consumption from causing false negatives. + Self::check_extrinsic_weight(info)?; + + Ok(ValidTransaction { priority: Self::get_priority(info), ..Default::default() }) + } +} + +impl SignedExtension for CheckWeight where + T::Call: Dispatchable +{ + type AccountId = T::AccountId; + type Call = T::Call; + type AdditionalSigned = (); + type Pre = (); + const IDENTIFIER: &'static str = "CheckWeight"; + + fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } + + fn pre_dispatch( + self, + _who: &Self::AccountId, + _call: &Self::Call, + info: &DispatchInfoOf, + len: usize, + ) -> Result<(), TransactionValidityError> { + if info.class == DispatchClass::Mandatory { + Err(InvalidTransaction::MandatoryDispatch)? + } + Self::do_pre_dispatch(info, len) + } + + fn validate( + &self, + _who: &Self::AccountId, + _call: &Self::Call, + info: &DispatchInfoOf, + len: usize, + ) -> TransactionValidity { + if info.class == DispatchClass::Mandatory { + Err(InvalidTransaction::MandatoryDispatch)? + } + Self::do_validate(info, len) + } + + fn pre_dispatch_unsigned( + _call: &Self::Call, + info: &DispatchInfoOf, + len: usize, + ) -> Result<(), TransactionValidityError> { + Self::do_pre_dispatch(info, len) + } + + fn validate_unsigned( + _call: &Self::Call, + info: &DispatchInfoOf, + len: usize, + ) -> TransactionValidity { + Self::do_validate(info, len) + } + + fn post_dispatch( + _pre: Self::Pre, + info: &DispatchInfoOf, + post_info: &PostDispatchInfoOf, + _len: usize, + result: &DispatchResult, + ) -> Result<(), TransactionValidityError> { + // Since mandatory dispatched do not get validated for being overweight, we are sensitive + // to them actually being useful. Block producers are thus not allowed to include mandatory + // extrinsics that result in error. + if let (DispatchClass::Mandatory, Err(e)) = (info.class, result) { + "Bad mandantory".print(); + e.print(); + + Err(InvalidTransaction::BadMandatory)? + } + + let unspent = post_info.calc_unspent(info); + if unspent > 0 { + crate::BlockWeight::mutate(|current_weight| { + current_weight.sub(unspent, info.class); + }) + } + + Ok(()) + } +} + +impl sp_std::fmt::Debug for CheckWeight { + #[cfg(feature = "std")] + fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + write!(f, "CheckWeight") + } + + #[cfg(not(feature = "std"))] + fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{BlockWeight, AllExtrinsicsLen}; + use crate::mock::{Test, CALL, new_test_ext, System}; + use sp_std::marker::PhantomData; + use frame_support::{assert_ok, assert_noop}; + use frame_support::weights::{Weight, Pays}; + + fn normal_weight_limit() -> Weight { + ::AvailableBlockRatio::get() * ::MaximumBlockWeight::get() + } + + fn normal_length_limit() -> u32 { + ::AvailableBlockRatio::get() * ::MaximumBlockLength::get() + } + + #[test] + fn mandatory_extrinsic_doesnt_care_about_limits() { + fn check(call: impl FnOnce(&DispatchInfo, usize)) { + new_test_ext().execute_with(|| { + let max = DispatchInfo { + weight: Weight::max_value(), + class: DispatchClass::Mandatory, + ..Default::default() + }; + let len = 0_usize; + + call(&max, len); + }); + } + + check(|max, len| { + assert_ok!(CheckWeight::::do_pre_dispatch(max, len)); + assert_eq!(System::block_weight().total(), Weight::max_value()); + assert!(System::block_weight().total() > ::MaximumBlockWeight::get()); + }); + check(|max, len| { + assert_ok!(CheckWeight::::do_validate(max, len)); + }); + } + + #[test] + fn normal_extrinsic_limited_by_maximum_extrinsic_weight() { + new_test_ext().execute_with(|| { + let max = DispatchInfo { + weight: ::MaximumExtrinsicWeight::get() + 1, + class: DispatchClass::Normal, + ..Default::default() + }; + let len = 0_usize; + + assert_noop!( + CheckWeight::::do_validate(&max, len), + InvalidTransaction::ExhaustsResources + ); + }); + } + + #[test] + fn operational_extrinsic_limited_by_operational_space_limit() { + new_test_ext().execute_with(|| { + let operational_limit = CheckWeight::::get_dispatch_limit_ratio( + DispatchClass::Operational + ) * ::MaximumBlockWeight::get(); + let base_weight = ::ExtrinsicBaseWeight::get(); + let block_base = ::BlockExecutionWeight::get(); + + let weight = operational_limit - base_weight - block_base; + let okay = DispatchInfo { + weight, + class: DispatchClass::Operational, + ..Default::default() + }; + let max = DispatchInfo { + weight: weight + 1, + class: DispatchClass::Operational, + ..Default::default() + }; + let len = 0_usize; + + assert_eq!( + CheckWeight::::do_validate(&okay, len), + Ok(ValidTransaction { + priority: CheckWeight::::get_priority(&okay), + ..Default::default() + }) + ); + assert_noop!( + CheckWeight::::do_validate(&max, len), + InvalidTransaction::ExhaustsResources + ); + }); + } + + #[test] + fn register_extra_weight_unchecked_doesnt_care_about_limits() { + new_test_ext().execute_with(|| { + System::register_extra_weight_unchecked(Weight::max_value(), DispatchClass::Normal); + assert_eq!(System::block_weight().total(), Weight::max_value()); + assert!(System::block_weight().total() > ::MaximumBlockWeight::get()); + }); + } + + #[test] + fn full_block_with_normal_and_operational() { + new_test_ext().execute_with(|| { + // Max block is 1024 + // Max normal is 768 (75%) + // 10 is taken for block execution weight + // So normal extrinsic can be 758 weight (-5 for base extrinsic weight) + // And Operational can be 256 to produce a full block (-5 for base) + let max_normal = DispatchInfo { weight: 753, ..Default::default() }; + let rest_operational = DispatchInfo { weight: 251, class: DispatchClass::Operational, ..Default::default() }; + + let len = 0_usize; + + assert_ok!(CheckWeight::::do_pre_dispatch(&max_normal, len)); + assert_eq!(System::block_weight().total(), 768); + assert_ok!(CheckWeight::::do_pre_dispatch(&rest_operational, len)); + assert_eq!(::MaximumBlockWeight::get(), 1024); + assert_eq!(System::block_weight().total(), ::MaximumBlockWeight::get()); + // Checking single extrinsic should not take current block weight into account. + assert_eq!(CheckWeight::::check_extrinsic_weight(&rest_operational), Ok(())); + }); + } + + #[test] + fn dispatch_order_does_not_effect_weight_logic() { + new_test_ext().execute_with(|| { + // We switch the order of `full_block_with_normal_and_operational` + let max_normal = DispatchInfo { weight: 753, ..Default::default() }; + let rest_operational = DispatchInfo { weight: 251, class: DispatchClass::Operational, ..Default::default() }; + + let len = 0_usize; + + assert_ok!(CheckWeight::::do_pre_dispatch(&rest_operational, len)); + // Extra 15 here from block execution + base extrinsic weight + assert_eq!(System::block_weight().total(), 266); + assert_ok!(CheckWeight::::do_pre_dispatch(&max_normal, len)); + assert_eq!(::MaximumBlockWeight::get(), 1024); + assert_eq!(System::block_weight().total(), ::MaximumBlockWeight::get()); + }); + } + + #[test] + fn operational_works_on_full_block() { + new_test_ext().execute_with(|| { + // An on_initialize takes up the whole block! (Every time!) + System::register_extra_weight_unchecked(Weight::max_value(), DispatchClass::Mandatory); + let dispatch_normal = DispatchInfo { weight: 251, class: DispatchClass::Normal, ..Default::default() }; + let dispatch_operational = DispatchInfo { weight: 251, class: DispatchClass::Operational, ..Default::default() }; + let len = 0_usize; + + assert_noop!( + CheckWeight::::do_pre_dispatch(&dispatch_normal, len), + InvalidTransaction::ExhaustsResources + ); + // Thank goodness we can still do an operational transaction to possibly save the blockchain. + assert_ok!(CheckWeight::::do_pre_dispatch(&dispatch_operational, len)); + // Not too much though + assert_noop!( + CheckWeight::::do_pre_dispatch(&dispatch_operational, len), + InvalidTransaction::ExhaustsResources + ); + // Even with full block, validity of single transaction should be correct. + assert_eq!(CheckWeight::::check_extrinsic_weight(&dispatch_operational), Ok(())); + }); + } + + #[test] + fn signed_ext_check_weight_works_operational_tx() { + new_test_ext().execute_with(|| { + let normal = DispatchInfo { weight: 100, ..Default::default() }; + let op = DispatchInfo { weight: 100, class: DispatchClass::Operational, pays_fee: Pays::Yes }; + let len = 0_usize; + let normal_limit = normal_weight_limit(); + + // given almost full block + BlockWeight::mutate(|current_weight| { + current_weight.put(normal_limit, DispatchClass::Normal) + }); + // will not fit. + assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &normal, len).is_err()); + // will fit. + assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &op, len).is_ok()); + + // likewise for length limit. + let len = 100_usize; + AllExtrinsicsLen::put(normal_length_limit()); + assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &normal, len).is_err()); + assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &op, len).is_ok()); + }) + } + + #[test] + fn signed_ext() { + new_test_ext().execute_with(|| { + let normal = DispatchInfo { weight: 100, class: DispatchClass::Normal, pays_fee: Pays::Yes }; + let op = DispatchInfo { weight: 100, class: DispatchClass::Operational, pays_fee: Pays::Yes }; + let len = 0_usize; + + let priority = CheckWeight::(PhantomData) + .validate(&1, CALL, &normal, len) + .unwrap() + .priority; + assert_eq!(priority, 100); + + let priority = CheckWeight::(PhantomData) + .validate(&1, CALL, &op, len) + .unwrap() + .priority; + assert_eq!(priority, u64::max_value() / 2); + }) + } + + #[test] + fn signed_ext_check_weight_block_size_works() { + new_test_ext().execute_with(|| { + let normal = DispatchInfo::default(); + let normal_limit = normal_weight_limit() as usize; + let reset_check_weight = |tx, s, f| { + AllExtrinsicsLen::put(0); + let r = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, tx, s); + if f { assert!(r.is_err()) } else { assert!(r.is_ok()) } + }; + + reset_check_weight(&normal, normal_limit - 1, false); + reset_check_weight(&normal, normal_limit, false); + reset_check_weight(&normal, normal_limit + 1, true); + + // Operational ones don't have this limit. + let op = DispatchInfo { weight: 0, class: DispatchClass::Operational, pays_fee: Pays::Yes }; + reset_check_weight(&op, normal_limit, false); + reset_check_weight(&op, normal_limit + 100, false); + reset_check_weight(&op, 1024, false); + reset_check_weight(&op, 1025, true); + }) + } + + + #[test] + fn signed_ext_check_weight_works_normal_tx() { + new_test_ext().execute_with(|| { + let normal_limit = normal_weight_limit(); + let small = DispatchInfo { weight: 100, ..Default::default() }; + let medium = DispatchInfo { + weight: normal_limit - ::ExtrinsicBaseWeight::get(), + ..Default::default() + }; + let big = DispatchInfo { + weight: normal_limit - ::ExtrinsicBaseWeight::get() + 1, + ..Default::default() + }; + let len = 0_usize; + + let reset_check_weight = |i, f, s| { + BlockWeight::mutate(|current_weight| { + current_weight.put(s, DispatchClass::Normal) + }); + let r = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, i, len); + if f { assert!(r.is_err()) } else { assert!(r.is_ok()) } + }; + + reset_check_weight(&small, false, 0); + reset_check_weight(&medium, false, 0); + reset_check_weight(&big, true, 1); + }) + } + + #[test] + fn signed_ext_check_weight_refund_works() { + new_test_ext().execute_with(|| { + // This is half of the max block weight + let info = DispatchInfo { weight: 512, ..Default::default() }; + let post_info = PostDispatchInfo { actual_weight: Some(128), }; + let len = 0_usize; + + // We allow 75% for normal transaction, so we put 25% - extrinsic base weight + BlockWeight::mutate(|current_weight| { + current_weight.put(256 - ::ExtrinsicBaseWeight::get(), DispatchClass::Normal) + }); + + let pre = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &info, len).unwrap(); + assert_eq!(BlockWeight::get().total(), info.weight + 256); + + assert!( + CheckWeight::::post_dispatch(pre, &info, &post_info, len, &Ok(())) + .is_ok() + ); + assert_eq!( + BlockWeight::get().total(), + post_info.actual_weight.unwrap() + 256, + ); + }) + } + + #[test] + fn signed_ext_check_weight_actual_weight_higher_than_max_is_capped() { + new_test_ext().execute_with(|| { + let info = DispatchInfo { weight: 512, ..Default::default() }; + let post_info = PostDispatchInfo { actual_weight: Some(700), }; + let len = 0_usize; + + BlockWeight::mutate(|current_weight| { + current_weight.put(128, DispatchClass::Normal) + }); + + let pre = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &info, len).unwrap(); + assert_eq!( + BlockWeight::get().total(), + info.weight + 128 + ::ExtrinsicBaseWeight::get(), + ); + + assert!( + CheckWeight::::post_dispatch(pre, &info, &post_info, len, &Ok(())) + .is_ok() + ); + assert_eq!( + BlockWeight::get().total(), + info.weight + 128 + ::ExtrinsicBaseWeight::get(), + ); + }) + } + + #[test] + fn zero_weight_extrinsic_still_has_base_weight() { + new_test_ext().execute_with(|| { + let free = DispatchInfo { weight: 0, ..Default::default() }; + let len = 0_usize; + + // Initial weight from `BlockExecutionWeight` + assert_eq!(System::block_weight().total(), ::BlockExecutionWeight::get()); + let r = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &free, len); + assert!(r.is_ok()); + assert_eq!( + System::block_weight().total(), + ::ExtrinsicBaseWeight::get() + ::BlockExecutionWeight::get() + ); + }) + } +} diff --git a/frame/system/src/extensions/mod.rs b/frame/system/src/extensions/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..ff61353e2d176fbc1660e2a6787e370c6f15328c --- /dev/null +++ b/frame/system/src/extensions/mod.rs @@ -0,0 +1,24 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod check_genesis; +pub mod check_mortality; +pub mod check_nonce; +pub mod check_spec_version; +pub mod check_tx_version; +pub mod check_weight; + diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index e2516740d37cb7455b3e1f4a55140c5db6b2b7e0..dc103b204d93f66d114861d6c34e539f3a78766c 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -102,18 +102,14 @@ use sp_std::marker::PhantomData; use sp_std::fmt::Debug; use sp_version::RuntimeVersion; use sp_runtime::{ - RuntimeDebug, Perbill, DispatchOutcome, DispatchError, DispatchResult, - generic::{self, Era}, - transaction_validity::{ - ValidTransaction, TransactionPriority, TransactionLongevity, TransactionValidityError, - InvalidTransaction, TransactionValidity, - }, + RuntimeDebug, Perbill, DispatchError, Either, generic, traits::{ - self, CheckEqual, AtLeast32Bit, Zero, SignedExtension, Lookup, LookupError, - SimpleBitOps, Hash, Member, MaybeDisplay, BadOrigin, SaturatedConversion, + self, CheckEqual, AtLeast32Bit, Zero, Lookup, LookupError, + SimpleBitOps, Hash, Member, MaybeDisplay, BadOrigin, MaybeSerialize, MaybeSerializeDeserialize, MaybeMallocSizeOf, StaticLookup, One, Bounded, - Dispatchable, DispatchInfoOf, PostDispatchInfoOf, + Dispatchable, AtLeast32BitUnsigned }, + offchain::storage_lock::BlockNumberProvider, }; use sp_core::{ChangesTrieConfiguration, storage::well_known_keys}; @@ -122,12 +118,13 @@ use frame_support::{ storage, traits::{ Contains, Get, ModuleToIndex, OnNewAccount, OnKilledAccount, IsDeadAccount, Happened, - StoredMap, EnsureOrigin, + StoredMap, EnsureOrigin, OriginTrait, Filter, }, weights::{ - Weight, RuntimeDbWeight, DispatchInfo, PostDispatchInfo, DispatchClass, - FunctionOf, Pays, - } + Weight, RuntimeDbWeight, DispatchInfo, DispatchClass, + extract_actual_weight, + }, + dispatch::DispatchResultWithPostInfo, }; use codec::{Encode, Decode, FullCodec, EncodeLike}; @@ -135,6 +132,21 @@ use codec::{Encode, Decode, FullCodec, EncodeLike}; use sp_io::TestExternalities; pub mod offchain; +#[cfg(test)] +pub(crate) mod mock; + +mod extensions; +mod weights; +#[cfg(test)] +mod tests; + +pub use extensions::{ + check_mortality::CheckMortality, check_genesis::CheckGenesis, check_nonce::CheckNonce, + check_spec_version::CheckSpecVersion, check_tx_version::CheckTxVersion, + check_weight::CheckWeight, +}; +// Backward compatible re-export. +pub use extensions::check_mortality::CheckMortality as CheckEra; /// Compute the trie root of a list of extrinsics. pub fn extrinsics_root(extrinsics: &[E]) -> H::Output { @@ -147,11 +159,16 @@ pub fn extrinsics_data_root(xts: Vec>) -> H::Output { } pub trait Trait: 'static + Eq + Clone { - /// The aggregated `Origin` type used by dispatchable calls. + /// The basic call filter to use in Origin. All origins are built with this filter as base, + /// except Root. + type BaseCallFilter: Filter; + + /// The `Origin` type used by dispatchable calls. type Origin: Into, Self::Origin>> + From> - + Clone; + + Clone + + OriginTrait; /// The aggregated `Call` type. type Call: Dispatchable + Debug; @@ -164,8 +181,9 @@ pub trait Trait: 'static + Eq + Clone { /// The block number type used by the runtime. type BlockNumber: - Parameter + Member + MaybeSerializeDeserialize + Debug + MaybeDisplay + AtLeast32Bit - + Default + Bounded + Copy + sp_std::hash::Hash + sp_std::str::FromStr + MaybeMallocSizeOf; + Parameter + Member + MaybeSerializeDeserialize + Debug + MaybeDisplay + + AtLeast32BitUnsigned + Default + Bounded + Copy + sp_std::hash::Hash + + sp_std::str::FromStr + MaybeMallocSizeOf; /// The output of the `Hashing` function. type Hash: @@ -211,6 +229,11 @@ pub trait Trait: 'static + Eq + Clone { /// The base weight of an Extrinsic in the block, independent of the of extrinsic being executed. type ExtrinsicBaseWeight: Get; + /// The maximal weight of a single Extrinsic. This should be set to at most + /// `MaximumBlockWeight - AverageOnInitializeWeight`. The limit only applies to extrinsics + /// containing `Normal` dispatch class calls. + type MaximumExtrinsicWeight: Get; + /// The maximum length of a block (in bytes). type MaximumBlockLength: Get; @@ -360,60 +383,6 @@ impl From for LastRuntimeUpgradeInfo { } } -/// An object to track the currently used extrinsic weight in a block. -#[derive(Clone, Eq, PartialEq, Default, RuntimeDebug, Encode, Decode)] -pub struct ExtrinsicsWeight { - normal: Weight, - operational: Weight, -} - -impl ExtrinsicsWeight { - /// Returns the total weight consumed by all extrinsics in the block. - pub fn total(&self) -> Weight { - self.normal.saturating_add(self.operational) - } - - /// Add some weight of a specific dispatch class, saturating at the numeric bounds of `Weight`. - pub fn add(&mut self, weight: Weight, class: DispatchClass) { - let value = self.get_mut(class); - *value = value.saturating_add(weight); - } - - /// Try to add some weight of a specific dispatch class, returning Err(()) if overflow would occur. - pub fn checked_add(&mut self, weight: Weight, class: DispatchClass) -> Result<(), ()> { - let value = self.get_mut(class); - *value = value.checked_add(weight).ok_or(())?; - Ok(()) - } - - /// Subtract some weight of a specific dispatch class, saturating at the numeric bounds of `Weight`. - pub fn sub(&mut self, weight: Weight, class: DispatchClass) { - let value = self.get_mut(class); - *value = value.saturating_sub(weight); - } - - /// Get the current weight of a specific dispatch class. - pub fn get(&self, class: DispatchClass) -> Weight { - match class { - DispatchClass::Operational => self.operational, - DispatchClass::Normal | DispatchClass::Mandatory => self.normal, - } - } - - /// Get a mutable reference to the current weight of a specific dispatch class. - fn get_mut(&mut self, class: DispatchClass) -> &mut Weight { - match class { - DispatchClass::Operational => &mut self.operational, - DispatchClass::Normal | DispatchClass::Mandatory => &mut self.normal, - } - } - - /// Set the weight of a specific dispatch class. - pub fn put(&mut self, new: Weight, class: DispatchClass) { - *self.get_mut(class) = new; - } -} - decl_storage! { trait Store for Module as System { /// The full account information for a particular account ID. @@ -423,8 +392,8 @@ decl_storage! { /// Total extrinsics count for the current block. ExtrinsicCount: Option; - /// Total weight for all extrinsics for the current block. - AllExtrinsicsWeight: ExtrinsicsWeight; + /// The current weight for the block. + BlockWeight get(fn block_weight): weights::ExtrinsicsWeight; /// Total length (in bytes) for all extrinsics put together, for the current block. AllExtrinsicsLen: Option; @@ -559,11 +528,7 @@ decl_module! { /// A dispatch that will fill the block weight up to the given ratio. // TODO: This should only be available for testing, rather than in general usage, but // that's not possible at present (since it's within the decl_module macro). - #[weight = FunctionOf( - |(ratio,): (&Perbill,)| *ratio * T::MaximumBlockWeight::get(), - DispatchClass::Operational, - Pays::Yes, - )] + #[weight = *_ratio * T::MaximumBlockWeight::get()] fn fill_block(origin, _ratio: Perbill) { ensure_root(origin)?; } @@ -662,13 +627,10 @@ decl_module! { /// - Base Weight: 0.568 * i µs /// - Writes: Number of items /// # - #[weight = FunctionOf( - |(items,): (&Vec,)| { - T::DbWeight::get().writes(items.len() as Weight) - .saturating_add((items.len() as Weight).saturating_mul(600_000)) - }, + #[weight = ( + T::DbWeight::get().writes(items.len() as Weight) + .saturating_add((items.len() as Weight).saturating_mul(600_000)), DispatchClass::Operational, - Pays::Yes, )] fn set_storage(origin, items: Vec) { ensure_root(origin)?; @@ -685,13 +647,10 @@ decl_module! { /// - Base Weight: .378 * i µs /// - Writes: Number of items /// # - #[weight = FunctionOf( - |(keys,): (&Vec,)| { - T::DbWeight::get().writes(keys.len() as Weight) - .saturating_add((keys.len() as Weight).saturating_mul(400_000)) - }, + #[weight = ( + T::DbWeight::get().writes(keys.len() as Weight) + .saturating_add((keys.len() as Weight).saturating_mul(400_000)), DispatchClass::Operational, - Pays::Yes, )] fn kill_storage(origin, keys: Vec) { ensure_root(origin)?; @@ -711,13 +670,10 @@ decl_module! { /// - Base Weight: 0.834 * P µs /// - Writes: Number of subkeys + 1 /// # - #[weight = FunctionOf( - |(_, &subkeys): (&Key, &u32)| { - T::DbWeight::get().writes(Weight::from(subkeys) + 1) - .saturating_add((Weight::from(subkeys) + 1).saturating_mul(850_000)) - }, + #[weight = ( + T::DbWeight::get().writes(Weight::from(*_subkeys) + 1) + .saturating_add((Weight::from(*_subkeys) + 1).saturating_mul(850_000)), DispatchClass::Operational, - Pays::Yes, )] fn kill_prefix(origin, prefix: Key, _subkeys: u32) { ensure_root(origin)?; @@ -735,12 +691,12 @@ decl_module! { /// No DB Read or Write operations because caller is already in overlay /// # #[weight = (10_000_000, DispatchClass::Operational)] - fn suicide(origin) { + pub fn suicide(origin) { let who = ensure_signed(origin)?; let account = Account::::get(&who); ensure!(account.refcount == 0, Error::::NonZeroRefCount); ensure!(account.data == T::AccountData::default(), Error::::NonDefaultComposite); - Account::::remove(who); + Self::kill_account(&who); } } } @@ -840,6 +796,30 @@ impl EnsureOrigin for EnsureNever { } } +/// The "OR gate" implementation of `EnsureOrigin`. +/// +/// Origin check will pass if `L` or `R` origin check passes. `L` is tested first. +pub struct EnsureOneOf(sp_std::marker::PhantomData<(AccountId, L, R)>); +impl< + AccountId, + O: Into, O>> + From>, + L: EnsureOrigin, + R: EnsureOrigin, +> EnsureOrigin for EnsureOneOf { + type Success = Either; + fn try_origin(o: O) -> Result { + L::try_origin(o).map_or_else( + |o| R::try_origin(o).map(|o| Either::Right(o)), + |o| Ok(Either::Left(o)), + ) + } + + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin() -> O { + L::successful_origin() + } +} + /// Ensure that the origin `o` represents a signed extrinsic (i.e. transaction). /// Returns `Ok` with the account that signed the extrinsic or an `Err` otherwise. pub fn ensure_signed(o: OuterOrigin) -> Result @@ -972,11 +952,6 @@ impl Module { ExtrinsicCount::get().unwrap_or_default() } - /// Gets the weight of all executed extrinsics. - pub fn all_extrinsics_weight() -> ExtrinsicsWeight { - AllExtrinsicsWeight::get() - } - pub fn all_extrinsics_len() -> u32 { AllExtrinsicsLen::get().unwrap_or_default() } @@ -997,7 +972,7 @@ impl Module { /// /// Another potential use-case could be for the `on_initialize` and `on_finalize` hooks. pub fn register_extra_weight_unchecked(weight: Weight, class: DispatchClass) { - AllExtrinsicsWeight::mutate(|current_weight| { + BlockWeight::mutate(|current_weight| { current_weight.add(weight, class); }); } @@ -1019,6 +994,10 @@ impl Module { >::insert(*number - One::one(), parent_hash); >::put(txs_root); + // Remove previous block data from storage + BlockWeight::kill(); + + // Kill inspectable storage entries in state when `InitKind::Full`. if let InitKind::Full = kind { >::kill(); EventCount::kill(); @@ -1030,7 +1009,6 @@ impl Module { pub fn finalize() -> T::Header { ExecutionPhase::kill(); ExtrinsicCount::kill(); - AllExtrinsicsWeight::kill(); AllExtrinsicsLen::kill(); let number = >::take(); @@ -1120,12 +1098,21 @@ impl Module { /// Set the current block weight. This should only be used in some integration tests. #[cfg(any(feature = "std", test))] pub fn set_block_limits(weight: Weight, len: usize) { - AllExtrinsicsWeight::mutate(|current_weight| { + BlockWeight::mutate(|current_weight| { current_weight.put(weight, DispatchClass::Normal) }); AllExtrinsicsLen::put(len as u32); } + /// Reset events. Can be used as an alternative to + /// `initialize` for tests that don't need to bother with the other environment entries. + #[cfg(any(feature = "std", feature = "runtime-benchmarks", test))] + pub fn reset_events() { + >::kill(); + EventCount::kill(); + >::remove_all(); + } + /// Return the chain's current runtime version. pub fn runtime_version() -> RuntimeVersion { T::Version::get() } @@ -1150,13 +1137,14 @@ impl Module { } /// To be called immediately after an extrinsic has been applied. - pub fn note_applied_extrinsic(r: &DispatchOutcome, _encoded_len: u32, info: DispatchInfo) { + pub fn note_applied_extrinsic(r: &DispatchResultWithPostInfo, mut info: DispatchInfo) { + info.weight = extract_actual_weight(r, &info); Self::deposit_event( match r { - Ok(()) => RawEvent::ExtrinsicSuccess(info), + Ok(_) => RawEvent::ExtrinsicSuccess(info), Err(err) => { sp_runtime::print(err); - RawEvent::ExtrinsicFailed(err.clone(), info) + RawEvent::ExtrinsicFailed(err.error, info) }, } ); @@ -1216,8 +1204,8 @@ impl Module { "WARNING: Referenced account deleted. This is probably a bug." ); } - Module::::on_killed_account(who.clone()); } + Module::::on_killed_account(who.clone()); } /// Determine whether or not it is possible to update the code. @@ -1263,6 +1251,15 @@ impl Happened for CallKillAccount { } } +impl BlockNumberProvider for Module +{ + type BlockNumber = ::BlockNumber; + + fn current_block_number() -> Self::BlockNumber { + Module::::block_number() + } +} + // Implement StoredMap for a simple single-item, kill-account-on-remove system. This works fine for // storing a single item which is required to not be empty/default for the account to exist. // Anything more complex will need more sophisticated logic. @@ -1332,314 +1329,6 @@ pub fn split_inner(option: Option, splitter: impl FnOnce(T) -> (R, S } } -/// resource limit check. -#[derive(Encode, Decode, Clone, Eq, PartialEq)] -pub struct CheckWeight(PhantomData); - -impl CheckWeight where - T::Call: Dispatchable -{ - /// Get the quota ratio of each dispatch class type. This indicates that all operational and mandatory - /// dispatches can use the full capacity of any resource, while user-triggered ones can consume - /// a portion. - fn get_dispatch_limit_ratio(class: DispatchClass) -> Perbill { - match class { - DispatchClass::Operational | DispatchClass::Mandatory - => ::one(), - DispatchClass::Normal => T::AvailableBlockRatio::get(), - } - } - - /// Checks if the current extrinsic can fit into the block with respect to block weight limits. - /// - /// Upon successes, it returns the new block weight as a `Result`. - fn check_weight( - info: &DispatchInfoOf, - ) -> Result { - let maximum_weight = T::MaximumBlockWeight::get(); - let mut all_weight = Module::::all_extrinsics_weight(); - match info.class { - // If we have a dispatch that must be included in the block, it ignores all the limits. - DispatchClass::Mandatory => { - let extrinsic_weight = info.weight.saturating_add(T::ExtrinsicBaseWeight::get()); - all_weight.add(extrinsic_weight, DispatchClass::Mandatory); - Ok(all_weight) - }, - // If we have a normal dispatch, we follow all the normal rules and limits. - DispatchClass::Normal => { - let normal_limit = Self::get_dispatch_limit_ratio(DispatchClass::Normal) * maximum_weight; - let extrinsic_weight = info.weight.checked_add(T::ExtrinsicBaseWeight::get()) - .ok_or(InvalidTransaction::ExhaustsResources)?; - all_weight.checked_add(extrinsic_weight, DispatchClass::Normal) - .map_err(|_| InvalidTransaction::ExhaustsResources)?; - if all_weight.get(DispatchClass::Normal) > normal_limit { - Err(InvalidTransaction::ExhaustsResources.into()) - } else { - Ok(all_weight) - } - }, - // If we have an operational dispatch, allow it if we have not used our full - // "operational space" (independent of existing fullness). - DispatchClass::Operational => { - let operational_limit = Self::get_dispatch_limit_ratio(DispatchClass::Operational) * maximum_weight; - let normal_limit = Self::get_dispatch_limit_ratio(DispatchClass::Normal) * maximum_weight; - let operational_space = operational_limit.saturating_sub(normal_limit); - - let extrinsic_weight = info.weight.checked_add(T::ExtrinsicBaseWeight::get()) - .ok_or(InvalidTransaction::ExhaustsResources)?; - all_weight.checked_add(extrinsic_weight, DispatchClass::Operational) - .map_err(|_| InvalidTransaction::ExhaustsResources)?; - - // If it would fit in normally, its okay - if all_weight.total() <= maximum_weight || - // If we have not used our operational space - all_weight.get(DispatchClass::Operational) <= operational_space { - Ok(all_weight) - } else { - Err(InvalidTransaction::ExhaustsResources.into()) - } - } - } - } - - /// Checks if the current extrinsic can fit into the block with respect to block length limits. - /// - /// Upon successes, it returns the new block length as a `Result`. - fn check_block_length( - info: &DispatchInfoOf, - len: usize, - ) -> Result { - let current_len = Module::::all_extrinsics_len(); - let maximum_len = T::MaximumBlockLength::get(); - let limit = Self::get_dispatch_limit_ratio(info.class) * maximum_len; - let added_len = len as u32; - let next_len = current_len.saturating_add(added_len); - if next_len > limit { - Err(InvalidTransaction::ExhaustsResources.into()) - } else { - Ok(next_len) - } - } - - /// get the priority of an extrinsic denoted by `info`. - fn get_priority(info: &DispatchInfoOf) -> TransactionPriority { - match info.class { - DispatchClass::Normal => info.weight.into(), - DispatchClass::Operational => Bounded::max_value(), - // Mandatory extrinsics are only for inherents; never transactions. - DispatchClass::Mandatory => Bounded::min_value(), - } - } - - /// Creates new `SignedExtension` to check weight of the extrinsic. - pub fn new() -> Self { - Self(PhantomData) - } - - /// Do the pre-dispatch checks. This can be applied to both signed and unsigned. - /// - /// It checks and notes the new weight and length. - fn do_pre_dispatch( - info: &DispatchInfoOf, - len: usize, - ) -> Result<(), TransactionValidityError> { - let next_len = Self::check_block_length(info, len)?; - let next_weight = Self::check_weight(info)?; - AllExtrinsicsLen::put(next_len); - AllExtrinsicsWeight::put(next_weight); - Ok(()) - } - - /// Do the validate checks. This can be applied to both signed and unsigned. - /// - /// It only checks that the block weight and length limit will not exceed. - fn do_validate( - info: &DispatchInfoOf, - len: usize, - ) -> TransactionValidity { - // ignore the next weight and length. If they return `Ok`, then it is below the limit. - let _ = Self::check_block_length(info, len)?; - let _ = Self::check_weight(info)?; - - Ok(ValidTransaction { priority: Self::get_priority(info), ..Default::default() }) - } -} - -impl SignedExtension for CheckWeight where - T::Call: Dispatchable -{ - type AccountId = T::AccountId; - type Call = T::Call; - type AdditionalSigned = (); - type Pre = (); - const IDENTIFIER: &'static str = "CheckWeight"; - - fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } - - fn pre_dispatch( - self, - _who: &Self::AccountId, - _call: &Self::Call, - info: &DispatchInfoOf, - len: usize, - ) -> Result<(), TransactionValidityError> { - if info.class == DispatchClass::Mandatory { - Err(InvalidTransaction::MandatoryDispatch)? - } - Self::do_pre_dispatch(info, len) - } - - fn validate( - &self, - _who: &Self::AccountId, - _call: &Self::Call, - info: &DispatchInfoOf, - len: usize, - ) -> TransactionValidity { - if info.class == DispatchClass::Mandatory { - Err(InvalidTransaction::MandatoryDispatch)? - } - Self::do_validate(info, len) - } - - fn pre_dispatch_unsigned( - _call: &Self::Call, - info: &DispatchInfoOf, - len: usize, - ) -> Result<(), TransactionValidityError> { - Self::do_pre_dispatch(info, len) - } - - fn validate_unsigned( - _call: &Self::Call, - info: &DispatchInfoOf, - len: usize, - ) -> TransactionValidity { - Self::do_validate(info, len) - } - - fn post_dispatch( - _pre: Self::Pre, - info: &DispatchInfoOf, - post_info: &PostDispatchInfoOf, - _len: usize, - result: &DispatchResult, - ) -> Result<(), TransactionValidityError> { - // Since mandatory dispatched do not get validated for being overweight, we are sensitive - // to them actually being useful. Block producers are thus not allowed to include mandatory - // extrinsics that result in error. - if info.class == DispatchClass::Mandatory && result.is_err() { - Err(InvalidTransaction::BadMandatory)? - } - - let unspent = post_info.calc_unspent(info); - if unspent > 0 { - AllExtrinsicsWeight::mutate(|current_weight| { - current_weight.sub(unspent, info.class); - }) - } - - Ok(()) - } -} - -impl Debug for CheckWeight { - #[cfg(feature = "std")] - fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - write!(f, "CheckWeight") - } - - #[cfg(not(feature = "std"))] - fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - Ok(()) - } -} - -/// Nonce check and increment to give replay protection for transactions. -#[derive(Encode, Decode, Clone, Eq, PartialEq)] -pub struct CheckNonce(#[codec(compact)] T::Index); - -impl CheckNonce { - /// utility constructor. Used only in client/factory code. - pub fn from(nonce: T::Index) -> Self { - Self(nonce) - } -} - -impl Debug for CheckNonce { - #[cfg(feature = "std")] - fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - write!(f, "CheckNonce({})", self.0) - } - - #[cfg(not(feature = "std"))] - fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - Ok(()) - } -} - -impl SignedExtension for CheckNonce where - T::Call: Dispatchable -{ - type AccountId = T::AccountId; - type Call = T::Call; - type AdditionalSigned = (); - type Pre = (); - const IDENTIFIER: &'static str = "CheckNonce"; - - fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } - - fn pre_dispatch( - self, - who: &Self::AccountId, - _call: &Self::Call, - _info: &DispatchInfoOf, - _len: usize, - ) -> Result<(), TransactionValidityError> { - let mut account = Account::::get(who); - if self.0 != account.nonce { - return Err( - if self.0 < account.nonce { - InvalidTransaction::Stale - } else { - InvalidTransaction::Future - }.into() - ) - } - account.nonce += T::Index::one(); - Account::::insert(who, account); - Ok(()) - } - - fn validate( - &self, - who: &Self::AccountId, - _call: &Self::Call, - info: &DispatchInfoOf, - _len: usize, - ) -> TransactionValidity { - // check index - let account = Account::::get(who); - if self.0 < account.nonce { - return InvalidTransaction::Stale.into() - } - - let provides = vec![Encode::encode(&(who, self.0))]; - let requires = if account.nonce < self.0 { - vec![Encode::encode(&(who, self.0 - One::one()))] - } else { - vec![] - }; - - Ok(ValidTransaction { - priority: info.weight as TransactionPriority, - requires, - provides, - longevity: TransactionLongevity::max_value(), - propagate: true, - }) - } -} impl IsDeadAccount for Module { fn is_dead_account(who: &T::AccountId) -> bool { @@ -1647,167 +1336,6 @@ impl IsDeadAccount for Module { } } -/// Check for transaction mortality. -#[derive(Encode, Decode, Clone, Eq, PartialEq)] -pub struct CheckEra(Era, sp_std::marker::PhantomData); - -impl CheckEra { - /// utility constructor. Used only in client/factory code. - pub fn from(era: Era) -> Self { - Self(era, sp_std::marker::PhantomData) - } -} - -impl Debug for CheckEra { - #[cfg(feature = "std")] - fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - write!(f, "CheckEra({:?})", self.0) - } - - #[cfg(not(feature = "std"))] - fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - Ok(()) - } -} - -impl SignedExtension for CheckEra { - type AccountId = T::AccountId; - type Call = T::Call; - type AdditionalSigned = T::Hash; - type Pre = (); - const IDENTIFIER: &'static str = "CheckEra"; - - fn validate( - &self, - _who: &Self::AccountId, - _call: &Self::Call, - _info: &DispatchInfoOf, - _len: usize, - ) -> TransactionValidity { - let current_u64 = >::block_number().saturated_into::(); - let valid_till = self.0.death(current_u64); - Ok(ValidTransaction { - longevity: valid_till.saturating_sub(current_u64), - ..Default::default() - }) - } - - fn additional_signed(&self) -> Result { - let current_u64 = >::block_number().saturated_into::(); - let n = self.0.birth(current_u64).saturated_into::(); - if !>::contains_key(n) { - Err(InvalidTransaction::AncientBirthBlock.into()) - } else { - Ok(>::block_hash(n)) - } - } -} - -/// Nonce check and increment to give replay protection for transactions. -#[derive(Encode, Decode, Clone, Eq, PartialEq)] -pub struct CheckGenesis(sp_std::marker::PhantomData); - -impl Debug for CheckGenesis { - #[cfg(feature = "std")] - fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - write!(f, "CheckGenesis") - } - - #[cfg(not(feature = "std"))] - fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - Ok(()) - } -} - -impl CheckGenesis { - /// Creates new `SignedExtension` to check genesis hash. - pub fn new() -> Self { - Self(sp_std::marker::PhantomData) - } -} - -impl SignedExtension for CheckGenesis { - type AccountId = T::AccountId; - type Call = ::Call; - type AdditionalSigned = T::Hash; - type Pre = (); - const IDENTIFIER: &'static str = "CheckGenesis"; - - fn additional_signed(&self) -> Result { - Ok(>::block_hash(T::BlockNumber::zero())) - } -} - -/// Ensure the transaction version registered in the transaction is the same as at present. -#[derive(Encode, Decode, Clone, Eq, PartialEq)] -pub struct CheckTxVersion(sp_std::marker::PhantomData); - -impl Debug for CheckTxVersion { - #[cfg(feature = "std")] - fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - write!(f, "CheckTxVersion") - } - - #[cfg(not(feature = "std"))] - fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - Ok(()) - } -} - -impl CheckTxVersion { - /// Create new `SignedExtension` to check transaction version. - pub fn new() -> Self { - Self(sp_std::marker::PhantomData) - } -} - -impl SignedExtension for CheckTxVersion { - type AccountId = T::AccountId; - type Call = ::Call; - type AdditionalSigned = u32; - type Pre = (); - const IDENTIFIER: &'static str = "CheckTxVersion"; - - fn additional_signed(&self) -> Result { - Ok(>::runtime_version().transaction_version) - } -} - -/// Ensure the runtime version registered in the transaction is the same as at present. -#[derive(Encode, Decode, Clone, Eq, PartialEq)] -pub struct CheckSpecVersion(sp_std::marker::PhantomData); - -impl Debug for CheckSpecVersion { - #[cfg(feature = "std")] - fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - write!(f, "CheckSpecVersion") - } - - #[cfg(not(feature = "std"))] - fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - Ok(()) - } -} - -impl CheckSpecVersion { - /// Create new `SignedExtension` to check runtime version. - pub fn new() -> Self { - Self(sp_std::marker::PhantomData) - } -} - -impl SignedExtension for CheckSpecVersion { - type AccountId = T::AccountId; - type Call = ::Call; - type AdditionalSigned = u32; - type Pre = (); - const IDENTIFIER: &'static str = "CheckSpecVersion"; - - fn additional_signed(&self) -> Result { - Ok(>::runtime_version().spec_version) - } -} - pub struct ChainContext(sp_std::marker::PhantomData); impl Default for ChainContext { fn default() -> Self { @@ -1823,700 +1351,3 @@ impl Lookup for ChainContext { ::lookup(s) } } - -#[cfg(test)] -pub(crate) mod tests { - use super::*; - use sp_std::cell::RefCell; - use sp_core::H256; - use sp_runtime::{traits::{BlakeTwo256, IdentityLookup, SignedExtension}, testing::Header, DispatchError}; - use frame_support::{impl_outer_origin, parameter_types, assert_ok, assert_noop}; - - impl_outer_origin! { - pub enum Origin for Test where system = super {} - } - - #[derive(Clone, Eq, PartialEq, Debug)] - pub struct Test; - - parameter_types! { - pub const BlockHashCount: u64 = 10; - pub const MaximumBlockWeight: Weight = 1024; - pub const AvailableBlockRatio: Perbill = Perbill::from_percent(75); - pub const MaximumBlockLength: u32 = 1024; - pub const Version: RuntimeVersion = RuntimeVersion { - spec_name: sp_version::create_runtime_str!("test"), - impl_name: sp_version::create_runtime_str!("system-test"), - authoring_version: 1, - spec_version: 1, - impl_version: 1, - apis: sp_version::create_apis_vec!([]), - transaction_version: 1, - }; - pub const BlockExecutionWeight: Weight = 10; - pub const ExtrinsicBaseWeight: Weight = 5; - pub const DbWeight: RuntimeDbWeight = RuntimeDbWeight { - read: 10, - write: 100, - }; - } - - thread_local!{ - pub static KILLED: RefCell> = RefCell::new(vec![]); - } - - pub struct RecordKilled; - impl OnKilledAccount for RecordKilled { - fn on_killed_account(who: &u64) { KILLED.with(|r| r.borrow_mut().push(*who)) } - } - - #[derive(Debug, codec::Encode, codec::Decode)] - pub struct Call; - - impl Dispatchable for Call { - type Origin = (); - type Trait = (); - type Info = DispatchInfo; - type PostInfo = PostDispatchInfo; - fn dispatch(self, _origin: Self::Origin) - -> sp_runtime::DispatchResultWithInfo { - panic!("Do not use dummy implementation for dispatch."); - } - } - - impl Trait for Test { - type Origin = Origin; - type Call = Call; - type Index = u64; - type BlockNumber = u64; - type Hash = H256; - type Hashing = BlakeTwo256; - type AccountId = u64; - type Lookup = IdentityLookup; - type Header = Header; - type Event = u16; - type BlockHashCount = BlockHashCount; - type MaximumBlockWeight = MaximumBlockWeight; - type DbWeight = DbWeight; - type BlockExecutionWeight = BlockExecutionWeight; - type ExtrinsicBaseWeight = ExtrinsicBaseWeight; - type AvailableBlockRatio = AvailableBlockRatio; - type MaximumBlockLength = MaximumBlockLength; - type Version = Version; - type ModuleToIndex = (); - type AccountData = u32; - type OnNewAccount = (); - type OnKilledAccount = RecordKilled; - } - - impl From> for u16 { - fn from(e: Event) -> u16 { - match e { - Event::::ExtrinsicSuccess(..) => 100, - Event::::ExtrinsicFailed(..) => 101, - Event::::CodeUpdated => 102, - _ => 103, - } - } - } - - type System = Module; - - const CALL: &::Call = &Call; - - fn new_test_ext() -> sp_io::TestExternalities { - let mut ext: sp_io::TestExternalities = GenesisConfig::default().build_storage::().unwrap().into(); - // Add to each test the initial weight of a block - ext.execute_with(|| System::register_extra_weight_unchecked(::BlockExecutionWeight::get(), DispatchClass::Mandatory)); - ext - } - - fn normal_weight_limit() -> Weight { - ::AvailableBlockRatio::get() * ::MaximumBlockWeight::get() - } - - fn normal_length_limit() -> u32 { - ::AvailableBlockRatio::get() * ::MaximumBlockLength::get() - } - - #[test] - fn origin_works() { - let o = Origin::from(RawOrigin::::Signed(1u64)); - let x: Result, Origin> = o.into(); - assert_eq!(x, Ok(RawOrigin::::Signed(1u64))); - } - - #[test] - fn stored_map_works() { - new_test_ext().execute_with(|| { - System::insert(&0, 42); - assert!(System::allow_death(&0)); - - System::inc_ref(&0); - assert!(!System::allow_death(&0)); - - System::insert(&0, 69); - assert!(!System::allow_death(&0)); - - System::dec_ref(&0); - assert!(System::allow_death(&0)); - - assert!(KILLED.with(|r| r.borrow().is_empty())); - System::kill_account(&0); - assert_eq!(KILLED.with(|r| r.borrow().clone()), vec![0u64]); - }); - } - - #[test] - fn deposit_event_should_work() { - new_test_ext().execute_with(|| { - System::initialize( - &1, - &[0u8; 32].into(), - &[0u8; 32].into(), - &Default::default(), - InitKind::Full, - ); - System::note_finished_extrinsics(); - System::deposit_event(1u16); - System::finalize(); - assert_eq!( - System::events(), - vec![ - EventRecord { - phase: Phase::Finalization, - event: 1u16, - topics: vec![], - } - ] - ); - - System::initialize( - &2, - &[0u8; 32].into(), - &[0u8; 32].into(), - &Default::default(), - InitKind::Full, - ); - System::deposit_event(32u16); - System::note_finished_initialize(); - System::deposit_event(42u16); - System::note_applied_extrinsic(&Ok(()), 0, Default::default()); - System::note_applied_extrinsic(&Err(DispatchError::BadOrigin), 0, Default::default()); - System::note_finished_extrinsics(); - System::deposit_event(3u16); - System::finalize(); - assert_eq!( - System::events(), - vec![ - EventRecord { phase: Phase::Initialization, event: 32u16, topics: vec![] }, - EventRecord { phase: Phase::ApplyExtrinsic(0), event: 42u16, topics: vec![] }, - EventRecord { phase: Phase::ApplyExtrinsic(0), event: 100u16, topics: vec![] }, - EventRecord { phase: Phase::ApplyExtrinsic(1), event: 101u16, topics: vec![] }, - EventRecord { phase: Phase::Finalization, event: 3u16, topics: vec![] } - ] - ); - }); - } - - #[test] - fn deposit_event_topics() { - new_test_ext().execute_with(|| { - const BLOCK_NUMBER: u64 = 1; - - System::initialize( - &BLOCK_NUMBER, - &[0u8; 32].into(), - &[0u8; 32].into(), - &Default::default(), - InitKind::Full, - ); - System::note_finished_extrinsics(); - - let topics = vec![ - H256::repeat_byte(1), - H256::repeat_byte(2), - H256::repeat_byte(3), - ]; - - // We deposit a few events with different sets of topics. - System::deposit_event_indexed(&topics[0..3], 1u16); - System::deposit_event_indexed(&topics[0..1], 2u16); - System::deposit_event_indexed(&topics[1..2], 3u16); - - System::finalize(); - - // Check that topics are reflected in the event record. - assert_eq!( - System::events(), - vec![ - EventRecord { - phase: Phase::Finalization, - event: 1u16, - topics: topics[0..3].to_vec(), - }, - EventRecord { - phase: Phase::Finalization, - event: 2u16, - topics: topics[0..1].to_vec(), - }, - EventRecord { - phase: Phase::Finalization, - event: 3u16, - topics: topics[1..2].to_vec(), - } - ] - ); - - // Check that the topic-events mapping reflects the deposited topics. - // Note that these are indexes of the events. - assert_eq!( - System::event_topics(&topics[0]), - vec![(BLOCK_NUMBER, 0), (BLOCK_NUMBER, 1)], - ); - assert_eq!( - System::event_topics(&topics[1]), - vec![(BLOCK_NUMBER, 0), (BLOCK_NUMBER, 2)], - ); - assert_eq!( - System::event_topics(&topics[2]), - vec![(BLOCK_NUMBER, 0)], - ); - }); - } - - #[test] - fn prunes_block_hash_mappings() { - new_test_ext().execute_with(|| { - // simulate import of 15 blocks - for n in 1..=15 { - System::initialize( - &n, - &[n as u8 - 1; 32].into(), - &[0u8; 32].into(), - &Default::default(), - InitKind::Full, - ); - - System::finalize(); - } - - // first 5 block hashes are pruned - for n in 0..5 { - assert_eq!( - System::block_hash(n), - H256::zero(), - ); - } - - // the remaining 10 are kept - for n in 5..15 { - assert_eq!( - System::block_hash(n), - [n as u8; 32].into(), - ); - } - }) - } - - #[test] - fn signed_ext_check_nonce_works() { - new_test_ext().execute_with(|| { - Account::::insert(1, AccountInfo { nonce: 1, refcount: 0, data: 0 }); - let info = DispatchInfo::default(); - let len = 0_usize; - // stale - assert!(CheckNonce::(0).validate(&1, CALL, &info, len).is_err()); - assert!(CheckNonce::(0).pre_dispatch(&1, CALL, &info, len).is_err()); - // correct - assert!(CheckNonce::(1).validate(&1, CALL, &info, len).is_ok()); - assert!(CheckNonce::(1).pre_dispatch(&1, CALL, &info, len).is_ok()); - // future - assert!(CheckNonce::(5).validate(&1, CALL, &info, len).is_ok()); - assert!(CheckNonce::(5).pre_dispatch(&1, CALL, &info, len).is_err()); - }) - } - - #[test] - fn signed_ext_check_weight_works_normal_tx() { - new_test_ext().execute_with(|| { - let normal_limit = normal_weight_limit(); - let small = DispatchInfo { weight: 100, ..Default::default() }; - let medium = DispatchInfo { - weight: normal_limit - ::ExtrinsicBaseWeight::get(), - ..Default::default() - }; - let big = DispatchInfo { - weight: normal_limit - ::ExtrinsicBaseWeight::get() + 1, - ..Default::default() - }; - let len = 0_usize; - - let reset_check_weight = |i, f, s| { - AllExtrinsicsWeight::mutate(|current_weight| { - current_weight.put(s, DispatchClass::Normal) - }); - let r = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, i, len); - if f { assert!(r.is_err()) } else { assert!(r.is_ok()) } - }; - - reset_check_weight(&small, false, 0); - reset_check_weight(&medium, false, 0); - reset_check_weight(&big, true, 1); - }) - } - - #[test] - fn signed_ext_check_weight_refund_works() { - new_test_ext().execute_with(|| { - // This is half of the max block weight - let info = DispatchInfo { weight: 512, ..Default::default() }; - let post_info = PostDispatchInfo { actual_weight: Some(128), }; - let len = 0_usize; - - // We allow 75% for normal transaction, so we put 25% - extrinsic base weight - AllExtrinsicsWeight::mutate(|current_weight| { - current_weight.put(256 - ::ExtrinsicBaseWeight::get(), DispatchClass::Normal) - }); - - let pre = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &info, len).unwrap(); - assert_eq!(AllExtrinsicsWeight::get().total(), info.weight + 256); - - assert!( - CheckWeight::::post_dispatch(pre, &info, &post_info, len, &Ok(())) - .is_ok() - ); - assert_eq!( - AllExtrinsicsWeight::get().total(), - post_info.actual_weight.unwrap() + 256, - ); - }) - } - - #[test] - fn signed_ext_check_weight_actual_weight_higher_than_max_is_capped() { - new_test_ext().execute_with(|| { - let info = DispatchInfo { weight: 512, ..Default::default() }; - let post_info = PostDispatchInfo { actual_weight: Some(700), }; - let len = 0_usize; - - AllExtrinsicsWeight::mutate(|current_weight| { - current_weight.put(128, DispatchClass::Normal) - }); - - let pre = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &info, len).unwrap(); - assert_eq!( - AllExtrinsicsWeight::get().total(), - info.weight + 128 + ::ExtrinsicBaseWeight::get(), - ); - - assert!( - CheckWeight::::post_dispatch(pre, &info, &post_info, len, &Ok(())) - .is_ok() - ); - assert_eq!( - AllExtrinsicsWeight::get().total(), - info.weight + 128 + ::ExtrinsicBaseWeight::get(), - ); - }) - } - - #[test] - fn zero_weight_extrinsic_still_has_base_weight() { - new_test_ext().execute_with(|| { - let free = DispatchInfo { weight: 0, ..Default::default() }; - let len = 0_usize; - - // Initial weight from `BlockExecutionWeight` - assert_eq!(System::all_extrinsics_weight().total(), ::BlockExecutionWeight::get()); - let r = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &free, len); - assert!(r.is_ok()); - assert_eq!( - System::all_extrinsics_weight().total(), - ::ExtrinsicBaseWeight::get() + ::BlockExecutionWeight::get() - ); - }) - } - - #[test] - fn mandatory_extrinsic_doesnt_care_about_limits() { - new_test_ext().execute_with(|| { - let max = DispatchInfo { - weight: Weight::max_value(), - class: DispatchClass::Mandatory, - ..Default::default() - }; - let len = 0_usize; - - assert_ok!(CheckWeight::::do_pre_dispatch(&max, len)); - assert_eq!(System::all_extrinsics_weight().total(), Weight::max_value()); - assert!(System::all_extrinsics_weight().total() > ::MaximumBlockWeight::get()); - }); - } - - #[test] - fn register_extra_weight_unchecked_doesnt_care_about_limits() { - new_test_ext().execute_with(|| { - System::register_extra_weight_unchecked(Weight::max_value(), DispatchClass::Normal); - assert_eq!(System::all_extrinsics_weight().total(), Weight::max_value()); - assert!(System::all_extrinsics_weight().total() > ::MaximumBlockWeight::get()); - }); - } - - #[test] - fn full_block_with_normal_and_operational() { - new_test_ext().execute_with(|| { - // Max block is 1024 - // Max normal is 768 (75%) - // 10 is taken for block execution weight - // So normal extrinsic can be 758 weight (-5 for base extrinsic weight) - // And Operational can be 256 to produce a full block (-5 for base) - let max_normal = DispatchInfo { weight: 753, ..Default::default() }; - let rest_operational = DispatchInfo { weight: 251, class: DispatchClass::Operational, ..Default::default() }; - - let len = 0_usize; - - assert_ok!(CheckWeight::::do_pre_dispatch(&max_normal, len)); - assert_eq!(System::all_extrinsics_weight().total(), 768); - assert_ok!(CheckWeight::::do_pre_dispatch(&rest_operational, len)); - assert_eq!(::MaximumBlockWeight::get(), 1024); - assert_eq!(System::all_extrinsics_weight().total(), ::MaximumBlockWeight::get()); - }); - } - - #[test] - fn dispatch_order_does_not_effect_weight_logic() { - new_test_ext().execute_with(|| { - // We switch the order of `full_block_with_normal_and_operational` - let max_normal = DispatchInfo { weight: 753, ..Default::default() }; - let rest_operational = DispatchInfo { weight: 251, class: DispatchClass::Operational, ..Default::default() }; - - let len = 0_usize; - - assert_ok!(CheckWeight::::do_pre_dispatch(&rest_operational, len)); - // Extra 15 here from block execution + base extrinsic weight - assert_eq!(System::all_extrinsics_weight().total(), 266); - assert_ok!(CheckWeight::::do_pre_dispatch(&max_normal, len)); - assert_eq!(::MaximumBlockWeight::get(), 1024); - assert_eq!(System::all_extrinsics_weight().total(), ::MaximumBlockWeight::get()); - }); - } - - #[test] - fn operational_works_on_full_block() { - new_test_ext().execute_with(|| { - // An on_initialize takes up the whole block! (Every time!) - System::register_extra_weight_unchecked(Weight::max_value(), DispatchClass::Mandatory); - let dispatch_normal = DispatchInfo { weight: 251, class: DispatchClass::Normal, ..Default::default() }; - let dispatch_operational = DispatchInfo { weight: 251, class: DispatchClass::Operational, ..Default::default() }; - let len = 0_usize; - - assert_noop!(CheckWeight::::do_pre_dispatch(&dispatch_normal, len), InvalidTransaction::ExhaustsResources); - // Thank goodness we can still do an operational transaction to possibly save the blockchain. - assert_ok!(CheckWeight::::do_pre_dispatch(&dispatch_operational, len)); - // Not too much though - assert_noop!(CheckWeight::::do_pre_dispatch(&dispatch_operational, len), InvalidTransaction::ExhaustsResources); - }); - } - - #[test] - fn signed_ext_check_weight_works_operational_tx() { - new_test_ext().execute_with(|| { - let normal = DispatchInfo { weight: 100, ..Default::default() }; - let op = DispatchInfo { weight: 100, class: DispatchClass::Operational, pays_fee: Pays::Yes }; - let len = 0_usize; - let normal_limit = normal_weight_limit(); - - // given almost full block - AllExtrinsicsWeight::mutate(|current_weight| { - current_weight.put(normal_limit, DispatchClass::Normal) - }); - // will not fit. - assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &normal, len).is_err()); - // will fit. - assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &op, len).is_ok()); - - // likewise for length limit. - let len = 100_usize; - AllExtrinsicsLen::put(normal_length_limit()); - assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &normal, len).is_err()); - assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &op, len).is_ok()); - }) - } - - #[test] - fn signed_ext_check_weight_priority_works() { - new_test_ext().execute_with(|| { - let normal = DispatchInfo { weight: 100, class: DispatchClass::Normal, pays_fee: Pays::Yes }; - let op = DispatchInfo { weight: 100, class: DispatchClass::Operational, pays_fee: Pays::Yes }; - let len = 0_usize; - - let priority = CheckWeight::(PhantomData) - .validate(&1, CALL, &normal, len) - .unwrap() - .priority; - assert_eq!(priority, 100); - - let priority = CheckWeight::(PhantomData) - .validate(&1, CALL, &op, len) - .unwrap() - .priority; - assert_eq!(priority, u64::max_value()); - }) - } - - #[test] - fn signed_ext_check_weight_block_size_works() { - new_test_ext().execute_with(|| { - let normal = DispatchInfo::default(); - let normal_limit = normal_weight_limit() as usize; - let reset_check_weight = |tx, s, f| { - AllExtrinsicsLen::put(0); - let r = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, tx, s); - if f { assert!(r.is_err()) } else { assert!(r.is_ok()) } - }; - - reset_check_weight(&normal, normal_limit - 1, false); - reset_check_weight(&normal, normal_limit, false); - reset_check_weight(&normal, normal_limit + 1, true); - - // Operational ones don't have this limit. - let op = DispatchInfo { weight: 0, class: DispatchClass::Operational, pays_fee: Pays::Yes }; - reset_check_weight(&op, normal_limit, false); - reset_check_weight(&op, normal_limit + 100, false); - reset_check_weight(&op, 1024, false); - reset_check_weight(&op, 1025, true); - }) - } - - #[test] - fn signed_ext_check_era_should_work() { - new_test_ext().execute_with(|| { - // future - assert_eq!( - CheckEra::::from(Era::mortal(4, 2)).additional_signed().err().unwrap(), - InvalidTransaction::AncientBirthBlock.into(), - ); - - // correct - System::set_block_number(13); - >::insert(12, H256::repeat_byte(1)); - assert!(CheckEra::::from(Era::mortal(4, 12)).additional_signed().is_ok()); - }) - } - - #[test] - fn signed_ext_check_era_should_change_longevity() { - new_test_ext().execute_with(|| { - let normal = DispatchInfo { weight: 100, class: DispatchClass::Normal, pays_fee: Pays::Yes }; - let len = 0_usize; - let ext = ( - CheckWeight::(PhantomData), - CheckEra::::from(Era::mortal(16, 256)), - ); - System::set_block_number(17); - >::insert(16, H256::repeat_byte(1)); - - assert_eq!(ext.validate(&1, CALL, &normal, len).unwrap().longevity, 15); - }) - } - - - #[test] - fn set_code_checks_works() { - struct CallInWasm(Vec); - - impl sp_core::traits::CallInWasm for CallInWasm { - fn call_in_wasm( - &self, - _: &[u8], - _: Option>, - _: &str, - _: &[u8], - _: &mut dyn sp_externalities::Externalities, - _: sp_core::traits::MissingHostFunctions, - ) -> Result, String> { - Ok(self.0.clone()) - } - } - - let test_data = vec![ - ("test", 1, 2, Err(Error::::SpecVersionNeedsToIncrease)), - ("test", 1, 1, Err(Error::::SpecVersionNeedsToIncrease)), - ("test2", 1, 1, Err(Error::::InvalidSpecName)), - ("test", 2, 1, Ok(())), - ("test", 0, 1, Err(Error::::SpecVersionNeedsToIncrease)), - ("test", 1, 0, Err(Error::::SpecVersionNeedsToIncrease)), - ]; - - for (spec_name, spec_version, impl_version, expected) in test_data.into_iter() { - let version = RuntimeVersion { - spec_name: spec_name.into(), - spec_version, - impl_version, - ..Default::default() - }; - let call_in_wasm = CallInWasm(version.encode()); - - let mut ext = new_test_ext(); - ext.register_extension(sp_core::traits::CallInWasmExt::new(call_in_wasm)); - ext.execute_with(|| { - let res = System::set_code( - RawOrigin::Root.into(), - vec![1, 2, 3, 4], - ); - - assert_eq!(expected.map_err(DispatchError::from), res); - }); - } - } - - #[test] - fn set_code_with_real_wasm_blob() { - let executor = substrate_test_runtime_client::new_native_executor(); - let mut ext = new_test_ext(); - ext.register_extension(sp_core::traits::CallInWasmExt::new(executor)); - ext.execute_with(|| { - System::set_block_number(1); - System::set_code( - RawOrigin::Root.into(), - substrate_test_runtime_client::runtime::WASM_BINARY.to_vec(), - ).unwrap(); - - assert_eq!( - System::events(), - vec![EventRecord { phase: Phase::Initialization, event: 102u16, topics: vec![] }], - ); - }); - } - - #[test] - fn runtime_upgraded_with_set_storage() { - let executor = substrate_test_runtime_client::new_native_executor(); - let mut ext = new_test_ext(); - ext.register_extension(sp_core::traits::CallInWasmExt::new(executor)); - ext.execute_with(|| { - System::set_storage( - RawOrigin::Root.into(), - vec![( - well_known_keys::CODE.to_vec(), - substrate_test_runtime_client::runtime::WASM_BINARY.to_vec() - )], - ).unwrap(); - }); - } - - #[test] - fn events_not_emitted_during_genesis() { - new_test_ext().execute_with(|| { - // Block Number is zero at genesis - assert!(System::block_number().is_zero()); - System::on_created_account(Default::default()); - assert!(System::events().is_empty()); - // Events will be emitted starting on block 1 - System::set_block_number(1); - System::on_created_account(Default::default()); - assert!(System::events().len() == 1); - }); - } -} diff --git a/frame/system/src/mock.rs b/frame/system/src/mock.rs new file mode 100644 index 0000000000000000000000000000000000000000..0484b34ba3e3cdebd18d3c7452959a799ebb37a1 --- /dev/null +++ b/frame/system/src/mock.rs @@ -0,0 +1,124 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::*; +use sp_std::cell::RefCell; +use sp_core::H256; +use sp_runtime::{ + traits::{BlakeTwo256, IdentityLookup}, + testing::Header, +}; +use frame_support::{ + impl_outer_origin, parameter_types, + weights::PostDispatchInfo, +}; + +impl_outer_origin! { + pub enum Origin for Test where system = super {} +} + +#[derive(Clone, Eq, PartialEq, Debug, Default)] +pub struct Test; + +parameter_types! { + pub const BlockHashCount: u64 = 10; + pub const MaximumBlockWeight: Weight = 1024; + pub const MaximumExtrinsicWeight: Weight = 768; + pub const AvailableBlockRatio: Perbill = Perbill::from_percent(75); + pub const MaximumBlockLength: u32 = 1024; + pub Version: RuntimeVersion = RuntimeVersion { + spec_name: sp_version::create_runtime_str!("test"), + impl_name: sp_version::create_runtime_str!("system-test"), + authoring_version: 1, + spec_version: 1, + impl_version: 1, + apis: sp_version::create_apis_vec!([]), + transaction_version: 1, + }; + pub const BlockExecutionWeight: Weight = 10; + pub const ExtrinsicBaseWeight: Weight = 5; + pub const DbWeight: RuntimeDbWeight = RuntimeDbWeight { + read: 10, + write: 100, + }; +} + +thread_local!{ + pub static KILLED: RefCell> = RefCell::new(vec![]); +} + +pub struct RecordKilled; +impl OnKilledAccount for RecordKilled { + fn on_killed_account(who: &u64) { KILLED.with(|r| r.borrow_mut().push(*who)) } +} + +#[derive(Debug, codec::Encode, codec::Decode)] +pub struct Call; + +impl Dispatchable for Call { + type Origin = Origin; + type Trait = (); + type Info = DispatchInfo; + type PostInfo = PostDispatchInfo; + fn dispatch(self, _origin: Self::Origin) + -> sp_runtime::DispatchResultWithInfo { + panic!("Do not use dummy implementation for dispatch."); + } +} + +impl Trait for Test { + type BaseCallFilter = (); + type Origin = Origin; + type Call = Call; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = BlockHashCount; + type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = DbWeight; + type BlockExecutionWeight = BlockExecutionWeight; + type ExtrinsicBaseWeight = ExtrinsicBaseWeight; + type MaximumExtrinsicWeight = MaximumExtrinsicWeight; + type AvailableBlockRatio = AvailableBlockRatio; + type MaximumBlockLength = MaximumBlockLength; + type Version = Version; + type ModuleToIndex = (); + type AccountData = u32; + type OnNewAccount = (); + type OnKilledAccount = RecordKilled; +} + +pub type System = Module; +pub type SysEvent = ::Event; + +pub const CALL: &::Call = &Call; + +/// Create new externalities for `System` module tests. +pub fn new_test_ext() -> sp_io::TestExternalities { + let mut ext: sp_io::TestExternalities = GenesisConfig::default().build_storage::().unwrap().into(); + // Add to each test the initial weight of a block + ext.execute_with(|| System::register_extra_weight_unchecked( + ::BlockExecutionWeight::get(), + DispatchClass::Mandatory + )); + ext +} diff --git a/frame/system/src/offchain.rs b/frame/system/src/offchain.rs index 42699362a36a30aba55985cb213bcfc6d1ab0c92..1290ca6378eb88aa87c41eddd423fb8c02d78700 100644 --- a/frame/system/src/offchain.rs +++ b/frame/system/src/offchain.rs @@ -638,7 +638,7 @@ pub trait SignedPayload: Encode { mod tests { use super::*; use codec::Decode; - use crate::tests::{Test as TestRuntime, Call}; + use crate::mock::{Test as TestRuntime, Call}; use sp_core::offchain::{testing, TransactionPoolExt}; use sp_runtime::testing::{UintAuthorityId, TestSignature, TestXt}; diff --git a/frame/system/src/tests.rs b/frame/system/src/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..2f93dc858f10bb72b1d4265a6f9bd1233ccd78f2 --- /dev/null +++ b/frame/system/src/tests.rs @@ -0,0 +1,424 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::*; +use mock::{*, Origin}; +use sp_core::H256; +use sp_runtime::DispatchError; +use frame_support::weights::WithPostDispatchInfo; + +#[test] +fn origin_works() { + let o = Origin::from(RawOrigin::::Signed(1u64)); + let x: Result, Origin> = o.into(); + assert_eq!(x.unwrap(), RawOrigin::::Signed(1u64)); +} + +#[test] +fn stored_map_works() { + new_test_ext().execute_with(|| { + System::insert(&0, 42); + assert!(System::allow_death(&0)); + + System::inc_ref(&0); + assert!(!System::allow_death(&0)); + + System::insert(&0, 69); + assert!(!System::allow_death(&0)); + + System::dec_ref(&0); + assert!(System::allow_death(&0)); + + assert!(KILLED.with(|r| r.borrow().is_empty())); + System::kill_account(&0); + assert_eq!(KILLED.with(|r| r.borrow().clone()), vec![0u64]); + }); +} + +#[test] +fn deposit_event_should_work() { + new_test_ext().execute_with(|| { + System::initialize( + &1, + &[0u8; 32].into(), + &[0u8; 32].into(), + &Default::default(), + InitKind::Full, + ); + System::note_finished_extrinsics(); + System::deposit_event(SysEvent::CodeUpdated); + System::finalize(); + assert_eq!( + System::events(), + vec![ + EventRecord { + phase: Phase::Finalization, + event: SysEvent::CodeUpdated, + topics: vec![], + } + ] + ); + + System::initialize( + &2, + &[0u8; 32].into(), + &[0u8; 32].into(), + &Default::default(), + InitKind::Full, + ); + System::deposit_event(SysEvent::NewAccount(32)); + System::note_finished_initialize(); + System::deposit_event(SysEvent::KilledAccount(42)); + System::note_applied_extrinsic(&Ok(().into()), Default::default()); + System::note_applied_extrinsic( + &Err(DispatchError::BadOrigin.into()), + Default::default() + ); + System::note_finished_extrinsics(); + System::deposit_event(SysEvent::NewAccount(3)); + System::finalize(); + assert_eq!( + System::events(), + vec![ + EventRecord { + phase: Phase::Initialization, + event: SysEvent::NewAccount(32), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: SysEvent::KilledAccount(42), + topics: vec![] + }, + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: SysEvent::ExtrinsicSuccess(Default::default()), + topics: vec![] + }, + EventRecord { + phase: Phase::ApplyExtrinsic(1), + event: SysEvent::ExtrinsicFailed( + DispatchError::BadOrigin.into(), + Default::default() + ), + topics: vec![] + }, + EventRecord { + phase: Phase::Finalization, + event: SysEvent::NewAccount(3), + topics: vec![] + }, + ] + ); + }); +} + +#[test] +fn deposit_event_uses_actual_weight() { + new_test_ext().execute_with(|| { + System::initialize( + &1, + &[0u8; 32].into(), + &[0u8; 32].into(), + &Default::default(), + InitKind::Full, + ); + System::note_finished_initialize(); + + let pre_info = DispatchInfo { + weight: 1000, + .. Default::default() + }; + System::note_applied_extrinsic( + &Ok(Some(300).into()), + pre_info, + ); + System::note_applied_extrinsic( + &Ok(Some(1000).into()), + pre_info, + ); + System::note_applied_extrinsic( + // values over the pre info should be capped at pre dispatch value + &Ok(Some(1200).into()), + pre_info, + ); + System::note_applied_extrinsic( + &Err(DispatchError::BadOrigin.with_weight(999)), + pre_info, + ); + + assert_eq!( + System::events(), + vec![ + EventRecord { + phase: Phase::ApplyExtrinsic(0), + event: SysEvent::ExtrinsicSuccess( + DispatchInfo { + weight: 300, + .. Default::default() + }, + ), + topics: vec![] + }, + EventRecord { + phase: Phase::ApplyExtrinsic(1), + event: SysEvent::ExtrinsicSuccess( + DispatchInfo { + weight: 1000, + .. Default::default() + }, + ), + topics: vec![] + }, + EventRecord { + phase: Phase::ApplyExtrinsic(2), + event: SysEvent::ExtrinsicSuccess( + DispatchInfo { + weight: 1000, + .. Default::default() + }, + ), + topics: vec![] + }, + EventRecord { + phase: Phase::ApplyExtrinsic(3), + event: SysEvent::ExtrinsicFailed( + DispatchError::BadOrigin.into(), + DispatchInfo { + weight: 999, + .. Default::default() + }, + ), + topics: vec![] + }, + ] + ); + }); +} + +#[test] +fn deposit_event_topics() { + new_test_ext().execute_with(|| { + const BLOCK_NUMBER: u64 = 1; + + System::initialize( + &BLOCK_NUMBER, + &[0u8; 32].into(), + &[0u8; 32].into(), + &Default::default(), + InitKind::Full, + ); + System::note_finished_extrinsics(); + + let topics = vec![ + H256::repeat_byte(1), + H256::repeat_byte(2), + H256::repeat_byte(3), + ]; + + // We deposit a few events with different sets of topics. + System::deposit_event_indexed(&topics[0..3], SysEvent::NewAccount(1)); + System::deposit_event_indexed(&topics[0..1], SysEvent::NewAccount(2)); + System::deposit_event_indexed(&topics[1..2], SysEvent::NewAccount(3)); + + System::finalize(); + + // Check that topics are reflected in the event record. + assert_eq!( + System::events(), + vec![ + EventRecord { + phase: Phase::Finalization, + event: SysEvent::NewAccount(1), + topics: topics[0..3].to_vec(), + }, + EventRecord { + phase: Phase::Finalization, + event: SysEvent::NewAccount(2), + topics: topics[0..1].to_vec(), + }, + EventRecord { + phase: Phase::Finalization, + event: SysEvent::NewAccount(3), + topics: topics[1..2].to_vec(), + } + ] + ); + + // Check that the topic-events mapping reflects the deposited topics. + // Note that these are indexes of the events. + assert_eq!( + System::event_topics(&topics[0]), + vec![(BLOCK_NUMBER, 0), (BLOCK_NUMBER, 1)], + ); + assert_eq!( + System::event_topics(&topics[1]), + vec![(BLOCK_NUMBER, 0), (BLOCK_NUMBER, 2)], + ); + assert_eq!( + System::event_topics(&topics[2]), + vec![(BLOCK_NUMBER, 0)], + ); + }); +} + +#[test] +fn prunes_block_hash_mappings() { + new_test_ext().execute_with(|| { + // simulate import of 15 blocks + for n in 1..=15 { + System::initialize( + &n, + &[n as u8 - 1; 32].into(), + &[0u8; 32].into(), + &Default::default(), + InitKind::Full, + ); + + System::finalize(); + } + + // first 5 block hashes are pruned + for n in 0..5 { + assert_eq!( + System::block_hash(n), + H256::zero(), + ); + } + + // the remaining 10 are kept + for n in 5..15 { + assert_eq!( + System::block_hash(n), + [n as u8; 32].into(), + ); + } + }) +} + +#[test] +fn set_code_checks_works() { + struct CallInWasm(Vec); + + impl sp_core::traits::CallInWasm for CallInWasm { + fn call_in_wasm( + &self, + _: &[u8], + _: Option>, + _: &str, + _: &[u8], + _: &mut dyn sp_externalities::Externalities, + _: sp_core::traits::MissingHostFunctions, + ) -> Result, String> { + Ok(self.0.clone()) + } + } + + let test_data = vec![ + ("test", 1, 2, Err(Error::::SpecVersionNeedsToIncrease)), + ("test", 1, 1, Err(Error::::SpecVersionNeedsToIncrease)), + ("test2", 1, 1, Err(Error::::InvalidSpecName)), + ("test", 2, 1, Ok(())), + ("test", 0, 1, Err(Error::::SpecVersionNeedsToIncrease)), + ("test", 1, 0, Err(Error::::SpecVersionNeedsToIncrease)), + ]; + + for (spec_name, spec_version, impl_version, expected) in test_data.into_iter() { + let version = RuntimeVersion { + spec_name: spec_name.into(), + spec_version, + impl_version, + ..Default::default() + }; + let call_in_wasm = CallInWasm(version.encode()); + + let mut ext = new_test_ext(); + ext.register_extension(sp_core::traits::CallInWasmExt::new(call_in_wasm)); + ext.execute_with(|| { + let res = System::set_code( + RawOrigin::Root.into(), + vec![1, 2, 3, 4], + ); + + assert_eq!(expected.map_err(DispatchError::from), res); + }); + } +} + +#[test] +fn set_code_with_real_wasm_blob() { + let executor = substrate_test_runtime_client::new_native_executor(); + let mut ext = new_test_ext(); + ext.register_extension(sp_core::traits::CallInWasmExt::new(executor)); + ext.execute_with(|| { + System::set_block_number(1); + System::set_code( + RawOrigin::Root.into(), + substrate_test_runtime_client::runtime::WASM_BINARY.to_vec(), + ).unwrap(); + + assert_eq!( + System::events(), + vec![EventRecord { + phase: Phase::Initialization, + event: SysEvent::CodeUpdated, + topics: vec![], + }], + ); + }); +} + +#[test] +fn runtime_upgraded_with_set_storage() { + let executor = substrate_test_runtime_client::new_native_executor(); + let mut ext = new_test_ext(); + ext.register_extension(sp_core::traits::CallInWasmExt::new(executor)); + ext.execute_with(|| { + System::set_storage( + RawOrigin::Root.into(), + vec![( + well_known_keys::CODE.to_vec(), + substrate_test_runtime_client::runtime::WASM_BINARY.to_vec() + )], + ).unwrap(); + }); +} + +#[test] +fn events_not_emitted_during_genesis() { + new_test_ext().execute_with(|| { + // Block Number is zero at genesis + assert!(System::block_number().is_zero()); + System::on_created_account(Default::default()); + assert!(System::events().is_empty()); + // Events will be emitted starting on block 1 + System::set_block_number(1); + System::on_created_account(Default::default()); + assert!(System::events().len() == 1); + }); +} + +#[test] +fn ensure_one_of_works() { + fn ensure_root_or_signed(o: RawOrigin) -> Result, Origin> { + EnsureOneOf::, EnsureSigned>::try_origin(o.into()) + } + + assert_eq!(ensure_root_or_signed(RawOrigin::Root).unwrap(), Either::Left(())); + assert_eq!(ensure_root_or_signed(RawOrigin::Signed(0)).unwrap(), Either::Right(0)); + assert!(ensure_root_or_signed(RawOrigin::None).is_err()) +} diff --git a/frame/system/src/weights.rs b/frame/system/src/weights.rs new file mode 100644 index 0000000000000000000000000000000000000000..93295093c4fb88aaff70c259d889a6ce1e352a04 --- /dev/null +++ b/frame/system/src/weights.rs @@ -0,0 +1,76 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use codec::{Encode, Decode}; +use frame_support::weights::{Weight, DispatchClass}; +use sp_runtime::RuntimeDebug; + +/// An object to track the currently used extrinsic weight in a block. +#[derive(Clone, Eq, PartialEq, Default, RuntimeDebug, Encode, Decode)] +pub struct ExtrinsicsWeight { + normal: Weight, + operational: Weight, +} + +impl ExtrinsicsWeight { + /// Returns the total weight consumed by all extrinsics in the block. + pub fn total(&self) -> Weight { + self.normal.saturating_add(self.operational) + } + + /// Add some weight of a specific dispatch class, saturating at the numeric bounds of `Weight`. + pub fn add(&mut self, weight: Weight, class: DispatchClass) { + let value = self.get_mut(class); + *value = value.saturating_add(weight); + } + + /// Try to add some weight of a specific dispatch class, returning Err(()) if overflow would + /// occur. + pub fn checked_add(&mut self, weight: Weight, class: DispatchClass) -> Result<(), ()> { + let value = self.get_mut(class); + *value = value.checked_add(weight).ok_or(())?; + Ok(()) + } + + /// Subtract some weight of a specific dispatch class, saturating at the numeric bounds of + /// `Weight`. + pub fn sub(&mut self, weight: Weight, class: DispatchClass) { + let value = self.get_mut(class); + *value = value.saturating_sub(weight); + } + + /// Get the current weight of a specific dispatch class. + pub fn get(&self, class: DispatchClass) -> Weight { + match class { + DispatchClass::Operational => self.operational, + DispatchClass::Normal | DispatchClass::Mandatory => self.normal, + } + } + + /// Get a mutable reference to the current weight of a specific dispatch class. + fn get_mut(&mut self, class: DispatchClass) -> &mut Weight { + match class { + DispatchClass::Operational => &mut self.operational, + DispatchClass::Normal | DispatchClass::Mandatory => &mut self.normal, + } + } + + /// Set the weight of a specific dispatch class. + pub fn put(&mut self, new: Weight, class: DispatchClass) { + *self.get_mut(class) = new; + } +} diff --git a/frame/timestamp/Cargo.toml b/frame/timestamp/Cargo.toml index e5ac4f1aa7e9244255c4cbc8d89dbfefce2a4d52..2c2ad68b96bfd16df299dd28e6eef4f8c01548d0 100644 --- a/frame/timestamp/Cargo.toml +++ b/frame/timestamp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-timestamp" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -15,20 +15,20 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io", optional = true } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/inherents" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../benchmarking", optional = true } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -sp-timestamp = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/timestamp" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io", optional = true } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/inherents" } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +sp-timestamp = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/timestamp" } impl-trait-for-tuples = "0.1.3" [dev-dependencies] -sp-io ={ version = "2.0.0-alpha.8", path = "../../primitives/io" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } +sp-io ={ version = "2.0.0-rc4", path = "../../primitives/io" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } [features] default = ["std"] diff --git a/frame/timestamp/src/lib.rs b/frame/timestamp/src/lib.rs index e886f8079fb4c6560115e4b33453f3a8ba7d3828..63456100a5e02e94e31b40452ee0c21b026c4210 100644 --- a/frame/timestamp/src/lib.rs +++ b/frame/timestamp/src/lib.rs @@ -314,6 +314,7 @@ mod tests { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -329,6 +330,7 @@ mod tests { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); @@ -351,7 +353,7 @@ mod tests { fn timestamp_works() { new_test_ext().execute_with(|| { Timestamp::set_timestamp(42); - assert_ok!(Timestamp::dispatch(Call::set(69), Origin::NONE)); + assert_ok!(Timestamp::set(Origin::none(), 69)); assert_eq!(Timestamp::now(), 69); }); } @@ -361,8 +363,8 @@ mod tests { fn double_timestamp_should_fail() { new_test_ext().execute_with(|| { Timestamp::set_timestamp(42); - assert_ok!(Timestamp::dispatch(Call::set(69), Origin::NONE)); - let _ = Timestamp::dispatch(Call::set(70), Origin::NONE); + assert_ok!(Timestamp::set(Origin::none(), 69)); + let _ = Timestamp::set(Origin::none(), 70); }); } @@ -371,7 +373,7 @@ mod tests { fn block_period_minimum_enforced() { new_test_ext().execute_with(|| { Timestamp::set_timestamp(42); - let _ = Timestamp::dispatch(Call::set(46), Origin::NONE); + let _ = Timestamp::set(Origin::none(), 46); }); } } diff --git a/frame/transaction-payment/Cargo.toml b/frame/transaction-payment/Cargo.toml index 866d2ceda569d7dc907b6737198bb68f2784c4d3..c1409c2675c36b41fbc9613edf6c25f496098c6f 100644 --- a/frame/transaction-payment/Cargo.toml +++ b/frame/transaction-payment/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-transaction-payment" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,22 +12,25 @@ description = "FRAME pallet to manage transaction payments" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-alpha.8", default-features = false, path = "./rpc/runtime-api" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +serde = { version = "1.0.101", optional = true } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-rc4", default-features = false, path = "./rpc/runtime-api" } +smallvec = "1.4.0" [dev-dependencies] -sp-io = { version = "2.0.0-alpha.8", path = "../../primitives/io" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } -sp-storage = { version = "2.0.0-alpha.8", path = "../../primitives/storage" } +sp-io = { version = "2.0.0-rc4", path = "../../primitives/io" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } +sp-storage = { version = "2.0.0-rc4", path = "../../primitives/storage" } [features] default = ["std"] std = [ + "serde", "codec/std", "sp-std/std", "sp-runtime/std", diff --git a/frame/transaction-payment/rpc/Cargo.toml b/frame/transaction-payment/rpc/Cargo.toml index f66e2a9f9d1ea7bdef5fbca16fe5d05795ee8177..f26f6044714effc0eedb854612b4dbe22dccb686 100644 --- a/frame/transaction-payment/rpc/Cargo.toml +++ b/frame/transaction-payment/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-transaction-payment-rpc" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,14 +12,14 @@ description = "RPC interface for the transaction payment module." targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0" } -jsonrpc-core = "14.0.3" -jsonrpc-core-client = "14.0.5" -jsonrpc-derive = "14.0.3" -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sp-rpc = { version = "2.0.0-alpha.8", path = "../../../primitives/rpc" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +jsonrpc-core = "14.2.0" +jsonrpc-core-client = "14.2.0" +jsonrpc-derive = "14.2.1" +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sp-rpc = { version = "2.0.0-rc4", path = "../../../primitives/rpc" } serde = { version = "1.0.101", features = ["derive"] } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.8", path = "../../../primitives/api" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } -pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-alpha.8", path = "./runtime-api" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-rc4", path = "../../../primitives/api" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } +pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-rc4", path = "./runtime-api" } diff --git a/frame/transaction-payment/rpc/runtime-api/Cargo.toml b/frame/transaction-payment/rpc/runtime-api/Cargo.toml index 909e526ed316c7783ba8dff57358982fe5a9ed64..2cd9977704cf5bae7e4becb7e2e74a7366029a35 100644 --- a/frame/transaction-payment/rpc/runtime-api/Cargo.toml +++ b/frame/transaction-payment/rpc/runtime-api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-transaction-payment-rpc-runtime-api" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,11 +13,11 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-api = { version = "2.0.0-alpha.8", default-features = false, path = "../../../../primitives/api" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../../../support" } +sp-api = { version = "2.0.0-rc4", default-features = false, path = "../../../../primitives/api" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../../../primitives/std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../../../primitives/runtime" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../../../support" } [dev-dependencies] serde_json = "1.0.41" diff --git a/frame/transaction-payment/rpc/runtime-api/src/lib.rs b/frame/transaction-payment/rpc/runtime-api/src/lib.rs index 43928073389c003e856f7adbb134ddd9f1ba9d58..17a8bcdf44e87a695e703515801a56f03ffc4d8e 100644 --- a/frame/transaction-payment/rpc/runtime-api/src/lib.rs +++ b/frame/transaction-payment/rpc/runtime-api/src/lib.rs @@ -26,7 +26,7 @@ use codec::{Encode, Codec, Decode}; use serde::{Serialize, Deserialize, Serializer, Deserializer}; use sp_runtime::traits::{MaybeDisplay, MaybeFromStr}; -/// Some information related to a dispatchable that can be queried from the runtime. +/// Information related to a dispatchable's class, weight, and fee that can be queried from the runtime. #[derive(Eq, PartialEq, Encode, Decode, Default)] #[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] #[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] @@ -35,8 +35,8 @@ pub struct RuntimeDispatchInfo { pub weight: Weight, /// Class of this dispatch. pub class: DispatchClass, - /// The partial inclusion fee of this dispatch. This does not include tip or anything else which - /// is dependent on the signature (aka. depends on a `SignedExtension`). + /// The inclusion fee of this dispatch. This does not include a tip or anything else that + /// depends on the signature (i.e. depends on a `SignedExtension`). #[cfg_attr(feature = "std", serde(bound(serialize = "Balance: std::fmt::Display")))] #[cfg_attr(feature = "std", serde(serialize_with = "serialize_as_string"))] #[cfg_attr(feature = "std", serde(bound(deserialize = "Balance: std::str::FromStr")))] diff --git a/frame/transaction-payment/src/lib.rs b/frame/transaction-payment/src/lib.rs index 42004021d6cad2cbdb4a3475f0626f83cda8a785..b993a85da3df3f9ccd3caa413583661d75827a26 100644 --- a/frame/transaction-payment/src/lib.rs +++ b/frame/transaction-payment/src/lib.rs @@ -37,28 +37,145 @@ use codec::{Encode, Decode}; use frame_support::{ decl_storage, decl_module, traits::{Currency, Get, OnUnbalanced, ExistenceRequirement, WithdrawReason, Imbalance}, - weights::{Weight, DispatchInfo, PostDispatchInfo, GetDispatchInfo, Pays}, + weights::{ + Weight, DispatchInfo, PostDispatchInfo, GetDispatchInfo, Pays, WeightToFeePolynomial, + WeightToFeeCoefficient, + }, dispatch::DispatchResult, }; use sp_runtime::{ - Fixed128, + FixedU128, FixedPointNumber, FixedPointOperand, Perquintill, RuntimeDebug, transaction_validity::{ TransactionPriority, ValidTransaction, InvalidTransaction, TransactionValidityError, TransactionValidity, }, traits::{ Zero, Saturating, SignedExtension, SaturatedConversion, Convert, Dispatchable, - DispatchInfoOf, PostDispatchInfoOf, UniqueSaturatedFrom, UniqueSaturatedInto, + DispatchInfoOf, PostDispatchInfoOf, }, }; use pallet_transaction_payment_rpc_runtime_api::RuntimeDispatchInfo; -type Multiplier = Fixed128; +/// Fee multiplier. +pub type Multiplier = FixedU128; + type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; type NegativeImbalanceOf = <::Currency as Currency<::AccountId>>::NegativeImbalance; +/// A struct to update the weight multiplier per block. It implements `Convert`, meaning that it can convert the previous multiplier to the next one. This should +/// be called on `on_finalize` of a block, prior to potentially cleaning the weight data from the +/// system module. +/// +/// given: +/// s = previous block weight +/// s'= ideal block weight +/// m = maximum block weight +/// diff = (s - s')/m +/// v = 0.00001 +/// t1 = (v * diff) +/// t2 = (v * diff)^2 / 2 +/// then: +/// next_multiplier = prev_multiplier * (1 + t1 + t2) +/// +/// Where `(s', v)` must be given as the `Get` implementation of the `T` generic type. Moreover, `M` +/// must provide the minimum allowed value for the multiplier. Note that a runtime should ensure +/// with tests that the combination of this `M` and `V` is not such that the multiplier can drop to +/// zero and never recover. +/// +/// note that `s'` is interpreted as a portion in the _normal transaction_ capacity of the block. +/// For example, given `s' == 0.25` and `AvailableBlockRatio = 0.75`, then the target fullness is +/// _0.25 of the normal capacity_ and _0.1875 of the entire block_. +/// +/// This implementation implies the bound: +/// - `v ≤ p / k * (s − s')` +/// - or, solving for `p`: `p >= v * k * (s - s')` +/// +/// where `p` is the amount of change over `k` blocks. +/// +/// Hence: +/// - in a fully congested chain: `p >= v * k * (1 - s')`. +/// - in an empty chain: `p >= v * k * (-s')`. +/// +/// For example, when all blocks are full and there are 28800 blocks per day (default in `substrate-node`) +/// and v == 0.00001, s' == 0.1875, we'd have: +/// +/// p >= 0.00001 * 28800 * 0.8125 +/// p >= 0.234 +/// +/// Meaning that fees can change by around ~23% per day, given extreme congestion. +/// +/// More info can be found at: +/// https://w3f-research.readthedocs.io/en/latest/polkadot/Token%20Economics.html +pub struct TargetedFeeAdjustment(sp_std::marker::PhantomData<(T, S, V, M)>); + +impl Convert for TargetedFeeAdjustment + where T: frame_system::Trait, S: Get, V: Get, M: Get, +{ + fn convert(previous: Multiplier) -> Multiplier { + // Defensive only. The multiplier in storage should always be at most positive. Nonetheless + // we recover here in case of errors, because any value below this would be stale and can + // never change. + let min_multiplier = M::get(); + let previous = previous.max(min_multiplier); + + // the computed ratio is only among the normal class. + let normal_max_weight = + ::AvailableBlockRatio::get() * + ::MaximumBlockWeight::get(); + let normal_block_weight = + >::block_weight() + .get(frame_support::weights::DispatchClass::Normal) + .min(normal_max_weight); + + let s = S::get(); + let v = V::get(); + + let target_weight = (s * normal_max_weight) as u128; + let block_weight = normal_block_weight as u128; + + // determines if the first_term is positive + let positive = block_weight >= target_weight; + let diff_abs = block_weight.max(target_weight) - block_weight.min(target_weight); + + // defensive only, a test case assures that the maximum weight diff can fit in Multiplier + // without any saturation. + let diff = Multiplier::saturating_from_rational(diff_abs, normal_max_weight.max(1)); + let diff_squared = diff.saturating_mul(diff); + + let v_squared_2 = v.saturating_mul(v) / Multiplier::saturating_from_integer(2); + + let first_term = v.saturating_mul(diff); + let second_term = v_squared_2.saturating_mul(diff_squared); + + if positive { + let excess = first_term.saturating_add(second_term).saturating_mul(previous); + previous.saturating_add(excess).max(min_multiplier) + } else { + // Defensive-only: first_term > second_term. Safe subtraction. + let negative = first_term.saturating_sub(second_term).saturating_mul(previous); + previous.saturating_sub(negative).max(min_multiplier) + } + } +} + +/// Storage releases of the module. +#[derive(Encode, Decode, Clone, Copy, PartialEq, Eq, RuntimeDebug)] +enum Releases { + /// Original version of the module. + V1Ancient, + /// One that bumps the usage to FixedU128 from FixedI128. + V2, +} + +impl Default for Releases { + fn default() -> Self { + Releases::V1Ancient + } +} + pub trait Trait: frame_system::Trait { /// The currency type in which fees will be paid. type Currency: Currency + Send + Sync; @@ -72,7 +189,7 @@ pub trait Trait: frame_system::Trait { type TransactionByteFee: Get>; /// Convert a weight value into a deductible fee based on the currency type. - type WeightToFee: Convert>; + type WeightToFee: WeightToFeePolynomial>; /// Update the multiplier of the next block, based on the previous block's weight. type FeeMultiplierUpdate: Convert; @@ -80,7 +197,9 @@ pub trait Trait: frame_system::Trait { decl_storage! { trait Store for Module as TransactionPayment { - pub NextFeeMultiplier get(fn next_fee_multiplier): Multiplier = Multiplier::from_parts(0); + pub NextFeeMultiplier get(fn next_fee_multiplier): Multiplier = Multiplier::saturating_from_integer(1); + + StorageVersion build(|_: &GenesisConfig| Releases::V2): Releases; } } @@ -89,19 +208,39 @@ decl_module! { /// The fee to be paid for making a transaction; the per-byte portion. const TransactionByteFee: BalanceOf = T::TransactionByteFee::get(); + /// The polynomial that is applied in order to derive fee from weight. + const WeightToFee: Vec>> = + T::WeightToFee::polynomial().to_vec(); + fn on_finalize() { NextFeeMultiplier::mutate(|fm| { - *fm = T::FeeMultiplierUpdate::convert(*fm) + *fm = T::FeeMultiplierUpdate::convert(*fm); }); } + + fn integrity_test() { + // given weight == u64, we build multipliers from `diff` of two weight values, which can + // at most be MaximumBlockWeight. Make sure that this can fit in a multiplier without + // loss. + use sp_std::convert::TryInto; + assert!( + ::max_value() >= + Multiplier::checked_from_integer( + ::MaximumBlockWeight::get().try_into().unwrap() + ).unwrap(), + ); + } } } -impl Module { +impl Module where + BalanceOf: FixedPointOperand +{ /// Query the data that we know about the fee of a given `call`. /// - /// As this module is not and cannot be aware of the internals of a signed extension, it only - /// interprets them as some encoded value and takes their length into account. + /// This module is not and cannot be aware of the internals of a signed extension, for example + /// a tip. It only interprets the extrinsic as some encoded value and accounts for its weight + /// and length, the runtime's extrinsic base weight, and the current fee multiplier. /// /// All dispatchables must be annotated with weight and will have some fee info. This function /// always returns. @@ -130,17 +269,24 @@ impl Module { /// Compute the final fee value for a particular transaction. /// /// The final fee is composed of: - /// - _base_fee_: This is the minimum amount a user pays for a transaction. - /// - _len_fee_: This is the amount paid merely to pay for size of the transaction. - /// - _weight_fee_: This amount is computed based on the weight of the transaction. Unlike - /// size-fee, this is not input dependent and reflects the _complexity_ of the execution - /// and the time it consumes. - /// - _targeted_fee_adjustment_: This is a multiplier that can tune the final fee based on + /// - `base_fee`: This is the minimum amount a user pays for a transaction. It is declared + /// as a base _weight_ in the runtime and converted to a fee using `WeightToFee`. + /// - `len_fee`: The length fee, the amount paid for the encoded length (in bytes) of the + /// transaction. + /// - `weight_fee`: This amount is computed based on the weight of the transaction. Weight + /// accounts for the execution time of a transaction. + /// - `targeted_fee_adjustment`: This is a multiplier that can tune the final fee based on /// the congestion of the network. - /// - (optional) _tip_: if included in the transaction, it will be added on top. Only signed - /// transactions can have a tip. + /// - (Optional) `tip`: If included in the transaction, the tip will be added on top. Only + /// signed transactions can have a tip. /// - /// final_fee = base_fee + targeted_fee_adjustment(len_fee + weight_fee) + tip; + /// The base fee and adjusted weight and length fees constitute the _inclusion fee,_ which is + /// the minimum fee for a transaction to be included in a block. + /// + /// ```ignore + /// inclusion_fee = base_fee + len_fee + [targeted_fee_adjustment * weight_fee]; + /// final_fee = inclusion_fee + tip; + /// ``` pub fn compute_fee( len: u32, info: &DispatchInfoOf, @@ -148,42 +294,72 @@ impl Module { ) -> BalanceOf where T::Call: Dispatchable, { - if info.pays_fee == Pays::Yes { + Self::compute_fee_raw(len, info.weight, tip, info.pays_fee) + } + + /// Compute the actual post dispatch fee for a particular transaction. + /// + /// Identical to `compute_fee` with the only difference that the post dispatch corrected + /// weight is used for the weight fee calculation. + pub fn compute_actual_fee( + len: u32, + info: &DispatchInfoOf, + post_info: &PostDispatchInfoOf, + tip: BalanceOf, + ) -> BalanceOf where + T::Call: Dispatchable, + { + Self::compute_fee_raw(len, post_info.calc_actual_weight(info), tip, info.pays_fee) + } + + fn compute_fee_raw( + len: u32, + weight: Weight, + tip: BalanceOf, + pays_fee: Pays, + ) -> BalanceOf { + if pays_fee == Pays::Yes { let len = >::from(len); let per_byte = T::TransactionByteFee::get(); - let len_fee = per_byte.saturating_mul(len); - let unadjusted_weight_fee = Self::weight_to_fee(info.weight); - // the adjustable part of the fee - let adjustable_fee = len_fee.saturating_add(unadjusted_weight_fee); - let targeted_fee_adjustment = NextFeeMultiplier::get(); - let adjusted_fee = targeted_fee_adjustment.saturated_multiply_accumulate(adjustable_fee.saturated_into()); + // length fee. this is not adjusted. + let fixed_len_fee = per_byte.saturating_mul(len); + + // the adjustable part of the fee. + let unadjusted_weight_fee = Self::weight_to_fee(weight); + let multiplier = Self::next_fee_multiplier(); + // final adjusted weight fee. + let adjusted_weight_fee = multiplier.saturating_mul_int(unadjusted_weight_fee); let base_fee = Self::weight_to_fee(T::ExtrinsicBaseWeight::get()); - base_fee.saturating_add(adjusted_fee.saturated_into()).saturating_add(tip) + base_fee + .saturating_add(fixed_len_fee) + .saturating_add(adjusted_weight_fee) + .saturating_add(tip) } else { tip } } - /// Compute the fee for the specified weight. - /// - /// This fee is already adjusted by the per block fee adjustment factor and is therefore - /// the share that the weight contributes to the overall fee of a transaction. - pub fn weight_to_fee_with_adjustment(weight: Weight) -> Balance where - Balance: UniqueSaturatedFrom - { - let fee = UniqueSaturatedInto::::unique_saturated_into(Self::weight_to_fee(weight)); - UniqueSaturatedFrom::unique_saturated_from( - NextFeeMultiplier::get().saturated_multiply_accumulate(fee) - ) - } - fn weight_to_fee(weight: Weight) -> BalanceOf { // cap the weight to the maximum defined in runtime, otherwise it will be the // `Bounded` maximum of its data type, which is not desired. let capped_weight = weight.min(::MaximumBlockWeight::get()); - T::WeightToFee::convert(capped_weight) + T::WeightToFee::calc(&capped_weight) + } +} + +impl Convert> for Module where + T: Trait, + BalanceOf: FixedPointOperand, +{ + /// Compute the fee for the specified weight. + /// + /// This fee is already adjusted by the per block fee adjustment factor and is therefore the + /// share that the weight contributes to the overall fee of a transaction. It is mainly + /// for informational purposes and not used in the actual fee calculation. + fn convert(weight: Weight) -> BalanceOf { + NextFeeMultiplier::get().saturating_mul_int(Self::weight_to_fee(weight)) } } @@ -194,7 +370,7 @@ pub struct ChargeTransactionPayment(#[codec(compact)] Ba impl ChargeTransactionPayment where T::Call: Dispatchable, - BalanceOf: Send + Sync, + BalanceOf: Send + Sync + FixedPointOperand, { /// utility constructor. Used only in client/factory code. pub fn from(fee: BalanceOf) -> Self { @@ -243,14 +419,14 @@ impl sp_std::fmt::Debug for ChargeTransactionPayment } impl SignedExtension for ChargeTransactionPayment where - BalanceOf: Send + Sync + From, + BalanceOf: Send + Sync + From + FixedPointOperand, T::Call: Dispatchable, { const IDENTIFIER: &'static str = "ChargeTransactionPayment"; type AccountId = T::AccountId; type Call = T::Call; type AdditionalSigned = (); - type Pre = (BalanceOf, Self::AccountId, Option>); + type Pre = (BalanceOf, Self::AccountId, Option>, BalanceOf); fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } fn validate( @@ -276,20 +452,26 @@ impl SignedExtension for ChargeTransactionPayment whe info: &DispatchInfoOf, len: usize ) -> Result { - let (_, imbalance) = self.withdraw_fee(who, info, len)?; - Ok((self.0, who.clone(), imbalance)) + let (fee, imbalance) = self.withdraw_fee(who, info, len)?; + Ok((self.0, who.clone(), imbalance, fee)) } fn post_dispatch( pre: Self::Pre, info: &DispatchInfoOf, post_info: &PostDispatchInfoOf, - _len: usize, + len: usize, _result: &DispatchResult, ) -> Result<(), TransactionValidityError> { - let (tip, who, imbalance) = pre; + let (tip, who, imbalance, fee) = pre; if let Some(payed) = imbalance { - let refund = Module::::weight_to_fee_with_adjustment(post_info.calc_unspent(info)); + let actual_fee = Module::::compute_actual_fee( + len as u32, + info, + post_info, + tip, + ); + let refund = fee.saturating_sub(actual_fee); let actual_payment = match T::Currency::deposit_into_existing(&who, refund) { Ok(refund_imbalance) => { // The refund cannot be larger than the up front payed max weight. @@ -314,11 +496,13 @@ impl SignedExtension for ChargeTransactionPayment whe #[cfg(test)] mod tests { use super::*; - use core::num::NonZeroI128; use codec::Encode; use frame_support::{ - impl_outer_dispatch, impl_outer_origin, parameter_types, - weights::{DispatchClass, DispatchInfo, PostDispatchInfo, GetDispatchInfo, Weight}, + impl_outer_dispatch, impl_outer_origin, impl_outer_event, parameter_types, + weights::{ + DispatchClass, DispatchInfo, PostDispatchInfo, GetDispatchInfo, Weight, + WeightToFeePolynomial, WeightToFeeCoefficients, WeightToFeeCoefficient, + }, }; use pallet_balances::Call as BalancesCall; use pallet_transaction_payment_rpc_runtime_api::RuntimeDispatchInfo; @@ -329,6 +513,7 @@ mod tests { Perbill, }; use std::cell::RefCell; + use smallvec::smallvec; const CALL: &::Call = &Call::Balances(BalancesCall::transfer(2, 69)); @@ -340,6 +525,13 @@ mod tests { } } + impl_outer_event! { + pub enum Event for Runtime { + system, + pallet_balances, + } + } + #[derive(Clone, PartialEq, Eq, Debug)] pub struct Runtime; @@ -365,6 +557,7 @@ mod tests { } impl frame_system::Trait for Runtime { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -374,12 +567,13 @@ mod tests { type AccountId = u64; type Lookup = IdentityLookup; type Header = Header; - type Event = (); + type Event = Event; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = ExtrinsicBaseWeight; + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -395,7 +589,7 @@ mod tests { impl pallet_balances::Trait for Runtime { type Balance = u64; - type Event = (); + type Event = Event; type DustRemoval = (); type ExistentialDeposit = ExistentialDeposit; type AccountStore = System; @@ -410,10 +604,17 @@ mod tests { fn get() -> u64 { TRANSACTION_BYTE_FEE.with(|v| *v.borrow()) } } - pub struct WeightToFee(u64); - impl Convert for WeightToFee { - fn convert(t: Weight) -> u64 { - WEIGHT_TO_FEE.with(|v| *v.borrow() * (t as u64)) + pub struct WeightToFee; + impl WeightToFeePolynomial for WeightToFee { + type Balance = u64; + + fn polynomial() -> WeightToFeeCoefficients { + smallvec![WeightToFeeCoefficient { + degree: 1, + coeff_frac: Perbill::zero(), + coeff_integer: WEIGHT_TO_FEE.with(|v| *v.borrow()), + negative: false, + }] } } @@ -492,7 +693,7 @@ mod tests { /// create a transaction info struct from weight. Handy to avoid building the whole struct. pub fn info_from_weight(w: Weight) -> DispatchInfo { - // pays: yes -- class: normal + // pays_fee: Pays::Yes -- class: DispatchClass::Normal DispatchInfo { weight: w, ..Default::default() } } @@ -548,21 +749,21 @@ mod tests { .execute_with(|| { let len = 10; - NextFeeMultiplier::put(Fixed128::from_rational(1, NonZeroI128::new(2).unwrap())); + NextFeeMultiplier::put(Multiplier::saturating_from_rational(3, 2)); let pre = ChargeTransactionPayment::::from(5 /* tipped */) .pre_dispatch(&2, CALL, &info_from_weight(100), len) .unwrap(); - // 5 base fee, 3/2 * 10 byte fee, 3/2 * 100 weight fee, 5 tip - assert_eq!(Balances::free_balance(2), 200 - 5 - 15 - 150 - 5); + // 5 base fee, 10 byte fee, 3/2 * 100 weight fee, 5 tip + assert_eq!(Balances::free_balance(2), 200 - 5 - 10 - 150 - 5); assert!( ChargeTransactionPayment:: ::post_dispatch(pre, &info_from_weight(100), &post_info_from_weight(50), len, &Ok(())) .is_ok() ); - // 75 (3/2 of the returned 50 units of weight ) is refunded - assert_eq!(Balances::free_balance(2), 200 - 5 - 15 - 75 - 5); + // 75 (3/2 of the returned 50 units of weight) is refunded + assert_eq!(Balances::free_balance(2), 200 - 5 - 10 - 75 - 5); }); } @@ -636,7 +837,7 @@ mod tests { .execute_with(|| { // all fees should be x1.5 - NextFeeMultiplier::put(Fixed128::from_rational(1, NonZeroI128::new(2).unwrap())); + NextFeeMultiplier::put(Multiplier::saturating_from_rational(3, 2)); let len = 10; assert!( @@ -644,7 +845,14 @@ mod tests { .pre_dispatch(&1, CALL, &info_from_weight(3), len) .is_ok() ); - assert_eq!(Balances::free_balance(1), 100 - 10 - 5 - (10 + 3) * 3 / 2); + assert_eq!( + Balances::free_balance(1), + 100 // original + - 10 // tip + - 5 // base + - 10 // len + - (3 * 3 / 2) // adjusted weight + ); }) } @@ -664,7 +872,7 @@ mod tests { .execute_with(|| { // all fees should be x1.5 - NextFeeMultiplier::put(Fixed128::from_rational(1, NonZeroI128::new(2).unwrap())); + NextFeeMultiplier::put(Multiplier::saturating_from_rational(3, 2)); assert_eq!( TransactionPayment::query_info(xt, len), @@ -673,10 +881,8 @@ mod tests { class: info.class, partial_fee: 5 * 2 /* base * weight_fee */ - + ( - len as u64 /* len * 1 */ - + info.weight.min(MaximumBlockWeight::get()) as u64 * 2 /* weight * weight_to_fee */ - ) * 3 / 2 + + len as u64 /* len * 1 */ + + info.weight.min(MaximumBlockWeight::get()) as u64 * 2 * 3 / 2 /* weight */ }, ); @@ -693,7 +899,7 @@ mod tests { .execute_with(|| { // Next fee multiplier is zero - assert_eq!(NextFeeMultiplier::get(), Fixed128::from_natural(0)); + assert_eq!(NextFeeMultiplier::get(), Multiplier::one()); // Tip only, no fees works let dispatch_info = DispatchInfo { @@ -732,8 +938,8 @@ mod tests { .build() .execute_with(|| { - // Add a next fee multiplier - NextFeeMultiplier::put(Fixed128::from_rational(1, NonZeroI128::new(2).unwrap())); // = 1/2 = .5 + // Add a next fee multiplier. Fees will be x3/2. + NextFeeMultiplier::put(Multiplier::saturating_from_rational(3, 2)); // Base fee is unaffected by multiplier let dispatch_info = DispatchInfo { weight: 0, @@ -749,10 +955,44 @@ mod tests { pays_fee: Pays::Yes, }; // 123 weight, 456 length, 100 base - // adjustable fee = (123 * 1) + (456 * 10) = 4683 - // adjusted fee = (4683 * .5) + 4683 = 7024.5 -> 7024 - // final fee = 100 + 7024 + 789 tip = 7913 - assert_eq!(Module::::compute_fee(456, &dispatch_info, 789), 7913); + assert_eq!( + Module::::compute_fee(456, &dispatch_info, 789), + 100 + (3 * 123 / 2) + 4560 + 789, + ); + }); + } + + #[test] + fn compute_fee_works_with_negative_multiplier() { + ExtBuilder::default() + .base_weight(100) + .byte_fee(10) + .balance_factor(0) + .build() + .execute_with(|| + { + // Add a next fee multiplier. All fees will be x1/2. + NextFeeMultiplier::put(Multiplier::saturating_from_rational(1, 2)); + + // Base fee is unaffected by multiplier. + let dispatch_info = DispatchInfo { + weight: 0, + class: DispatchClass::Operational, + pays_fee: Pays::Yes, + }; + assert_eq!(Module::::compute_fee(0, &dispatch_info, 0), 100); + + // Everything works together. + let dispatch_info = DispatchInfo { + weight: 123, + class: DispatchClass::Operational, + pays_fee: Pays::Yes, + }; + // 123 weight, 456 length, 100 base + assert_eq!( + Module::::compute_fee(456, &dispatch_info, 789), + 100 + (123 / 2) + 4560 + 789, + ); }); } @@ -790,6 +1030,8 @@ mod tests { .build() .execute_with(|| { + // So events are emitted + System::set_block_number(10); let len = 10; let pre = ChargeTransactionPayment::::from(5 /* tipped */) .pre_dispatch(&2, CALL, &info_from_weight(100), len) @@ -806,6 +1048,14 @@ mod tests { .is_ok() ); assert_eq!(Balances::free_balance(2), 0); + // Transfer Event + assert!(System::events().iter().any(|event| { + event.event == Event::pallet_balances(pallet_balances::RawEvent::Transfer(2, 3, 80)) + })); + // Killed Event + assert!(System::events().iter().any(|event| { + event.event == Event::system(system::RawEvent::KilledAccount(2)) + })); }); } @@ -831,4 +1081,70 @@ mod tests { assert_eq!(Balances::free_balance(2), 200 - 5 - 10 - 100 - 5); }); } + + #[test] + fn zero_transfer_on_free_transaction() { + ExtBuilder::default() + .balance_factor(10) + .base_weight(5) + .build() + .execute_with(|| + { + // So events are emitted + System::set_block_number(10); + let len = 10; + let dispatch_info = DispatchInfo { + weight: 100, + pays_fee: Pays::No, + class: DispatchClass::Normal, + }; + let user = 69; + let pre = ChargeTransactionPayment::::from(0) + .pre_dispatch(&user, CALL, &dispatch_info, len) + .unwrap(); + assert_eq!(Balances::total_balance(&user), 0); + assert!( + ChargeTransactionPayment:: + ::post_dispatch(pre, &dispatch_info, &default_post_info(), len, &Ok(())) + .is_ok() + ); + assert_eq!(Balances::total_balance(&user), 0); + // No events for such a scenario + assert_eq!(System::events().len(), 0); + }); + } + + #[test] + fn refund_consistent_with_actual_weight() { + ExtBuilder::default() + .balance_factor(10) + .base_weight(7) + .build() + .execute_with(|| + { + let info = info_from_weight(100); + let post_info = post_info_from_weight(33); + let prev_balance = Balances::free_balance(2); + let len = 10; + let tip = 5; + + NextFeeMultiplier::put(Multiplier::saturating_from_rational(5, 4)); + + let pre = ChargeTransactionPayment::::from(tip) + .pre_dispatch(&2, CALL, &info, len) + .unwrap(); + + ChargeTransactionPayment:: + ::post_dispatch(pre, &info, &post_info, len, &Ok(())) + .unwrap(); + + let refund_based_fee = prev_balance - Balances::free_balance(2); + let actual_fee = Module:: + ::compute_actual_fee(len as u32, &info, &post_info, tip); + + // 33 weight, 10 length, 7 base, 5 tip + assert_eq!(actual_fee, 7 + 10 + (33 * 5 / 4) + 5); + assert_eq!(refund_based_fee, actual_fee); + }); + } } diff --git a/frame/treasury/Cargo.toml b/frame/treasury/Cargo.toml index 9d4a6fb5d2c20d2e6c14e5b5481c0f46d4bfd5f8..dfab1aca43b2831ecdf70e9c49c76084030d1be9 100644 --- a/frame/treasury/Cargo.toml +++ b/frame/treasury/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-treasury" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,18 +13,19 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -pallet-balances = { version = "2.0.0-alpha.8", default-features = false, path = "../balances" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +pallet-balances = { version = "2.0.0-rc4", default-features = false, path = "../balances" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../benchmarking", optional = true } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-io ={ version = "2.0.0-alpha.8", path = "../../primitives/io" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } +sp-io ={ version = "2.0.0-rc4", path = "../../primitives/io" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-storage = { version = "2.0.0-rc4", path = "../../primitives/storage" } [features] default = ["std"] diff --git a/frame/treasury/src/lib.rs b/frame/treasury/src/lib.rs index d1fed8fa2860a78d0888a52870f1920c1cb1a387..bb139c4cc6442f13cace9ae984e2887dfed2982a 100644 --- a/frame/treasury/src/lib.rs +++ b/frame/treasury/src/lib.rs @@ -102,7 +102,7 @@ use sp_runtime::{Permill, ModuleId, Percent, RuntimeDebug, traits::{ use frame_support::weights::{Weight, DispatchClass}; use frame_support::traits::{Contains, ContainsLengthBound, EnsureOrigin}; use codec::{Encode, Decode}; -use frame_system::{self as system, ensure_signed, ensure_root}; +use frame_system::{self as system, ensure_signed}; mod tests; mod benchmarking; @@ -192,13 +192,17 @@ pub struct OpenTip< reason: Hash, /// The account to be tipped. who: AccountId, - /// The account who began this tip and the amount held on deposit. - finder: Option<(AccountId, Balance)>, + /// The account who began this tip. + finder: AccountId, + /// The amount held on deposit for this tip. + deposit: Balance, /// The block number at which this tip will close if `Some`. If `None`, then no closing is /// scheduled. closes: Option, /// The members who have voted for this tip. Sorted by AccountId. tips: Vec<(AccountId, Balance)>, + /// Whether this tip should result in the finder taking a fee. + finders_fee: bool, } decl_storage! { @@ -316,7 +320,7 @@ decl_module! { /// The amount held on deposit per byte within the tip report reason. const TipReportDepositPerByte: BalanceOf = T::TipReportDepositPerByte::get(); - + /// The treasury's module id, used for deriving its sovereign account ID. const ModuleId: ModuleId = T::ModuleId::get(); @@ -355,6 +359,8 @@ decl_module! { /// Reject a proposed spend. The original deposit will be slashed. /// + /// May only be called from `T::RejectOrigin`. + /// /// # /// - Complexity: O(1) /// - DbReads: `Proposals`, `rejected proposer account` @@ -362,9 +368,7 @@ decl_module! { /// # #[weight = (130_000_000 + T::DbWeight::get().reads_writes(2, 2), DispatchClass::Operational)] fn reject_proposal(origin, #[compact] proposal_id: ProposalIndex) { - T::RejectOrigin::try_origin(origin) - .map(|_| ()) - .or_else(ensure_root)?; + T::RejectOrigin::ensure_origin(origin)?; let proposal = >::take(&proposal_id).ok_or(Error::::InvalidProposalIndex)?; let value = proposal.bond; @@ -377,6 +381,8 @@ decl_module! { /// Approve a proposal. At a later time, the proposal will be allocated to the beneficiary /// and the original deposit will be returned. /// + /// May only be called from `T::ApproveOrigin`. + /// /// # /// - Complexity: O(1). /// - DbReads: `Proposals`, `Approvals` @@ -384,9 +390,7 @@ decl_module! { /// # #[weight = (34_000_000 + T::DbWeight::get().reads_writes(2, 1), DispatchClass::Operational)] fn approve_proposal(origin, #[compact] proposal_id: ProposalIndex) { - T::ApproveOrigin::try_origin(origin) - .map(|_| ()) - .or_else(ensure_root)?; + T::ApproveOrigin::ensure_origin(origin)?; ensure!(>::contains_key(proposal_id), Error::::InvalidProposalIndex); Approvals::mutate(|v| v.push(proposal_id)); @@ -428,8 +432,15 @@ decl_module! { T::Currency::reserve(&finder, deposit)?; Reasons::::insert(&reason_hash, &reason); - let finder = Some((finder, deposit)); - let tip = OpenTip { reason: reason_hash, who, finder, closes: None, tips: vec![] }; + let tip = OpenTip { + reason: reason_hash, + who, + finder, + deposit, + closes: None, + tips: vec![], + finders_fee: true + }; Tips::::insert(&hash, tip); Self::deposit_event(RawEvent::NewTip(hash)); } @@ -457,12 +468,13 @@ decl_module! { fn retract_tip(origin, hash: T::Hash) { let who = ensure_signed(origin)?; let tip = Tips::::get(&hash).ok_or(Error::::UnknownTip)?; - let (finder, deposit) = tip.finder.ok_or(Error::::NotFinder)?; - ensure!(finder == who, Error::::NotFinder); + ensure!(tip.finder == who, Error::::NotFinder); Reasons::::remove(&tip.reason); Tips::::remove(&hash); - let _ = T::Currency::unreserve(&who, deposit); + if !tip.deposit.is_zero() { + let _ = T::Currency::unreserve(&who, tip.deposit); + } Self::deposit_event(RawEvent::TipRetracted(hash)); } @@ -501,8 +513,16 @@ decl_module! { Reasons::::insert(&reason_hash, &reason); Self::deposit_event(RawEvent::NewTip(hash.clone())); - let tips = vec![(tipper, tip_value)]; - let tip = OpenTip { reason: reason_hash, who, finder: None, closes: None, tips }; + let tips = vec![(tipper.clone(), tip_value)]; + let tip = OpenTip { + reason: reason_hash, + who, + finder: tipper, + deposit: Zero::zero(), + closes: None, + tips, + finders_fee: false, + }; Tips::::insert(&hash, tip); } @@ -667,15 +687,17 @@ impl Module { let treasury = Self::account_id(); let max_payout = Self::pot(); let mut payout = tips[tips.len() / 2].1.min(max_payout); - if let Some((finder, deposit)) = tip.finder { - let _ = T::Currency::unreserve(&finder, deposit); - if finder != tip.who { + if !tip.deposit.is_zero() { + let _ = T::Currency::unreserve(&tip.finder, tip.deposit); + } + if tip.finders_fee { + if tip.finder != tip.who { // pay out the finder's fee. let finders_fee = T::TipFindersFee::get() * payout; payout -= finders_fee; // this should go through given we checked it's at most the free balance, but still // we only make a best-effort. - let _ = T::Currency::transfer(&treasury, &finder, finders_fee, KeepAlive); + let _ = T::Currency::transfer(&treasury, &tip.finder, finders_fee, KeepAlive); } } // same as above: best-effort only. @@ -753,6 +775,55 @@ impl Module { // Must never be less than 0 but better be safe. .saturating_sub(T::Currency::minimum_balance()) } + + pub fn migrate_retract_tip_for_tip_new() { + /// An open tipping "motion". Retains all details of a tip including information on the finder + /// and the members who have voted. + #[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug)] + pub struct OldOpenTip< + AccountId: Parameter, + Balance: Parameter, + BlockNumber: Parameter, + Hash: Parameter, + > { + /// The hash of the reason for the tip. The reason should be a human-readable UTF-8 encoded string. A URL would be + /// sensible. + reason: Hash, + /// The account to be tipped. + who: AccountId, + /// The account who began this tip and the amount held on deposit. + finder: Option<(AccountId, Balance)>, + /// The block number at which this tip will close if `Some`. If `None`, then no closing is + /// scheduled. + closes: Option, + /// The members who have voted for this tip. Sorted by AccountId. + tips: Vec<(AccountId, Balance)>, + } + + use frame_support::{Twox64Concat, migration::StorageKeyIterator}; + + for (hash, old_tip) in StorageKeyIterator::< + T::Hash, + OldOpenTip, T::BlockNumber, T::Hash>, + Twox64Concat, + >::new(b"Treasury", b"Tips").drain() + { + let (finder, deposit, finders_fee) = match old_tip.finder { + Some((finder, deposit)) => (finder, deposit, true), + None => (T::AccountId::default(), Zero::zero(), false), + }; + let new_tip = OpenTip { + reason: old_tip.reason, + who: old_tip.who, + finder, + deposit, + closes: old_tip.closes, + tips: old_tip.tips, + finders_fee + }; + Tips::::insert(hash, new_tip) + } + } } impl OnUnbalanced> for Module { diff --git a/frame/treasury/src/tests.rs b/frame/treasury/src/tests.rs index 606abf7765372b8d5e4ae4ff0b30d3f99dae9a66..68820ffd5d2c9aa66675f642fe00c63ca2006209 100644 --- a/frame/treasury/src/tests.rs +++ b/frame/treasury/src/tests.rs @@ -60,6 +60,7 @@ parameter_types! { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -75,6 +76,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); @@ -270,7 +272,7 @@ fn close_tip_works() { assert_noop!(Treasury::close_tip(Origin::signed(0), h.into()), Error::::Premature); System::set_block_number(2); - assert_noop!(Treasury::close_tip(Origin::NONE, h.into()), BadOrigin); + assert_noop!(Treasury::close_tip(Origin::none(), h.into()), BadOrigin); assert_ok!(Treasury::close_tip(Origin::signed(0), h.into())); assert_eq!(Balances::free_balance(3), 10); @@ -291,6 +293,7 @@ fn close_tip_works() { #[test] fn retract_tip_works() { new_test_ext().execute_with(|| { + // with report awesome Balances::make_free_balance_be(&Treasury::account_id(), 101); assert_ok!(Treasury::report_awesome(Origin::signed(0), b"awesome.dot".to_vec(), 3)); let h = tip_hash(); @@ -301,6 +304,17 @@ fn retract_tip_works() { assert_ok!(Treasury::retract_tip(Origin::signed(0), h.clone())); System::set_block_number(2); assert_noop!(Treasury::close_tip(Origin::signed(0), h.into()), Error::::UnknownTip); + + // with tip new + Balances::make_free_balance_be(&Treasury::account_id(), 101); + assert_ok!(Treasury::tip_new(Origin::signed(10), b"awesome.dot".to_vec(), 3, 10)); + let h = tip_hash(); + assert_ok!(Treasury::tip(Origin::signed(11), h.clone(), 10)); + assert_ok!(Treasury::tip(Origin::signed(12), h.clone(), 10)); + assert_noop!(Treasury::retract_tip(Origin::signed(0), h.clone()), Error::::NotFinder); + assert_ok!(Treasury::retract_tip(Origin::signed(10), h.clone())); + System::set_block_number(2); + assert_noop!(Treasury::close_tip(Origin::signed(10), h.into()), Error::::UnknownTip); }); } @@ -380,7 +394,7 @@ fn accepted_spend_proposal_ignored_outside_spend_period() { Balances::make_free_balance_be(&Treasury::account_id(), 101); assert_ok!(Treasury::propose_spend(Origin::signed(0), 100, 3)); - assert_ok!(Treasury::approve_proposal(Origin::ROOT, 0)); + assert_ok!(Treasury::approve_proposal(Origin::root(), 0)); >::on_initialize(1); assert_eq!(Balances::free_balance(3), 0); @@ -407,7 +421,7 @@ fn rejected_spend_proposal_ignored_on_spend_period() { Balances::make_free_balance_be(&Treasury::account_id(), 101); assert_ok!(Treasury::propose_spend(Origin::signed(0), 100, 3)); - assert_ok!(Treasury::reject_proposal(Origin::ROOT, 0)); + assert_ok!(Treasury::reject_proposal(Origin::root(), 0)); >::on_initialize(2); assert_eq!(Balances::free_balance(3), 0); @@ -421,22 +435,22 @@ fn reject_already_rejected_spend_proposal_fails() { Balances::make_free_balance_be(&Treasury::account_id(), 101); assert_ok!(Treasury::propose_spend(Origin::signed(0), 100, 3)); - assert_ok!(Treasury::reject_proposal(Origin::ROOT, 0)); - assert_noop!(Treasury::reject_proposal(Origin::ROOT, 0), Error::::InvalidProposalIndex); + assert_ok!(Treasury::reject_proposal(Origin::root(), 0)); + assert_noop!(Treasury::reject_proposal(Origin::root(), 0), Error::::InvalidProposalIndex); }); } #[test] fn reject_non_existent_spend_proposal_fails() { new_test_ext().execute_with(|| { - assert_noop!(Treasury::reject_proposal(Origin::ROOT, 0), Error::::InvalidProposalIndex); + assert_noop!(Treasury::reject_proposal(Origin::root(), 0), Error::::InvalidProposalIndex); }); } #[test] fn accept_non_existent_spend_proposal_fails() { new_test_ext().execute_with(|| { - assert_noop!(Treasury::approve_proposal(Origin::ROOT, 0), Error::::InvalidProposalIndex); + assert_noop!(Treasury::approve_proposal(Origin::root(), 0), Error::::InvalidProposalIndex); }); } @@ -446,8 +460,8 @@ fn accept_already_rejected_spend_proposal_fails() { Balances::make_free_balance_be(&Treasury::account_id(), 101); assert_ok!(Treasury::propose_spend(Origin::signed(0), 100, 3)); - assert_ok!(Treasury::reject_proposal(Origin::ROOT, 0)); - assert_noop!(Treasury::approve_proposal(Origin::ROOT, 0), Error::::InvalidProposalIndex); + assert_ok!(Treasury::reject_proposal(Origin::root(), 0)); + assert_noop!(Treasury::approve_proposal(Origin::root(), 0), Error::::InvalidProposalIndex); }); } @@ -458,7 +472,7 @@ fn accepted_spend_proposal_enacted_on_spend_period() { assert_eq!(Treasury::pot(), 100); assert_ok!(Treasury::propose_spend(Origin::signed(0), 100, 3)); - assert_ok!(Treasury::approve_proposal(Origin::ROOT, 0)); + assert_ok!(Treasury::approve_proposal(Origin::root(), 0)); >::on_initialize(2); assert_eq!(Balances::free_balance(3), 100); @@ -473,7 +487,7 @@ fn pot_underflow_should_not_diminish() { assert_eq!(Treasury::pot(), 100); assert_ok!(Treasury::propose_spend(Origin::signed(0), 150, 3)); - assert_ok!(Treasury::approve_proposal(Origin::ROOT, 0)); + assert_ok!(Treasury::approve_proposal(Origin::root(), 0)); >::on_initialize(2); assert_eq!(Treasury::pot(), 100); // Pot hasn't changed @@ -495,13 +509,13 @@ fn treasury_account_doesnt_get_deleted() { let treasury_balance = Balances::free_balance(&Treasury::account_id()); assert_ok!(Treasury::propose_spend(Origin::signed(0), treasury_balance, 3)); - assert_ok!(Treasury::approve_proposal(Origin::ROOT, 0)); + assert_ok!(Treasury::approve_proposal(Origin::root(), 0)); >::on_initialize(2); assert_eq!(Treasury::pot(), 100); // Pot hasn't changed assert_ok!(Treasury::propose_spend(Origin::signed(0), Treasury::pot(), 3)); - assert_ok!(Treasury::approve_proposal(Origin::ROOT, 1)); + assert_ok!(Treasury::approve_proposal(Origin::root(), 1)); >::on_initialize(4); assert_eq!(Treasury::pot(), 0); // Pot is emptied @@ -525,9 +539,9 @@ fn inexistent_account_works() { assert_eq!(Treasury::pot(), 0); // Pot is empty assert_ok!(Treasury::propose_spend(Origin::signed(0), 99, 3)); - assert_ok!(Treasury::approve_proposal(Origin::ROOT, 0)); + assert_ok!(Treasury::approve_proposal(Origin::root(), 0)); assert_ok!(Treasury::propose_spend(Origin::signed(0), 1, 3)); - assert_ok!(Treasury::approve_proposal(Origin::ROOT, 1)); + assert_ok!(Treasury::approve_proposal(Origin::root(), 1)); >::on_initialize(2); assert_eq!(Treasury::pot(), 0); // Pot hasn't changed assert_eq!(Balances::free_balance(3), 0); // Balance of `3` hasn't changed @@ -542,3 +556,97 @@ fn inexistent_account_works() { assert_eq!(Balances::free_balance(3), 99); // Balance of `3` has changed }); } + +#[test] +fn test_last_reward_migration() { + use sp_storage::Storage; + + let mut s = Storage::default(); + + #[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug)] + pub struct OldOpenTip< + AccountId: Parameter, + Balance: Parameter, + BlockNumber: Parameter, + Hash: Parameter, + > { + /// The hash of the reason for the tip. The reason should be a human-readable UTF-8 encoded string. A URL would be + /// sensible. + reason: Hash, + /// The account to be tipped. + who: AccountId, + /// The account who began this tip and the amount held on deposit. + finder: Option<(AccountId, Balance)>, + /// The block number at which this tip will close if `Some`. If `None`, then no closing is + /// scheduled. + closes: Option, + /// The members who have voted for this tip. Sorted by AccountId. + tips: Vec<(AccountId, Balance)>, + } + + let reason1 = BlakeTwo256::hash(b"reason1"); + let hash1 = BlakeTwo256::hash_of(&(reason1, 10u64)); + + let old_tip_finder = OldOpenTip:: { + reason: reason1, + who: 10, + finder: Some((20, 30)), + closes: Some(13), + tips: vec![(40, 50), (60, 70)] + }; + + let reason2 = BlakeTwo256::hash(b"reason2"); + let hash2 = BlakeTwo256::hash_of(&(reason2, 20u64)); + + let old_tip_no_finder = OldOpenTip:: { + reason: reason2, + who: 20, + finder: None, + closes: Some(13), + tips: vec![(40, 50), (60, 70)] + }; + + let data = vec![ + ( + Tips::::hashed_key_for(hash1), + old_tip_finder.encode().to_vec() + ), + ( + Tips::::hashed_key_for(hash2), + old_tip_no_finder.encode().to_vec() + ), + ]; + + s.top = data.into_iter().collect(); + sp_io::TestExternalities::new(s).execute_with(|| { + Treasury::migrate_retract_tip_for_tip_new(); + + // Test w/ finder + assert_eq!( + Tips::::get(hash1), + Some(OpenTip { + reason: reason1, + who: 10, + finder: 20, + deposit: 30, + closes: Some(13), + tips: vec![(40, 50), (60, 70)], + finders_fee: true, + }) + ); + + // Test w/o finder + assert_eq!( + Tips::::get(hash2), + Some(OpenTip { + reason: reason2, + who: 20, + finder: Default::default(), + deposit: 0, + closes: Some(13), + tips: vec![(40, 50), (60, 70)], + finders_fee: false, + }) + ); + }); +} diff --git a/frame/utility/Cargo.toml b/frame/utility/Cargo.toml index 6fc23eb91e86b6e66b00208d34dafb97fa6caa6b..e4dbfdfff73ec728ee1acd3e71a46371347b7a90 100644 --- a/frame/utility/Cargo.toml +++ b/frame/utility/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-utility" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,19 +13,19 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../benchmarking", optional = true } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } [features] default = ["std"] diff --git a/frame/utility/src/benchmarking.rs b/frame/utility/src/benchmarking.rs index 4e77c8e44a676f02377cd7c54e789dafa60efafb..155a279807afc2bc185f94cba6aae6639be5b448 100644 --- a/frame/utility/src/benchmarking.rs +++ b/frame/utility/src/benchmarking.rs @@ -20,27 +20,17 @@ #![cfg(feature = "runtime-benchmarks")] use super::*; -use frame_system::RawOrigin; +use frame_system::{RawOrigin, EventRecord}; use frame_benchmarking::{benchmarks, account}; -use sp_runtime::traits::Saturating; - -use crate::Module as Utility; const SEED: u32 = 0; -fn setup_multi(s: u32, z: u32) -> Result<(Vec, Box<::Call>), &'static str>{ - let mut signatories: Vec = Vec::new(); - for i in 0 .. s { - let signatory = account("signatory", i, SEED); - // Give them some balance for a possible deposit - let deposit = T::MultisigDepositBase::get() + T::MultisigDepositFactor::get() * s.into(); - let balance = T::Currency::minimum_balance().saturating_mul(100.into()) + deposit; - T::Currency::make_free_balance_be(&signatory, balance); - signatories.push(signatory); - } - signatories.sort(); - let call: Box<::Call> = Box::new(frame_system::Call::remark(vec![0; z as usize]).into()); - return Ok((signatories, call)) +fn assert_last_event(generic_event: ::Event) { + let events = frame_system::Module::::events(); + let system_event: ::Event = generic_event.into(); + // compare to the last event record + let EventRecord { event, .. } = &events[events.len() - 1]; + assert_eq!(event, &system_event); } benchmarks! { @@ -55,97 +45,15 @@ benchmarks! { } let caller = account("caller", 0, SEED); }: _(RawOrigin::Signed(caller), calls) + verify { + assert_last_event::(Event::BatchCompleted.into()) + } - as_sub { + as_derivative { let u in 0 .. 1000; let caller = account("caller", u, SEED); let call = Box::new(frame_system::Call::remark(vec![]).into()); }: _(RawOrigin::Signed(caller), u as u16, call) - - as_multi_create { - // Signatories, need at least 2 total people - let s in 2 .. T::MaxSignatories::get() as u32; - // Transaction Length - let z in 0 .. 10_000; - let (mut signatories, call) = setup_multi::(s, z)?; - let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; - }: as_multi(RawOrigin::Signed(caller), s as u16, signatories, None, call) - - as_multi_approve { - // Signatories, need at least 2 people - let s in 2 .. T::MaxSignatories::get() as u32; - // Transaction Length - let z in 0 .. 10_000; - let (mut signatories, call) = setup_multi::(s, z)?; - let mut signatories2 = signatories.clone(); - let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; - // before the call, get the timepoint - let timepoint = Utility::::timepoint(); - // Create the multi - Utility::::as_multi(RawOrigin::Signed(caller).into(), s as u16, signatories, None, call.clone())?; - let caller2 = signatories2.remove(0); - }: as_multi(RawOrigin::Signed(caller2), s as u16, signatories2, Some(timepoint), call) - - as_multi_complete { - // Signatories, need at least 2 people - let s in 2 .. T::MaxSignatories::get() as u32; - // Transaction Length - let z in 0 .. 10_000; - let (mut signatories, call) = setup_multi::(s, z)?; - let mut signatories2 = signatories.clone(); - let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; - // before the call, get the timepoint - let timepoint = Utility::::timepoint(); - // Create the multi - Utility::::as_multi(RawOrigin::Signed(caller).into(), s as u16, signatories, None, call.clone())?; - // Everyone except the first person approves - for i in 1 .. s - 1 { - let mut signatories_loop = signatories2.clone(); - let caller_loop = signatories_loop.remove(i as usize); - Utility::::as_multi(RawOrigin::Signed(caller_loop).into(), s as u16, signatories_loop, Some(timepoint), call.clone())?; - } - let caller2 = signatories2.remove(0); - }: as_multi(RawOrigin::Signed(caller2), s as u16, signatories2, Some(timepoint), call) - - approve_as_multi_create { - // Signatories, need at least 2 people - let s in 2 .. T::MaxSignatories::get() as u32; - // Transaction Length - let z in 0 .. 10_000; - let (mut signatories, call) = setup_multi::(s, z)?; - let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; - let call_hash = call.using_encoded(blake2_256); - // Create the multi - }: approve_as_multi(RawOrigin::Signed(caller), s as u16, signatories, None, call_hash) - - approve_as_multi_approve { - // Signatories, need at least 2 people - let s in 2 .. T::MaxSignatories::get() as u32; - // Transaction Length - let z in 0 .. 10_000; - let (mut signatories, call) = setup_multi::(s, z)?; - let mut signatories2 = signatories.clone(); - let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; - let call_hash = call.using_encoded(blake2_256); - // before the call, get the timepoint - let timepoint = Utility::::timepoint(); - // Create the multi - Utility::::as_multi(RawOrigin::Signed(caller).into(), s as u16, signatories, None, call.clone())?; - let caller2 = signatories2.remove(0); - }: approve_as_multi(RawOrigin::Signed(caller2), s as u16, signatories2, Some(timepoint), call_hash) - - cancel_as_multi { - // Signatories, need at least 2 people - let s in 2 .. T::MaxSignatories::get() as u32; - // Transaction Length - let z in 0 .. 10_000; - let (mut signatories, call) = setup_multi::(s, z)?; - let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; - let call_hash = call.using_encoded(blake2_256); - let timepoint = Utility::::timepoint(); - // Create the multi - Utility::::as_multi(RawOrigin::Signed(caller.clone()).into(), s as u16, signatories.clone(), None, call.clone())?; - }: _(RawOrigin::Signed(caller), s as u16, signatories, timepoint, call_hash) } #[cfg(test)] @@ -158,13 +66,7 @@ mod tests { fn test_benchmarks() { new_test_ext().execute_with(|| { assert_ok!(test_benchmark_batch::()); - assert_ok!(test_benchmark_as_sub::()); - assert_ok!(test_benchmark_as_multi_create::()); - assert_ok!(test_benchmark_as_multi_approve::()); - assert_ok!(test_benchmark_as_multi_complete::()); - assert_ok!(test_benchmark_approve_as_multi_create::()); - assert_ok!(test_benchmark_approve_as_multi_approve::()); - assert_ok!(test_benchmark_cancel_as_multi::()); + assert_ok!(test_benchmark_as_derivative::()); }); } } diff --git a/frame/utility/src/lib.rs b/frame/utility/src/lib.rs index 6692848609dde949dcaa3ca837a29585393860a5..47ca4f13e7c55ae5ab28e06678af4ea03a3818b6 100644 --- a/frame/utility/src/lib.rs +++ b/frame/utility/src/lib.rs @@ -16,29 +16,28 @@ // limitations under the License. //! # Utility Module -//! A module with helpers for dispatch management. +//! A stateless module with helpers for dispatch management which does no re-authentication. //! //! - [`utility::Trait`](./trait.Trait.html) //! - [`Call`](./enum.Call.html) //! //! ## Overview //! -//! This module contains three basic pieces of functionality, two of which are stateless: +//! This module contains two basic pieces of functionality: //! - Batch dispatch: A stateless operation, allowing any origin to execute multiple calls in a //! single dispatch. This can be useful to amalgamate proposals, combining `set_code` with //! corresponding `set_storage`s, for efficient multiple payouts with just a single signature //! verify, or in combination with one of the other two dispatch functionality. //! - Pseudonymal dispatch: A stateless operation, allowing a signed origin to execute a call from -//! an alternative signed origin. Each account has 2**16 possible "pseudonyms" (alternative +//! an alternative signed origin. Each account has 2 * 2**16 possible "pseudonyms" (alternative //! account IDs) and these can be stacked. This can be useful as a key management tool, where you //! need multiple distinct accounts (e.g. as controllers for many staking accounts), but where //! it's perfectly fine to have each of them controlled by the same underlying keypair. -//! - Multisig dispatch (stateful): A potentially stateful operation, allowing multiple signed -//! origins (accounts) to coordinate and dispatch a call from a well-known origin, derivable -//! deterministically from the set of account IDs and the threshold number of accounts from the -//! set that must approve it. In the case that the threshold is just one then this is a stateless -//! operation. This is useful for multisig wallets where cryptographic threshold signatures are -//! not available or desired. +//! Derivative accounts are, for the purposes of proxy filtering considered exactly the same as +//! the oigin and are thus hampered with the origin's filters. +//! +//! Since proxy filters are respected in all dispatches of this module, it should never need to be +//! filtered by any proxy. //! //! ## Interface //! @@ -48,13 +47,7 @@ //! * `batch` - Dispatch multiple calls from the sender's origin. //! //! #### For pseudonymal dispatch -//! * `as_sub` - Dispatch a call from a secondary ("sub") signed origin. -//! -//! #### For multisig dispatch -//! * `as_multi` - Approve and if possible dispatch a call from a composite origin formed from a -//! number of signed origins. -//! * `approve_as_multi` - Approve a call from a composite origin. -//! * `cancel_as_multi` - Cancel a call from a composite origin. +//! * `as_derivative` - Dispatch a call from a derivative signed origin. //! //! [`Call`]: ./enum.Call.html //! [`Trait`]: ./trait.Trait.html @@ -66,131 +59,40 @@ use sp_std::prelude::*; use codec::{Encode, Decode}; use sp_core::TypeId; use sp_io::hashing::blake2_256; -use frame_support::{decl_module, decl_event, decl_error, decl_storage, Parameter, ensure, RuntimeDebug}; -use frame_support::{traits::{Get, ReservableCurrency, Currency}, - weights::{Weight, GetDispatchInfo, DispatchClass, FunctionOf, Pays}, - dispatch::{DispatchResultWithPostInfo, DispatchErrorWithPostInfo, PostDispatchInfo}, +use frame_support::{decl_module, decl_event, decl_storage, Parameter}; +use frame_support::{ + traits::{OriginTrait, UnfilteredDispatchable}, + weights::{Weight, GetDispatchInfo, DispatchClass}, dispatch::PostDispatchInfo, }; -use frame_system::{self as system, ensure_signed}; +use frame_system::{self as system, ensure_signed, ensure_root}; use sp_runtime::{DispatchError, DispatchResult, traits::Dispatchable}; mod tests; mod benchmarking; -type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; - /// Configuration trait. pub trait Trait: frame_system::Trait { /// The overarching event type. - type Event: From> + Into<::Event>; + type Event: From + Into<::Event>; /// The overarching call type. - type Call: Parameter + Dispatchable + GetDispatchInfo + From>; - - /// The currency mechanism. - type Currency: ReservableCurrency; - - /// The base amount of currency needed to reserve for creating a multisig execution. - /// - /// This is held for an additional storage item whose value size is - /// `4 + sizeof((BlockNumber, Balance, AccountId))` bytes. - type MultisigDepositBase: Get>; - - /// The amount of currency needed per unit threshold when creating a multisig execution. - /// - /// This is held for adding 32 bytes more into a pre-existing storage value. - type MultisigDepositFactor: Get>; - - /// The maximum amount of signatories allowed in the multisig. - type MaxSignatories: Get; -} - -/// A global extrinsic index, formed as the extrinsic index within a block, together with that -/// block's height. This allows a transaction in which a multisig operation of a particular -/// composite was created to be uniquely identified. -#[derive(Copy, Clone, Eq, PartialEq, Encode, Decode, Default, RuntimeDebug)] -pub struct Timepoint { - /// The height of the chain at the point in time. - height: BlockNumber, - /// The index of the extrinsic at the point in time. - index: u32, -} - -/// An open multisig operation. -#[derive(Clone, Eq, PartialEq, Encode, Decode, Default, RuntimeDebug)] -pub struct Multisig { - /// The extrinsic when the multisig operation was opened. - when: Timepoint, - /// The amount held in reserve of the `depositor`, to be returned once the operation ends. - deposit: Balance, - /// The account who opened it (i.e. the first to approve it). - depositor: AccountId, - /// The approvals achieved so far, including the depositor. Always sorted. - approvals: Vec, + type Call: Parameter + Dispatchable + + GetDispatchInfo + From> + + UnfilteredDispatchable; } decl_storage! { - trait Store for Module as Utility { - /// The set of open multisig operations. - pub Multisigs: double_map - hasher(twox_64_concat) T::AccountId, hasher(blake2_128_concat) [u8; 32] - => Option, T::AccountId>>; - } -} - -decl_error! { - pub enum Error for Module { - /// Threshold is too low (zero). - ZeroThreshold, - /// Call is already approved by this signatory. - AlreadyApproved, - /// Call doesn't need any (more) approvals. - NoApprovalsNeeded, - /// There are too few signatories in the list. - TooFewSignatories, - /// There are too many signatories in the list. - TooManySignatories, - /// The signatories were provided out of order; they should be ordered. - SignatoriesOutOfOrder, - /// The sender was contained in the other signatories; it shouldn't be. - SenderInSignatories, - /// Multisig operation not found when attempting to cancel. - NotFound, - /// Only the account that originally created the multisig is able to cancel it. - NotOwner, - /// No timepoint was given, yet the multisig operation is already underway. - NoTimepoint, - /// A different timepoint was given to the multisig operation that is underway. - WrongTimepoint, - /// A timepoint was given, yet no multisig operation is underway. - UnexpectedTimepoint, - } + trait Store for Module as Utility {} } decl_event! { /// Events type. - pub enum Event where - AccountId = ::AccountId, - BlockNumber = ::BlockNumber, - CallHash = [u8; 32] - { + pub enum Event { /// Batch of dispatches did not complete fully. Index of first failing dispatch given, as /// well as the error. BatchInterrupted(u32, DispatchError), /// Batch of dispatches completed fully with no error. BatchCompleted, - /// A new multisig operation has begun. First param is the account that is approving, - /// second is the multisig account, third is hash of the call. - NewMultisig(AccountId, AccountId, CallHash), - /// A multisig operation has been approved by someone. First param is the account that is - /// approving, third is the multisig account, fourth is hash of the call. - MultisigApproval(AccountId, Timepoint, AccountId, CallHash), - /// A multisig operation has been executed. First param is the account that is - /// approving, third is the multisig account, fourth is hash of the call to be executed. - MultisigExecuted(AccountId, Timepoint, AccountId, CallHash, DispatchResult), - /// A multisig operation has been cancelled. First param is the account that is - /// cancelling, third is the multisig account, fourth is hash of the call. - MultisigCancelled(AccountId, Timepoint, AccountId, CallHash), } } @@ -202,40 +104,20 @@ impl TypeId for IndexedUtilityModuleId { const TYPE_ID: [u8; 4] = *b"suba"; } -mod weight_of { - use super::*; - - /// - Base Weight: - /// - Create: 46.55 + 0.089 * S µs - /// - Approve: 34.03 + .112 * S µs - /// - Complete: 40.36 + .225 * S µs - /// - DB Weight: - /// - Reads: Multisig Storage, [Caller Account] - /// - Writes: Multisig Storage, [Caller Account] - /// - Plus Call Weight - pub fn as_multi(other_sig_len: usize, call_weight: Weight) -> Weight { - call_weight - .saturating_add(45_000_000) - .saturating_add((other_sig_len as Weight).saturating_mul(250_000)) - .saturating_add(T::DbWeight::get().reads_writes(1, 1)) - } -} - decl_module! { pub struct Module for enum Call where origin: T::Origin { - type Error = Error; - /// Deposit one of this module's events by using the default implementation. fn deposit_event() = default; /// Send a batch of dispatch calls. /// - /// This will execute until the first one fails and then stop. - /// /// May be called from any origin. /// /// - `calls`: The calls to be dispatched from the same origin. /// + /// If origin is root then call are dispatch without checking origin filter. (This includes + /// bypassing `frame_system::Trait::BaseCallFilter`). + /// /// # /// - Base weight: 14.39 + .987 * c µs /// - Plus the sum of the weights of the `calls`. @@ -247,14 +129,12 @@ decl_module! { /// `BatchInterrupted` event is deposited, along with the number of successful calls made /// and the error of the failed call. If all were successful, then the `BatchCompleted` /// event is deposited. - #[weight = FunctionOf( - |args: (&Vec<::Call>,)| { - args.0.iter() - .map(|call| call.get_dispatch_info().weight) - .fold(15_000_000, |a: Weight, n| a.saturating_add(n).saturating_add(1_000_000)) - }, - |args: (&Vec<::Call>,)| { - let all_operational = args.0.iter() + #[weight = ( + calls.iter() + .map(|call| call.get_dispatch_info().weight) + .fold(15_000_000, |a: Weight, n| a.saturating_add(n).saturating_add(1_000_000)), + { + let all_operational = calls.iter() .map(|call| call.get_dispatch_info().class) .all(|class| class == DispatchClass::Operational); if all_operational { @@ -263,383 +143,59 @@ decl_module! { DispatchClass::Normal } }, - Pays::Yes, )] fn batch(origin, calls: Vec<::Call>) { + let is_root = ensure_root(origin.clone()).is_ok(); for (index, call) in calls.into_iter().enumerate() { - let result = call.dispatch(origin.clone()); + let result = if is_root { + call.dispatch_bypass_filter(origin.clone()) + } else { + call.dispatch(origin.clone()) + }; if let Err(e) = result { - Self::deposit_event(Event::::BatchInterrupted(index as u32, e.error)); + Self::deposit_event(Event::BatchInterrupted(index as u32, e.error)); return Ok(()); } } - Self::deposit_event(Event::::BatchCompleted); + Self::deposit_event(Event::BatchCompleted); } /// Send a call through an indexed pseudonym of the sender. /// - /// The dispatch origin for this call must be _Signed_. - /// - /// # - /// - Base weight: 2.861 µs - /// - Plus the weight of the `call` - /// # - #[weight = FunctionOf( - |args: (&u16, &Box<::Call>)| { - args.1.get_dispatch_info().weight.saturating_add(3_000_000) - }, - |args: (&u16, &Box<::Call>)| args.1.get_dispatch_info().class, - Pays::Yes, - )] - fn as_sub(origin, index: u16, call: Box<::Call>) -> DispatchResult { - let who = ensure_signed(origin)?; - let pseudonym = Self::sub_account_id(who, index); - call.dispatch(frame_system::RawOrigin::Signed(pseudonym).into()) - .map(|_| ()).map_err(|e| e.error) - } - - /// Register approval for a dispatch to be made from a deterministic composite account if - /// approved by a total of `threshold - 1` of `other_signatories`. + /// Filter from origin are passed along. The call will be dispatched with an origin which + /// use the same filter as the origin of this call. /// - /// If there are enough, then dispatch the call. + /// NOTE: If you need to ensure that any account-based filtering is not honored (i.e. + /// because you expect `proxy` to have been used prior in the call stack and you do not want + /// the call restrictions to apply to any sub-accounts), then use `as_multi_threshold_1` + /// in the Multisig pallet instead. /// - /// Payment: `MultisigDepositBase` will be reserved if this is the first approval, plus - /// `threshold` times `MultisigDepositFactor`. It is returned once this dispatch happens or - /// is cancelled. + /// NOTE: Prior to version *12, this was called `as_limited_sub`. /// /// The dispatch origin for this call must be _Signed_. /// - /// - `threshold`: The total number of approvals for this dispatch before it is executed. - /// - `other_signatories`: The accounts (other than the sender) who can approve this - /// dispatch. May not be empty. - /// - `maybe_timepoint`: If this is the first approval, then this must be `None`. If it is - /// not the first approval, then it must be `Some`, with the timepoint (block number and - /// transaction index) of the first approval transaction. - /// - `call`: The call to be executed. - /// - /// NOTE: Unless this is the final approval, you will generally want to use - /// `approve_as_multi` instead, since it only requires a hash of the call. - /// - /// Result is equivalent to the dispatched result if `threshold` is exactly `1`. Otherwise - /// on success, result is `Ok` and the result from the interior call, if it was executed, - /// may be found in the deposited `MultisigExecuted` event. - /// /// # - /// - `O(S + Z + Call)`. - /// - Up to one balance-reserve or unreserve operation. - /// - One passthrough operation, one insert, both `O(S)` where `S` is the number of - /// signatories. `S` is capped by `MaxSignatories`, with weight being proportional. - /// - One call encode & hash, both of complexity `O(Z)` where `Z` is tx-len. - /// - One encode & hash, both of complexity `O(S)`. - /// - Up to one binary search and insert (`O(logS + S)`). - /// - I/O: 1 read `O(S)`, up to 1 mutate `O(S)`. Up to one remove. - /// - One event. - /// - The weight of the `call`. - /// - Storage: inserts one item, value size bounded by `MaxSignatories`, with a - /// deposit taken for its lifetime of - /// `MultisigDepositBase + threshold * MultisigDepositFactor`. - /// ------------------------------- - /// - Base Weight: - /// - Create: 46.55 + 0.089 * S µs - /// - Approve: 34.03 + .112 * S µs - /// - Complete: 40.36 + .225 * S µs - /// - DB Weight: - /// - Reads: Multisig Storage, [Caller Account] - /// - Writes: Multisig Storage, [Caller Account] - /// - Plus Call Weight - /// # - #[weight = FunctionOf( - |args: (&u16, &Vec, &Option>, &Box<::Call>)| { - weight_of::as_multi::(args.1.len(),args.3.get_dispatch_info().weight) - }, - |args: (&u16, &Vec, &Option>, &Box<::Call>)| { - args.3.get_dispatch_info().class - }, - Pays::Yes, - )] - fn as_multi(origin, - threshold: u16, - other_signatories: Vec, - maybe_timepoint: Option>, - call: Box<::Call>, - ) -> DispatchResultWithPostInfo { - let who = ensure_signed(origin)?; - ensure!(threshold >= 1, Error::::ZeroThreshold); - let max_sigs = T::MaxSignatories::get() as usize; - ensure!(!other_signatories.is_empty(), Error::::TooFewSignatories); - let other_signatories_len = other_signatories.len(); - ensure!(other_signatories_len < max_sigs, Error::::TooManySignatories); - let signatories = Self::ensure_sorted_and_insert(other_signatories, who.clone())?; - - let id = Self::multi_account_id(&signatories, threshold); - let call_hash = call.using_encoded(blake2_256); - - if let Some(mut m) = >::get(&id, call_hash) { - let timepoint = maybe_timepoint.ok_or(Error::::NoTimepoint)?; - ensure!(m.when == timepoint, Error::::WrongTimepoint); - if let Err(pos) = m.approvals.binary_search(&who) { - // we know threshold is greater than zero from the above ensure. - if (m.approvals.len() as u16) < threshold - 1 { - m.approvals.insert(pos, who.clone()); - >::insert(&id, call_hash, m); - Self::deposit_event(RawEvent::MultisigApproval(who, timepoint, id, call_hash)); - // Call is not made, so the actual weight does not include call - return Ok(Some(weight_of::as_multi::(other_signatories_len, 0)).into()) - } - } else { - if (m.approvals.len() as u16) < threshold { - Err(Error::::AlreadyApproved)? - } - } - - let result = call.dispatch(frame_system::RawOrigin::Signed(id.clone()).into()); - let _ = T::Currency::unreserve(&m.depositor, m.deposit); - >::remove(&id, call_hash); - Self::deposit_event(RawEvent::MultisigExecuted( - who, timepoint, id, call_hash, result.map(|_| ()).map_err(|e| e.error) - )); - return Ok(None.into()) - } else { - ensure!(maybe_timepoint.is_none(), Error::::UnexpectedTimepoint); - if threshold > 1 { - let deposit = T::MultisigDepositBase::get() - + T::MultisigDepositFactor::get() * threshold.into(); - T::Currency::reserve(&who, deposit)?; - >::insert(&id, call_hash, Multisig { - when: Self::timepoint(), - deposit, - depositor: who.clone(), - approvals: vec![who.clone()], - }); - Self::deposit_event(RawEvent::NewMultisig(who, id, call_hash)); - // Call is not made, so we can return that weight - return Ok(Some(weight_of::as_multi::(other_signatories_len, 0)).into()) - } else { - let result = call.dispatch(frame_system::RawOrigin::Signed(id).into()); - match result { - Ok(post_dispatch_info) => { - match post_dispatch_info.actual_weight { - Some(actual_weight) => return Ok(Some(weight_of::as_multi::(other_signatories_len, actual_weight)).into()), - None => return Ok(None.into()), - } - }, - Err(err) => { - match err.post_info.actual_weight { - Some(actual_weight) => { - let weight_used = weight_of::as_multi::(other_signatories_len, actual_weight); - return Err(DispatchErrorWithPostInfo { post_info: Some(weight_used).into(), error: err.error.into() }) - }, - None => { - return Err(err) - } - } - } - } - } - } - } - - /// Register approval for a dispatch to be made from a deterministic composite account if - /// approved by a total of `threshold - 1` of `other_signatories`. - /// - /// Payment: `MultisigDepositBase` will be reserved if this is the first approval, plus - /// `threshold` times `MultisigDepositFactor`. It is returned once this dispatch happens or - /// is cancelled. - /// - /// The dispatch origin for this call must be _Signed_. - /// - /// - `threshold`: The total number of approvals for this dispatch before it is executed. - /// - `other_signatories`: The accounts (other than the sender) who can approve this - /// dispatch. May not be empty. - /// - `maybe_timepoint`: If this is the first approval, then this must be `None`. If it is - /// not the first approval, then it must be `Some`, with the timepoint (block number and - /// transaction index) of the first approval transaction. - /// - `call_hash`: The hash of the call to be executed. - /// - /// NOTE: If this is the final approval, you will want to use `as_multi` instead. - /// - /// # - /// - `O(S)`. - /// - Up to one balance-reserve or unreserve operation. - /// - One passthrough operation, one insert, both `O(S)` where `S` is the number of - /// signatories. `S` is capped by `MaxSignatories`, with weight being proportional. - /// - One encode & hash, both of complexity `O(S)`. - /// - Up to one binary search and insert (`O(logS + S)`). - /// - I/O: 1 read `O(S)`, up to 1 mutate `O(S)`. Up to one remove. - /// - One event. - /// - Storage: inserts one item, value size bounded by `MaxSignatories`, with a - /// deposit taken for its lifetime of - /// `MultisigDepositBase + threshold * MultisigDepositFactor`. - /// ---------------------------------- - /// - Base Weight: - /// - Create: 44.71 + 0.088 * S - /// - Approve: 31.48 + 0.116 * S - /// - DB Weight: - /// - Read: Multisig Storage, [Caller Account] - /// - Write: Multisig Storage, [Caller Account] - /// # - #[weight = FunctionOf( - |args: (&u16, &Vec, &Option>, &[u8; 32])| { - T::DbWeight::get().reads_writes(1, 1) - .saturating_add(45_000_000) - .saturating_add((args.1.len() as Weight).saturating_mul(120_000)) - }, - DispatchClass::Normal, - Pays::Yes, - )] - fn approve_as_multi(origin, - threshold: u16, - other_signatories: Vec, - maybe_timepoint: Option>, - call_hash: [u8; 32], - ) -> DispatchResult { - let who = ensure_signed(origin)?; - ensure!(threshold >= 1, Error::::ZeroThreshold); - let max_sigs = T::MaxSignatories::get() as usize; - ensure!(!other_signatories.is_empty(), Error::::TooFewSignatories); - ensure!(other_signatories.len() < max_sigs, Error::::TooManySignatories); - let signatories = Self::ensure_sorted_and_insert(other_signatories, who.clone())?; - - let id = Self::multi_account_id(&signatories, threshold); - - if let Some(mut m) = >::get(&id, call_hash) { - let timepoint = maybe_timepoint.ok_or(Error::::NoTimepoint)?; - ensure!(m.when == timepoint, Error::::WrongTimepoint); - ensure!(m.approvals.len() < threshold as usize, Error::::NoApprovalsNeeded); - if let Err(pos) = m.approvals.binary_search(&who) { - m.approvals.insert(pos, who.clone()); - >::insert(&id, call_hash, m); - Self::deposit_event(RawEvent::MultisigApproval(who, timepoint, id, call_hash)); - } else { - Err(Error::::AlreadyApproved)? - } - } else { - if threshold > 1 { - ensure!(maybe_timepoint.is_none(), Error::::UnexpectedTimepoint); - let deposit = T::MultisigDepositBase::get() - + T::MultisigDepositFactor::get() * threshold.into(); - T::Currency::reserve(&who, deposit)?; - >::insert(&id, call_hash, Multisig { - when: Self::timepoint(), - deposit, - depositor: who.clone(), - approvals: vec![who.clone()], - }); - Self::deposit_event(RawEvent::NewMultisig(who, id, call_hash)); - } else { - Err(Error::::NoApprovalsNeeded)? - } - } - Ok(()) - } - - /// Cancel a pre-existing, on-going multisig transaction. Any deposit reserved previously - /// for this operation will be unreserved on success. - /// - /// The dispatch origin for this call must be _Signed_. - /// - /// - `threshold`: The total number of approvals for this dispatch before it is executed. - /// - `other_signatories`: The accounts (other than the sender) who can approve this - /// dispatch. May not be empty. - /// - `timepoint`: The timepoint (block number and transaction index) of the first approval - /// transaction for this dispatch. - /// - `call_hash`: The hash of the call to be executed. - /// - /// # - /// - `O(S)`. - /// - Up to one balance-reserve or unreserve operation. - /// - One passthrough operation, one insert, both `O(S)` where `S` is the number of - /// signatories. `S` is capped by `MaxSignatories`, with weight being proportional. - /// - One encode & hash, both of complexity `O(S)`. - /// - One event. - /// - I/O: 1 read `O(S)`, one remove. - /// - Storage: removes one item. - /// ---------------------------------- - /// - Base Weight: 37.6 + 0.084 * S - /// - DB Weight: - /// - Read: Multisig Storage, [Caller Account] - /// - Write: Multisig Storage, [Caller Account] + /// - Base weight: 2.861 µs + /// - Plus the weight of the `call` /// # - #[weight = FunctionOf( - |args: (&u16, &Vec, &Timepoint, &[u8; 32])| { - T::DbWeight::get().reads_writes(1, 1) - .saturating_add(40_000_000) - .saturating_add((args.1.len() as Weight).saturating_mul(100_000)) - }, - DispatchClass::Normal, - Pays::Yes, + #[weight = ( + call.get_dispatch_info().weight.saturating_add(3_000_000), + call.get_dispatch_info().class, )] - fn cancel_as_multi(origin, - threshold: u16, - other_signatories: Vec, - timepoint: Timepoint, - call_hash: [u8; 32], - ) -> DispatchResult { - let who = ensure_signed(origin)?; - ensure!(threshold >= 1, Error::::ZeroThreshold); - let max_sigs = T::MaxSignatories::get() as usize; - ensure!(!other_signatories.is_empty(), Error::::TooFewSignatories); - ensure!(other_signatories.len() < max_sigs, Error::::TooManySignatories); - let signatories = Self::ensure_sorted_and_insert(other_signatories, who.clone())?; - - let id = Self::multi_account_id(&signatories, threshold); - - let m = >::get(&id, call_hash) - .ok_or(Error::::NotFound)?; - ensure!(m.when == timepoint, Error::::WrongTimepoint); - ensure!(m.depositor == who, Error::::NotOwner); - - let _ = T::Currency::unreserve(&m.depositor, m.deposit); - >::remove(&id, call_hash); - - Self::deposit_event(RawEvent::MultisigCancelled(who, timepoint, id, call_hash)); - Ok(()) + fn as_derivative(origin, index: u16, call: Box<::Call>) -> DispatchResult { + let mut origin = origin; + let who = ensure_signed(origin.clone())?; + let pseudonym = Self::derivative_account_id(who, index); + origin.set_caller_from(frame_system::RawOrigin::Signed(pseudonym)); + call.dispatch(origin).map(|_| ()).map_err(|e| e.error) } } } impl Module { - /// Derive a sub-account ID from the owner account and the sub-account index. - pub fn sub_account_id(who: T::AccountId, index: u16) -> T::AccountId { + /// Derive a derivative account ID from the owner account and the sub-account index. + pub fn derivative_account_id(who: T::AccountId, index: u16) -> T::AccountId { let entropy = (b"modlpy/utilisuba", who, index).using_encoded(blake2_256); T::AccountId::decode(&mut &entropy[..]).unwrap_or_default() } - - /// Derive a multi-account ID from the sorted list of accounts and the threshold that are - /// required. - /// - /// NOTE: `who` must be sorted. If it is not, then you'll get the wrong answer. - pub fn multi_account_id(who: &[T::AccountId], threshold: u16) -> T::AccountId { - let entropy = (b"modlpy/utilisuba", who, threshold).using_encoded(blake2_256); - T::AccountId::decode(&mut &entropy[..]).unwrap_or_default() - } - - /// The current `Timepoint`. - pub fn timepoint() -> Timepoint { - Timepoint { - height: >::block_number(), - index: >::extrinsic_index().unwrap_or_default(), - } - } - - /// Check that signatories is sorted and doesn't contain sender, then insert sender. - fn ensure_sorted_and_insert(other_signatories: Vec, who: T::AccountId) - -> Result, DispatchError> - { - let mut signatories = other_signatories; - let mut maybe_last = None; - let mut index = 0; - for item in signatories.iter() { - if let Some(last) = maybe_last { - ensure!(last < item, Error::::SignatoriesOutOfOrder); - } - if item <= &who { - ensure!(item != &who, Error::::SenderInSignatories); - index += 1; - } - maybe_last = Some(item); - } - signatories.insert(index, who); - Ok(signatories) - } } diff --git a/frame/utility/src/tests.rs b/frame/utility/src/tests.rs index 7dfc58350de49c81f8ebc68d1f97557d703a2523..349d748a378348b18f00b33172f6d24acdc85006 100644 --- a/frame/utility/src/tests.rs +++ b/frame/utility/src/tests.rs @@ -23,7 +23,7 @@ use super::*; use frame_support::{ assert_ok, assert_noop, impl_outer_origin, parameter_types, impl_outer_dispatch, - weights::Weight, impl_outer_event + weights::Weight, impl_outer_event, dispatch::DispatchError, traits::Filter, storage, }; use sp_core::H256; use sp_runtime::{Perbill, traits::{BlakeTwo256, IdentityLookup}, testing::Header}; @@ -32,12 +32,11 @@ use crate as utility; impl_outer_origin! { pub enum Origin for Test where system = frame_system {} } - impl_outer_event! { pub enum TestEvent for Test { system, pallet_balances, - utility, + utility, } } impl_outer_dispatch! { @@ -60,6 +59,7 @@ parameter_types! { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = TestBaseCallFilter; type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -75,6 +75,7 @@ impl frame_system::Trait for Test { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -88,8 +89,8 @@ parameter_types! { } impl pallet_balances::Trait for Test { type Balance = u64; - type Event = TestEvent; type DustRemoval = (); + type Event = TestEvent; type ExistentialDeposit = ExistentialDeposit; type AccountStore = System; } @@ -98,13 +99,20 @@ parameter_types! { pub const MultisigDepositFactor: u64 = 1; pub const MaxSignatories: u16 = 3; } +pub struct TestBaseCallFilter; +impl Filter for TestBaseCallFilter { + fn filter(c: &Call) -> bool { + match *c { + Call::Balances(_) => true, + // For benchmarking, this acts as a noop call + Call::System(frame_system::Call::remark(..)) => true, + _ => false, + } + } +} impl Trait for Test { type Event = TestEvent; type Call = Call; - type Currency = Balances; - type MultisigDepositBase = MultisigDepositBase; - type MultisigDepositFactor = MultisigDepositFactor; - type MaxSignatories = MaxSignatories; } type System = frame_system::Module; type Balances = pallet_balances::Module; @@ -116,7 +124,7 @@ use pallet_balances::Error as BalancesError; pub fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { - balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 10)], + balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 2)], }.assimilate_storage(&mut t).unwrap(); let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); @@ -131,264 +139,17 @@ fn expect_event>(e: E) { assert_eq!(last_event(), e.into()); } -fn now() -> Timepoint { - Utility::timepoint() -} - -#[test] -fn multisig_deposit_is_taken_and_returned() { - new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call.clone())); - assert_eq!(Balances::free_balance(1), 2); - assert_eq!(Balances::reserved_balance(1), 3); - - assert_ok!(Utility::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), call)); - assert_eq!(Balances::free_balance(1), 5); - assert_eq!(Balances::reserved_balance(1), 0); - }); -} - -#[test] -fn cancel_multisig_returns_deposit() { - new_test_ext().execute_with(|| { - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - let hash = call.using_encoded(blake2_256); - assert_ok!(Utility::approve_as_multi(Origin::signed(1), 3, vec![2, 3], None, hash.clone())); - assert_ok!(Utility::approve_as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), hash.clone())); - assert_eq!(Balances::free_balance(1), 6); - assert_eq!(Balances::reserved_balance(1), 4); - assert_ok!( - Utility::cancel_as_multi(Origin::signed(1), 3, vec![2, 3], now(), hash.clone()), - ); - assert_eq!(Balances::free_balance(1), 10); - assert_eq!(Balances::reserved_balance(1), 0); - }); -} - #[test] -fn timepoint_checking_works() { +fn as_derivative_works() { new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - let hash = call.using_encoded(blake2_256); - - assert_noop!( - Utility::approve_as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), hash.clone()), - Error::::UnexpectedTimepoint, - ); - - assert_ok!(Utility::approve_as_multi(Origin::signed(1), 2, vec![2, 3], None, hash)); - - assert_noop!( - Utility::as_multi(Origin::signed(2), 2, vec![1, 3], None, call.clone()), - Error::::NoTimepoint, - ); - let later = Timepoint { index: 1, .. now() }; - assert_noop!( - Utility::as_multi(Origin::signed(2), 2, vec![1, 3], Some(later), call.clone()), - Error::::WrongTimepoint, - ); - }); -} - -#[test] -fn multisig_2_of_3_works() { - new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - let hash = call.using_encoded(blake2_256); - assert_ok!(Utility::approve_as_multi(Origin::signed(1), 2, vec![2, 3], None, hash)); - assert_eq!(Balances::free_balance(6), 0); - - assert_ok!(Utility::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), call)); - assert_eq!(Balances::free_balance(6), 15); - }); -} - -#[test] -fn multisig_3_of_3_works() { - new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 3); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - let hash = call.using_encoded(blake2_256); - assert_ok!(Utility::approve_as_multi(Origin::signed(1), 3, vec![2, 3], None, hash.clone())); - assert_ok!(Utility::approve_as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), hash.clone())); - assert_eq!(Balances::free_balance(6), 0); - - assert_ok!(Utility::as_multi(Origin::signed(3), 3, vec![1, 2], Some(now()), call)); - assert_eq!(Balances::free_balance(6), 15); - }); -} - -#[test] -fn cancel_multisig_works() { - new_test_ext().execute_with(|| { - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - let hash = call.using_encoded(blake2_256); - assert_ok!(Utility::approve_as_multi(Origin::signed(1), 3, vec![2, 3], None, hash.clone())); - assert_ok!(Utility::approve_as_multi(Origin::signed(2), 3, vec![1, 3], Some(now()), hash.clone())); - assert_noop!( - Utility::cancel_as_multi(Origin::signed(2), 3, vec![1, 3], now(), hash.clone()), - Error::::NotOwner, - ); - assert_ok!( - Utility::cancel_as_multi(Origin::signed(1), 3, vec![2, 3], now(), hash.clone()), - ); - }); -} - -#[test] -fn multisig_2_of_3_as_multi_works() { - new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call.clone())); - assert_eq!(Balances::free_balance(6), 0); - - assert_ok!(Utility::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), call)); - assert_eq!(Balances::free_balance(6), 15); - }); -} - -#[test] -fn multisig_2_of_3_as_multi_with_many_calls_works() { - new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call1 = Box::new(Call::Balances(BalancesCall::transfer(6, 10))); - let call2 = Box::new(Call::Balances(BalancesCall::transfer(7, 5))); - - assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call1.clone())); - assert_ok!(Utility::as_multi(Origin::signed(2), 2, vec![1, 3], None, call2.clone())); - assert_ok!(Utility::as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), call2)); - assert_ok!(Utility::as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), call1)); - - assert_eq!(Balances::free_balance(6), 10); - assert_eq!(Balances::free_balance(7), 5); - }); -} - -#[test] -fn multisig_2_of_3_cannot_reissue_same_call() { - new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 2); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 10))); - assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call.clone())); - assert_ok!(Utility::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), call.clone())); - assert_eq!(Balances::free_balance(multi), 5); - - assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call.clone())); - assert_ok!(Utility::as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), call.clone())); - - let err = DispatchError::from(BalancesError::::InsufficientBalance).stripped(); - expect_event(RawEvent::MultisigExecuted(3, now(), multi, call.using_encoded(blake2_256), Err(err))); - }); -} - -#[test] -fn zero_threshold_fails() { - new_test_ext().execute_with(|| { - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - assert_noop!( - Utility::as_multi(Origin::signed(1), 0, vec![2], None, call), - Error::::ZeroThreshold, - ); - }); -} - -#[test] -fn too_many_signatories_fails() { - new_test_ext().execute_with(|| { - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - assert_noop!( - Utility::as_multi(Origin::signed(1), 2, vec![2, 3, 4], None, call.clone()), - Error::::TooManySignatories, - ); - }); -} - -#[test] -fn duplicate_approvals_are_ignored() { - new_test_ext().execute_with(|| { - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - let hash = call.using_encoded(blake2_256); - assert_ok!(Utility::approve_as_multi(Origin::signed(1), 2, vec![2, 3], None, hash.clone())); - assert_noop!( - Utility::approve_as_multi(Origin::signed(1), 2, vec![2, 3], Some(now()), hash.clone()), - Error::::AlreadyApproved, - ); - assert_ok!(Utility::approve_as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), hash.clone())); - assert_noop!( - Utility::approve_as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), hash.clone()), - Error::::NoApprovalsNeeded, - ); - }); -} - -#[test] -fn multisig_1_of_3_works() { - new_test_ext().execute_with(|| { - let multi = Utility::multi_account_id(&[1, 2, 3][..], 1); - assert_ok!(Balances::transfer(Origin::signed(1), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(2), multi, 5)); - assert_ok!(Balances::transfer(Origin::signed(3), multi, 5)); - - let call = Box::new(Call::Balances(BalancesCall::transfer(6, 15))); - let hash = call.using_encoded(blake2_256); - assert_noop!( - Utility::approve_as_multi(Origin::signed(1), 1, vec![2, 3], None, hash.clone()), - Error::::NoApprovalsNeeded, - ); - assert_noop!( - Utility::as_multi(Origin::signed(4), 1, vec![2, 3], None, call.clone()), - BalancesError::::InsufficientBalance, - ); - assert_ok!(Utility::as_multi(Origin::signed(1), 1, vec![2, 3], None, call)); - - assert_eq!(Balances::free_balance(6), 15); - }); -} - -#[test] -fn as_sub_works() { - new_test_ext().execute_with(|| { - let sub_1_0 = Utility::sub_account_id(1, 0); + let sub_1_0 = Utility::derivative_account_id(1, 0); assert_ok!(Balances::transfer(Origin::signed(1), sub_1_0, 5)); - assert_noop!(Utility::as_sub( + assert_noop!(Utility::as_derivative( Origin::signed(1), 1, Box::new(Call::Balances(BalancesCall::transfer(6, 3))), ), BalancesError::::InsufficientBalance); - assert_ok!(Utility::as_sub( + assert_ok!(Utility::as_derivative( Origin::signed(1), 0, Box::new(Call::Balances(BalancesCall::transfer(2, 3))), @@ -398,17 +159,33 @@ fn as_sub_works() { }); } +#[test] +fn as_derivative_filters() { + new_test_ext().execute_with(|| { + assert_noop!(Utility::as_derivative( + Origin::signed(1), + 1, + Box::new(Call::System(frame_system::Call::suicide())), + ), DispatchError::BadOrigin); + }); +} + #[test] fn batch_with_root_works() { new_test_ext().execute_with(|| { + let k = b"a".to_vec(); + let call = Call::System(frame_system::Call::set_storage(vec![(k.clone(), k.clone())])); + assert!(!TestBaseCallFilter::filter(&call)); assert_eq!(Balances::free_balance(1), 10); assert_eq!(Balances::free_balance(2), 10); - assert_ok!(Utility::batch(Origin::ROOT, vec![ + assert_ok!(Utility::batch(Origin::root(), vec![ + Call::Balances(BalancesCall::force_transfer(1, 2, 5)), Call::Balances(BalancesCall::force_transfer(1, 2, 5)), - Call::Balances(BalancesCall::force_transfer(1, 2, 5)) + call, // Check filters are correctly bypassed ])); assert_eq!(Balances::free_balance(1), 0); assert_eq!(Balances::free_balance(2), 20); + assert_eq!(storage::unhashed::get_raw(&k), Some(k)); }); } @@ -428,6 +205,18 @@ fn batch_with_signed_works() { }); } +#[test] +fn batch_with_signed_filters() { + new_test_ext().execute_with(|| { + assert_ok!( + Utility::batch(Origin::signed(1), vec![ + Call::System(frame_system::Call::suicide()) + ]), + ); + expect_event(Event::BatchInterrupted(0, DispatchError::BadOrigin)); + }); +} + #[test] fn batch_early_exit_works() { new_test_ext().execute_with(|| { diff --git a/frame/vesting/Cargo.toml b/frame/vesting/Cargo.toml index 94924c07e285f619e0671bbed9cb08aea418b4c0..aa5f0731f21575c4004280996a62f604fcf5a075 100644 --- a/frame/vesting/Cargo.toml +++ b/frame/vesting/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-vesting" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,19 +13,19 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } enumflags2 = { version = "0.6.2" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../system" } -frame-benchmarking = { version = "2.0.0-alpha.8", default-features = false, path = "../benchmarking", optional = true } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../system" } +frame-benchmarking = { version = "2.0.0-rc4", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.8", path = "../balances" } -sp-storage = { version = "2.0.0-alpha.8", path = "../../primitives/storage" } +sp-io = { version = "2.0.0-rc4", path = "../../primitives/io" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-rc4", path = "../balances" } +sp-storage = { version = "2.0.0-rc4", path = "../../primitives/storage" } hex-literal = "0.2.1" [features] @@ -34,7 +34,6 @@ std = [ "serde", "codec/std", "sp-std/std", - "sp-io/std", "sp-runtime/std", "frame-support/std", "frame-system/std", diff --git a/frame/vesting/src/lib.rs b/frame/vesting/src/lib.rs index a6748b15ce6466000e54ccc48bf0db65d565d035..5e11c8af953da5302a5862eea651f5bae1e83adc 100644 --- a/frame/vesting/src/lib.rs +++ b/frame/vesting/src/lib.rs @@ -51,15 +51,14 @@ use sp_std::prelude::*; use sp_std::fmt::Debug; use codec::{Encode, Decode}; use sp_runtime::{DispatchResult, RuntimeDebug, traits::{ - StaticLookup, Zero, AtLeast32Bit, MaybeSerializeDeserialize, Convert + StaticLookup, Zero, AtLeast32BitUnsigned, MaybeSerializeDeserialize, Convert }}; use frame_support::{decl_module, decl_event, decl_storage, decl_error, ensure}; use frame_support::traits::{ Currency, LockableCurrency, VestingSchedule, WithdrawReason, LockIdentifier, ExistenceRequirement, Get }; - -use frame_system::{self as system, ensure_signed}; +use frame_system::{self as system, ensure_signed, ensure_root}; mod benchmarking; @@ -93,8 +92,8 @@ pub struct VestingInfo { } impl< - Balance: AtLeast32Bit + Copy, - BlockNumber: AtLeast32Bit + Copy, + Balance: AtLeast32BitUnsigned + Copy, + BlockNumber: AtLeast32BitUnsigned + Copy, > VestingInfo { /// Amount locked at block `n`. pub fn locked_at< @@ -267,6 +266,47 @@ decl_module! { Ok(()) } + + /// Force a vested transfer. + /// + /// The dispatch origin for this call must be _Root_. + /// + /// - `source`: The account whose funds should be transferred. + /// - `target`: The account that should be transferred the vested funds. + /// - `amount`: The amount of funds to transfer and will be vested. + /// - `schedule`: The vesting schedule attached to the transfer. + /// + /// Emits `VestingCreated`. + /// + /// # + /// - `O(1)`. + /// - DbWeight: 4 Reads, 4 Writes + /// - Reads: Vesting Storage, Balances Locks, Target Account, Source Account + /// - Writes: Vesting Storage, Balances Locks, Target Account, Source Account + /// - Benchmark: 100.3 + .365 * l µs (min square analysis) + /// - Using 100 µs fixed. Assuming less than 50 locks on any user, else we may want factor in number of locks. + /// # + #[weight = 100_000_000 + T::DbWeight::get().reads_writes(4, 4)] + pub fn force_vested_transfer( + origin, + source: ::Source, + target: ::Source, + schedule: VestingInfo, T::BlockNumber>, + ) -> DispatchResult { + ensure_root(origin)?; + ensure!(schedule.locked >= T::MinVestedTransfer::get(), Error::::AmountLow); + + let target = T::Lookup::lookup(target)?; + let source = T::Lookup::lookup(source)?; + ensure!(!Vesting::::contains_key(&target), Error::::ExistingVestingSchedule); + + T::Currency::transfer(&source, &target, schedule.locked, ExistenceRequirement::AllowDeath)?; + + Self::add_vesting_schedule(&target, schedule.locked, schedule.per_block, schedule.starting_block) + .expect("user does not have an existing vesting schedule; q.e.d."); + + Ok(()) + } } } @@ -362,8 +402,9 @@ mod tests { use sp_runtime::{ Perbill, testing::Header, - traits::{BlakeTwo256, IdentityLookup, Identity}, + traits::{BlakeTwo256, IdentityLookup, Identity, BadOrigin}, }; + use frame_system::RawOrigin; impl_outer_origin! { pub enum Origin for Test where system = frame_system {} @@ -381,6 +422,7 @@ mod tests { pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Trait for Test { + type BaseCallFilter = (); type Origin = Origin; type Index = u64; type BlockNumber = u64; @@ -396,6 +438,7 @@ mod tests { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -717,4 +760,94 @@ mod tests { assert_eq!(user4_free_balance, 256 * 40); }); } + + #[test] + fn force_vested_transfer_works() { + ExtBuilder::default() + .existential_deposit(256) + .build() + .execute_with(|| { + let user3_free_balance = Balances::free_balance(&3); + let user4_free_balance = Balances::free_balance(&4); + assert_eq!(user3_free_balance, 256 * 30); + assert_eq!(user4_free_balance, 256 * 40); + // Account 4 should not have any vesting yet. + assert_eq!(Vesting::vesting(&4), None); + // Make the schedule for the new transfer. + let new_vesting_schedule = VestingInfo { + locked: 256 * 5, + per_block: 64, // Vesting over 20 blocks + starting_block: 10, + }; + assert_noop!(Vesting::force_vested_transfer(Some(4).into(), 3, 4, new_vesting_schedule), BadOrigin); + assert_ok!(Vesting::force_vested_transfer(RawOrigin::Root.into(), 3, 4, new_vesting_schedule)); + // Now account 4 should have vesting. + assert_eq!(Vesting::vesting(&4), Some(new_vesting_schedule)); + // Ensure the transfer happened correctly. + let user3_free_balance_updated = Balances::free_balance(&3); + assert_eq!(user3_free_balance_updated, 256 * 25); + let user4_free_balance_updated = Balances::free_balance(&4); + assert_eq!(user4_free_balance_updated, 256 * 45); + // Account 4 has 5 * 256 locked. + assert_eq!(Vesting::vesting_balance(&4), Some(256 * 5)); + + System::set_block_number(20); + assert_eq!(System::block_number(), 20); + + // Account 4 has 5 * 64 units vested by block 20. + assert_eq!(Vesting::vesting_balance(&4), Some(10 * 64)); + + System::set_block_number(30); + assert_eq!(System::block_number(), 30); + + // Account 4 has fully vested. + assert_eq!(Vesting::vesting_balance(&4), Some(0)); + }); + } + + #[test] + fn force_vested_transfer_correctly_fails() { + ExtBuilder::default() + .existential_deposit(256) + .build() + .execute_with(|| { + let user2_free_balance = Balances::free_balance(&2); + let user4_free_balance = Balances::free_balance(&4); + assert_eq!(user2_free_balance, 256 * 20); + assert_eq!(user4_free_balance, 256 * 40); + // Account 2 should already have a vesting schedule. + let user2_vesting_schedule = VestingInfo { + locked: 256 * 20, + per_block: 256, // Vesting over 20 blocks + starting_block: 10, + }; + assert_eq!(Vesting::vesting(&2), Some(user2_vesting_schedule)); + + // The vesting schedule we will try to create, fails due to pre-existence of schedule. + let new_vesting_schedule = VestingInfo { + locked: 256 * 5, + per_block: 64, // Vesting over 20 blocks + starting_block: 10, + }; + assert_noop!( + Vesting::force_vested_transfer(RawOrigin::Root.into(), 4, 2, new_vesting_schedule), + Error::::ExistingVestingSchedule, + ); + + // Fails due to too low transfer amount. + let new_vesting_schedule_too_low = VestingInfo { + locked: 256 * 1, + per_block: 64, + starting_block: 10, + }; + assert_noop!( + Vesting::force_vested_transfer(RawOrigin::Root.into(), 3, 4, new_vesting_schedule_too_low), + Error::::AmountLow, + ); + + // Verify no currency transfer happened. + assert_eq!(user2_free_balance, 256 * 20); + assert_eq!(user4_free_balance, 256 * 40); + }); + } } diff --git a/primitives/allocator/Cargo.toml b/primitives/allocator/Cargo.toml index e3f64cc6c90a26a2d36b59b5f8f74603b858b0cc..ba0aed938723336accd84adce227105dce34b140 100644 --- a/primitives/allocator/Cargo.toml +++ b/primitives/allocator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-allocator" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,9 +13,9 @@ documentation = "https://docs.rs/sp-allocator" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-std = { version = "2.0.0-alpha.8", path = "../std", default-features = false } -sp-core = { version = "2.0.0-alpha.8", path = "../core", default-features = false } -sp-wasm-interface = { version = "2.0.0-alpha.8", path = "../wasm-interface", default-features = false } +sp-std = { version = "2.0.0-rc4", path = "../std", default-features = false } +sp-core = { version = "2.0.0-rc4", path = "../core", default-features = false } +sp-wasm-interface = { version = "2.0.0-rc4", path = "../wasm-interface", default-features = false } log = { version = "0.4.8", optional = true } derive_more = { version = "0.99.2", optional = true } diff --git a/primitives/api/Cargo.toml b/primitives/api/Cargo.toml index 8c490673b2b9aa2924433507922c1d3ae2f133ad..8fe0a6d910bd92973e189a2fe81f6259554f8c81 100644 --- a/primitives/api/Cargo.toml +++ b/primitives/api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-api" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,17 +12,17 @@ description = "Substrate runtime api primitives" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-api-proc-macro = { version = "2.0.0-alpha.8", path = "proc-macro" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../core" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../runtime" } -sp-version = { version = "2.0.0-alpha.8", default-features = false, path = "../version" } -sp-state-machine = { version = "0.8.0-alpha.8", optional = true, path = "../../primitives/state-machine" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +sp-api-proc-macro = { version = "2.0.0-rc4", path = "proc-macro" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../core" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../runtime" } +sp-version = { version = "2.0.0-rc4", default-features = false, path = "../version" } +sp-state-machine = { version = "0.8.0-rc4", optional = true, path = "../../primitives/state-machine" } hash-db = { version = "0.15.2", optional = true } [dev-dependencies] -sp-test-primitives = { version = "2.0.0-alpha.8", path = "../test-primitives" } +sp-test-primitives = { version = "2.0.0-rc4", path = "../test-primitives" } [features] default = [ "std" ] diff --git a/primitives/api/proc-macro/Cargo.toml b/primitives/api/proc-macro/Cargo.toml index 724a46500838c2a4b7451f1450db85d3e28b84c0..fb426fde885e27a75657f36d1e84328d1f55fc27 100644 --- a/primitives/api/proc-macro/Cargo.toml +++ b/primitives/api/proc-macro/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-api-proc-macro" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" diff --git a/primitives/api/proc-macro/src/decl_runtime_apis.rs b/primitives/api/proc-macro/src/decl_runtime_apis.rs index 7e1391b7b57bd3dcfd5db10feaa9de8aa08dd09b..93ec09d0e615b8fa778ee3e786b414f08b2fe04f 100644 --- a/primitives/api/proc-macro/src/decl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/decl_runtime_apis.rs @@ -191,7 +191,8 @@ fn generate_native_call_generators(decl: &ItemTrait) -> Result { input: &I, error_desc: &'static str, ) -> std::result::Result { - ::decode( + ::decode_with_depth_limit( + #crate_::MAX_EXTRINSIC_DEPTH, &mut &#crate_::Encode::encode(input)[..], ).map_err(|e| format!("{} {}", error_desc, e.what())) } diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index 2878bd2c136837a7fed3dbb03e99802f32f0078d..a4c35dcf42991b3df5337c90bcde0ac8dc557a32 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -34,7 +34,7 @@ use syn::{ fold::{self, Fold}, parse_quote, }; -use std::{collections::HashSet, iter}; +use std::collections::HashSet; /// Unique identifier used to make the hidden includes unique for this macro. const HIDDEN_INCLUDES_ID: &str = "IMPL_RUNTIME_APIS"; @@ -71,10 +71,8 @@ fn generate_impl_call( let params = extract_parameter_names_types_and_borrows(signature, AllowSelfRefInParameters::No)?; let c = generate_crate_access(HIDDEN_INCLUDES_ID); - let c_iter = iter::repeat(&c); let fn_name = &signature.ident; - let fn_name_str = iter::repeat(fn_name.to_string()); - let input = iter::repeat(input); + let fn_name_str = fn_name.to_string(); let pnames = params.iter().map(|v| &v.0); let pnames2 = params.iter().map(|v| &v.0); let ptypes = params.iter().map(|v| &v.1); @@ -82,12 +80,14 @@ fn generate_impl_call( Ok( quote!( - #( - let #pnames : #ptypes = match #c_iter::Decode::decode(&mut #input) { - Ok(input) => input, + let (#( #pnames ),*) : ( #( #ptypes ),* ) = + match #c::DecodeLimit::decode_all_with_depth_limit( + #c::MAX_EXTRINSIC_DEPTH, + &#input, + ) { + Ok(res) => res, Err(e) => panic!("Bad input data provided to {}: {}", #fn_name_str, e.what()), }; - )* #[allow(deprecated)] <#runtime as #impl_trait>::#fn_name(#( #pborrow #pnames2 ),*) @@ -135,7 +135,7 @@ fn generate_impl_calls( /// Generate the dispatch function that is used in native to call into the runtime. fn generate_dispatch_function(impls: &[ItemImpl]) -> Result { - let data = Ident::new("data", Span::call_site()); + let data = Ident::new("__sp_api__input_data", Span::call_site()); let c = generate_crate_access(HIDDEN_INCLUDES_ID); let impl_calls = generate_impl_calls(impls, &data)? .into_iter() @@ -257,6 +257,7 @@ fn generate_runtime_api_base_structures() -> Result { &self, map_call: F, ) -> std::result::Result where Self: Sized { + self.changes.borrow_mut().start_transaction(); *self.commit_on_success.borrow_mut() = false; let res = map_call(self); *self.commit_on_success.borrow_mut() = true; @@ -366,6 +367,9 @@ fn generate_runtime_api_base_structures() -> Result { &self, call_api_at: F, ) -> std::result::Result<#crate_::NativeOrEncoded, E> { + if *self.commit_on_success.borrow() { + self.changes.borrow_mut().start_transaction(); + } let res = call_api_at( &self.call, self, @@ -381,11 +385,16 @@ fn generate_runtime_api_base_structures() -> Result { } fn commit_on_ok(&self, res: &std::result::Result) { + let proof = "\ + We only close a transaction when we opened one ourself. + Other parts of the runtime that make use of transactions (state-machine) + also balance their transactions. The runtime cannot close client initiated + transactions. qed"; if *self.commit_on_success.borrow() { if res.is_err() { - self.changes.borrow_mut().discard_prospective(); + self.changes.borrow_mut().rollback_transaction().expect(proof); } else { - self.changes.borrow_mut().commit_prospective(); + self.changes.borrow_mut().commit_transaction().expect(proof); } } } diff --git a/primitives/api/src/lib.rs b/primitives/api/src/lib.rs index ec15c1eae71d068217b0f5d2846bd0455786ceda..0aaf72e2d2b2440579a81a9375553c41c6adb7e6 100644 --- a/primitives/api/src/lib.rs +++ b/primitives/api/src/lib.rs @@ -69,11 +69,14 @@ pub use sp_std::{slice, mem}; #[cfg(feature = "std")] use sp_std::result; #[doc(hidden)] -pub use codec::{Encode, Decode}; +pub use codec::{Encode, Decode, DecodeLimit}; use sp_core::OpaqueMetadata; #[cfg(feature = "std")] use std::{panic::UnwindSafe, cell::RefCell}; +/// Maximum nesting level for extrinsics. +pub const MAX_EXTRINSIC_DEPTH: u32 = 256; + /// Declares given traits as runtime apis. /// /// The macro will create two declarations, one for using on the client side and one for using diff --git a/primitives/api/test/Cargo.toml b/primitives/api/test/Cargo.toml index 8aa77b01eaef7a028436262f38cb7c8f405f0565..cf8f0ce47e17de4a5a32e316312d1c5fde1a0e4c 100644 --- a/primitives/api/test/Cargo.toml +++ b/primitives/api/test/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-api-test" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,22 +12,22 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-api = { version = "2.0.0-alpha.8", path = "../" } -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../../test-utils/runtime/client" } -sp-version = { version = "2.0.0-alpha.8", path = "../../version" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../runtime" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../blockchain" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/common" } -sc-block-builder = { version = "0.8.0-alpha.8", path = "../../../client/block-builder" } -codec = { package = "parity-scale-codec", version = "1.3.0" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../../primitives/state-machine" } +sp-api = { version = "2.0.0-rc4", path = "../" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../../test-utils/runtime/client" } +sp-version = { version = "2.0.0-rc4", path = "../../version" } +sp-runtime = { version = "2.0.0-rc4", path = "../../runtime" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../blockchain" } +sp-consensus = { version = "0.8.0-rc4", path = "../../../primitives/consensus/common" } +sc-block-builder = { version = "0.8.0-rc4", path = "../../../client/block-builder" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../../primitives/state-machine" } trybuild = "1.0.17" rustversion = "1.0.0" [dev-dependencies] criterion = "0.3.0" -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../../test-utils/runtime/client" } -sp-core = { version = "2.0.0-alpha.8", path = "../../core" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../../test-utils/runtime/client" } +sp-core = { version = "2.0.0-rc4", path = "../../core" } [[bench]] name = "bench" diff --git a/primitives/api/test/tests/runtime_calls.rs b/primitives/api/test/tests/runtime_calls.rs index 555104446ae2ef4a9e01e91c8b9ea4b375835960..6717ab7a3bb92756275cee2928e2799b9d00ee94 100644 --- a/primitives/api/test/tests/runtime_calls.rs +++ b/primitives/api/test/tests/runtime_calls.rs @@ -207,3 +207,14 @@ fn record_proof_works() { &runtime_code, ).expect("Executes block while using the proof backend"); } + +#[test] +fn call_runtime_api_with_multiple_arguments() { + let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build(); + + let data = vec![1, 2, 4, 5, 6, 7, 8, 8, 10, 12]; + let block_id = BlockId::Number(client.chain_info().best_number); + client.runtime_api() + .test_multiple_arguments(&block_id, data.clone(), data.clone(), data.len() as u32) + .unwrap(); +} diff --git a/primitives/api/test/tests/ui/declaring_old_block.rs b/primitives/api/test/tests/ui/declaring_old_block.rs index ba98bf9bf688317b4a8d6dd940b61112fba82485..fb741590282249c692af3b9233f59a95fa3b0b5b 100644 --- a/primitives/api/test/tests/ui/declaring_old_block.rs +++ b/primitives/api/test/tests/ui/declaring_old_block.rs @@ -1,5 +1,3 @@ -use sp_runtime::traits::Block as BlockT; - sp_api::decl_runtime_apis! { pub trait Api { fn test(); diff --git a/primitives/api/test/tests/ui/declaring_old_block.stderr b/primitives/api/test/tests/ui/declaring_old_block.stderr index 448dc38594321e5b7f11c0ccfeb4f4d6e6414862..50dd37582c603d5d355472e2003b1813d18ecb67 100644 --- a/primitives/api/test/tests/ui/declaring_old_block.stderr +++ b/primitives/api/test/tests/ui/declaring_old_block.stderr @@ -1,19 +1,11 @@ error: `Block: BlockT` generic parameter will be added automatically by the `decl_runtime_apis!` macro! If you try to use a different trait than the substrate `Block` trait, please rename it locally. - --> $DIR/declaring_old_block.rs:4:23 + --> $DIR/declaring_old_block.rs:2:23 | -4 | pub trait Api { +2 | pub trait Api { | ^^^^^^ error: `Block: BlockT` generic parameter will be added automatically by the `decl_runtime_apis!` macro! - --> $DIR/declaring_old_block.rs:4:16 + --> $DIR/declaring_old_block.rs:2:16 | -4 | pub trait Api { +2 | pub trait Api { | ^^^^^ - -warning: unused import: `sp_runtime::traits::Block as BlockT` - --> $DIR/declaring_old_block.rs:1:5 - | -1 | use sp_runtime::traits::Block as BlockT; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` on by default diff --git a/primitives/api/test/tests/ui/declaring_own_block_with_different_name.rs b/primitives/api/test/tests/ui/declaring_own_block_with_different_name.rs index 67bb9cab10502cc7ef33fe9106d95221547a9a99..e3c7ae8a39ab748557c5d160f35e24dfcd0a353d 100644 --- a/primitives/api/test/tests/ui/declaring_own_block_with_different_name.rs +++ b/primitives/api/test/tests/ui/declaring_own_block_with_different_name.rs @@ -1,5 +1,3 @@ -use sp_runtime::traits::Block as BlockT; - sp_api::decl_runtime_apis! { pub trait Api { fn test(); diff --git a/primitives/api/test/tests/ui/declaring_own_block_with_different_name.stderr b/primitives/api/test/tests/ui/declaring_own_block_with_different_name.stderr index fe445b822dd88a25e340beb75fc834a5ff85febf..778de3c9d15f7ff9caf94b858182af78ff260bd1 100644 --- a/primitives/api/test/tests/ui/declaring_own_block_with_different_name.stderr +++ b/primitives/api/test/tests/ui/declaring_own_block_with_different_name.stderr @@ -1,13 +1,5 @@ error: `Block: BlockT` generic parameter will be added automatically by the `decl_runtime_apis!` macro! If you try to use a different trait than the substrate `Block` trait, please rename it locally. - --> $DIR/declaring_own_block_with_different_name.rs:4:19 + --> $DIR/declaring_own_block_with_different_name.rs:2:19 | -4 | pub trait Api { +2 | pub trait Api { | ^^^^^^ - -warning: unused import: `sp_runtime::traits::Block as BlockT` - --> $DIR/declaring_own_block_with_different_name.rs:1:5 - | -1 | use sp_runtime::traits::Block as BlockT; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` on by default diff --git a/primitives/api/test/tests/ui/impl_two_traits_with_same_name.rs b/primitives/api/test/tests/ui/impl_two_traits_with_same_name.rs index 2122a6991ae5a9c6f38a0a9de9abdcc05b0641ca..76555a825dc8415c591455398ad931d43d6d7d5f 100644 --- a/primitives/api/test/tests/ui/impl_two_traits_with_same_name.rs +++ b/primitives/api/test/tests/ui/impl_two_traits_with_same_name.rs @@ -15,8 +15,6 @@ sp_api::decl_runtime_apis! { } mod second { - use super::*; - sp_api::decl_runtime_apis! { pub trait Api { fn test2(data: u64); diff --git a/primitives/api/test/tests/ui/impl_two_traits_with_same_name.stderr b/primitives/api/test/tests/ui/impl_two_traits_with_same_name.stderr index 24ee66f84b314d345dd7b5d86638293b140eedb1..17ee56d409a66e34fcaec32af8644385fe5ef520 100644 --- a/primitives/api/test/tests/ui/impl_two_traits_with_same_name.stderr +++ b/primitives/api/test/tests/ui/impl_two_traits_with_same_name.stderr @@ -1,13 +1,5 @@ error: Two traits with the same name detected! The trait name is used to generate its ID. Please rename one trait at the declaration! - --> $DIR/impl_two_traits_with_same_name.rs:32:15 + --> $DIR/impl_two_traits_with_same_name.rs:30:15 | -32 | impl second::Api for Runtime { +30 | impl second::Api for Runtime { | ^^^ - -warning: unused import: `super::*` - --> $DIR/impl_two_traits_with_same_name.rs:18:6 - | -18 | use super::*; - | ^^^^^^^^ - | - = note: `#[warn(unused_imports)]` on by default diff --git a/primitives/api/test/tests/ui/mock_only_one_block_type.rs b/primitives/api/test/tests/ui/mock_only_one_block_type.rs index 969b21d7378d75b6b2bb28c1ca118aa041015784..f49cafd23a00122bd53976c0656e0779f3d3fb5d 100644 --- a/primitives/api/test/tests/ui/mock_only_one_block_type.rs +++ b/primitives/api/test/tests/ui/mock_only_one_block_type.rs @@ -1,5 +1,3 @@ -use substrate_test_runtime_client::runtime::Block; - struct Block2; sp_api::decl_runtime_apis! { diff --git a/primitives/api/test/tests/ui/mock_only_one_block_type.stderr b/primitives/api/test/tests/ui/mock_only_one_block_type.stderr index 1abc8db726a160e151cb47abf6b745d5630b1851..1831d0485b473a3d33ca1bafce6c2fa67f6b78a2 100644 --- a/primitives/api/test/tests/ui/mock_only_one_block_type.stderr +++ b/primitives/api/test/tests/ui/mock_only_one_block_type.stderr @@ -1,19 +1,11 @@ error: Block type should be the same between all runtime apis. - --> $DIR/mock_only_one_block_type.rs:22:12 + --> $DIR/mock_only_one_block_type.rs:20:12 | -22 | impl Api2 for MockApi { +20 | impl Api2 for MockApi { | ^^^^^^ error: First block type found here - --> $DIR/mock_only_one_block_type.rs:18:11 + --> $DIR/mock_only_one_block_type.rs:16:11 | -18 | impl Api for MockApi { +16 | impl Api for MockApi { | ^^^^^ - -warning: unused import: `substrate_test_runtime_client::runtime::Block` - --> $DIR/mock_only_one_block_type.rs:1:5 - | -1 | use substrate_test_runtime_client::runtime::Block; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` on by default diff --git a/primitives/api/test/tests/ui/mock_only_one_error_type.stderr b/primitives/api/test/tests/ui/mock_only_one_error_type.stderr index b190c2134fafa6d4312b856f391c905a1d5c72b3..65d05e83a7f69149947a4628b98224976a9e2659 100644 --- a/primitives/api/test/tests/ui/mock_only_one_error_type.stderr +++ b/primitives/api/test/tests/ui/mock_only_one_error_type.stderr @@ -5,21 +5,26 @@ error: Error type can not change between runtime apis | ^^^^ error[E0277]: the trait bound `u32: std::convert::From` is not satisfied - --> $DIR/mock_only_one_error_type.rs:15:1 - | -15 | / sp_api::mock_impl_runtime_apis! { -16 | | impl Api for MockApi { -17 | | type Error = u32; -18 | | -... | -26 | | } -27 | | } - | |_^ the trait `std::convert::From` is not implemented for `u32` - | - = help: the following implementations were found: - > - > - > - > - and 18 others - = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) + --> $DIR/mock_only_one_error_type.rs:15:1 + | +15 | / sp_api::mock_impl_runtime_apis! { +16 | | impl Api for MockApi { +17 | | type Error = u32; +18 | | +... | +26 | | } +27 | | } + | |_^ the trait `std::convert::From` is not implemented for `u32` + | + ::: $WORKSPACE/primitives/api/src/lib.rs:350:35 + | +350 | type Error: std::fmt::Debug + From; + | ------------ required by this bound in `sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ApiErrorExt` + | + = help: the following implementations were found: + > + > + > + > + and 18 others + = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/primitives/api/test/tests/ui/mock_only_one_self_type.rs b/primitives/api/test/tests/ui/mock_only_one_self_type.rs index 4b29ec2a6ab07e7a08689549423e03687501543d..617031b4d5f1a1f37c974ed8d826f43e49003233 100644 --- a/primitives/api/test/tests/ui/mock_only_one_self_type.rs +++ b/primitives/api/test/tests/ui/mock_only_one_self_type.rs @@ -1,5 +1,3 @@ -use substrate_test_runtime_client::runtime::Block; - sp_api::decl_runtime_apis! { pub trait Api { fn test(data: u64); diff --git a/primitives/api/test/tests/ui/mock_only_one_self_type.stderr b/primitives/api/test/tests/ui/mock_only_one_self_type.stderr index 996d1d44c044c151bc5e57e93e83d3749e86a80c..5f1e29b281c8504d39a355028882099b40298810 100644 --- a/primitives/api/test/tests/ui/mock_only_one_self_type.stderr +++ b/primitives/api/test/tests/ui/mock_only_one_self_type.stderr @@ -1,19 +1,11 @@ error: Self type should not change between runtime apis - --> $DIR/mock_only_one_self_type.rs:21:23 + --> $DIR/mock_only_one_self_type.rs:19:23 | -21 | impl Api2 for MockApi2 { +19 | impl Api2 for MockApi2 { | ^^^^^^^^ error: First self type found here - --> $DIR/mock_only_one_self_type.rs:17:22 + --> $DIR/mock_only_one_self_type.rs:15:22 | -17 | impl Api for MockApi { +15 | impl Api for MockApi { | ^^^^^^^ - -warning: unused import: `substrate_test_runtime_client::runtime::Block` - --> $DIR/mock_only_one_self_type.rs:1:5 - | -1 | use substrate_test_runtime_client::runtime::Block; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: `#[warn(unused_imports)]` on by default diff --git a/primitives/application-crypto/Cargo.toml b/primitives/application-crypto/Cargo.toml index b48b61a99f67ab1df8f407a270e96240522deb28..8e9c9225091dda4cad054f4b241b5bc167149efc 100644 --- a/primitives/application-crypto/Cargo.toml +++ b/primitives/application-crypto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-application-crypto" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" description = "Provides facilities for generating application specific crypto wrapper types." @@ -14,11 +14,11 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../core" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../core" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } [features] default = [ "std" ] diff --git a/primitives/application-crypto/test/Cargo.toml b/primitives/application-crypto/test/Cargo.toml index b50d035546e56f52fff92175d8708ab8475f069d..d3b336d92a6a7947ea87031b89cd63ac884824ce 100644 --- a/primitives/application-crypto/test/Cargo.toml +++ b/primitives/application-crypto/test/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-application-crypto-test" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" description = "Integration tests for application-crypto" @@ -13,8 +13,8 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../core" } -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../../test-utils/runtime/client" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../runtime" } -sp-api = { version = "2.0.0-alpha.8", path = "../../api" } -sp-application-crypto = { version = "2.0.0-alpha.8", path = "../" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../core" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../../test-utils/runtime/client" } +sp-runtime = { version = "2.0.0-rc4", path = "../../runtime" } +sp-api = { version = "2.0.0-rc4", path = "../../api" } +sp-application-crypto = { version = "2.0.0-rc4", path = "../" } diff --git a/primitives/arithmetic/Cargo.toml b/primitives/arithmetic/Cargo.toml index 0cab74a7b1b3ab686de4c1150ac66c2595ad8b56..c3bef60d1a13541ffc25f028f9be29b5ca8ac668 100644 --- a/primitives/arithmetic/Cargo.toml +++ b/primitives/arithmetic/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-arithmetic" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -14,18 +14,18 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } integer-sqrt = "0.1.2" num-traits = { version = "0.2.8", default-features = false } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../std" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../std" } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-debug-derive = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/debug-derive" } -primitive-types = { version = "0.7.0", default-features = false } +sp-debug-derive = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/debug-derive" } [dev-dependencies] rand = "0.7.2" criterion = "0.3" serde_json = "1.0" +primitive-types = "0.7.0" [features] default = ["std"] @@ -35,7 +35,6 @@ std = [ "sp-std/std", "serde", "sp-debug-derive/std", - "primitive-types/std", ] [[bench]] diff --git a/primitives/arithmetic/fuzzer/Cargo.lock b/primitives/arithmetic/fuzzer/Cargo.lock deleted file mode 100644 index 3a4187437ae7312ec7016cb9e41638a018cfcbbf..0000000000000000000000000000000000000000 --- a/primitives/arithmetic/fuzzer/Cargo.lock +++ /dev/null @@ -1,401 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -[[package]] -name = "arbitrary" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cf76cb6e2222ed0ea86b2b0ee2f71c96ec6edd5af42e84d59160e91b836ec4" - -[[package]] -name = "arrayvec" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" - -[[package]] -name = "autocfg" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" - -[[package]] -name = "bitvec" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a993f74b4c99c1908d156b8d2e0fb6277736b0ecbd833982fd1241d39b2766a6" - -[[package]] -name = "byte-slice-cast" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0a5e3906bcbf133e33c1d4d95afc664ad37fbdb9f6568d8043e7ea8c27d93d3" - -[[package]] -name = "byteorder" -version = "1.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" - -[[package]] -name = "c2-chacha" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" -dependencies = [ - "ppv-lite86", -] - -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - -[[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - -[[package]] -name = "fixed-hash" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3367952ceb191f4ab95dd5685dc163ac539e36202f9fcfd0cb22f9f9c542fefc" -dependencies = [ - "byteorder", - "rand", - "rustc-hex", - "static_assertions", -] - -[[package]] -name = "getrandom" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "honggfuzz" -version = "0.5.45" -dependencies = [ - "arbitrary", - "lazy_static", - "memmap", -] - -[[package]] -name = "impl-codec" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1be51a921b067b0eaca2fad532d9400041561aa922221cc65f95a85641c6bf53" -dependencies = [ - "parity-scale-codec", -] - -[[package]] -name = "integer-sqrt" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65877bf7d44897a473350b1046277941cee20b263397e90869c50b6e766088b" - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "libc" -version = "0.2.67" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb147597cdf94ed43ab7a9038716637d2d1bf2bc571da995d0028dec06bd3018" - -[[package]] -name = "memmap" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "num-bigint" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-integer" -version = "0.1.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" -dependencies = [ - "autocfg", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" -dependencies = [ - "autocfg", -] - -[[package]] -name = "parity-scale-codec" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f509c5e67ca0605ee17dcd3f91ef41cadd685c75a298fb6261b781a5acb3f910" -dependencies = [ - "arrayvec", - "bitvec", - "byte-slice-cast", - "parity-scale-codec-derive", - "serde", -] - -[[package]] -name = "parity-scale-codec-derive" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a0ec292e92e8ec7c58e576adacc1e3f399c597c8f263c42f18420abe58e7245" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" - -[[package]] -name = "primitive-types" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4336f4f5d5524fa60bcbd6fe626f9223d8142a50e7053e979acdf0da41ab975" -dependencies = [ - "fixed-hash", - "impl-codec", - "uint", -] - -[[package]] -name = "proc-macro-crate" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10d4b51f154c8a7fb96fd6dad097cb74b863943ec010ac94b9fd1be8861fe1e" -dependencies = [ - "toml", -] - -[[package]] -name = "proc-macro2" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c09721c6781493a2a492a96b5a5bf19b65917fe6728884e7c44dd0c60ca3435" -dependencies = [ - "unicode-xid", -] - -[[package]] -name = "quote" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom", - "libc", - "rand_chacha", - "rand_core", - "rand_hc", -] - -[[package]] -name = "rand_chacha" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853" -dependencies = [ - "c2-chacha", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core", -] - -[[package]] -name = "rustc-hex" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" - -[[package]] -name = "serde" -version = "1.0.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "sp-arithmetic" -version = "2.0.0-alpha.3" -dependencies = [ - "integer-sqrt", - "num-traits", - "parity-scale-codec", - "serde", - "sp-debug-derive", - "sp-std", -] - -[[package]] -name = "sp-arithmetic-fuzzer" -version = "2.0.0" -dependencies = [ - "honggfuzz", - "num-bigint", - "num-traits", - "primitive-types", - "sp-arithmetic", -] - -[[package]] -name = "sp-debug-derive" -version = "2.0.0-alpha.3" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "sp-std" -version = "2.0.0-alpha.3" - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "syn" -version = "1.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "123bd9499cfb380418d509322d7a6d52e5315f064fe4b3ad18a53d6b92c07859" -dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", -] - -[[package]] -name = "toml" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a" -dependencies = [ - "serde", -] - -[[package]] -name = "uint" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75a4cdd7b87b28840dba13c483b9a88ee6bbf16ba5c951ee1ecfcf723078e0d" -dependencies = [ - "byteorder", - "crunchy", - "rustc-hex", - "static_assertions", -] - -[[package]] -name = "unicode-xid" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" - -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - -[[package]] -name = "winapi" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/primitives/arithmetic/fuzzer/Cargo.toml b/primitives/arithmetic/fuzzer/Cargo.toml index 9180414a3add0c696512d787c724f4475e4faf6e..c7e5485a19fb5eeb3dab9d7a21ea2ff5fb20e6c4 100644 --- a/primitives/arithmetic/fuzzer/Cargo.toml +++ b/primitives/arithmetic/fuzzer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-arithmetic-fuzzer" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -14,8 +14,8 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-arithmetic = { version = "2.0.0-alpha.8", path = ".." } -honggfuzz = "0.5" +sp-arithmetic = { version = "2.0.0-rc4", path = ".." } +honggfuzz = "0.5.49" primitive-types = "0.7.0" num-bigint = "0.2" num-traits = "0.2" @@ -24,6 +24,10 @@ num-traits = "0.2" name = "biguint" path = "src/biguint.rs" +[[bin]] +name = "normalize" +path = "src/normalize.rs" + [[bin]] name = "per_thing_rational" path = "src/per_thing_rational.rs" @@ -31,3 +35,7 @@ path = "src/per_thing_rational.rs" [[bin]] name = "rational128" path = "src/rational128.rs" + +[[bin]] +name = "fixed_point" +path = "src/fixed_point.rs" diff --git a/primitives/arithmetic/fuzzer/src/fixed_point.rs b/primitives/arithmetic/fuzzer/src/fixed_point.rs new file mode 100644 index 0000000000000000000000000000000000000000..9a88197ac32adfbd1ada4ecc1a85805dc291cd91 --- /dev/null +++ b/primitives/arithmetic/fuzzer/src/fixed_point.rs @@ -0,0 +1,82 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Running +//! Running this fuzzer can be done with `cargo hfuzz run fixed_point`. `honggfuzz` CLI options can +//! be used by setting `HFUZZ_RUN_ARGS`, such as `-n 4` to use 4 threads. +//! +//! # Debugging a panic +//! Once a panic is found, it can be debugged with +//! `cargo hfuzz run-debug fixed_point hfuzz_workspace/fixed_point/*.fuzz`. +//! +//! # More information +//! More information about `honggfuzz` can be found +//! [here](https://docs.rs/honggfuzz/). + +use honggfuzz::fuzz; +use sp_arithmetic::{FixedPointNumber, FixedI64, traits::Saturating}; + +fn main() { + loop { + fuzz!(|data: (i32, i32)| { + let x: i128 = data.0.into(); + let y: i128 = data.1.into(); + + // Check `from_rational` and division are consistent. + if y != 0 { + let f1 = FixedI64::saturating_from_integer(x) / FixedI64::saturating_from_integer(y); + let f2 = FixedI64::saturating_from_rational(x, y); + assert_eq!(f1.into_inner(), f2.into_inner()); + } + + // Check `saturating_mul`. + let a = FixedI64::saturating_from_rational(2, 5); + let b = a.saturating_mul(FixedI64::saturating_from_integer(x)); + let n = b.into_inner() as i128; + let m = 2i128 * x * FixedI64::accuracy() as i128 / 5i128; + assert_eq!(n, m); + + // Check `saturating_mul` and division are inverse. + if x != 0 { + assert_eq!(a, b / FixedI64::saturating_from_integer(x)); + } + + // Check `reciprocal`. + let r = a.reciprocal().unwrap().reciprocal().unwrap(); + assert_eq!(a, r); + + // Check addition. + let a = FixedI64::saturating_from_integer(x); + let b = FixedI64::saturating_from_integer(y); + let c = FixedI64::saturating_from_integer(x.saturating_add(y)); + assert_eq!(a.saturating_add(b), c); + + // Check substraction. + let a = FixedI64::saturating_from_integer(x); + let b = FixedI64::saturating_from_integer(y); + let c = FixedI64::saturating_from_integer(x.saturating_sub(y)); + assert_eq!(a.saturating_sub(b), c); + + // Check `saturating_mul_acc_int`. + let a = FixedI64::saturating_from_rational(2, 5); + let b = a.saturating_mul_acc_int(x); + let xx = FixedI64::saturating_from_integer(x); + let d = a.saturating_mul(xx).saturating_add(xx).into_inner() as i128 / FixedI64::accuracy() as i128; + assert_eq!(b, d); + }); + } +} diff --git a/primitives/arithmetic/fuzzer/src/normalize.rs b/primitives/arithmetic/fuzzer/src/normalize.rs new file mode 100644 index 0000000000000000000000000000000000000000..34c4ef9cb0ab51e125c4a9a2a2535d8770a8b342 --- /dev/null +++ b/primitives/arithmetic/fuzzer/src/normalize.rs @@ -0,0 +1,62 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +//! # Running +//! Running this fuzzer can be done with `cargo hfuzz run normalize`. `honggfuzz` CLI options can +//! be used by setting `HFUZZ_RUN_ARGS`, such as `-n 4` to use 4 threads. +//! +//! # Debugging a panic +//! Once a panic is found, it can be debugged with +//! `cargo hfuzz run-debug normalize hfuzz_workspace/normalize/*.fuzz`. + +use honggfuzz::fuzz; +use sp_arithmetic::Normalizable; +use std::convert::TryInto; + +fn main() { + let sum_limit = u32::max_value() as u128; + let len_limit: usize = u32::max_value().try_into().unwrap(); + + loop { + fuzz!(|data: (Vec, u32)| { + let (data, norm) = data; + if data.len() == 0 { return; } + let pre_sum: u128 = data.iter().map(|x| *x as u128).sum(); + + let normalized = data.normalize(norm); + // error cases. + if pre_sum > sum_limit || data.len() > len_limit { + assert!(normalized.is_err()) + } else { + if let Ok(normalized) = normalized { + // if sum goes beyond u128, panic. + let sum: u128 = normalized.iter().map(|x| *x as u128).sum(); + + // if this function returns Ok(), then it will ALWAYS be accurate. + assert_eq!( + sum, + norm as u128, + "sums don't match {:?}, {}", + normalized, + norm, + ); + } + } + }) + } +} diff --git a/primitives/arithmetic/fuzzer/src/per_thing_rational.rs b/primitives/arithmetic/fuzzer/src/per_thing_rational.rs index 0820a35100aee1217f37f75b0657f9761b6926c7..fc22eacc9e499fc6a75489e6830f42d63f09f163 100644 --- a/primitives/arithmetic/fuzzer/src/per_thing_rational.rs +++ b/primitives/arithmetic/fuzzer/src/per_thing_rational.rs @@ -114,7 +114,7 @@ fn main() { } } -fn assert_per_thing_equal_error(a: T, b: T, err: u128) { +fn assert_per_thing_equal_error(a: P, b: P, err: u128) { let a_abs = a.deconstruct().saturated_into::(); let b_abs = b.deconstruct().saturated_into::(); let diff = a_abs.max(b_abs) - a_abs.min(b_abs); diff --git a/primitives/arithmetic/src/biguint.rs b/primitives/arithmetic/src/biguint.rs index a1e5ea266620526025c198993afc320fe03db59b..41e2c759a59679a25eda9d42655eee83559cd4b4 100644 --- a/primitives/arithmetic/src/biguint.rs +++ b/primitives/arithmetic/src/biguint.rs @@ -151,7 +151,7 @@ impl BigUint { // has the ability to cause this. There is nothing to do if the number already has 1 // limb only. call it a day and return. if self.len().is_zero() { return; } - let index = self.digits.iter().position(|&elem| elem != 0).unwrap_or(0); + let index = self.digits.iter().position(|&elem| elem != 0).unwrap_or(self.len() - 1); if index > 0 { self.digits = self.digits[index..].to_vec() @@ -581,19 +581,19 @@ pub mod tests { fn strip_works() { let mut a = BigUint::from_limbs(&[0, 1, 0]); a.lstrip(); - assert_eq!(a, BigUint { digits: vec![1, 0] }); + assert_eq!(a.digits, vec![1, 0]); let mut a = BigUint::from_limbs(&[0, 0, 1]); a.lstrip(); - assert_eq!(a, BigUint { digits: vec![1] }); + assert_eq!(a.digits, vec![1]); let mut a = BigUint::from_limbs(&[0, 0]); a.lstrip(); - assert_eq!(a, BigUint { digits: vec![0] }); + assert_eq!(a.digits, vec![0]); let mut a = BigUint::from_limbs(&[0, 0, 0]); a.lstrip(); - assert_eq!(a, BigUint { digits: vec![0] }); + assert_eq!(a.digits, vec![0]); } #[test] diff --git a/primitives/arithmetic/src/fixed128.rs b/primitives/arithmetic/src/fixed128.rs deleted file mode 100644 index 6682b37949467ffddffd625118cadce89f475d67..0000000000000000000000000000000000000000 --- a/primitives/arithmetic/src/fixed128.rs +++ /dev/null @@ -1,732 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2020 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use codec::{Decode, Encode}; -use primitive_types::U256; -use crate::{ - traits::{Bounded, Saturating, UniqueSaturatedInto, SaturatedConversion}, - PerThing, Perquintill, -}; -use sp_std::{ - convert::{Into, TryFrom, TryInto}, - fmt, ops, - num::NonZeroI128, -}; - -#[cfg(feature = "std")] -use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; - -/// A signed fixed-point number. -/// Can hold any value in the range [-170_141_183_460_469_231_731, 170_141_183_460_469_231_731] -/// with fixed-point accuracy of 10 ** 18. -#[derive(Encode, Decode, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct Fixed128(i128); - -const DIV: i128 = 1_000_000_000_000_000_000; - -impl Fixed128 { - /// Create self from a natural number. - /// - /// Note that this might be lossy. - pub fn from_natural(int: i128) -> Self { - Self(int.saturating_mul(DIV)) - } - - /// Accuracy of `Fixed128`. - pub const fn accuracy() -> i128 { - DIV - } - - /// Raw constructor. Equal to `parts / DIV`. - pub const fn from_parts(parts: i128) -> Self { - Self(parts) - } - - /// Creates self from a rational number. Equal to `n/d`. - /// - /// Note that this might be lossy. Only use this if you are sure that `n * DIV` can fit into an - /// i128. - pub fn from_rational>(n: N, d: NonZeroI128) -> Self { - let n = n.unique_saturated_into(); - Self(n.saturating_mul(DIV.into()) / d.get()) - } - - /// Consume self and return the inner raw `i128` value. - /// - /// Note this is a low level function, as the returned value is represented with accuracy. - pub fn deconstruct(self) -> i128 { - self.0 - } - - /// Takes the reciprocal(inverse) of Fixed128, 1/x - pub fn recip(&self) -> Option { - Self::from_natural(1i128).checked_div(self) - } - - /// Checked add. Same semantic to `num_traits::CheckedAdd`. - pub fn checked_add(&self, rhs: &Self) -> Option { - self.0.checked_add(rhs.0).map(Self) - } - - /// Checked sub. Same semantic to `num_traits::CheckedSub`. - pub fn checked_sub(&self, rhs: &Self) -> Option { - self.0.checked_sub(rhs.0).map(Self) - } - - /// Checked mul. Same semantic to `num_traits::CheckedMul`. - pub fn checked_mul(&self, rhs: &Self) -> Option { - let signum = self.0.signum() * rhs.0.signum(); - let mut lhs = self.0; - if lhs.is_negative() { - lhs = lhs.saturating_mul(-1); - } - let mut rhs: i128 = rhs.0.saturated_into(); - if rhs.is_negative() { - rhs = rhs.saturating_mul(-1); - } - - U256::from(lhs) - .checked_mul(U256::from(rhs)) - .and_then(|n| n.checked_div(U256::from(DIV))) - .and_then(|n| TryInto::::try_into(n).ok()) - .map(|n| Self(n * signum)) - } - - /// Checked div. Same semantic to `num_traits::CheckedDiv`. - pub fn checked_div(&self, rhs: &Self) -> Option { - if rhs.0.signum() == 0 { - return None; - } - if self.0 == 0 { - return Some(*self); - } - - let signum = self.0.signum() / rhs.0.signum(); - let mut lhs: i128 = self.0; - if lhs.is_negative() { - lhs = lhs.saturating_mul(-1); - } - let mut rhs: i128 = rhs.0.saturated_into(); - if rhs.is_negative() { - rhs = rhs.saturating_mul(-1); - } - - U256::from(lhs) - .checked_mul(U256::from(DIV)) - .and_then(|n| n.checked_div(U256::from(rhs))) - .and_then(|n| TryInto::::try_into(n).ok()) - .map(|n| Self(n * signum)) - } - - /// Checked mul for int type `N`. - pub fn checked_mul_int(&self, other: &N) -> Option - where - N: Copy + TryFrom + TryInto, - { - N::try_into(*other).ok().and_then(|rhs| { - let mut lhs = self.0; - if lhs.is_negative() { - lhs = lhs.saturating_mul(-1); - } - let mut rhs: i128 = rhs.saturated_into(); - let signum = self.0.signum() * rhs.signum(); - if rhs.is_negative() { - rhs = rhs.saturating_mul(-1); - } - - U256::from(lhs) - .checked_mul(U256::from(rhs)) - .and_then(|n| n.checked_div(U256::from(DIV))) - .and_then(|n| TryInto::::try_into(n).ok()) - .and_then(|n| TryInto::::try_into(n * signum).ok()) - }) - } - - /// Checked mul for int type `N`. - pub fn saturating_mul_int(&self, other: &N) -> N - where - N: Copy + TryFrom + TryInto + Bounded, - { - self.checked_mul_int(other).unwrap_or_else(|| { - N::try_into(*other) - .map(|n| n.signum()) - .map(|n| n * self.0.signum()) - .map(|signum| { - if signum.is_negative() { - Bounded::min_value() - } else { - Bounded::max_value() - } - }) - .unwrap_or(Bounded::max_value()) - }) - } - - /// Checked div for int type `N`. - pub fn checked_div_int(&self, other: &N) -> Option - where - N: Copy + TryFrom + TryInto, - { - N::try_into(*other) - .ok() - .and_then(|n| self.0.checked_div(n)) - .and_then(|n| n.checked_div(DIV)) - .and_then(|n| TryInto::::try_into(n).ok()) - } - - pub fn zero() -> Self { - Self(0) - } - - pub fn is_zero(&self) -> bool { - self.0 == 0 - } - - /// Saturating absolute value. Returning MAX if `parts` == i128::MIN instead of overflowing. - pub fn saturating_abs(&self) -> Self { - if self.0 == i128::min_value() { - return Fixed128::max_value(); - } - - if self.0.is_negative() { - Fixed128::from_parts(self.0 * -1) - } else { - *self - } - } - - pub fn is_positive(&self) -> bool { - self.0.is_positive() - } - - pub fn is_negative(&self) -> bool { - self.0.is_negative() - } - - /// Performs a saturated multiply and accumulate by unsigned number. - /// - /// Returns a saturated `int + (self * int)`. - pub fn saturated_multiply_accumulate(self, int: N) -> N - where - N: TryFrom + From + UniqueSaturatedInto + Bounded + Clone + Saturating + - ops::Rem + ops::Div + ops::Mul + - ops::Add, - { - let div = DIV as u128; - let positive = self.0 > 0; - // safe to convert as absolute value. - let parts = self.0.checked_abs().map(|v| v as u128).unwrap_or(i128::max_value() as u128 + 1); - - - // will always fit. - let natural_parts = parts / div; - // might saturate. - let natural_parts: N = natural_parts.saturated_into(); - // fractional parts can always fit into u64. - let perquintill_parts = (parts % div) as u64; - - let n = int.clone().saturating_mul(natural_parts); - let p = Perquintill::from_parts(perquintill_parts) * int.clone(); - - // everything that needs to be either added or subtracted from the original weight. - let excess = n.saturating_add(p); - - if positive { - int.saturating_add(excess) - } else { - int.saturating_sub(excess) - } - } -} - -/// Note that this is a standard, _potentially-panicking_, implementation. Use `Saturating` trait -/// for safe addition. -impl ops::Add for Fixed128 { - type Output = Self; - - fn add(self, rhs: Self) -> Self::Output { - Self(self.0 + rhs.0) - } -} - -/// Note that this is a standard, _potentially-panicking_, implementation. Use `Saturating` trait -/// for safe subtraction. -impl ops::Sub for Fixed128 { - type Output = Self; - - fn sub(self, rhs: Self) -> Self::Output { - Self(self.0 - rhs.0) - } -} - -impl Saturating for Fixed128 { - fn saturating_add(self, rhs: Self) -> Self { - Self(self.0.saturating_add(rhs.0)) - } - - fn saturating_sub(self, rhs: Self) -> Self { - Self(self.0.saturating_sub(rhs.0)) - } - - fn saturating_mul(self, rhs: Self) -> Self { - self.checked_mul(&rhs).unwrap_or_else(|| { - if (self.0.signum() * rhs.0.signum()).is_negative() { - Bounded::min_value() - } else { - Bounded::max_value() - } - }) - } - - fn saturating_pow(self, exp: usize) -> Self { - if exp == 0 { - return Self::from_natural(1); - } - - let exp = exp as u64; - let msb_pos = 64 - exp.leading_zeros(); - - let mut result = Self::from_natural(1); - let mut pow_val = self; - for i in 0..msb_pos { - if ((1 << i) & exp) > 0 { - result = result.saturating_mul(pow_val); - } - pow_val = pow_val.saturating_mul(pow_val); - } - result - } -} - -impl Bounded for Fixed128 { - fn min_value() -> Self { - Self(Bounded::min_value()) - } - - fn max_value() -> Self { - Self(Bounded::max_value()) - } -} - -impl fmt::Debug for Fixed128 { - #[cfg(feature = "std")] - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let integral = { - let int = self.0 / DIV; - let signum_for_zero = if int == 0 && self.is_negative() { "-" } else { "" }; - format!("{}{}", signum_for_zero, int) - }; - let fractional = format!("{:0>18}", (self.0 % DIV).abs()); - write!(f, "Fixed128({}.{})", integral, fractional) - } - - #[cfg(not(feature = "std"))] - fn fmt(&self, _: &mut fmt::Formatter) -> fmt::Result { - Ok(()) - } -} - -impl From

for Fixed128 { - fn from(val: P) -> Self { - let accuracy = P::ACCURACY.saturated_into().max(1) as i128; - let value = val.deconstruct().saturated_into() as i128; - Fixed128::from_rational(value, NonZeroI128::new(accuracy).unwrap()) - } -} - -#[cfg(feature = "std")] -impl Fixed128 { - fn i128_str(&self) -> String { - format!("{}", &self.0) - } - - fn try_from_i128_str(s: &str) -> Result { - let parts: i128 = s.parse().map_err(|_| "invalid string input")?; - Ok(Self::from_parts(parts)) - } -} - -// Manual impl `Serialize` as serde_json does not support i128. -// TODO: remove impl if issue https://github.com/serde-rs/json/issues/548 fixed. -#[cfg(feature = "std")] -impl Serialize for Fixed128 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&self.i128_str()) - } -} - -// Manual impl `Serialize` as serde_json does not support i128. -// TODO: remove impl if issue https://github.com/serde-rs/json/issues/548 fixed. -#[cfg(feature = "std")] -impl<'de> Deserialize<'de> for Fixed128 { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - Fixed128::try_from_i128_str(&s).map_err(|err_str| de::Error::custom(err_str)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{Perbill, Percent, Permill, Perquintill}; - - fn max() -> Fixed128 { - Fixed128::max_value() - } - - fn min() -> Fixed128 { - Fixed128::min_value() - } - - #[test] - fn fixed128_semantics() { - let a = Fixed128::from_rational(5, NonZeroI128::new(2).unwrap()); - let b = Fixed128::from_rational(10, NonZeroI128::new(4).unwrap()); - assert_eq!(a.0, 5 * DIV / 2); - assert_eq!(a, b); - - let a = Fixed128::from_rational(-5, NonZeroI128::new(1).unwrap()); - assert_eq!(a, Fixed128::from_natural(-5)); - - let a = Fixed128::from_rational(5, NonZeroI128::new(-1).unwrap()); - assert_eq!(a, Fixed128::from_natural(-5)); - - // biggest value that can be created. - assert_ne!(max(), Fixed128::from_natural(170_141_183_460_469_231_731)); - assert_eq!(max(), Fixed128::from_natural(170_141_183_460_469_231_732)); - - // the smallest value that can be created. - assert_ne!(min(), Fixed128::from_natural(-170_141_183_460_469_231_731)); - assert_eq!(min(), Fixed128::from_natural(-170_141_183_460_469_231_732)); - } - - #[test] - fn fixed128_operation() { - let a = Fixed128::from_natural(2); - let b = Fixed128::from_natural(1); - assert_eq!(a.checked_add(&b), Some(Fixed128::from_natural(1 + 2))); - assert_eq!(a.checked_sub(&b), Some(Fixed128::from_natural(2 - 1))); - assert_eq!(a.checked_mul(&b), Some(Fixed128::from_natural(1 * 2))); - assert_eq!( - a.checked_div(&b), - Some(Fixed128::from_rational(2, NonZeroI128::new(1).unwrap())) - ); - - let a = Fixed128::from_rational(5, NonZeroI128::new(2).unwrap()); - let b = Fixed128::from_rational(3, NonZeroI128::new(2).unwrap()); - assert_eq!( - a.checked_add(&b), - Some(Fixed128::from_rational(8, NonZeroI128::new(2).unwrap())) - ); - assert_eq!( - a.checked_sub(&b), - Some(Fixed128::from_rational(2, NonZeroI128::new(2).unwrap())) - ); - assert_eq!( - a.checked_mul(&b), - Some(Fixed128::from_rational(15, NonZeroI128::new(4).unwrap())) - ); - assert_eq!( - a.checked_div(&b), - Some(Fixed128::from_rational(10, NonZeroI128::new(6).unwrap())) - ); - - let a = Fixed128::from_natural(120); - assert_eq!(a.checked_div_int(&2i32), Some(60)); - - let a = Fixed128::from_rational(20, NonZeroI128::new(1).unwrap()); - assert_eq!(a.checked_div_int(&2i32), Some(10)); - - let a = Fixed128::from_natural(120); - assert_eq!(a.checked_mul_int(&2i32), Some(240)); - - let a = Fixed128::from_rational(1, NonZeroI128::new(2).unwrap()); - assert_eq!(a.checked_mul_int(&20i32), Some(10)); - - let a = Fixed128::from_rational(-1, NonZeroI128::new(2).unwrap()); - assert_eq!(a.checked_mul_int(&20i32), Some(-10)); - } - - #[test] - fn saturating_mul_should_work() { - let a = Fixed128::from_natural(-1); - assert_eq!(min().saturating_mul(a), max()); - - assert_eq!(Fixed128::from_natural(125).saturating_mul(a).deconstruct(), -125 * DIV); - - let a = Fixed128::from_rational(1, NonZeroI128::new(5).unwrap()); - assert_eq!(Fixed128::from_natural(125).saturating_mul(a).deconstruct(), 25 * DIV); - } - - #[test] - fn saturating_mul_int_works() { - let a = Fixed128::from_rational(10, NonZeroI128::new(1).unwrap()); - assert_eq!(a.saturating_mul_int(&i32::max_value()), i32::max_value()); - - let a = Fixed128::from_rational(-10, NonZeroI128::new(1).unwrap()); - assert_eq!(a.saturating_mul_int(&i32::max_value()), i32::min_value()); - - let a = Fixed128::from_rational(3, NonZeroI128::new(1).unwrap()); - assert_eq!(a.saturating_mul_int(&100i8), i8::max_value()); - - let a = Fixed128::from_rational(10, NonZeroI128::new(1).unwrap()); - assert_eq!(a.saturating_mul_int(&123i128), 1230); - - let a = Fixed128::from_rational(-10, NonZeroI128::new(1).unwrap()); - assert_eq!(a.saturating_mul_int(&123i128), -1230); - - assert_eq!(max().saturating_mul_int(&2i128), 340_282_366_920_938_463_463); - - assert_eq!(max().saturating_mul_int(&i128::min_value()), i128::min_value()); - - assert_eq!(min().saturating_mul_int(&i128::max_value()), i128::min_value()); - - assert_eq!(min().saturating_mul_int(&i128::min_value()), i128::max_value()); - } - - #[test] - fn zero_works() { - assert_eq!(Fixed128::zero(), Fixed128::from_natural(0)); - } - - #[test] - fn is_zero_works() { - assert!(Fixed128::zero().is_zero()); - assert!(!Fixed128::from_natural(1).is_zero()); - } - - #[test] - fn checked_div_with_zero_should_be_none() { - let a = Fixed128::from_natural(1); - let b = Fixed128::from_natural(0); - assert_eq!(a.checked_div(&b), None); - assert_eq!(b.checked_div(&a), Some(b)); - } - - #[test] - fn checked_div_int_with_zero_should_be_none() { - let a = Fixed128::from_natural(1); - assert_eq!(a.checked_div_int(&0i32), None); - let a = Fixed128::from_natural(0); - assert_eq!(a.checked_div_int(&1i32), Some(0)); - } - - #[test] - fn checked_div_with_zero_dividend_should_be_zero() { - let a = Fixed128::zero(); - let b = Fixed128::from_parts(1); - - assert_eq!(a.checked_div(&b), Some(Fixed128::zero())); - } - - #[test] - fn under_flow_should_be_none() { - let b = Fixed128::from_natural(1); - assert_eq!(min().checked_sub(&b), None); - } - - #[test] - fn over_flow_should_be_none() { - let a = Fixed128::from_parts(i128::max_value() - 1); - let b = Fixed128::from_parts(2); - assert_eq!(a.checked_add(&b), None); - - let a = Fixed128::max_value(); - let b = Fixed128::from_rational(2, NonZeroI128::new(1).unwrap()); - assert_eq!(a.checked_mul(&b), None); - - let a = Fixed128::from_natural(255); - let b = 2u8; - assert_eq!(a.checked_mul_int(&b), None); - - let a = Fixed128::from_natural(256); - let b = 1u8; - assert_eq!(a.checked_div_int(&b), None); - - let a = Fixed128::from_natural(256); - let b = -1i8; - assert_eq!(a.checked_div_int(&b), None); - } - - #[test] - fn checked_div_int_should_work() { - // 256 / 10 = 25 (25.6 as int = 25) - let a = Fixed128::from_natural(256); - let result = a.checked_div_int(&10i128).unwrap(); - assert_eq!(result, 25); - - // 256 / 100 = 2 (2.56 as int = 2) - let a = Fixed128::from_natural(256); - let result = a.checked_div_int(&100i128).unwrap(); - assert_eq!(result, 2); - - // 256 / 1000 = 0 (0.256 as int = 0) - let a = Fixed128::from_natural(256); - let result = a.checked_div_int(&1000i128).unwrap(); - assert_eq!(result, 0); - - // 256 / -1 = -256 - let a = Fixed128::from_natural(256); - let result = a.checked_div_int(&-1i128).unwrap(); - assert_eq!(result, -256); - - // -256 / -1 = 256 - let a = Fixed128::from_natural(-256); - let result = a.checked_div_int(&-1i128).unwrap(); - assert_eq!(result, 256); - - // 10 / -5 = -2 - let a = Fixed128::from_rational(20, NonZeroI128::new(2).unwrap()); - let result = a.checked_div_int(&-5i128).unwrap(); - assert_eq!(result, -2); - - // -170_141_183_460_469_231_731 / -2 = 85_070_591_730_234_615_865 - let result = min().checked_div_int(&-2i128).unwrap(); - assert_eq!(result, 85_070_591_730_234_615_865); - - // 85_070_591_730_234_615_865 * -2 = -170_141_183_460_469_231_730 - let result = Fixed128::from_natural(result).checked_mul_int(&-2i128).unwrap(); - assert_eq!(result, -170_141_183_460_469_231_730); - } - - #[test] - fn perthing_into_fixed_i128() { - let ten_percent_percent: Fixed128 = Percent::from_percent(10).into(); - assert_eq!(ten_percent_percent.deconstruct(), DIV / 10); - - let ten_percent_permill: Fixed128 = Permill::from_percent(10).into(); - assert_eq!(ten_percent_permill.deconstruct(), DIV / 10); - - let ten_percent_perbill: Fixed128 = Perbill::from_percent(10).into(); - assert_eq!(ten_percent_perbill.deconstruct(), DIV / 10); - - let ten_percent_perquintill: Fixed128 = Perquintill::from_percent(10).into(); - assert_eq!(ten_percent_perquintill.deconstruct(), DIV / 10); - } - - #[test] - fn recip_should_work() { - let a = Fixed128::from_natural(2); - assert_eq!( - a.recip(), - Some(Fixed128::from_rational(1, NonZeroI128::new(2).unwrap())) - ); - - let a = Fixed128::from_natural(2); - assert_eq!(a.recip().unwrap().checked_mul_int(&4i32), Some(2i32)); - - let a = Fixed128::from_rational(100, NonZeroI128::new(121).unwrap()); - assert_eq!( - a.recip(), - Some(Fixed128::from_rational(121, NonZeroI128::new(100).unwrap())) - ); - - let a = Fixed128::from_rational(1, NonZeroI128::new(2).unwrap()); - assert_eq!(a.recip().unwrap().checked_mul(&a), Some(Fixed128::from_natural(1))); - - let a = Fixed128::from_natural(0); - assert_eq!(a.recip(), None); - - let a = Fixed128::from_rational(-1, NonZeroI128::new(2).unwrap()); - assert_eq!(a.recip(), Some(Fixed128::from_natural(-2))); - } - - #[test] - fn serialize_deserialize_should_work() { - let two_point_five = Fixed128::from_rational(5, NonZeroI128::new(2).unwrap()); - let serialized = serde_json::to_string(&two_point_five).unwrap(); - assert_eq!(serialized, "\"2500000000000000000\""); - let deserialized: Fixed128 = serde_json::from_str(&serialized).unwrap(); - assert_eq!(deserialized, two_point_five); - - let minus_two_point_five = Fixed128::from_rational(-5, NonZeroI128::new(2).unwrap()); - let serialized = serde_json::to_string(&minus_two_point_five).unwrap(); - assert_eq!(serialized, "\"-2500000000000000000\""); - let deserialized: Fixed128 = serde_json::from_str(&serialized).unwrap(); - assert_eq!(deserialized, minus_two_point_five); - } - - #[test] - fn saturating_abs_should_work() { - // normal - assert_eq!(Fixed128::from_parts(1).saturating_abs(), Fixed128::from_parts(1)); - assert_eq!(Fixed128::from_parts(-1).saturating_abs(), Fixed128::from_parts(1)); - - // saturating - assert_eq!(Fixed128::min_value().saturating_abs(), Fixed128::max_value()); - } - - #[test] - fn is_positive_negative_should_work() { - let positive = Fixed128::from_parts(1); - assert!(positive.is_positive()); - assert!(!positive.is_negative()); - - let negative = Fixed128::from_parts(-1); - assert!(!negative.is_positive()); - assert!(negative.is_negative()); - - let zero = Fixed128::zero(); - assert!(!zero.is_positive()); - assert!(!zero.is_negative()); - } - - #[test] - fn fmt_should_work() { - let positive = Fixed128::from_parts(1000000000000000001); - assert_eq!(format!("{:?}", positive), "Fixed128(1.000000000000000001)"); - let negative = Fixed128::from_parts(-1000000000000000001); - assert_eq!(format!("{:?}", negative), "Fixed128(-1.000000000000000001)"); - - let positive_fractional = Fixed128::from_parts(1); - assert_eq!(format!("{:?}", positive_fractional), "Fixed128(0.000000000000000001)"); - let negative_fractional = Fixed128::from_parts(-1); - assert_eq!(format!("{:?}", negative_fractional), "Fixed128(-0.000000000000000001)"); - - let zero = Fixed128::zero(); - assert_eq!(format!("{:?}", zero), "Fixed128(0.000000000000000000)"); - } - - #[test] - fn saturating_pow_should_work() { - assert_eq!(Fixed128::from_natural(2).saturating_pow(0), Fixed128::from_natural(1)); - assert_eq!(Fixed128::from_natural(2).saturating_pow(1), Fixed128::from_natural(2)); - assert_eq!(Fixed128::from_natural(2).saturating_pow(2), Fixed128::from_natural(4)); - assert_eq!(Fixed128::from_natural(2).saturating_pow(3), Fixed128::from_natural(8)); - assert_eq!(Fixed128::from_natural(2).saturating_pow(50), Fixed128::from_natural(1125899906842624)); - - assert_eq!(Fixed128::from_natural(1).saturating_pow(1000), Fixed128::from_natural(1)); - assert_eq!(Fixed128::from_natural(-1).saturating_pow(1000), Fixed128::from_natural(1)); - assert_eq!(Fixed128::from_natural(-1).saturating_pow(1001), Fixed128::from_natural(-1)); - assert_eq!(Fixed128::from_natural(1).saturating_pow(usize::max_value()), Fixed128::from_natural(1)); - assert_eq!(Fixed128::from_natural(-1).saturating_pow(usize::max_value()), Fixed128::from_natural(-1)); - assert_eq!(Fixed128::from_natural(-1).saturating_pow(usize::max_value() - 1), Fixed128::from_natural(1)); - - assert_eq!(Fixed128::from_natural(114209).saturating_pow(4), Fixed128::from_natural(170137997018538053761)); - assert_eq!(Fixed128::from_natural(114209).saturating_pow(5), Fixed128::max_value()); - - assert_eq!(Fixed128::from_natural(1).saturating_pow(usize::max_value()), Fixed128::from_natural(1)); - assert_eq!(Fixed128::from_natural(0).saturating_pow(usize::max_value()), Fixed128::from_natural(0)); - assert_eq!(Fixed128::from_natural(2).saturating_pow(usize::max_value()), Fixed128::max_value()); - } -} diff --git a/primitives/arithmetic/src/fixed64.rs b/primitives/arithmetic/src/fixed64.rs deleted file mode 100644 index 14baad543cf73c5ab115f79416b84ba7d6a67c7f..0000000000000000000000000000000000000000 --- a/primitives/arithmetic/src/fixed64.rs +++ /dev/null @@ -1,382 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use sp_std::{ - ops, prelude::*, - convert::{TryFrom, TryInto}, -}; -use codec::{Encode, Decode}; -use crate::{ - Perbill, - traits::{ - SaturatedConversion, CheckedSub, CheckedAdd, CheckedDiv, Bounded, UniqueSaturatedInto, Saturating - } -}; - -/// An unsigned fixed point number. Can hold any value in the range [-9_223_372_036, 9_223_372_036] -/// with fixed point accuracy of one billion. -#[derive(Encode, Decode, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct Fixed64(i64); - -/// The accuracy of the `Fixed64` type. -const DIV: i64 = 1_000_000_000; - -impl Fixed64 { - /// creates self from a natural number. - /// - /// Note that this might be lossy. - pub fn from_natural(int: i64) -> Self { - Self(int.saturating_mul(DIV)) - } - - /// Return the accuracy of the type. Given that this function returns the value `X`, it means - /// that an instance composed of `X` parts (`Fixed64::from_parts(X)`) is equal to `1`. - pub fn accuracy() -> i64 { - DIV - } - - /// Consume self and return the inner value. - pub fn into_inner(self) -> i64 { self.0 } - - /// Raw constructor. Equal to `parts / 1_000_000_000`. - pub fn from_parts(parts: i64) -> Self { - Self(parts) - } - - /// creates self from a rational number. Equal to `n/d`. - /// - /// Note that this might be lossy. - pub fn from_rational(n: i64, d: u64) -> Self { - Self( - (i128::from(n).saturating_mul(i128::from(DIV)) / i128::from(d).max(1)) - .try_into() - .unwrap_or_else(|_| Bounded::max_value()) - ) - } - - /// Performs a saturated multiply and accumulate by unsigned number. - /// - /// Returns a saturated `int + (self * int)`. - pub fn saturated_multiply_accumulate(self, int: N) -> N - where - N: TryFrom + From + UniqueSaturatedInto + Bounded + Clone + Saturating + - ops::Rem + ops::Div + ops::Mul + - ops::Add, - { - let div = DIV as u64; - let positive = self.0 > 0; - // safe to convert as absolute value. - let parts = self.0.checked_abs().map(|v| v as u64).unwrap_or(i64::max_value() as u64 + 1); - - - // will always fit. - let natural_parts = parts / div; - // might saturate. - let natural_parts: N = natural_parts.saturated_into(); - // fractional parts can always fit into u32. - let perbill_parts = (parts % div) as u32; - - let n = int.clone().saturating_mul(natural_parts); - let p = Perbill::from_parts(perbill_parts) * int.clone(); - - // everything that needs to be either added or subtracted from the original weight. - let excess = n.saturating_add(p); - - if positive { - int.saturating_add(excess) - } else { - int.saturating_sub(excess) - } - } - - pub fn is_negative(&self) -> bool { - self.0.is_negative() - } -} - -impl Saturating for Fixed64 { - fn saturating_add(self, rhs: Self) -> Self { - Self(self.0.saturating_add(rhs.0)) - } - - fn saturating_mul(self, rhs: Self) -> Self { - let a = self.0 as i128; - let b = rhs.0 as i128; - let res = a * b / DIV as i128; - Self(res.saturated_into()) - } - - fn saturating_sub(self, rhs: Self) -> Self { - Self(self.0.saturating_sub(rhs.0)) - } - - fn saturating_pow(self, exp: usize) -> Self { - if exp == 0 { - return Self::from_natural(1); - } - - let exp = exp as u64; - let msb_pos = 64 - exp.leading_zeros(); - - let mut result = Self::from_natural(1); - let mut pow_val = self; - for i in 0..msb_pos { - if ((1 << i) & exp) > 0 { - result = result.saturating_mul(pow_val); - } - pow_val = pow_val.saturating_mul(pow_val); - } - result - } -} - -/// Use `Saturating` trait for safe addition. -impl ops::Add for Fixed64 { - type Output = Self; - - fn add(self, rhs: Self) -> Self::Output { - Self(self.0 + rhs.0) - } -} - -/// Use `Saturating` trait for safe subtraction. -impl ops::Sub for Fixed64 { - type Output = Self; - - fn sub(self, rhs: Self) -> Self::Output { - Self(self.0 - rhs.0) - } -} - -/// Use `CheckedDiv` trait for safe division. -impl ops::Div for Fixed64 { - type Output = Self; - - fn div(self, rhs: Self) -> Self::Output { - if rhs.0 == 0 { - panic!("attempt to divide by zero"); - } - let (n, d) = if rhs.0 < 0 { - (-self.0, rhs.0.abs() as u64) - } else { - (self.0, rhs.0 as u64) - }; - Fixed64::from_rational(n, d) - } -} - -impl CheckedSub for Fixed64 { - fn checked_sub(&self, rhs: &Self) -> Option { - self.0.checked_sub(rhs.0).map(Self) - } -} - -impl CheckedAdd for Fixed64 { - fn checked_add(&self, rhs: &Self) -> Option { - self.0.checked_add(rhs.0).map(Self) - } -} - -impl CheckedDiv for Fixed64 { - fn checked_div(&self, rhs: &Self) -> Option { - if rhs.0 == 0 { - None - } else { - Some(*self / *rhs) - } - } -} - -impl sp_std::fmt::Debug for Fixed64 { - #[cfg(feature = "std")] - fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - let integral = { - let int = self.0 / DIV; - let signum_for_zero = if int == 0 && self.is_negative() { "-" } else { "" }; - format!("{}{}", signum_for_zero, int) - }; - let fractional = format!("{:0>9}", (self.0 % DIV).abs()); - write!(f, "Fixed64({}.{})", integral, fractional) - } - - #[cfg(not(feature = "std"))] - fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn max() -> Fixed64 { - Fixed64::from_parts(i64::max_value()) - } - - #[test] - fn fixed64_semantics() { - assert_eq!(Fixed64::from_rational(5, 2).0, 5 * 1_000_000_000 / 2); - assert_eq!(Fixed64::from_rational(5, 2), Fixed64::from_rational(10, 4)); - assert_eq!(Fixed64::from_rational(5, 0), Fixed64::from_rational(5, 1)); - - // biggest value that can be created. - assert_ne!(max(), Fixed64::from_natural(9_223_372_036)); - assert_eq!(max(), Fixed64::from_natural(9_223_372_037)); - } - - #[test] - fn fixed_64_growth_decrease_curve() { - let test_set = vec![0u32, 1, 10, 1000, 1_000_000_000]; - - // negative (1/2) - let mut fm = Fixed64::from_rational(-1, 2); - test_set.clone().into_iter().for_each(|i| { - assert_eq!(fm.saturated_multiply_accumulate(i) as i32, i as i32 - i as i32 / 2); - }); - - // unit (1) multiplier - fm = Fixed64::from_parts(0); - test_set.clone().into_iter().for_each(|i| { - assert_eq!(fm.saturated_multiply_accumulate(i), i); - }); - - // i.5 multiplier - fm = Fixed64::from_rational(1, 2); - test_set.clone().into_iter().for_each(|i| { - assert_eq!(fm.saturated_multiply_accumulate(i), i * 3 / 2); - }); - - // dual multiplier - fm = Fixed64::from_rational(1, 1); - test_set.clone().into_iter().for_each(|i| { - assert_eq!(fm.saturated_multiply_accumulate(i), i * 2); - }); - } - - macro_rules! saturating_mul_acc_test { - ($num_type:tt) => { - assert_eq!( - Fixed64::from_rational(100, 1).saturated_multiply_accumulate(10 as $num_type), - 1010, - ); - assert_eq!( - Fixed64::from_rational(100, 2).saturated_multiply_accumulate(10 as $num_type), - 510, - ); - assert_eq!( - Fixed64::from_rational(100, 3).saturated_multiply_accumulate(0 as $num_type), - 0, - ); - assert_eq!( - Fixed64::from_rational(5, 1).saturated_multiply_accumulate($num_type::max_value()), - $num_type::max_value() - ); - assert_eq!( - max().saturated_multiply_accumulate($num_type::max_value()), - $num_type::max_value() - ); - } - } - - #[test] - fn fixed64_multiply_accumulate_works() { - saturating_mul_acc_test!(u32); - saturating_mul_acc_test!(u64); - saturating_mul_acc_test!(u128); - } - - #[test] - fn div_works() { - let a = Fixed64::from_rational(12, 10); - let b = Fixed64::from_rational(10, 1); - assert_eq!(a / b, Fixed64::from_rational(12, 100)); - - let a = Fixed64::from_rational(12, 10); - let b = Fixed64::from_rational(1, 100); - assert_eq!(a / b, Fixed64::from_rational(120, 1)); - - let a = Fixed64::from_rational(12, 100); - let b = Fixed64::from_rational(10, 1); - assert_eq!(a / b, Fixed64::from_rational(12, 1000)); - - let a = Fixed64::from_rational(12, 100); - let b = Fixed64::from_rational(1, 100); - assert_eq!(a / b, Fixed64::from_rational(12, 1)); - - let a = Fixed64::from_rational(-12, 10); - let b = Fixed64::from_rational(10, 1); - assert_eq!(a / b, Fixed64::from_rational(-12, 100)); - - let a = Fixed64::from_rational(12, 10); - let b = Fixed64::from_rational(-10, 1); - assert_eq!(a / b, Fixed64::from_rational(-12, 100)); - - let a = Fixed64::from_rational(-12, 10); - let b = Fixed64::from_rational(-10, 1); - assert_eq!(a / b, Fixed64::from_rational(12, 100)); - } - - #[test] - #[should_panic(expected = "attempt to divide by zero")] - fn div_zero() { - let a = Fixed64::from_rational(12, 10); - let b = Fixed64::from_natural(0); - let _ = a / b; - } - - #[test] - fn checked_div_zero() { - let a = Fixed64::from_rational(12, 10); - let b = Fixed64::from_natural(0); - assert_eq!(a.checked_div(&b), None); - } - - #[test] - fn checked_div_non_zero() { - let a = Fixed64::from_rational(12, 10); - let b = Fixed64::from_rational(1, 100); - assert_eq!(a.checked_div(&b), Some(Fixed64::from_rational(120, 1))); - } - - #[test] - fn saturating_mul_should_work() { - assert_eq!(Fixed64::from_natural(100).saturating_mul(Fixed64::from_natural(100)), Fixed64::from_natural(10000)); - } - - #[test] - fn saturating_pow_should_work() { - assert_eq!(Fixed64::from_natural(2).saturating_pow(0), Fixed64::from_natural(1)); - assert_eq!(Fixed64::from_natural(2).saturating_pow(1), Fixed64::from_natural(2)); - assert_eq!(Fixed64::from_natural(2).saturating_pow(2), Fixed64::from_natural(4)); - assert_eq!(Fixed64::from_natural(2).saturating_pow(3), Fixed64::from_natural(8)); - assert_eq!(Fixed64::from_natural(2).saturating_pow(20), Fixed64::from_natural(1048576)); - - assert_eq!(Fixed64::from_natural(1).saturating_pow(1000), Fixed64::from_natural(1)); - assert_eq!(Fixed64::from_natural(-1).saturating_pow(1000), Fixed64::from_natural(1)); - assert_eq!(Fixed64::from_natural(-1).saturating_pow(1001), Fixed64::from_natural(-1)); - assert_eq!(Fixed64::from_natural(1).saturating_pow(usize::max_value()), Fixed64::from_natural(1)); - assert_eq!(Fixed64::from_natural(-1).saturating_pow(usize::max_value()), Fixed64::from_natural(-1)); - assert_eq!(Fixed64::from_natural(-1).saturating_pow(usize::max_value() - 1), Fixed64::from_natural(1)); - - assert_eq!(Fixed64::from_natural(309).saturating_pow(4), Fixed64::from_natural(9_116_621_361)); - assert_eq!(Fixed64::from_natural(309).saturating_pow(5), Fixed64::from_parts(i64::max_value())); - - assert_eq!(Fixed64::from_natural(1).saturating_pow(usize::max_value()), Fixed64::from_natural(1)); - assert_eq!(Fixed64::from_natural(0).saturating_pow(usize::max_value()), Fixed64::from_natural(0)); - assert_eq!(Fixed64::from_natural(2).saturating_pow(usize::max_value()), Fixed64::from_parts(i64::max_value())); - } -} diff --git a/primitives/arithmetic/src/fixed_point.rs b/primitives/arithmetic/src/fixed_point.rs new file mode 100644 index 0000000000000000000000000000000000000000..8653ee2c8f7c8cb743dae10082c547fde14873db --- /dev/null +++ b/primitives/arithmetic/src/fixed_point.rs @@ -0,0 +1,1622 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Decimal Fixed Point implementations for Substrate runtime. + +use sp_std::{ops::{self, Add, Sub, Mul, Div}, fmt::Debug, prelude::*, convert::{TryInto, TryFrom}}; +use codec::{Encode, Decode}; +use crate::{ + helpers_128bit::multiply_by_rational, PerThing, + traits::{ + SaturatedConversion, CheckedSub, CheckedAdd, CheckedMul, CheckedDiv, CheckedNeg, + Bounded, Saturating, UniqueSaturatedInto, Zero, One + }, +}; + +#[cfg(feature = "std")] +use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; + +/// Integer types that can be used to interact with `FixedPointNumber` implementations. +pub trait FixedPointOperand: Copy + Clone + Bounded + Zero + Saturating + + PartialOrd + UniqueSaturatedInto + TryFrom + CheckedNeg {} + +impl FixedPointOperand for i128 {} +impl FixedPointOperand for u128 {} +impl FixedPointOperand for i64 {} +impl FixedPointOperand for u64 {} +impl FixedPointOperand for i32 {} +impl FixedPointOperand for u32 {} +impl FixedPointOperand for i16 {} +impl FixedPointOperand for u16 {} +impl FixedPointOperand for i8 {} +impl FixedPointOperand for u8 {} + +/// Something that implements a decimal fixed point number. +/// +/// The precision is given by `Self::DIV`, i.e. `1 / DIV` can be represented. +/// +/// Each type can store numbers from `Self::Inner::min_value() / Self::DIV` +/// to `Self::Inner::max_value() / Self::DIV`. +/// This is also referred to as the _accuracy_ of the type in the documentation. +pub trait FixedPointNumber: + Sized + Copy + Default + Debug + + Saturating + Bounded + + Eq + PartialEq + Ord + PartialOrd + + CheckedSub + CheckedAdd + CheckedMul + CheckedDiv + + Add + Sub + Div + Mul +{ + /// The underlying data type used for this fixed point number. + type Inner: Debug + One + CheckedMul + CheckedDiv + FixedPointOperand; + + /// Precision of this fixed point implementation. It should be a power of `10`. + const DIV: Self::Inner; + + /// Indicates if this fixed point implementation is signed or not. + const SIGNED: bool; + + /// Precision of this fixed point implementation. + fn accuracy() -> Self::Inner { + Self::DIV + } + + /// Builds this type from an integer number. + fn from_inner(int: Self::Inner) -> Self; + + /// Consumes `self` and returns the inner raw value. + fn into_inner(self) -> Self::Inner; + + /// Creates self from an integer number `int`. + /// + /// Returns `Self::max` or `Self::min` if `int` exceeds accuracy. + fn saturating_from_integer(int: N) -> Self { + let mut n: I129 = int.into(); + n.value = n.value.saturating_mul(Self::DIV.saturated_into()); + Self::from_inner(from_i129(n).unwrap_or(to_bound(int, 0))) + } + + /// Creates `self` from an integer number `int`. + /// + /// Returns `None` if `int` exceeds accuracy. + fn checked_from_integer(int: Self::Inner) -> Option { + int.checked_mul(&Self::DIV).map(|inner| Self::from_inner(inner)) + } + + /// Creates `self` from a rational number. Equal to `n / d`. + /// + /// Panics if `d = 0`. Returns `Self::max` or `Self::min` if `n / d` exceeds accuracy. + fn saturating_from_rational(n: N, d: D) -> Self { + if d == D::zero() { + panic!("attempt to divide by zero") + } + Self::checked_from_rational(n, d).unwrap_or(to_bound(n, d)) + } + + /// Creates `self` from a rational number. Equal to `n / d`. + /// + /// Returns `None` if `d == 0` or `n / d` exceeds accuracy. + fn checked_from_rational(n: N, d: D) -> Option { + if d == D::zero() { + return None + } + + let n: I129 = n.into(); + let d: I129 = d.into(); + let negative = n.negative != d.negative; + + multiply_by_rational(n.value, Self::DIV.unique_saturated_into(), d.value).ok() + .and_then(|value| from_i129(I129 { value, negative })) + .map(|inner| Self::from_inner(inner)) + } + + /// Checked multiplication for integer type `N`. Equal to `self * n`. + /// + /// Returns `None` if the result does not fit in `N`. + fn checked_mul_int(self, n: N) -> Option { + let lhs: I129 = self.into_inner().into(); + let rhs: I129 = n.into(); + let negative = lhs.negative != rhs.negative; + + multiply_by_rational(lhs.value, rhs.value, Self::DIV.unique_saturated_into()).ok() + .and_then(|value| from_i129(I129 { value, negative })) + } + + /// Saturating multiplication for integer type `N`. Equal to `self * n`. + /// + /// Returns `N::min` or `N::max` if the result does not fit in `N`. + fn saturating_mul_int(self, n: N) -> N { + self.checked_mul_int(n).unwrap_or(to_bound(self.into_inner(), n)) + } + + /// Checked division for integer type `N`. Equal to `self / d`. + /// + /// Returns `None` if the result does not fit in `N` or `d == 0`. + fn checked_div_int(self, d: N) -> Option { + let lhs: I129 = self.into_inner().into(); + let rhs: I129 = d.into(); + let negative = lhs.negative != rhs.negative; + + lhs.value.checked_div(rhs.value) + .and_then(|n| n.checked_div(Self::DIV.unique_saturated_into())) + .and_then(|value| from_i129(I129 { value, negative })) + } + + /// Saturating division for integer type `N`. Equal to `self / d`. + /// + /// Panics if `d == 0`. Returns `N::min` or `N::max` if the result does not fit in `N`. + fn saturating_div_int(self, d: N) -> N { + if d == N::zero() { + panic!("attempt to divide by zero") + } + self.checked_div_int(d).unwrap_or(to_bound(self.into_inner(), d)) + } + + /// Saturating multiplication for integer type `N`, adding the result back. + /// Equal to `self * n + n`. + /// + /// Returns `N::min` or `N::max` if the multiplication or final result does not fit in `N`. + fn saturating_mul_acc_int(self, n: N) -> N { + if self.is_negative() && n > N::zero() { + n.saturating_sub(Self::zero().saturating_sub(self).saturating_mul_int(n)) + } else { + self.saturating_mul_int(n).saturating_add(n) + } + } + + /// Saturating absolute value. + /// + /// Returns `Self::max` if `self == Self::min`. + fn saturating_abs(self) -> Self { + let inner = self.into_inner(); + if inner >= Self::Inner::zero() { + self + } else { + Self::from_inner(inner.checked_neg().unwrap_or(Self::Inner::max_value())) + } + } + + /// Takes the reciprocal (inverse). Equal to `1 / self`. + /// + /// Returns `None` if `self = 0`. + fn reciprocal(self) -> Option { + Self::one().checked_div(&self) + } + + /// Returns zero. + fn zero() -> Self { + Self::from_inner(Self::Inner::zero()) + } + + /// Checks if the number is zero. + fn is_zero(&self) -> bool { + self.into_inner() == Self::Inner::zero() + } + + /// Returns one. + fn one() -> Self { + Self::from_inner(Self::DIV) + } + + /// Checks if the number is one. + fn is_one(&self) -> bool { + self.into_inner() == Self::Inner::one() + } + + /// Returns `true` if `self` is positive and `false` if the number is zero or negative. + fn is_positive(self) -> bool { + self.into_inner() > Self::Inner::zero() + } + + /// Returns `true` if `self` is negative and `false` if the number is zero or positive. + fn is_negative(self) -> bool { + self.into_inner() < Self::Inner::zero() + } + + /// Returns the integer part. + fn trunc(self) -> Self { + self.into_inner().checked_div(&Self::DIV) + .expect("panics only if DIV is zero, DIV is not zero; qed") + .checked_mul(&Self::DIV) + .map(|inner| Self::from_inner(inner)) + .expect("can not overflow since fixed number is >= integer part") + } + + /// Returns the fractional part. + /// + /// Note: the returned fraction will be non-negative for negative numbers, + /// except in the case where the integer part is zero. + fn frac(self) -> Self { + let integer = self.trunc(); + let fractional = self.saturating_sub(integer); + if integer == Self::zero() { + fractional + } else { + fractional.saturating_abs() + } + } + + /// Returns the smallest integer greater than or equal to a number. + /// + /// Saturates to `Self::max` (truncated) if the result does not fit. + fn ceil(self) -> Self { + if self.is_negative() { + self.trunc() + } else { + if self.frac() == Self::zero() { + self + } else { + self.saturating_add(Self::one()).trunc() + } + } + } + + /// Returns the largest integer less than or equal to a number. + /// + /// Saturates to `Self::min` (truncated) if the result does not fit. + fn floor(self) -> Self { + if self.is_negative() { + self.saturating_sub(Self::one()).trunc() + } else { + self.trunc() + } + } + + /// Returns the number rounded to the nearest integer. Rounds half-way cases away from 0.0. + /// + /// Saturates to `Self::min` or `Self::max` (truncated) if the result does not fit. + fn round(self) -> Self { + let n = self.frac().saturating_mul(Self::saturating_from_integer(10)); + if n < Self::saturating_from_integer(5) { + self.trunc() + } else { + if self.is_positive() { + self.saturating_add(Self::one()).trunc() + } else { + self.saturating_sub(Self::one()).trunc() + } + } + } +} + +/// Data type used as intermediate storage in some computations to avoid overflow. +struct I129 { + value: u128, + negative: bool, +} + +impl From for I129 { + fn from(n: N) -> I129 { + if n < N::zero() { + let value: u128 = n.checked_neg() + .map(|n| n.unique_saturated_into()) + .unwrap_or(N::max_value().unique_saturated_into().saturating_add(1)); + I129 { value, negative: true } + } else { + I129 { value: n.unique_saturated_into(), negative: false } + } + } +} + +/// Transforms an `I129` to `N` if it is possible. +fn from_i129(n: I129) -> Option { + let max_plus_one: u128 = N::max_value().unique_saturated_into().saturating_add(1); + if n.negative && N::min_value() < N::zero() && n.value == max_plus_one { + Some(N::min_value()) + } else { + let unsigned_inner: N = n.value.try_into().ok()?; + let inner = if n.negative { unsigned_inner.checked_neg()? } else { unsigned_inner }; + Some(inner) + } +} + +/// Returns `R::max` if the sign of `n * m` is positive, `R::min` otherwise. +fn to_bound(n: N, m: D) -> R { + if (n < N::zero()) != (m < D::zero()) { + R::min_value() + } else { + R::max_value() + } +} + +macro_rules! implement_fixed { + ( + $name:ident, + $test_mod:ident, + $inner_type:ty, + $signed:tt, + $div:tt, + $title:expr $(,)? + ) => { + /// A fixed point number representation in the range. + /// + #[doc = $title] + #[derive(Encode, Decode, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] + pub struct $name($inner_type); + + impl From<$inner_type> for $name { + fn from(int: $inner_type) -> Self { + $name::saturating_from_integer(int) + } + } + + impl From<(N, D)> for $name { + fn from(r: (N, D)) -> Self { + $name::saturating_from_rational(r.0, r.1) + } + } + + impl FixedPointNumber for $name { + type Inner = $inner_type; + + const DIV: Self::Inner = $div; + const SIGNED: bool = $signed; + + fn from_inner(inner: Self::Inner) -> Self { + Self(inner) + } + + fn into_inner(self) -> Self::Inner { + self.0 + } + } + + impl $name { + /// const version of `FixedPointNumber::from_inner`. + pub const fn from_inner(inner: $inner_type) -> Self { + Self(inner) + } + + #[cfg(any(feature = "std", test))] + pub fn from_fraction(x: f64) -> Self { + Self((x * (::DIV as f64)) as $inner_type) + } + + #[cfg(any(feature = "std", test))] + pub fn to_fraction(self) -> f64 { + self.0 as f64 / ::DIV as f64 + } + } + + impl Saturating for $name { + fn saturating_add(self, rhs: Self) -> Self { + Self(self.0.saturating_add(rhs.0)) + } + + fn saturating_sub(self, rhs: Self) -> Self { + Self(self.0.saturating_sub(rhs.0)) + } + + fn saturating_mul(self, rhs: Self) -> Self { + self.checked_mul(&rhs).unwrap_or(to_bound(self.0, rhs.0)) + } + + fn saturating_pow(self, exp: usize) -> Self { + if exp == 0 { + return Self::saturating_from_integer(1); + } + + let exp = exp as u32; + let msb_pos = 32 - exp.leading_zeros(); + + let mut result = Self::saturating_from_integer(1); + let mut pow_val = self; + for i in 0..msb_pos { + if ((1 << i) & exp) > 0 { + result = result.saturating_mul(pow_val); + } + pow_val = pow_val.saturating_mul(pow_val); + } + result + } + } + + impl ops::Neg for $name { + type Output = Self; + + fn neg(self) -> Self::Output { + Self(::Inner::zero() - self.0) + } + } + + impl ops::Add for $name { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } + } + + impl ops::Sub for $name { + type Output = Self; + + fn sub(self, rhs: Self) -> Self::Output { + Self(self.0 - rhs.0) + } + } + + impl ops::Mul for $name { + type Output = Self; + + fn mul(self, rhs: Self) -> Self::Output { + self.checked_mul(&rhs) + .unwrap_or_else(|| panic!("attempt to multiply with overflow")) + } + } + + impl ops::Div for $name { + type Output = Self; + + fn div(self, rhs: Self) -> Self::Output { + if rhs.0 == 0 { + panic!("attempt to divide by zero") + } + self.checked_div(&rhs) + .unwrap_or_else(|| panic!("attempt to divide with overflow")) + } + } + + impl CheckedSub for $name { + fn checked_sub(&self, rhs: &Self) -> Option { + self.0.checked_sub(rhs.0).map(Self) + } + } + + impl CheckedAdd for $name { + fn checked_add(&self, rhs: &Self) -> Option { + self.0.checked_add(rhs.0).map(Self) + } + } + + impl CheckedDiv for $name { + fn checked_div(&self, other: &Self) -> Option { + if other.0 == 0 { + return None + } + + let lhs: I129 = self.0.into(); + let rhs: I129 = other.0.into(); + let negative = lhs.negative != rhs.negative; + + multiply_by_rational(lhs.value, Self::DIV as u128, rhs.value).ok() + .and_then(|value| from_i129(I129 { value, negative })) + .map(Self) + } + } + + impl CheckedMul for $name { + fn checked_mul(&self, other: &Self) -> Option { + let lhs: I129 = self.0.into(); + let rhs: I129 = other.0.into(); + let negative = lhs.negative != rhs.negative; + + multiply_by_rational(lhs.value, rhs.value, Self::DIV as u128).ok() + .and_then(|value| from_i129(I129 { value, negative })) + .map(Self) + } + } + + impl Bounded for $name { + fn min_value() -> Self { + Self(::Inner::min_value()) + } + + fn max_value() -> Self { + Self(::Inner::max_value()) + } + } + + impl sp_std::fmt::Debug for $name { + #[cfg(feature = "std")] + fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + let integral = { + let int = self.0 / Self::accuracy(); + let signum_for_zero = if int == 0 && self.is_negative() { "-" } else { "" }; + format!("{}{}", signum_for_zero, int) + }; + let precision = (Self::accuracy() as f64).log10() as usize; + let fractional = format!("{:0>weight$}", ((self.0 % Self::accuracy()) as i128).abs(), weight=precision); + write!(f, "{}({}.{})", stringify!($name), integral, fractional) + } + + #[cfg(not(feature = "std"))] + fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + Ok(()) + } + } + + impl From

for $name { + fn from(p: P) -> Self { + let accuracy = P::ACCURACY.saturated_into(); + let value = p.deconstruct().saturated_into(); + $name::saturating_from_rational(value, accuracy) + } + } + + #[cfg(feature = "std")] + impl sp_std::fmt::Display for $name { + fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + write!(f, "{}", self.0) + } + } + + #[cfg(feature = "std")] + impl sp_std::str::FromStr for $name { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + let inner: ::Inner = s.parse() + .map_err(|_| "invalid string input for fixed point number")?; + Ok(Self::from_inner(inner)) + } + } + + // Manual impl `Serialize` as serde_json does not support i128. + // TODO: remove impl if issue https://github.com/serde-rs/json/issues/548 fixed. + #[cfg(feature = "std")] + impl Serialize for $name { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&self.to_string()) + } + } + + // Manual impl `Deserialize` as serde_json does not support i128. + // TODO: remove impl if issue https://github.com/serde-rs/json/issues/548 fixed. + #[cfg(feature = "std")] + impl<'de> Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + use sp_std::str::FromStr; + let s = String::deserialize(deserializer)?; + $name::from_str(&s).map_err(|err_str| de::Error::custom(err_str)) + } + } + + #[cfg(test)] + mod $test_mod { + use super::*; + use crate::{Perbill, Percent, Permill, Perquintill}; + + fn max() -> $name { + $name::max_value() + } + + fn min() -> $name { + $name::min_value() + } + + fn precision() -> usize { + ($name::accuracy() as f64).log10() as usize + } + + #[test] + fn macro_preconditions() { + assert!($name::DIV > 0); + } + + #[test] + fn from_i129_works() { + let a = I129 { + value: 1, + negative: true, + }; + + // Can't convert negative number to unsigned. + assert_eq!(from_i129::(a), None); + + let a = I129 { + value: u128::max_value() - 1, + negative: false, + }; + + // Max - 1 value fits. + assert_eq!(from_i129::(a), Some(u128::max_value() - 1)); + + let a = I129 { + value: u128::max_value(), + negative: false, + }; + + // Max value fits. + assert_eq!(from_i129::(a), Some(u128::max_value())); + + let a = I129 { + value: i128::max_value() as u128 + 1, + negative: true, + }; + + // Min value fits. + assert_eq!(from_i129::(a), Some(i128::min_value())); + + let a = I129 { + value: i128::max_value() as u128 + 1, + negative: false, + }; + + // Max + 1 does not fit. + assert_eq!(from_i129::(a), None); + + let a = I129 { + value: i128::max_value() as u128, + negative: false, + }; + + // Max value fits. + assert_eq!(from_i129::(a), Some(i128::max_value())); + } + + #[test] + fn to_bound_works() { + let a = 1i32; + let b = 1i32; + + // Pos + Pos => Max. + assert_eq!(to_bound::<_, _, i32>(a, b), i32::max_value()); + + let a = -1i32; + let b = -1i32; + + // Neg + Neg => Max. + assert_eq!(to_bound::<_, _, i32>(a, b), i32::max_value()); + + let a = 1i32; + let b = -1i32; + + // Pos + Neg => Min. + assert_eq!(to_bound::<_, _, i32>(a, b), i32::min_value()); + + let a = -1i32; + let b = 1i32; + + // Neg + Pos => Min. + assert_eq!(to_bound::<_, _, i32>(a, b), i32::min_value()); + + let a = 1i32; + let b = -1i32; + + // Pos + Neg => Min (unsigned). + assert_eq!(to_bound::<_, _, u32>(a, b), 0); + } + + #[test] + fn op_neg_works() { + let a = $name::zero(); + let b = -a; + + // Zero. + assert_eq!(a, b); + + if $name::SIGNED { + let a = $name::saturating_from_integer(5); + let b = -a; + + // Positive. + assert_eq!($name::saturating_from_integer(-5), b); + + let a = $name::saturating_from_integer(-5); + let b = -a; + + // Negative + assert_eq!($name::saturating_from_integer(5), b); + + let a = $name::max_value(); + let b = -a; + + // Max. + assert_eq!($name::min_value() + $name::from_inner(1), b); + + let a = $name::min_value() + $name::from_inner(1); + let b = -a; + + // Min. + assert_eq!($name::max_value(), b); + + } + } + + #[test] + fn op_checked_add_overflow_works() { + let a = $name::max_value(); + let b = 1.into(); + assert!(a.checked_add(&b).is_none()); + } + + #[test] + fn op_add_works() { + let a = $name::saturating_from_rational(5, 2); + let b = $name::saturating_from_rational(1, 2); + + // Positive case: 6/2 = 3. + assert_eq!($name::saturating_from_integer(3), a + b); + + if $name::SIGNED { + // Negative case: 4/2 = 2. + let b = $name::saturating_from_rational(1, -2); + assert_eq!($name::saturating_from_integer(2), a + b); + } + } + + #[test] + fn op_checked_sub_underflow_works() { + let a = $name::min_value(); + let b = 1.into(); + assert!(a.checked_sub(&b).is_none()); + } + + #[test] + fn op_sub_works() { + let a = $name::saturating_from_rational(5, 2); + let b = $name::saturating_from_rational(1, 2); + + assert_eq!($name::saturating_from_integer(2), a - b); + assert_eq!($name::saturating_from_integer(-2), b.saturating_sub(a)); + } + + #[test] + fn op_checked_mul_overflow_works() { + let a = $name::max_value(); + let b = 2.into(); + assert!(a.checked_mul(&b).is_none()); + } + + #[test] + fn op_mul_works() { + let a = $name::saturating_from_integer(42); + let b = $name::saturating_from_integer(2); + assert_eq!($name::saturating_from_integer(84), a * b); + + let a = $name::saturating_from_integer(42); + let b = $name::saturating_from_integer(-2); + assert_eq!($name::saturating_from_integer(-84), a * b); + } + + #[test] + #[should_panic(expected = "attempt to divide by zero")] + fn op_div_panics_on_zero_divisor() { + let a = $name::saturating_from_integer(1); + let b = 0.into(); + let _c = a / b; + } + + #[test] + fn op_checked_div_overflow_works() { + if $name::SIGNED { + let a = $name::min_value(); + let b = $name::zero().saturating_sub($name::one()); + assert!(a.checked_div(&b).is_none()); + } + } + + #[test] + fn op_div_works() { + let a = $name::saturating_from_integer(42); + let b = $name::saturating_from_integer(2); + assert_eq!($name::saturating_from_integer(21), a / b); + + if $name::SIGNED { + let a = $name::saturating_from_integer(42); + let b = $name::saturating_from_integer(-2); + assert_eq!($name::saturating_from_integer(-21), a / b); + } + } + + #[test] + fn saturating_from_integer_works() { + let inner_max = <$name as FixedPointNumber>::Inner::max_value(); + let inner_min = <$name as FixedPointNumber>::Inner::min_value(); + let accuracy = $name::accuracy(); + + // Cases where integer fits. + let a = $name::saturating_from_integer(42); + assert_eq!(a.into_inner(), 42 * accuracy); + + let a = $name::saturating_from_integer(-42); + assert_eq!(a.into_inner(), 0.saturating_sub(42 * accuracy)); + + // Max/min integers that fit. + let a = $name::saturating_from_integer(inner_max / accuracy); + assert_eq!(a.into_inner(), (inner_max / accuracy) * accuracy); + + let a = $name::saturating_from_integer(inner_min / accuracy); + assert_eq!(a.into_inner(), (inner_min / accuracy) * accuracy); + + // Cases where integer doesn't fit, so it saturates. + let a = $name::saturating_from_integer(inner_max / accuracy + 1); + assert_eq!(a.into_inner(), inner_max); + + let a = $name::saturating_from_integer((inner_min / accuracy).saturating_sub(1)); + assert_eq!(a.into_inner(), inner_min); + } + + #[test] + fn checked_from_integer_works() { + let inner_max = <$name as FixedPointNumber>::Inner::max_value(); + let inner_min = <$name as FixedPointNumber>::Inner::min_value(); + let accuracy = $name::accuracy(); + + // Case where integer fits. + let a = $name::checked_from_integer(42) + .expect("42 * accuracy <= inner_max; qed"); + assert_eq!(a.into_inner(), 42 * accuracy); + + // Max integer that fit. + let a = $name::checked_from_integer(inner_max / accuracy) + .expect("(inner_max / accuracy) * accuracy <= inner_max; qed"); + assert_eq!(a.into_inner(), (inner_max / accuracy) * accuracy); + + // Case where integer doesn't fit, so it returns `None`. + let a = $name::checked_from_integer(inner_max / accuracy + 1); + assert_eq!(a, None); + + if $name::SIGNED { + // Case where integer fits. + let a = $name::checked_from_integer(0.saturating_sub(42)) + .expect("-42 * accuracy >= inner_min; qed"); + assert_eq!(a.into_inner(), 0 - 42 * accuracy); + + // Min integer that fit. + let a = $name::checked_from_integer(inner_min / accuracy) + .expect("(inner_min / accuracy) * accuracy <= inner_min; qed"); + assert_eq!(a.into_inner(), (inner_min / accuracy) * accuracy); + + // Case where integer doesn't fit, so it returns `None`. + let a = $name::checked_from_integer(inner_min / accuracy - 1); + assert_eq!(a, None); + } + } + + #[test] + fn from_inner_works() { + let inner_max = <$name as FixedPointNumber>::Inner::max_value(); + let inner_min = <$name as FixedPointNumber>::Inner::min_value(); + + assert_eq!(max(), $name::from_inner(inner_max)); + assert_eq!(min(), $name::from_inner(inner_min)); + } + + #[test] + #[should_panic(expected = "attempt to divide by zero")] + fn saturating_from_rational_panics_on_zero_divisor() { + let _ = $name::saturating_from_rational(1, 0); + } + + #[test] + fn saturating_from_rational_works() { + let inner_max = <$name as FixedPointNumber>::Inner::max_value(); + let inner_min = <$name as FixedPointNumber>::Inner::min_value(); + let accuracy = $name::accuracy(); + + let a = $name::saturating_from_rational(5, 2); + + // Positive case: 2.5 + assert_eq!(a.into_inner(), 25 * accuracy / 10); + + // Max - 1. + let a = $name::saturating_from_rational(inner_max - 1, accuracy); + assert_eq!(a.into_inner(), inner_max - 1); + + // Min + 1. + let a = $name::saturating_from_rational(inner_min + 1, accuracy); + assert_eq!(a.into_inner(), inner_min + 1); + + // Max. + let a = $name::saturating_from_rational(inner_max, accuracy); + assert_eq!(a.into_inner(), inner_max); + + // Min. + let a = $name::saturating_from_rational(inner_min, accuracy); + assert_eq!(a.into_inner(), inner_min); + + // Zero. + let a = $name::saturating_from_rational(0, 1); + assert_eq!(a.into_inner(), 0); + + if $name::SIGNED { + // Negative case: -2.5 + let a = $name::saturating_from_rational(-5, 2); + assert_eq!(a.into_inner(), 0 - 25 * accuracy / 10); + + // Other negative case: -2.5 + let a = $name::saturating_from_rational(5, -2); + assert_eq!(a.into_inner(), 0 - 25 * accuracy / 10); + + // Other positive case: 2.5 + let a = $name::saturating_from_rational(-5, -2); + assert_eq!(a.into_inner(), 25 * accuracy / 10); + + // Max + 1, saturates. + let a = $name::saturating_from_rational(inner_max as u128 + 1, accuracy); + assert_eq!(a.into_inner(), inner_max); + + // Min - 1, saturates. + let a = $name::saturating_from_rational(inner_max as u128 + 2, 0 - accuracy); + assert_eq!(a.into_inner(), inner_min); + + let a = $name::saturating_from_rational(inner_max, 0 - accuracy); + assert_eq!(a.into_inner(), 0 - inner_max); + + let a = $name::saturating_from_rational(inner_min, 0 - accuracy); + assert_eq!(a.into_inner(), inner_max); + + let a = $name::saturating_from_rational(inner_min + 1, 0 - accuracy); + assert_eq!(a.into_inner(), inner_max); + + let a = $name::saturating_from_rational(inner_min, 0 - 1); + assert_eq!(a.into_inner(), inner_max); + + let a = $name::saturating_from_rational(inner_max, 0 - 1); + assert_eq!(a.into_inner(), inner_min); + + let a = $name::saturating_from_rational(inner_max, 0 - inner_max); + assert_eq!(a.into_inner(), 0 - accuracy); + + let a = $name::saturating_from_rational(0 - inner_max, inner_max); + assert_eq!(a.into_inner(), 0 - accuracy); + + let a = $name::saturating_from_rational(inner_max, 0 - 3 * accuracy); + assert_eq!(a.into_inner(), 0 - inner_max / 3); + + let a = $name::saturating_from_rational(inner_min, 0 - accuracy / 3); + assert_eq!(a.into_inner(), inner_max); + + let a = $name::saturating_from_rational(1, 0 - accuracy); + assert_eq!(a.into_inner(), 0.saturating_sub(1)); + + let a = $name::saturating_from_rational(inner_min, inner_min); + assert_eq!(a.into_inner(), accuracy); + + // Out of accuracy. + let a = $name::saturating_from_rational(1, 0 - accuracy - 1); + assert_eq!(a.into_inner(), 0); + } + + let a = $name::saturating_from_rational(inner_max - 1, accuracy); + assert_eq!(a.into_inner(), inner_max - 1); + + let a = $name::saturating_from_rational(inner_min + 1, accuracy); + assert_eq!(a.into_inner(), inner_min + 1); + + let a = $name::saturating_from_rational(inner_max, 1); + assert_eq!(a.into_inner(), inner_max); + + let a = $name::saturating_from_rational(inner_min, 1); + assert_eq!(a.into_inner(), inner_min); + + let a = $name::saturating_from_rational(inner_max, inner_max); + assert_eq!(a.into_inner(), accuracy); + + let a = $name::saturating_from_rational(inner_max, 3 * accuracy); + assert_eq!(a.into_inner(), inner_max / 3); + + let a = $name::saturating_from_rational(inner_min, 2 * accuracy); + assert_eq!(a.into_inner(), inner_min / 2); + + let a = $name::saturating_from_rational(inner_min, accuracy / 3); + assert_eq!(a.into_inner(), inner_min); + + let a = $name::saturating_from_rational(1, accuracy); + assert_eq!(a.into_inner(), 1); + + // Out of accuracy. + let a = $name::saturating_from_rational(1, accuracy + 1); + assert_eq!(a.into_inner(), 0); + } + + #[test] + fn checked_from_rational_works() { + let inner_max = <$name as FixedPointNumber>::Inner::max_value(); + let inner_min = <$name as FixedPointNumber>::Inner::min_value(); + let accuracy = $name::accuracy(); + + // Divide by zero => None. + let a = $name::checked_from_rational(1, 0); + assert_eq!(a, None); + + // Max - 1. + let a = $name::checked_from_rational(inner_max - 1, accuracy).unwrap(); + assert_eq!(a.into_inner(), inner_max - 1); + + // Min + 1. + let a = $name::checked_from_rational(inner_min + 1, accuracy).unwrap(); + assert_eq!(a.into_inner(), inner_min + 1); + + // Max. + let a = $name::checked_from_rational(inner_max, accuracy).unwrap(); + assert_eq!(a.into_inner(), inner_max); + + // Min. + let a = $name::checked_from_rational(inner_min, accuracy).unwrap(); + assert_eq!(a.into_inner(), inner_min); + + // Max + 1 => Overflow => None. + let a = $name::checked_from_rational(inner_min, 0.saturating_sub(accuracy)); + assert_eq!(a, None); + + if $name::SIGNED { + // Min - 1 => Underflow => None. + let a = $name::checked_from_rational(inner_max as u128 + 2, 0.saturating_sub(accuracy)); + assert_eq!(a, None); + + let a = $name::checked_from_rational(inner_max, 0 - 3 * accuracy).unwrap(); + assert_eq!(a.into_inner(), 0 - inner_max / 3); + + let a = $name::checked_from_rational(inner_min, 0 - accuracy / 3); + assert_eq!(a, None); + + let a = $name::checked_from_rational(1, 0 - accuracy).unwrap(); + assert_eq!(a.into_inner(), 0.saturating_sub(1)); + + let a = $name::checked_from_rational(1, 0 - accuracy - 1).unwrap(); + assert_eq!(a.into_inner(), 0); + + let a = $name::checked_from_rational(inner_min, accuracy / 3); + assert_eq!(a, None); + } + + let a = $name::checked_from_rational(inner_max, 3 * accuracy).unwrap(); + assert_eq!(a.into_inner(), inner_max / 3); + + let a = $name::checked_from_rational(inner_min, 2 * accuracy).unwrap(); + assert_eq!(a.into_inner(), inner_min / 2); + + let a = $name::checked_from_rational(1, accuracy).unwrap(); + assert_eq!(a.into_inner(), 1); + + let a = $name::checked_from_rational(1, accuracy + 1).unwrap(); + assert_eq!(a.into_inner(), 0); + } + + #[test] + fn checked_mul_int_works() { + let a = $name::saturating_from_integer(2); + // Max - 1. + assert_eq!(a.checked_mul_int((i128::max_value() - 1) / 2), Some(i128::max_value() - 1)); + // Max. + assert_eq!(a.checked_mul_int(i128::max_value() / 2), Some(i128::max_value() - 1)); + // Max + 1 => None. + assert_eq!(a.checked_mul_int(i128::max_value() / 2 + 1), None); + + if $name::SIGNED { + // Min - 1. + assert_eq!(a.checked_mul_int((i128::min_value() + 1) / 2), Some(i128::min_value() + 2)); + // Min. + assert_eq!(a.checked_mul_int(i128::min_value() / 2), Some(i128::min_value())); + // Min + 1 => None. + assert_eq!(a.checked_mul_int(i128::min_value() / 2 - 1), None); + + let b = $name::saturating_from_rational(1, -2); + assert_eq!(b.checked_mul_int(42i128), Some(-21)); + assert_eq!(b.checked_mul_int(u128::max_value()), None); + assert_eq!(b.checked_mul_int(i128::max_value()), Some(i128::max_value() / -2)); + assert_eq!(b.checked_mul_int(i128::min_value()), Some(i128::min_value() / -2)); + } + + let a = $name::saturating_from_rational(1, 2); + assert_eq!(a.checked_mul_int(42i128), Some(21)); + assert_eq!(a.checked_mul_int(i128::max_value()), Some(i128::max_value() / 2)); + assert_eq!(a.checked_mul_int(i128::min_value()), Some(i128::min_value() / 2)); + + let c = $name::saturating_from_integer(255); + assert_eq!(c.checked_mul_int(2i8), None); + assert_eq!(c.checked_mul_int(2i128), Some(510)); + assert_eq!(c.checked_mul_int(i128::max_value()), None); + assert_eq!(c.checked_mul_int(i128::min_value()), None); + } + + #[test] + fn saturating_mul_int_works() { + let a = $name::saturating_from_integer(2); + // Max - 1. + assert_eq!(a.saturating_mul_int((i128::max_value() - 1) / 2), i128::max_value() - 1); + // Max. + assert_eq!(a.saturating_mul_int(i128::max_value() / 2), i128::max_value() - 1); + // Max + 1 => saturates to max. + assert_eq!(a.saturating_mul_int(i128::max_value() / 2 + 1), i128::max_value()); + + // Min - 1. + assert_eq!(a.saturating_mul_int((i128::min_value() + 1) / 2), i128::min_value() + 2); + // Min. + assert_eq!(a.saturating_mul_int(i128::min_value() / 2), i128::min_value()); + // Min + 1 => saturates to min. + assert_eq!(a.saturating_mul_int(i128::min_value() / 2 - 1), i128::min_value()); + + if $name::SIGNED { + let b = $name::saturating_from_rational(1, -2); + assert_eq!(b.saturating_mul_int(42i32), -21); + assert_eq!(b.saturating_mul_int(i128::max_value()), i128::max_value() / -2); + assert_eq!(b.saturating_mul_int(i128::min_value()), i128::min_value() / -2); + assert_eq!(b.saturating_mul_int(u128::max_value()), u128::min_value()); + } + + let a = $name::saturating_from_rational(1, 2); + assert_eq!(a.saturating_mul_int(42i32), 21); + assert_eq!(a.saturating_mul_int(i128::max_value()), i128::max_value() / 2); + assert_eq!(a.saturating_mul_int(i128::min_value()), i128::min_value() / 2); + + let c = $name::saturating_from_integer(255); + assert_eq!(c.saturating_mul_int(2i8), i8::max_value()); + assert_eq!(c.saturating_mul_int(-2i8), i8::min_value()); + assert_eq!(c.saturating_mul_int(i128::max_value()), i128::max_value()); + assert_eq!(c.saturating_mul_int(i128::min_value()), i128::min_value()); + } + + #[test] + fn checked_mul_works() { + let inner_max = <$name as FixedPointNumber>::Inner::max_value(); + let inner_min = <$name as FixedPointNumber>::Inner::min_value(); + + let a = $name::saturating_from_integer(2); + + // Max - 1. + let b = $name::from_inner(inner_max - 1); + assert_eq!(a.checked_mul(&(b/2.into())), Some(b)); + + // Max. + let c = $name::from_inner(inner_max); + assert_eq!(a.checked_mul(&(c/2.into())), Some(b)); + + // Max + 1 => None. + let e = $name::from_inner(1); + assert_eq!(a.checked_mul(&(c/2.into()+e)), None); + + if $name::SIGNED { + // Min + 1. + let b = $name::from_inner(inner_min + 1) / 2.into(); + let c = $name::from_inner(inner_min + 2); + assert_eq!(a.checked_mul(&b), Some(c)); + + // Min. + let b = $name::from_inner(inner_min) / 2.into(); + let c = $name::from_inner(inner_min); + assert_eq!(a.checked_mul(&b), Some(c)); + + // Min - 1 => None. + let b = $name::from_inner(inner_min) / 2.into() - $name::from_inner(1); + assert_eq!(a.checked_mul(&b), None); + + let c = $name::saturating_from_integer(255); + let b = $name::saturating_from_rational(1, -2); + + assert_eq!(b.checked_mul(&42.into()), Some(0.saturating_sub(21).into())); + assert_eq!(b.checked_mul(&$name::max_value()), $name::max_value().checked_div(&0.saturating_sub(2).into())); + assert_eq!(b.checked_mul(&$name::min_value()), $name::min_value().checked_div(&0.saturating_sub(2).into())); + assert_eq!(c.checked_mul(&$name::min_value()), None); + } + + let a = $name::saturating_from_rational(1, 2); + let c = $name::saturating_from_integer(255); + + assert_eq!(a.checked_mul(&42.into()), Some(21.into())); + assert_eq!(c.checked_mul(&2.into()), Some(510.into())); + assert_eq!(c.checked_mul(&$name::max_value()), None); + assert_eq!(a.checked_mul(&$name::max_value()), $name::max_value().checked_div(&2.into())); + assert_eq!(a.checked_mul(&$name::min_value()), $name::min_value().checked_div(&2.into())); + } + + #[test] + fn checked_div_int_works() { + let inner_max = <$name as FixedPointNumber>::Inner::max_value(); + let inner_min = <$name as FixedPointNumber>::Inner::min_value(); + let accuracy = $name::accuracy(); + + let a = $name::from_inner(inner_max); + let b = $name::from_inner(inner_min); + let c = $name::zero(); + let d = $name::one(); + let e = $name::saturating_from_integer(6); + let f = $name::saturating_from_integer(5); + + assert_eq!(e.checked_div_int(2.into()), Some(3)); + assert_eq!(f.checked_div_int(2.into()), Some(2)); + + assert_eq!(a.checked_div_int(i128::max_value()), Some(0)); + assert_eq!(a.checked_div_int(2), Some(inner_max / (2 * accuracy))); + assert_eq!(a.checked_div_int(inner_max / accuracy), Some(1)); + assert_eq!(a.checked_div_int(1i8), None); + + if b < c { + // Not executed by unsigned inners. + assert_eq!(a.checked_div_int(0.saturating_sub(2)), Some(0.saturating_sub(inner_max / (2 * accuracy)))); + assert_eq!(a.checked_div_int(0.saturating_sub(inner_max / accuracy)), Some(0.saturating_sub(1))); + assert_eq!(b.checked_div_int(i128::min_value()), Some(0)); + assert_eq!(b.checked_div_int(inner_min / accuracy), Some(1)); + assert_eq!(b.checked_div_int(1i8), None); + assert_eq!(b.checked_div_int(0.saturating_sub(2)), Some(0.saturating_sub(inner_min / (2 * accuracy)))); + assert_eq!(b.checked_div_int(0.saturating_sub(inner_min / accuracy)), Some(0.saturating_sub(1))); + assert_eq!(c.checked_div_int(i128::min_value()), Some(0)); + assert_eq!(d.checked_div_int(i32::min_value()), Some(0)); + } + + assert_eq!(b.checked_div_int(2), Some(inner_min / (2 * accuracy))); + + assert_eq!(c.checked_div_int(1), Some(0)); + assert_eq!(c.checked_div_int(i128::max_value()), Some(0)); + assert_eq!(c.checked_div_int(1i8), Some(0)); + + assert_eq!(d.checked_div_int(1), Some(1)); + assert_eq!(d.checked_div_int(i32::max_value()), Some(0)); + assert_eq!(d.checked_div_int(1i8), Some(1)); + + assert_eq!(a.checked_div_int(0), None); + assert_eq!(b.checked_div_int(0), None); + assert_eq!(c.checked_div_int(0), None); + assert_eq!(d.checked_div_int(0), None); + } + + #[test] + #[should_panic(expected = "attempt to divide by zero")] + fn saturating_div_int_panics_when_divisor_is_zero() { + let _ = $name::one().saturating_div_int(0); + } + + #[test] + fn saturating_div_int_works() { + let inner_max = <$name as FixedPointNumber>::Inner::max_value(); + let inner_min = <$name as FixedPointNumber>::Inner::min_value(); + let accuracy = $name::accuracy(); + + let a = $name::saturating_from_integer(5); + assert_eq!(a.saturating_div_int(2), 2); + + let a = $name::min_value(); + assert_eq!(a.saturating_div_int(1i128), (inner_min / accuracy) as i128); + + if $name::SIGNED { + let a = $name::saturating_from_integer(5); + assert_eq!(a.saturating_div_int(-2), -2); + + let a = $name::min_value(); + assert_eq!(a.saturating_div_int(-1i128), (inner_max / accuracy) as i128); + } + } + + #[test] + fn saturating_abs_works() { + let inner_max = <$name as FixedPointNumber>::Inner::max_value(); + let inner_min = <$name as FixedPointNumber>::Inner::min_value(); + + assert_eq!($name::from_inner(inner_max).saturating_abs(), $name::max_value()); + assert_eq!($name::zero().saturating_abs(), 0.into()); + + if $name::SIGNED { + assert_eq!($name::from_inner(inner_min).saturating_abs(), $name::max_value()); + assert_eq!($name::saturating_from_rational(-1, 2).saturating_abs(), (1, 2).into()); + } + } + + #[test] + fn saturating_mul_acc_int_works() { + assert_eq!($name::zero().saturating_mul_acc_int(42i8), 42i8); + assert_eq!($name::one().saturating_mul_acc_int(42i8), 2 * 42i8); + + assert_eq!($name::one().saturating_mul_acc_int(i128::max_value()), i128::max_value()); + assert_eq!($name::one().saturating_mul_acc_int(i128::min_value()), i128::min_value()); + + assert_eq!($name::one().saturating_mul_acc_int(u128::max_value() / 2), u128::max_value() - 1); + assert_eq!($name::one().saturating_mul_acc_int(u128::min_value()), u128::min_value()); + + if $name::SIGNED { + let a = $name::saturating_from_rational(-1, 2); + assert_eq!(a.saturating_mul_acc_int(42i8), 21i8); + assert_eq!(a.saturating_mul_acc_int(42u8), 21u8); + assert_eq!(a.saturating_mul_acc_int(u128::max_value() - 1), u128::max_value() / 2); + } + } + + #[test] + fn saturating_pow_should_work() { + assert_eq!($name::saturating_from_integer(2).saturating_pow(0), $name::saturating_from_integer(1)); + assert_eq!($name::saturating_from_integer(2).saturating_pow(1), $name::saturating_from_integer(2)); + assert_eq!($name::saturating_from_integer(2).saturating_pow(2), $name::saturating_from_integer(4)); + assert_eq!($name::saturating_from_integer(2).saturating_pow(3), $name::saturating_from_integer(8)); + assert_eq!($name::saturating_from_integer(2).saturating_pow(50), + $name::saturating_from_integer(1125899906842624i64)); + + assert_eq!($name::saturating_from_integer(1).saturating_pow(1000), (1).into()); + assert_eq!($name::saturating_from_integer(1).saturating_pow(usize::max_value()), (1).into()); + + if $name::SIGNED { + // Saturating. + assert_eq!($name::saturating_from_integer(2).saturating_pow(68), $name::max_value()); + + assert_eq!($name::saturating_from_integer(-1).saturating_pow(1000), (1).into()); + assert_eq!($name::saturating_from_integer(-1).saturating_pow(1001), 0.saturating_sub(1).into()); + assert_eq!($name::saturating_from_integer(-1).saturating_pow(usize::max_value()), 0.saturating_sub(1).into()); + assert_eq!($name::saturating_from_integer(-1).saturating_pow(usize::max_value() - 1), (1).into()); + } + + assert_eq!($name::saturating_from_integer(114209).saturating_pow(5), $name::max_value()); + + assert_eq!($name::saturating_from_integer(1).saturating_pow(usize::max_value()), (1).into()); + assert_eq!($name::saturating_from_integer(0).saturating_pow(usize::max_value()), (0).into()); + assert_eq!($name::saturating_from_integer(2).saturating_pow(usize::max_value()), $name::max_value()); + } + + #[test] + fn checked_div_works() { + let inner_max = <$name as FixedPointNumber>::Inner::max_value(); + let inner_min = <$name as FixedPointNumber>::Inner::min_value(); + + let a = $name::from_inner(inner_max); + let b = $name::from_inner(inner_min); + let c = $name::zero(); + let d = $name::one(); + let e = $name::saturating_from_integer(6); + let f = $name::saturating_from_integer(5); + + assert_eq!(e.checked_div(&2.into()), Some(3.into())); + assert_eq!(f.checked_div(&2.into()), Some((5, 2).into())); + + assert_eq!(a.checked_div(&inner_max.into()), Some(1.into())); + assert_eq!(a.checked_div(&2.into()), Some($name::from_inner(inner_max / 2))); + assert_eq!(a.checked_div(&$name::max_value()), Some(1.into())); + assert_eq!(a.checked_div(&d), Some(a)); + + if b < c { + // Not executed by unsigned inners. + assert_eq!(a.checked_div(&0.saturating_sub(2).into()), Some($name::from_inner(0.saturating_sub(inner_max / 2)))); + assert_eq!(a.checked_div(&-$name::max_value()), Some(0.saturating_sub(1).into())); + assert_eq!(b.checked_div(&0.saturating_sub(2).into()), Some($name::from_inner(0.saturating_sub(inner_min / 2)))); + assert_eq!(c.checked_div(&$name::max_value()), Some(0.into())); + assert_eq!(b.checked_div(&b), Some($name::one())); + } + + assert_eq!(b.checked_div(&2.into()), Some($name::from_inner(inner_min / 2))); + assert_eq!(b.checked_div(&a), Some(0.saturating_sub(1).into())); + assert_eq!(c.checked_div(&1.into()), Some(0.into())); + assert_eq!(d.checked_div(&1.into()), Some(1.into())); + + assert_eq!(a.checked_div(&$name::one()), Some(a)); + assert_eq!(b.checked_div(&$name::one()), Some(b)); + assert_eq!(c.checked_div(&$name::one()), Some(c)); + assert_eq!(d.checked_div(&$name::one()), Some(d)); + + assert_eq!(a.checked_div(&$name::zero()), None); + assert_eq!(b.checked_div(&$name::zero()), None); + assert_eq!(c.checked_div(&$name::zero()), None); + assert_eq!(d.checked_div(&$name::zero()), None); + } + + #[test] + fn is_positive_negative_works() { + let one = $name::one(); + assert!(one.is_positive()); + assert!(!one.is_negative()); + + let zero = $name::zero(); + assert!(!zero.is_positive()); + assert!(!zero.is_negative()); + + if $signed { + let minus_one = $name::saturating_from_integer(-1); + assert!(minus_one.is_negative()); + assert!(!minus_one.is_positive()); + } + } + + #[test] + fn trunc_works() { + let n = $name::saturating_from_rational(5, 2).trunc(); + assert_eq!(n, $name::saturating_from_integer(2)); + + if $name::SIGNED { + let n = $name::saturating_from_rational(-5, 2).trunc(); + assert_eq!(n, $name::saturating_from_integer(-2)); + } + } + + #[test] + fn frac_works() { + let n = $name::saturating_from_rational(5, 2); + let i = n.trunc(); + let f = n.frac(); + + assert_eq!(n, i + f); + + let n = $name::saturating_from_rational(5, 2) + .frac() + .saturating_mul(10.into()); + assert_eq!(n, 5.into()); + + let n = $name::saturating_from_rational(1, 2) + .frac() + .saturating_mul(10.into()); + assert_eq!(n, 5.into()); + + if $name::SIGNED { + let n = $name::saturating_from_rational(-5, 2); + let i = n.trunc(); + let f = n.frac(); + assert_eq!(n, i - f); + + // The sign is attached to the integer part unless it is zero. + let n = $name::saturating_from_rational(-5, 2) + .frac() + .saturating_mul(10.into()); + assert_eq!(n, 5.into()); + + let n = $name::saturating_from_rational(-1, 2) + .frac() + .saturating_mul(10.into()); + assert_eq!(n, 0.saturating_sub(5).into()); + } + } + + #[test] + fn ceil_works() { + let n = $name::saturating_from_rational(5, 2); + assert_eq!(n.ceil(), 3.into()); + + let n = $name::saturating_from_rational(-5, 2); + assert_eq!(n.ceil(), 0.saturating_sub(2).into()); + + // On the limits: + let n = $name::max_value(); + assert_eq!(n.ceil(), n.trunc()); + + let n = $name::min_value(); + assert_eq!(n.ceil(), n.trunc()); + } + + #[test] + fn floor_works() { + let n = $name::saturating_from_rational(5, 2); + assert_eq!(n.floor(), 2.into()); + + let n = $name::saturating_from_rational(-5, 2); + assert_eq!(n.floor(), 0.saturating_sub(3).into()); + + // On the limits: + let n = $name::max_value(); + assert_eq!(n.floor(), n.trunc()); + + let n = $name::min_value(); + assert_eq!(n.floor(), n.trunc()); + } + + #[test] + fn round_works() { + let n = $name::zero(); + assert_eq!(n.round(), n); + + let n = $name::one(); + assert_eq!(n.round(), n); + + let n = $name::saturating_from_rational(5, 2); + assert_eq!(n.round(), 3.into()); + + let n = $name::saturating_from_rational(-5, 2); + assert_eq!(n.round(), 0.saturating_sub(3).into()); + + // Saturating: + let n = $name::max_value(); + assert_eq!(n.round(), n.trunc()); + + let n = $name::min_value(); + assert_eq!(n.round(), n.trunc()); + + // On the limit: + + // floor(max - 1) + 0.33.. + let n = $name::max_value() + .saturating_sub(1.into()) + .trunc() + .saturating_add((1, 3).into()); + + assert_eq!(n.round(), ($name::max_value() - 1.into()).trunc()); + + // floor(max - 1) + 0.5 + let n = $name::max_value() + .saturating_sub(1.into()) + .trunc() + .saturating_add((1, 2).into()); + + assert_eq!(n.round(), $name::max_value().trunc()); + + if $name::SIGNED { + // floor(min + 1) - 0.33.. + let n = $name::min_value() + .saturating_add(1.into()) + .trunc() + .saturating_sub((1, 3).into()); + + assert_eq!(n.round(), ($name::min_value() + 1.into()).trunc()); + + // floor(min + 1) - 0.5 + let n = $name::min_value() + .saturating_add(1.into()) + .trunc() + .saturating_sub((1, 2).into()); + + assert_eq!(n.round(), $name::min_value().trunc()); + } + } + + #[test] + fn perthing_into_works() { + let ten_percent_percent: $name = Percent::from_percent(10).into(); + assert_eq!(ten_percent_percent.into_inner(), $name::accuracy() / 10); + + let ten_percent_permill: $name = Permill::from_percent(10).into(); + assert_eq!(ten_percent_permill.into_inner(), $name::accuracy() / 10); + + let ten_percent_perbill: $name = Perbill::from_percent(10).into(); + assert_eq!(ten_percent_perbill.into_inner(), $name::accuracy() / 10); + + let ten_percent_perquintill: $name = Perquintill::from_percent(10).into(); + assert_eq!(ten_percent_perquintill.into_inner(), $name::accuracy() / 10); + } + + #[test] + fn fmt_should_work() { + let zero = $name::zero(); + assert_eq!(format!("{:?}", zero), format!("{}(0.{:0>weight$})", stringify!($name), 0, weight=precision())); + + let one = $name::one(); + assert_eq!(format!("{:?}", one), format!("{}(1.{:0>weight$})", stringify!($name), 0, weight=precision())); + + let frac = $name::saturating_from_rational(1, 2); + assert_eq!(format!("{:?}", frac), format!("{}(0.{:0weight$})", stringify!($name), 0, weight=precision())); + + let frac = $name::saturating_from_rational(-314, 100); + assert_eq!(format!("{:?}", frac), format!("{}(-3.{:0 { + /// Compare if `self` is `threshold` greater or less than `other`. + fn tcmp(&self, other: &T, epsilon: T) -> Ordering; +} + +impl ThresholdOrd for T +where + T: Ord + PartialOrd + Copy + Clone + traits::Zero + traits::Saturating, +{ + fn tcmp(&self, other: &T, threshold: T) -> Ordering { + // early exit. + if threshold.is_zero() { + return self.cmp(&other) + } + + let upper_bound = other.saturating_add(threshold); + let lower_bound = other.saturating_sub(threshold); + + if upper_bound <= lower_bound { + // defensive only. Can never happen. + self.cmp(&other) + } else { + // upper_bound is guaranteed now to be bigger than lower. + match (self.cmp(&lower_bound), self.cmp(&upper_bound)) { + (Ordering::Greater, Ordering::Greater) => Ordering::Greater, + (Ordering::Less, Ordering::Less) => Ordering::Less, + _ => Ordering::Equal, + } + } + + } +} + +/// A collection-like object that is made of values of type `T` and can normalize its individual +/// values around a centric point. +/// +/// Note that the order of items in the collection may affect the result. +pub trait Normalizable { + /// Normalize self around `targeted_sum`. + /// + /// Only returns `Ok` if the new sum of results is guaranteed to be equal to `targeted_sum`. + /// Else, returns an error explaining why it failed to do so. + fn normalize(&self, targeted_sum: T) -> Result, &'static str>; +} + +macro_rules! impl_normalize_for_numeric { + ($($numeric:ty),*) => { + $( + impl Normalizable<$numeric> for Vec<$numeric> { + fn normalize(&self, targeted_sum: $numeric) -> Result, &'static str> { + normalize(self.as_ref(), targeted_sum) + } + } + )* + }; +} + +impl_normalize_for_numeric!(u8, u16, u32, u64, u128); + +impl Normalizable

for Vec

{ + fn normalize(&self, targeted_sum: P) -> Result, &'static str> { + let inners = self.iter().map(|p| p.clone().deconstruct().into()).collect::>(); + let normalized = normalize(inners.as_ref(), targeted_sum.deconstruct().into())?; + Ok(normalized.into_iter().map(|i: UpperOf

| P::from_parts(i.saturated_into())).collect()) + } +} + + +/// Normalize `input` so that the sum of all elements reaches `targeted_sum`. +/// +/// This implementation is currently in a balanced position between being performant and accurate. +/// +/// 1. We prefer storing original indices, and sorting the `input` only once. This will save the +/// cost of sorting per round at the cost of a little bit of memory. +/// 2. The granularity of increment/decrements is determined by the number of elements in `input` +/// and their sum difference with `targeted_sum`, namely `diff = diff(sum(input), target_sum)`. +/// This value is then distributed into `per_round = diff / input.len()` and `leftover = diff % +/// round`. First, per_round is applied to all elements of input, and then we move to leftover, +/// in which case we add/subtract 1 by 1 until `leftover` is depleted. +/// +/// When the sum is less than the target, the above approach always holds. In this case, then each +/// individual element is also less than target. Thus, by adding `per_round` to each item, neither +/// of them can overflow the numeric bound of `T`. In fact, neither of the can go beyond +/// `target_sum`*. +/// +/// If sum is more than target, there is small twist. The subtraction of `per_round` +/// form each element might go below zero. In this case, we saturate and add the error to the +/// `leftover` value. This ensures that the result will always stay accurate, yet it might cause the +/// execution to become increasingly slow, since leftovers are applied one by one. +/// +/// All in all, the complicated case above is rare to happen in all substrate use cases, hence we +/// opt for it due to its simplicity. +/// +/// This function will return an error is if length of `input` cannot fit in `T`, or if `sum(input)` +/// cannot fit inside `T`. +/// +/// * This proof is used in the implementation as well. +pub fn normalize(input: &[T], targeted_sum: T) -> Result, &'static str> + where T: Clone + Copy + Ord + BaseArithmetic + Unsigned + Debug, +{ + // compute sum and return error if failed. + let mut sum = T::zero(); + for t in input.iter() { + sum = sum.checked_add(t).ok_or("sum of input cannot fit in `T`")?; + } + + // convert count and return error if failed. + let count = input.len(); + let count_t: T = count.try_into().map_err(|_| "length of `inputs` cannot fit in `T`")?; + + // Nothing to do here. + if count.is_zero() { + return Ok(Vec::::new()); + } + + let diff = targeted_sum.max(sum) - targeted_sum.min(sum); + if diff.is_zero() { + return Ok(input.to_vec()); + } + + let needs_bump = targeted_sum > sum; + let per_round = diff / count_t; + let mut leftover = diff % count_t; + + // sort output once based on diff. This will require more data transfer and saving original + // index, but we sort only twice instead: once now and once at the very end. + let mut output_with_idx = input.iter().cloned().enumerate().collect::>(); + output_with_idx.sort_unstable_by_key(|x| x.1); + + if needs_bump { + // must increase the values a bit. Bump from the min element. Index of minimum is now zero + // because we did a sort. If at any point the min goes greater or equal the `max_threshold`, + // we move to the next minimum. + let mut min_index = 0; + // at this threshold we move to next index. + let threshold = targeted_sum / count_t; + + if !per_round.is_zero() { + for _ in 0..count { + output_with_idx[min_index].1 = output_with_idx[min_index].1 + .checked_add(&per_round) + .expect("Proof provided in the module doc; qed."); + if output_with_idx[min_index].1 >= threshold { + min_index += 1; + min_index = min_index % count; + } + } + } + + // continue with the previous min_index + while !leftover.is_zero() { + output_with_idx[min_index].1 = output_with_idx[min_index].1 + .checked_add(&T::one()) + .expect("Proof provided in the module doc; qed."); + if output_with_idx[min_index].1 >= threshold { + min_index += 1; + min_index = min_index % count; + } + leftover -= One::one() + } + } else { + // must decrease the stakes a bit. decrement from the max element. index of maximum is now + // last. if at any point the max goes less or equal the `min_threshold`, we move to the next + // maximum. + let mut max_index = count - 1; + // at this threshold we move to next index. + let threshold = output_with_idx + .first() + .expect("length of input is greater than zero; it must have a first; qed") + .1; + + if !per_round.is_zero() { + for _ in 0..count { + output_with_idx[max_index].1 = output_with_idx[max_index].1 + .checked_sub(&per_round) + .unwrap_or_else(|| { + let remainder = per_round - output_with_idx[max_index].1; + leftover += remainder; + output_with_idx[max_index].1.saturating_sub(per_round) + }); + if output_with_idx[max_index].1 <= threshold { + max_index = max_index.checked_sub(1).unwrap_or(count - 1); + } + } + } + + // continue with the previous max_index + while !leftover.is_zero() { + if let Some(next) = output_with_idx[max_index].1.checked_sub(&One::one()) { + output_with_idx[max_index].1 = next; + if output_with_idx[max_index].1 <= threshold { + max_index = max_index.checked_sub(1).unwrap_or(count - 1); + } + leftover -= One::one() + } else { + max_index = max_index.checked_sub(1).unwrap_or(count - 1); + } + } + } + + debug_assert_eq!( + output_with_idx.iter().fold(T::zero(), |acc, (_, x)| acc + *x), + targeted_sum, + "sum({:?}) != {:?}", + output_with_idx, + targeted_sum, + ); + + // sort again based on the original index. + output_with_idx.sort_unstable_by_key(|x| x.0); + Ok(output_with_idx.into_iter().map(|(_, t)| t).collect()) +} + +#[cfg(test)] +mod normalize_tests { + use super::*; + + #[test] + fn work_for_all_types() { + macro_rules! test_for { + ($type:ty) => { + assert_eq!( + normalize(vec![8 as $type, 9, 7, 10].as_ref(), 40).unwrap(), + vec![10, 10, 10, 10], + ); + } + } + // it should work for all types as long as the length of vector can be converted to T. + test_for!(u128); + test_for!(u64); + test_for!(u32); + test_for!(u16); + test_for!(u8); + } + + #[test] + fn fails_on_if_input_sum_large() { + assert!(normalize(vec![1u8; 255].as_ref(), 10).is_ok()); + assert_eq!( + normalize(vec![1u8; 256].as_ref(), 10), + Err("sum of input cannot fit in `T`"), + ); + } + + #[test] + fn does_not_fail_on_subtraction_overflow() { + assert_eq!( + normalize(vec![1u8, 100, 100].as_ref(), 10).unwrap(), + vec![1, 9, 0], + ); + assert_eq!( + normalize(vec![1u8, 8, 9].as_ref(), 1).unwrap(), + vec![0, 1, 0], + ); + } + + #[test] + fn works_for_vec() { + assert_eq!(vec![8u32, 9, 7, 10].normalize(40).unwrap(), vec![10u32, 10, 10, 10]); + } + + #[test] + fn works_for_per_thing() { + assert_eq!( + vec![ + Perbill::from_percent(33), + Perbill::from_percent(33), + Perbill::from_percent(33) + ].normalize(Perbill::one()).unwrap(), + vec![ + Perbill::from_parts(333333334), + Perbill::from_parts(333333333), + Perbill::from_parts(333333333), + ] + ); + + assert_eq!( + vec![ + Perbill::from_percent(20), + Perbill::from_percent(15), + Perbill::from_percent(30) + ].normalize(Perbill::one()).unwrap(), + vec![ + Perbill::from_parts(316666668), + Perbill::from_parts(383333332), + Perbill::from_parts(300000000), + ] + ); + } + + #[test] + fn can_work_for_peru16() { + // Peru16 is a rather special case; since inner type is exactly the same as capacity, we + // could have a situation where the sum cannot be calculated in the inner type. Calculating + // using the upper type of the per_thing should assure this to be okay. + assert_eq!( + vec![ + PerU16::from_percent(40), + PerU16::from_percent(40), + PerU16::from_percent(40), + ].normalize(PerU16::one()).unwrap(), + vec![ + PerU16::from_parts(21845), // 33% + PerU16::from_parts(21845), // 33% + PerU16::from_parts(21845), // 33% + ] + ); + } + + #[test] + fn normalize_works_all_le() { + assert_eq!( + normalize(vec![8u32, 9, 7, 10].as_ref(), 40).unwrap(), + vec![10, 10, 10, 10], + ); + + assert_eq!( + normalize(vec![7u32, 7, 7, 7].as_ref(), 40).unwrap(), + vec![10, 10, 10, 10], + ); + + assert_eq!( + normalize(vec![7u32, 7, 7, 10].as_ref(), 40).unwrap(), + vec![11, 11, 8, 10], + ); + + assert_eq!( + normalize(vec![7u32, 8, 7, 10].as_ref(), 40).unwrap(), + vec![11, 8, 11, 10], + ); + + assert_eq!( + normalize(vec![7u32, 7, 8, 10].as_ref(), 40).unwrap(), + vec![11, 11, 8, 10], + ); + } + + #[test] + fn normalize_works_some_ge() { + assert_eq!( + normalize(vec![8u32, 11, 9, 10].as_ref(), 40).unwrap(), + vec![10, 11, 9, 10], + ); + } + + #[test] + fn always_inc_min() { + assert_eq!( + normalize(vec![10u32, 7, 10, 10].as_ref(), 40).unwrap(), + vec![10, 10, 10, 10], + ); + assert_eq!( + normalize(vec![10u32, 10, 7, 10].as_ref(), 40).unwrap(), + vec![10, 10, 10, 10], + ); + assert_eq!( + normalize(vec![10u32, 10, 10, 7].as_ref(), 40).unwrap(), + vec![10, 10, 10, 10], + ); + } + + #[test] + fn normalize_works_all_ge() { + assert_eq!( + normalize(vec![12u32, 11, 13, 10].as_ref(), 40).unwrap(), + vec![10, 10, 10, 10], + ); + + assert_eq!( + normalize(vec![13u32, 13, 13, 13].as_ref(), 40).unwrap(), + vec![10, 10, 10, 10], + ); + + assert_eq!( + normalize(vec![13u32, 13, 13, 10].as_ref(), 40).unwrap(), + vec![12, 9, 9, 10], + ); + + assert_eq!( + normalize(vec![13u32, 12, 13, 10].as_ref(), 40).unwrap(), + vec![9, 12, 9, 10], + ); + + assert_eq!( + normalize(vec![13u32, 13, 12, 10].as_ref(), 40).unwrap(), + vec![9, 9, 12, 10], + ); + } +} + #[cfg(test)] -mod tests { +mod threshold_compare_tests { use super::*; + use crate::traits::Saturating; + use sp_std::cmp::Ordering; + + #[test] + fn epsilon_ord_works() { + let b = 115u32; + let e = Perbill::from_percent(10).mul_ceil(b); + + // [115 - 11,5 (103,5), 115 + 11,5 (126,5)] is all equal + assert_eq!(103u32.tcmp(&b, e), Ordering::Equal); + assert_eq!(104u32.tcmp(&b, e), Ordering::Equal); + assert_eq!(115u32.tcmp(&b, e), Ordering::Equal); + assert_eq!(120u32.tcmp(&b, e), Ordering::Equal); + assert_eq!(126u32.tcmp(&b, e), Ordering::Equal); + assert_eq!(127u32.tcmp(&b, e), Ordering::Equal); + + assert_eq!(128u32.tcmp(&b, e), Ordering::Greater); + assert_eq!(102u32.tcmp(&b, e), Ordering::Less); + } + + #[test] + fn epsilon_ord_works_with_small_epc() { + let b = 115u32; + // way less than 1 percent. threshold will be zero. Result should be same as normal ord. + let e = Perbill::from_parts(100) * b; + + // [115 - 11,5 (103,5), 115 + 11,5 (126,5)] is all equal + assert_eq!(103u32.tcmp(&b, e), 103u32.cmp(&b)); + assert_eq!(104u32.tcmp(&b, e), 104u32.cmp(&b)); + assert_eq!(115u32.tcmp(&b, e), 115u32.cmp(&b)); + assert_eq!(120u32.tcmp(&b, e), 120u32.cmp(&b)); + assert_eq!(126u32.tcmp(&b, e), 126u32.cmp(&b)); + assert_eq!(127u32.tcmp(&b, e), 127u32.cmp(&b)); + + assert_eq!(128u32.tcmp(&b, e), 128u32.cmp(&b)); + assert_eq!(102u32.tcmp(&b, e), 102u32.cmp(&b)); + } #[test] fn peru16_rational_does_not_overflow() { @@ -56,4 +490,19 @@ mod tests { // maximum capacity of their type, e.g. PerU16. let _ = PerU16::from_rational_approximation(17424870u32, 17424870); } + + #[test] + fn saturating_mul_works() { + assert_eq!(Saturating::saturating_mul(2, i32::min_value()), i32::min_value()); + assert_eq!(Saturating::saturating_mul(2, i32::max_value()), i32::max_value()); + } + + #[test] + fn saturating_pow_works() { + assert_eq!(Saturating::saturating_pow(i32::min_value(), 0), 1); + assert_eq!(Saturating::saturating_pow(i32::max_value(), 0), 1); + assert_eq!(Saturating::saturating_pow(i32::min_value(), 3), i32::min_value()); + assert_eq!(Saturating::saturating_pow(i32::min_value(), 2), i32::max_value()); + assert_eq!(Saturating::saturating_pow(i32::max_value(), 2), i32::max_value()); + } } diff --git a/primitives/arithmetic/src/per_things.rs b/primitives/arithmetic/src/per_things.rs index b50b35f7af374f74d6aa978d4bf7ab2e8543389f..f809358446017a0ce8747f3ebe2d9bf36053d97c 100644 --- a/primitives/arithmetic/src/per_things.rs +++ b/primitives/arithmetic/src/per_things.rs @@ -21,21 +21,29 @@ use serde::{Serialize, Deserialize}; use sp_std::{ops, fmt, prelude::*, convert::TryInto}; use codec::{Encode, CompactAs}; use crate::traits::{ - SaturatedConversion, UniqueSaturatedInto, Saturating, BaseArithmetic, Bounded, Zero, + SaturatedConversion, UniqueSaturatedInto, Saturating, BaseArithmetic, Bounded, Zero, Unsigned, }; use sp_debug_derive::RuntimeDebug; +/// Get the inner type of a `PerThing`. +pub type InnerOf

=

::Inner; + +/// Get the upper type of a `PerThing`. +pub type UpperOf

=

::Upper; + /// Something that implements a fixed point ration with an arbitrary granularity `X`, as _parts per /// `X`_. pub trait PerThing: Sized + Saturating + Copy + Default + Eq + PartialEq + Ord + PartialOrd + Bounded + fmt::Debug { /// The data type used to build this per-thingy. - type Inner: BaseArithmetic + Copy + fmt::Debug; + type Inner: BaseArithmetic + Unsigned + Copy + fmt::Debug; /// A data type larger than `Self::Inner`, used to avoid overflow in some computations. /// It must be able to compute `ACCURACY^2`. - type Upper: BaseArithmetic + Copy + From + TryInto + fmt::Debug; + type Upper: + BaseArithmetic + Copy + From + TryInto + + UniqueSaturatedInto + Unsigned + fmt::Debug; /// The accuracy of this type. const ACCURACY: Self::Inner; @@ -86,7 +94,7 @@ pub trait PerThing: /// ``` fn mul_floor(self, b: N) -> N where N: Clone + From + UniqueSaturatedInto + ops::Rem + - ops::Div + ops::Mul + ops::Add + ops::Div + ops::Mul + ops::Add + Unsigned { overflow_prune_mul::(b, self.deconstruct(), Rounding::Down) } @@ -108,7 +116,7 @@ pub trait PerThing: /// ``` fn mul_ceil(self, b: N) -> N where N: Clone + From + UniqueSaturatedInto + ops::Rem + - ops::Div + ops::Mul + ops::Add + ops::Div + ops::Mul + ops::Add + Unsigned { overflow_prune_mul::(b, self.deconstruct(), Rounding::Up) } @@ -124,7 +132,8 @@ pub trait PerThing: /// ``` fn saturating_reciprocal_mul(self, b: N) -> N where N: Clone + From + UniqueSaturatedInto + ops::Rem + - ops::Div + ops::Mul + ops::Add + Saturating + ops::Div + ops::Mul + ops::Add + Saturating + + Unsigned { saturating_reciprocal_mul::(b, self.deconstruct(), Rounding::Nearest) } @@ -143,7 +152,8 @@ pub trait PerThing: /// ``` fn saturating_reciprocal_mul_floor(self, b: N) -> N where N: Clone + From + UniqueSaturatedInto + ops::Rem + - ops::Div + ops::Mul + ops::Add + Saturating + ops::Div + ops::Mul + ops::Add + Saturating + + Unsigned { saturating_reciprocal_mul::(b, self.deconstruct(), Rounding::Down) } @@ -162,7 +172,8 @@ pub trait PerThing: /// ``` fn saturating_reciprocal_mul_ceil(self, b: N) -> N where N: Clone + From + UniqueSaturatedInto + ops::Rem + - ops::Div + ops::Mul + ops::Add + Saturating + ops::Div + ops::Mul + ops::Add + Saturating + + Unsigned { saturating_reciprocal_mul::(b, self.deconstruct(), Rounding::Up) } @@ -190,14 +201,14 @@ pub trait PerThing: /// # fn main () { /// // 989/100 is technically closer to 99%. /// assert_eq!( - /// Percent::from_rational_approximation(989, 1000), + /// Percent::from_rational_approximation(989u64, 1000), /// Percent::from_parts(98), /// ); /// # } /// ``` fn from_rational_approximation(p: N, q: N) -> Self where N: Clone + Ord + From + TryInto + TryInto + - ops::Div + ops::Rem + ops::Add; + ops::Div + ops::Rem + ops::Add + Unsigned; } /// The rounding method to use. @@ -219,7 +230,7 @@ fn saturating_reciprocal_mul( ) -> N where N: Clone + From + UniqueSaturatedInto + ops::Div + ops::Mul + ops::Add + ops::Rem + Saturating, + Output=N> + ops::Add + ops::Rem + Saturating + Unsigned, P: PerThing, { let maximum: N = P::ACCURACY.into(); @@ -240,7 +251,7 @@ fn overflow_prune_mul( ) -> N where N: Clone + From + UniqueSaturatedInto + ops::Div + ops::Mul + ops::Add + ops::Rem, + Output=N> + ops::Add + ops::Rem + Unsigned, P: PerThing, { let maximum: N = P::ACCURACY.into(); @@ -266,7 +277,7 @@ fn rational_mul_correction( ) -> N where N: From + UniqueSaturatedInto + ops::Div + ops::Mul + ops::Add + ops::Rem, + Output=N> + ops::Add + ops::Rem + Unsigned, P: PerThing, { let numer_upper = P::Upper::from(numer); @@ -312,8 +323,7 @@ macro_rules! implement_per_thing { /// #[doc = $title] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] - #[derive(Encode, Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, - RuntimeDebug, CompactAs)] + #[derive(Encode, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, RuntimeDebug, CompactAs)] pub struct $name($type); impl PerThing for $name { @@ -328,14 +338,15 @@ macro_rules! implement_per_thing { /// Build this type from a number of parts per thing. fn from_parts(parts: Self::Inner) -> Self { Self(parts.min($max)) } + /// NOTE: saturate to 0 or 1 if x is beyond `[0, 1]` #[cfg(feature = "std")] fn from_fraction(x: f64) -> Self { - Self::from_parts((x * $max as f64) as Self::Inner) + Self::from_parts((x.max(0.).min(1.) * $max as f64) as Self::Inner) } fn from_rational_approximation(p: N, q: N) -> Self where N: Clone + Ord + From + TryInto + TryInto - + ops::Div + ops::Rem + ops::Add + + ops::Div + ops::Rem + ops::Add + Unsigned { let div_ceil = |x: N, f: N| -> N { let mut o = x.clone() / f.clone(); @@ -383,7 +394,6 @@ macro_rules! implement_per_thing { impl $name { /// From an explicitly defined number of parts per maximum of the type. /// - /// This can be called at compile time. // needed only for peru16. Since peru16 is the only type in which $max == // $type::max_value(), rustc is being a smart-a** here by warning that the comparison // is not needed. @@ -399,9 +409,9 @@ macro_rules! implement_per_thing { Self(([x, 100][(x > 100) as usize] as $upper_type * $max as $upper_type / 100) as $type) } - /// See [`PerThing::one`]. - pub fn one() -> Self { - ::one() + /// See [`PerThing::one`] + pub const fn one() -> Self { + Self::from_parts($max) } /// See [`PerThing::is_one`]. @@ -410,8 +420,8 @@ macro_rules! implement_per_thing { } /// See [`PerThing::zero`]. - pub fn zero() -> Self { - ::zero() + pub const fn zero() -> Self { + Self::from_parts(0) } /// See [`PerThing::is_zero`]. @@ -420,8 +430,8 @@ macro_rules! implement_per_thing { } /// See [`PerThing::deconstruct`]. - pub fn deconstruct(self) -> $type { - PerThing::deconstruct(self) + pub const fn deconstruct(self) -> $type { + self.0 } /// See [`PerThing::square`]. @@ -439,7 +449,8 @@ macro_rules! implement_per_thing { pub fn from_rational_approximation(p: N, q: N) -> Self where N: Clone + Ord + From<$type> + TryInto<$type> + TryInto<$upper_type> + ops::Div + ops::Rem + - ops::Add { + ops::Add + Unsigned + { ::from_rational_approximation(p, q) } @@ -447,7 +458,8 @@ macro_rules! implement_per_thing { pub fn mul_floor(self, b: N) -> N where N: Clone + From<$type> + UniqueSaturatedInto<$type> + ops::Rem + ops::Div + ops::Mul + - ops::Add { + ops::Add + Unsigned + { PerThing::mul_floor(self, b) } @@ -455,7 +467,8 @@ macro_rules! implement_per_thing { pub fn mul_ceil(self, b: N) -> N where N: Clone + From<$type> + UniqueSaturatedInto<$type> + ops::Rem + ops::Div + ops::Mul + - ops::Add { + ops::Add + Unsigned + { PerThing::mul_ceil(self, b) } @@ -463,7 +476,8 @@ macro_rules! implement_per_thing { pub fn saturating_reciprocal_mul(self, b: N) -> N where N: Clone + From<$type> + UniqueSaturatedInto<$type> + ops::Rem + ops::Div + ops::Mul + ops::Add + - Saturating { + Saturating + Unsigned + { PerThing::saturating_reciprocal_mul(self, b) } @@ -471,7 +485,8 @@ macro_rules! implement_per_thing { pub fn saturating_reciprocal_mul_floor(self, b: N) -> N where N: Clone + From<$type> + UniqueSaturatedInto<$type> + ops::Rem + ops::Div + ops::Mul + ops::Add + - Saturating { + Saturating + Unsigned + { PerThing::saturating_reciprocal_mul_floor(self, b) } @@ -479,7 +494,8 @@ macro_rules! implement_per_thing { pub fn saturating_reciprocal_mul_ceil(self, b: N) -> N where N: Clone + From<$type> + UniqueSaturatedInto<$type> + ops::Rem + ops::Div + ops::Mul + ops::Add + - Saturating { + Saturating + Unsigned + { PerThing::saturating_reciprocal_mul_ceil(self, b) } } @@ -567,13 +583,19 @@ macro_rules! implement_per_thing { } } + impl Default for $name { + fn default() -> Self { + ::zero() + } + } + /// Non-overflow multiplication. /// /// This is tailored to be used with a balance type. impl ops::Mul for $name where N: Clone + From<$type> + UniqueSaturatedInto<$type> + ops::Rem - + ops::Div + ops::Mul + ops::Add, + + ops::Div + ops::Mul + ops::Add + Unsigned, { type Output = N; fn mul(self, b: N) -> Self::Output { @@ -672,6 +694,8 @@ macro_rules! implement_per_thing { assert_eq!($name::from_fraction(0.0), $name::from_parts(Zero::zero())); assert_eq!($name::from_fraction(0.1), $name::from_parts($max / 10)); assert_eq!($name::from_fraction(1.0), $name::from_parts($max)); + assert_eq!($name::from_fraction(2.0), $name::from_parts($max)); + assert_eq!($name::from_fraction(-1.0), $name::from_parts(Zero::zero())); } macro_rules! u256ify { @@ -1130,6 +1154,18 @@ macro_rules! implement_per_thing { 1, ); } + + #[test] + #[allow(unused)] + fn const_fns_work() { + const C1: $name = $name::from_percent(50); + const C2: $name = $name::one(); + const C3: $name = $name::zero(); + const C4: $name = $name::from_parts(1); + + // deconstruct is also const, hence it can be called in const rhs. + const C5: bool = C1.deconstruct() == 0; + } } }; } diff --git a/primitives/arithmetic/src/rational128.rs b/primitives/arithmetic/src/rational128.rs index 9d0d10921d71747b710a4913b18c2ba0dfe06072..947c7bc537d19fc1906a4e2e29d0b3f9947338e5 100644 --- a/primitives/arithmetic/src/rational128.rs +++ b/primitives/arithmetic/src/rational128.rs @@ -215,7 +215,7 @@ mod tests { assert_eq!(r(MAX128 - 10, MAX128).to_den(10), Ok(r(10, 10))); assert_eq!(r(MAX128 / 2, MAX128).to_den(10), Ok(r(5, 10))); - // large to perbill. This is very well needed for phragmen. + // large to perbill. This is very well needed for npos-elections. assert_eq!( r(MAX128 / 2, MAX128).to_den(1000_000_000), Ok(r(500_000_000, 1000_000_000)) @@ -360,6 +360,15 @@ mod tests { multiply_by_rational(1_000_000_000, MAX128 / 8, MAX128 / 2).unwrap(), 250000000, ); + + assert_eq!( + multiply_by_rational( + 29459999999999999988000u128, + 1000000000000000000u128, + 10000000000000000000u128 + ).unwrap(), + 2945999999999999998800u128 + ); } #[test] diff --git a/primitives/arithmetic/src/traits.rs b/primitives/arithmetic/src/traits.rs index d8d789577ebf560d1c28eb9d38a1717414f2c647..ce645cfe65d94326c26fd5e4996324187e7913c7 100644 --- a/primitives/arithmetic/src/traits.rs +++ b/primitives/arithmetic/src/traits.rs @@ -21,8 +21,8 @@ use sp_std::{self, convert::{TryFrom, TryInto}}; use codec::HasCompact; pub use integer_sqrt::IntegerSquareRoot; pub use num_traits::{ - Zero, One, Bounded, CheckedAdd, CheckedSub, CheckedMul, CheckedDiv, - CheckedShl, CheckedShr, checked_pow + Zero, One, Bounded, CheckedAdd, CheckedSub, CheckedMul, CheckedDiv, CheckedNeg, + CheckedShl, CheckedShr, checked_pow, Signed, Unsigned, }; use sp_std::ops::{ Add, Sub, Mul, Div, Rem, AddAssign, SubAssign, MulAssign, DivAssign, @@ -79,6 +79,11 @@ pub trait AtLeast32Bit: BaseArithmetic + From + From {} impl + From> AtLeast32Bit for T {} +/// A meta trait for arithmetic. Same as [`AtLeast32Bit `], but also bounded to be unsigned. +pub trait AtLeast32BitUnsigned: AtLeast32Bit + Unsigned {} + +impl AtLeast32BitUnsigned for T {} + /// Just like `From` except that if the source value is too big to fit into the destination type /// then it'll saturate the destination. pub trait UniqueSaturatedFrom: Sized { @@ -145,7 +150,15 @@ impl Self { - checked_pow(self, exp).unwrap_or_else(Bounded::max_value) + let neg = self < T::zero() && exp % 2 != 0; + checked_pow(self, exp) + .unwrap_or_else(|| + if neg { + Bounded::min_value() + } else { + Bounded::max_value() + } + ) } } diff --git a/primitives/authority-discovery/Cargo.toml b/primitives/authority-discovery/Cargo.toml index e39e795d1b9f4851da7c8ed1e83b5e490cd8a67e..79b8a832fbe13ac0f184ceb4530d0735590caf37 100644 --- a/primitives/authority-discovery/Cargo.toml +++ b/primitives/authority-discovery/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-authority-discovery" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] description = "Authority discovery primitives" edition = "2018" @@ -12,11 +12,11 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.8", default-features = false, path = "../application-crypto" } -codec = { package = "parity-scale-codec", default-features = false, version = "1.3.0" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../std" } -sp-api = { version = "2.0.0-alpha.8", default-features = false, path = "../api" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../runtime" } +sp-application-crypto = { version = "2.0.0-rc4", default-features = false, path = "../application-crypto" } +codec = { package = "parity-scale-codec", default-features = false, version = "1.3.1" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../std" } +sp-api = { version = "2.0.0-rc4", default-features = false, path = "../api" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../runtime" } [features] default = ["std"] diff --git a/primitives/authorship/Cargo.toml b/primitives/authorship/Cargo.toml index 98c3a549de9eaa6e02690a5c290943833644e1cb..1c44b9aad7ab2ed740c8d142f5641096e6642687 100644 --- a/primitives/authorship/Cargo.toml +++ b/primitives/authorship/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-authorship" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] description = "Authorship primitives" edition = "2018" @@ -12,10 +12,10 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../inherents" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../runtime" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../std" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../inherents" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../runtime" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../std" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } [features] default = [ "std" ] diff --git a/primitives/block-builder/Cargo.toml b/primitives/block-builder/Cargo.toml index 1875bc08a86bd6c62999ca30a1777fe068a83b93..2b594640fd10fa97f0cb708c0d97dccc95cab9ea 100644 --- a/primitives/block-builder/Cargo.toml +++ b/primitives/block-builder/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-block-builder" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,11 +12,11 @@ description = "The block builder runtime api." targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../runtime" } -sp-api = { version = "2.0.0-alpha.8", default-features = false, path = "../api" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../std" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../inherents" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../runtime" } +sp-api = { version = "2.0.0-rc4", default-features = false, path = "../api" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../std" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../inherents" } [features] default = [ "std" ] diff --git a/primitives/blockchain/Cargo.toml b/primitives/blockchain/Cargo.toml index 3c6cd68170eb62dc8abbcb2f088bd6416c2bcd95..956ae1a8fc6b445863a2a69857d9ff3d21ccb560 100644 --- a/primitives/blockchain/Cargo.toml +++ b/primitives/blockchain/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-blockchain" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,14 +12,13 @@ documentation = "https://docs.rs/sp-blockchain" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] - [dependencies] log = "0.4.8" lru = "0.4.0" parking_lot = "0.10.0" derive_more = "0.99.2" -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-consensus = { version = "0.8.0-alpha.8", path = "../consensus/common" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../runtime" } -sp-block-builder = { version = "2.0.0-alpha.8", path = "../block-builder" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../state-machine" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-consensus = { version = "0.8.0-rc4", path = "../consensus/common" } +sp-runtime = { version = "2.0.0-rc4", path = "../runtime" } +sp-block-builder = { version = "2.0.0-rc4", path = "../block-builder" } +sp-state-machine = { version = "0.8.0-rc4", path = "../state-machine" } diff --git a/primitives/blockchain/src/header_metadata.rs b/primitives/blockchain/src/header_metadata.rs index a4df04b507fd1853a2e43cb8504ab30bf0f3a308..b8d9c5c9345812c36e9271e859a81b61ae21f2fc 100644 --- a/primitives/blockchain/src/header_metadata.rs +++ b/primitives/blockchain/src/header_metadata.rs @@ -137,7 +137,8 @@ pub fn tree_route>( from = backend.header_metadata(from.parent)?; } - // add the pivot block. and append the reversed to-branch (note that it's reverse order originals) + // add the pivot block. and append the reversed to-branch + // (note that it's reverse order originals) let pivot = from_branch.len(); from_branch.push(HashAndNumber { number: to.number, @@ -182,18 +183,24 @@ pub struct HashAndNumber { /// Tree route from C to E2. Retracted empty. Common is C, enacted [E1, E2] /// C -> E1 -> E2 /// ``` -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct TreeRoute { route: Vec>, pivot: usize, } impl TreeRoute { - /// Get a slice of all retracted blocks in reverse order (towards common ancestor) + /// Get a slice of all retracted blocks in reverse order (towards common ancestor). pub fn retracted(&self) -> &[HashAndNumber] { &self.route[..self.pivot] } + /// Convert into all retracted blocks in reverse order (towards common ancestor). + pub fn into_retracted(mut self) -> Vec> { + self.route.truncate(self.pivot); + self.route + } + /// Get the common ancestor block. This might be one of the two blocks of the /// route. pub fn common_block(&self) -> &HashAndNumber { @@ -213,8 +220,15 @@ pub trait HeaderMetadata { /// Error used in case the header metadata is not found. type Error; - fn header_metadata(&self, hash: Block::Hash) -> Result, Self::Error>; - fn insert_header_metadata(&self, hash: Block::Hash, header_metadata: CachedHeaderMetadata); + fn header_metadata( + &self, + hash: Block::Hash, + ) -> Result, Self::Error>; + fn insert_header_metadata( + &self, + hash: Block::Hash, + header_metadata: CachedHeaderMetadata, + ); fn remove_header_metadata(&self, hash: Block::Hash); } diff --git a/primitives/chain-spec/Cargo.toml b/primitives/chain-spec/Cargo.toml index 2bb950042b2e88d42e79a1443d2bb345f5403785..e091a59245541db3876a4274c29da49de5c4faac 100644 --- a/primitives/chain-spec/Cargo.toml +++ b/primitives/chain-spec/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-chain-spec" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" diff --git a/primitives/consensus/aura/Cargo.toml b/primitives/consensus/aura/Cargo.toml index f1f61f09575354087e1bbc0a83c50e16a3f964ff..10c7f5a2de0b570da7f99ded7848ef0af8c1694d 100644 --- a/primitives/consensus/aura/Cargo.toml +++ b/primitives/consensus/aura/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-consensus-aura" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "Primitives for Aura consensus" edition = "2018" @@ -12,13 +12,13 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.8", default-features = false, path = "../../application-crypto" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../std" } -sp-api = { version = "2.0.0-alpha.8", default-features = false, path = "../../api" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../runtime" } -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../../inherents" } -sp-timestamp = { version = "2.0.0-alpha.8", default-features = false, path = "../../timestamp" } +sp-application-crypto = { version = "2.0.0-rc4", default-features = false, path = "../../application-crypto" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../std" } +sp-api = { version = "2.0.0-rc4", default-features = false, path = "../../api" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../runtime" } +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../../inherents" } +sp-timestamp = { version = "2.0.0-rc4", default-features = false, path = "../../timestamp" } [features] default = ["std"] diff --git a/primitives/consensus/babe/Cargo.toml b/primitives/consensus/babe/Cargo.toml index f5b2bf0510ee57ae0a87a381ff4f300a68b67633..36492304683464b9700c49bc2d5cb551b3017bb9 100644 --- a/primitives/consensus/babe/Cargo.toml +++ b/primitives/consensus/babe/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-consensus-babe" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "Primitives for BABE consensus" edition = "2018" @@ -12,20 +12,22 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.8", default-features = false, path = "../../application-crypto" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } +sp-application-crypto = { version = "2.0.0-rc4", default-features = false, path = "../../application-crypto" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } merlin = { version = "2.0", default-features = false } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../std" } -sp-api = { version = "2.0.0-alpha.8", default-features = false, path = "../../api" } -sp-consensus = { version = "0.8.0-alpha.8", optional = true, path = "../common" } -sp-consensus-vrf = { version = "0.8.0-alpha.8", path = "../vrf", default-features = false } -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../../inherents" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../runtime" } -sp-timestamp = { version = "2.0.0-alpha.8", default-features = false, path = "../../timestamp" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../std" } +sp-api = { version = "2.0.0-rc4", default-features = false, path = "../../api" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../core" } +sp-consensus = { version = "0.8.0-rc4", optional = true, path = "../common" } +sp-consensus-vrf = { version = "0.8.0-rc4", path = "../vrf", default-features = false } +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../../inherents" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../runtime" } +sp-timestamp = { version = "2.0.0-rc4", default-features = false, path = "../../timestamp" } [features] default = ["std"] std = [ + "sp-core/std", "sp-application-crypto/std", "codec/std", "merlin/std", diff --git a/primitives/consensus/babe/src/lib.rs b/primitives/consensus/babe/src/lib.rs index 9848715a47f6fc3d3b6a1618715ad7faa0e6d7a7..10d4aa5ae50b7c99c2759eaf809fde47fa8738ea 100644 --- a/primitives/consensus/babe/src/lib.rs +++ b/primitives/consensus/babe/src/lib.rs @@ -31,6 +31,8 @@ pub use merlin::Transcript; use codec::{Encode, Decode}; use sp_std::vec::Vec; use sp_runtime::{ConsensusEngineId, RuntimeDebug}; +#[cfg(feature = "std")] +use sp_core::vrf::{VRFTranscriptData, VRFTranscriptValue}; use crate::digests::{NextEpochDescriptor, NextConfigDescriptor}; mod app { @@ -94,6 +96,23 @@ pub fn make_transcript( transcript } +/// Make a VRF transcript data container +#[cfg(feature = "std")] +pub fn make_transcript_data( + randomness: &Randomness, + slot_number: u64, + epoch: u64, +) -> VRFTranscriptData { + VRFTranscriptData { + label: &BABE_ENGINE_ID, + items: vec![ + ("slot number", VRFTranscriptValue::U64(slot_number)), + ("current epoch", VRFTranscriptValue::U64(epoch)), + ("chain randomness", VRFTranscriptValue::Bytes(&randomness[..])), + ] + } +} + /// An consensus log item for BABE. #[derive(Decode, Encode, Clone, PartialEq, Eq)] pub enum ConsensusLog { diff --git a/primitives/consensus/common/Cargo.toml b/primitives/consensus/common/Cargo.toml index ac17421f60392b18586f47abb8565c68bc8f937d..39c47545c2b08f68bb2a287d17b5652ecf4953e8 100644 --- a/primitives/consensus/common/Cargo.toml +++ b/primitives/consensus/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-consensus" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -15,24 +15,25 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] derive_more = "0.99.2" -libp2p = { version = "0.18.1", default-features = false } +libp2p = { version = "0.20.1", default-features = false } log = "0.4.8" -sp-core = { path= "../../core", version = "2.0.0-alpha.8"} -sp-inherents = { version = "2.0.0-alpha.8", path = "../../inherents" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../../primitives/state-machine" } +sp-core = { path= "../../core", version = "2.0.0-rc4"} +sp-inherents = { version = "2.0.0-rc4", path = "../../inherents" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../../primitives/state-machine" } futures = { version = "0.3.1", features = ["thread-pool"] } futures-timer = "3.0.1" -sp-std = { version = "2.0.0-alpha.8", path = "../../std" } -sp-version = { version = "2.0.0-alpha.8", path = "../../version" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../runtime" } -sp-utils = { version = "2.0.0-alpha.8", path = "../../utils" } -codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } +sp-std = { version = "2.0.0-rc4", path = "../../std" } +sp-version = { version = "2.0.0-rc4", path = "../../version" } +sp-runtime = { version = "2.0.0-rc4", path = "../../runtime" } +sp-utils = { version = "2.0.0-rc4", path = "../../utils" } +codec = { package = "parity-scale-codec", version = "1.3.1", features = ["derive"] } parking_lot = "0.10.0" serde = { version = "1.0", features = ["derive"] } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.8.0-alpha.8"} +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.8.0-rc4"} +wasm-timer = "0.2.4" [dev-dependencies] -sp-test-primitives = { version = "2.0.0-alpha.8", path = "../../test-primitives" } +sp-test-primitives = { version = "2.0.0-rc4", path = "../../test-primitives" } [features] default = [] diff --git a/primitives/consensus/common/src/block_validation.rs b/primitives/consensus/common/src/block_validation.rs index e8054f3ae40d7291b62604876ab9a639267f2632..66f960f16fff396fe40a6f3e38ff5a0d84ffc17e 100644 --- a/primitives/consensus/common/src/block_validation.rs +++ b/primitives/consensus/common/src/block_validation.rs @@ -36,7 +36,10 @@ impl, B: Block> Chain for Arc { #[derive(Debug, PartialEq, Eq)] pub enum Validation { /// Valid block announcement. - Success, + Success { + /// Is this the new best block of the node? + is_new_best: bool, + }, /// Invalid block announcement. Failure, } @@ -49,18 +52,10 @@ pub trait BlockAnnounceValidator { /// Default implementation of `BlockAnnounceValidator`. #[derive(Debug)] -pub struct DefaultBlockAnnounceValidator { - chain: C -} - -impl DefaultBlockAnnounceValidator { - pub fn new(chain: C) -> Self { - Self { chain } - } -} +pub struct DefaultBlockAnnounceValidator; -impl> BlockAnnounceValidator for DefaultBlockAnnounceValidator { +impl BlockAnnounceValidator for DefaultBlockAnnounceValidator { fn validate(&mut self, _h: &B::Header, _d: &[u8]) -> Result> { - Ok(Validation::Success) + Ok(Validation::Success { is_new_best: false }) } } diff --git a/primitives/consensus/common/src/import_queue.rs b/primitives/consensus/common/src/import_queue.rs index 85e82cc484af8ff963a1239ef0fabd4db06eb49e..94228a266385f6bc5469f0dda14ba1e2c89262e0 100644 --- a/primitives/consensus/common/src/import_queue.rs +++ b/primitives/consensus/common/src/import_queue.rs @@ -27,13 +27,17 @@ //! queues to be instantiated simply. use std::collections::HashMap; + use sp_runtime::{Justification, traits::{Block as BlockT, Header as _, NumberFor}}; -use crate::error::Error as ConsensusError; -use crate::block_import::{ - BlockImport, BlockOrigin, BlockImportParams, ImportedAux, JustificationImport, ImportResult, - BlockCheckParams, FinalityProofImport, -}; +use crate::{ + error::Error as ConsensusError, + block_import::{ + BlockImport, BlockOrigin, BlockImportParams, ImportedAux, JustificationImport, ImportResult, + BlockCheckParams, FinalityProofImport, + }, + metrics::Metrics, +}; pub use basic_queue::BasicQueue; mod basic_queue; @@ -186,6 +190,17 @@ pub fn import_single_block, Transaction>( block_origin: BlockOrigin, block: IncomingBlock, verifier: &mut V, +) -> Result>, BlockImportError> { + import_single_block_metered(import_handle, block_origin, block, verifier, None) +} + +/// Single block import function with metering. +pub(crate) fn import_single_block_metered, Transaction>( + import_handle: &mut dyn BlockImport, + block_origin: BlockOrigin, + block: IncomingBlock, + verifier: &mut V, + metrics: Option, ) -> Result>, BlockImportError> { let peer = block.origin; @@ -207,8 +222,8 @@ pub fn import_single_block, Transaction>( let hash = header.hash(); let parent_hash = header.parent_hash().clone(); - let import_error = |e| { - match e { + let import_handler = |import| { + match import { Ok(ImportResult::AlreadyInChain) => { trace!(target: "sync", "Block already in chain {}: {:?}", number, hash); Ok(BlockImportResult::ImportedKnown(number)) @@ -232,7 +247,8 @@ pub fn import_single_block, Transaction>( } } }; - match import_error(import_handle.check_block(BlockCheckParams { + + match import_handler(import_handle.check_block(BlockCheckParams { hash, number, parent_hash, @@ -243,6 +259,7 @@ pub fn import_single_block, Transaction>( r => return Ok(r), // Any other successful result means that the block is already imported. } + let started = wasm_timer::Instant::now(); let (mut import_block, maybe_keys) = verifier.verify(block_origin, header, justification, block.body) .map_err(|msg| { if let Some(ref peer) = peer { @@ -250,14 +267,21 @@ pub fn import_single_block, Transaction>( } else { trace!(target: "sync", "Verifying {}({}) failed: {}", number, hash, msg); } + if let Some(metrics) = metrics.as_ref() { + metrics.report_verification(false, started.elapsed()); + } BlockImportError::VerificationFailed(peer.clone(), msg) })?; + if let Some(metrics) = metrics.as_ref() { + metrics.report_verification(true, started.elapsed()); + } + let mut cache = HashMap::new(); if let Some(keys) = maybe_keys { cache.extend(keys.into_iter()); } import_block.allow_missing_state = block.allow_missing_state; - import_error(import_handle.import_block(import_block.convert_transaction(), cache)) + import_handler(import_handle.import_block(import_block.convert_transaction(), cache)) } diff --git a/primitives/consensus/common/src/import_queue/basic_queue.rs b/primitives/consensus/common/src/import_queue/basic_queue.rs index c63c73bb42182f23cdac93c54c21340ebc3b65c9..8eb194841f196f4e1fa6b073ee84307743d7de98 100644 --- a/primitives/consensus/common/src/import_queue/basic_queue.rs +++ b/primitives/consensus/common/src/import_queue/basic_queue.rs @@ -22,13 +22,15 @@ use sp_runtime::{Justification, traits::{Block as BlockT, Header as HeaderT, Num use sp_utils::mpsc::{TracingUnboundedSender, tracing_unbounded}; use prometheus_endpoint::Registry; -use crate::block_import::BlockOrigin; -use crate::metrics::Metrics; -use crate::import_queue::{ - BlockImportResult, BlockImportError, Verifier, BoxBlockImport, BoxFinalityProofImport, - BoxJustificationImport, ImportQueue, Link, Origin, - IncomingBlock, import_single_block, - buffered_link::{self, BufferedLinkSender, BufferedLinkReceiver} +use crate::{ + block_import::BlockOrigin, + import_queue::{ + BlockImportResult, BlockImportError, Verifier, BoxBlockImport, BoxFinalityProofImport, + BoxJustificationImport, ImportQueue, Link, Origin, + IncomingBlock, import_single_block_metered, + buffered_link::{self, BufferedLinkSender, BufferedLinkReceiver}, + }, + metrics::Metrics, }; /// Interface to a basic block import queue that is importing blocks sequentially in a separate @@ -59,7 +61,7 @@ impl BasicQueue { block_import: BoxBlockImport, justification_import: Option>, finality_proof_import: Option>, - spawner: &impl sp_core::traits::SpawnBlocking, + spawner: &impl sp_core::traits::SpawnNamed, prometheus_registry: Option<&Registry>, ) -> Self { let (result_sender, result_port) = buffered_link::buffered_link(); @@ -146,11 +148,6 @@ struct BlockImportWorker { _phantom: PhantomData, } -const METRIC_SUCCESS_FIELDS: [&'static str; 8] = [ - "success", "incomplete_header", "verification_failed", "bad_block", - "missing_state", "unknown_parent", "cancelled", "failed" -]; - impl BlockImportWorker { fn new>( result_sender: BufferedLinkSender, @@ -228,7 +225,7 @@ impl BlockImportWorker { // a `Future` into `importing`. let (bi, verif) = block_import_verifier.take() .expect("block_import_verifier is always Some; qed"); - importing = Some(worker.import_a_batch_of_blocks(bi, verif, origin, blocks)); + importing = Some(worker.import_batch(bi, verif, origin, blocks)); }, ToWorkerMsg::ImportFinalityProof(who, hash, number, proof) => { let (_, verif) = block_import_verifier.as_mut() @@ -250,39 +247,18 @@ impl BlockImportWorker { /// /// For lifetime reasons, the `BlockImport` implementation must be passed by value, and is /// yielded back in the output once the import is finished. - fn import_a_batch_of_blocks>( + fn import_batch>( &mut self, block_import: BoxBlockImport, verifier: V, origin: BlockOrigin, - blocks: Vec> + blocks: Vec>, ) -> impl Future, V)> { let mut result_sender = self.result_sender.clone(); let metrics = self.metrics.clone(); - import_many_blocks(block_import, origin, blocks, verifier, self.delay_between_blocks) + import_many_blocks(block_import, origin, blocks, verifier, self.delay_between_blocks, metrics) .then(move |(imported, count, results, block_import, verifier)| { - if let Some(metrics) = metrics { - let amounts = results.iter().fold([0u64; 8], |mut acc, result| { - match result.0 { - Ok(_) => acc[0] += 1, - Err(BlockImportError::IncompleteHeader(_)) => acc[1] += 1, - Err(BlockImportError::VerificationFailed(_,_)) => acc[2] += 1, - Err(BlockImportError::BadBlock(_)) => acc[3] += 1, - Err(BlockImportError::MissingState) => acc[4] += 1, - Err(BlockImportError::UnknownParent) => acc[5] += 1, - Err(BlockImportError::Cancelled) => acc[6] += 1, - Err(BlockImportError::Other(_)) => acc[7] += 1, - }; - acc - }); - for (idx, field) in METRIC_SUCCESS_FIELDS.iter().enumerate() { - let amount = amounts[idx]; - if amount > 0 { - metrics.import_queue_processed.with_label_values(&[&field]).inc_by(amount) - } - }; - } result_sender.blocks_processed(imported, count, results); future::ready((block_import, verifier)) }) @@ -352,6 +328,7 @@ fn import_many_blocks, Transaction>( blocks: Vec>, verifier: V, delay_between_blocks: Duration, + metrics: Option, ) -> impl Future< Output = ( usize, @@ -401,9 +378,9 @@ fn import_many_blocks, Transaction>( None => { // No block left to import, success! let import_handle = import_handle.take() - .expect("Future polled again after it has finished"); + .expect("Future polled again after it has finished (import handle is None)"); let verifier = verifier.take() - .expect("Future polled again after it has finished"); + .expect("Future polled again after it has finished (verifier handle is None)"); let results = mem::replace(&mut results, Vec::new()); return Poll::Ready((imported, count, results, import_handle, verifier)); }, @@ -413,9 +390,9 @@ fn import_many_blocks, Transaction>( // therefore `import_handle` and `verifier` are always `Some` here. It is illegal to poll // a `Future` again after it has ended. let import_handle = import_handle.as_mut() - .expect("Future polled again after it has finished"); + .expect("Future polled again after it has finished (import handle is None)"); let verifier = verifier.as_mut() - .expect("Future polled again after it has finished"); + .expect("Future polled again after it has finished (verifier handle is None)"); let block_number = block.header.as_ref().map(|h| h.number().clone()); let block_hash = block.hash; @@ -423,14 +400,19 @@ fn import_many_blocks, Transaction>( Err(BlockImportError::Cancelled) } else { // The actual import. - import_single_block( + import_single_block_metered( &mut **import_handle, blocks_origin.clone(), block, verifier, + metrics.clone(), ) }; + if let Some(metrics) = metrics.as_ref() { + metrics.report_import::(&import_result); + } + if import_result.is_ok() { trace!(target: "sync", "Block imported successfully {:?} ({})", block_number, block_hash); imported += 1; diff --git a/primitives/consensus/common/src/lib.rs b/primitives/consensus/common/src/lib.rs index 52b034ffdd66722afb0f9f00d6dbd4d32112d852..da23172783cdcba7d973a5c6f4c1dd96b6b86963 100644 --- a/primitives/consensus/common/src/lib.rs +++ b/primitives/consensus/common/src/lib.rs @@ -72,7 +72,9 @@ pub enum BlockStatus { Unknown, } -/// Environment producer for a Consensus instance. Creates proposer instance and communication streams. +/// Environment for a Consensus instance. +/// +/// Creates proposer instance. pub trait Environment { /// The proposer type this creates. type Proposer: Proposer + Send + 'static; @@ -155,7 +157,7 @@ pub trait Proposer { /// /// Returns a future that resolves to a [`Proposal`] or to [`Error`]. fn propose( - &mut self, + self, inherent_data: InherentData, inherent_digests: DigestFor, max_duration: Duration, diff --git a/primitives/consensus/common/src/metrics.rs b/primitives/consensus/common/src/metrics.rs index 90e01214d8d0b67c6e3f6ff4dd51d89b5c4083ca..90df85a2948ef39cfb391a9f09a168b0e8effbcd 100644 --- a/primitives/consensus/common/src/metrics.rs +++ b/primitives/consensus/common/src/metrics.rs @@ -16,12 +16,17 @@ //! Metering tools for consensus -use prometheus_endpoint::{register, U64, Registry, PrometheusError, Opts, CounterVec}; +use prometheus_endpoint::{register, U64, Registry, PrometheusError, Opts, CounterVec, HistogramVec, HistogramOpts}; + +use sp_runtime::traits::{Block as BlockT, NumberFor}; + +use crate::import_queue::{BlockImportResult, BlockImportError}; /// Generic Prometheus metrics for common consensus functionality. #[derive(Clone)] pub(crate) struct Metrics { pub import_queue_processed: CounterVec, + pub block_verification_time: HistogramVec, } impl Metrics { @@ -34,6 +39,42 @@ impl Metrics { )?, registry, )?, + block_verification_time: register( + HistogramVec::new( + HistogramOpts::new( + "block_verification_time", + "Histogram of time taken to import blocks", + ), + &["result"], + )?, + registry, + )?, }) } + + pub fn report_import( + &self, + result: &Result>, BlockImportError>, + ) { + let label = match result { + Ok(_) => "success", + Err(BlockImportError::IncompleteHeader(_)) => "incomplete_header", + Err(BlockImportError::VerificationFailed(_,_)) => "verification_failed", + Err(BlockImportError::BadBlock(_)) => "bad_block", + Err(BlockImportError::MissingState) => "missing_state", + Err(BlockImportError::UnknownParent) => "unknown_parent", + Err(BlockImportError::Cancelled) => "cancelled", + Err(BlockImportError::Other(_)) => "failed", + }; + + self.import_queue_processed.with_label_values( + &[label] + ).inc(); + } + + pub fn report_verification(&self, success: bool, time: std::time::Duration) { + self.block_verification_time.with_label_values( + &[if success { "success" } else { "verification_failed" }] + ).observe(time.as_secs_f64()); + } } diff --git a/primitives/consensus/common/src/offline_tracker.rs b/primitives/consensus/common/src/offline_tracker.rs index 9269640ffc8e46a1c0c21fe5318bbb4bf36bdb1d..b96498041f25d39fc55e12d9d5fd99cb3e974ce9 100644 --- a/primitives/consensus/common/src/offline_tracker.rs +++ b/primitives/consensus/common/src/offline_tracker.rs @@ -18,7 +18,8 @@ //! Tracks offline validators. use std::collections::HashMap; -use std::time::{Instant, Duration}; +use std::time::Duration; +use wasm_timer::Instant; // time before we report a validator. const REPORT_TIME: Duration = Duration::from_secs(60 * 5); diff --git a/primitives/consensus/pow/Cargo.toml b/primitives/consensus/pow/Cargo.toml index 6e3115258bb55c6028fedad546870b0ea3af6707..5e031235dc10a1cd1abd7eed8f9b747fef4b4218 100644 --- a/primitives/consensus/pow/Cargo.toml +++ b/primitives/consensus/pow/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-consensus-pow" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "Primitives for Aura consensus" edition = "2018" @@ -12,11 +12,11 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-api = { version = "2.0.0-alpha.8", default-features = false, path = "../../api" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../runtime" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../core" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +sp-api = { version = "2.0.0-rc4", default-features = false, path = "../../api" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../runtime" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../core" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } [features] default = ["std"] diff --git a/primitives/consensus/vrf/Cargo.toml b/primitives/consensus/vrf/Cargo.toml index 76de25ffd642eddf099bf9c18b3b9096daed6613..3c89c05bb1aebde21b1777eb30bbaaf299d1c318 100644 --- a/primitives/consensus/vrf/Cargo.toml +++ b/primitives/consensus/vrf/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-consensus-vrf" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "Primitives for VRF based consensus" edition = "2018" @@ -14,9 +14,9 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { version = "1.0.0", package = "parity-scale-codec", default-features = false } schnorrkel = { version = "0.9.1", features = ["preaudit_deprecated", "u64_backend"], default-features = false } -sp-std = { version = "2.0.0-alpha.8", path = "../../std", default-features = false } -sp-core = { version = "2.0.0-alpha.8", path = "../../core", default-features = false } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../runtime" } +sp-std = { version = "2.0.0-rc4", path = "../../std", default-features = false } +sp-core = { version = "2.0.0-rc4", path = "../../core", default-features = false } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../runtime" } [features] default = ["std"] diff --git a/primitives/core/Cargo.toml b/primitives/core/Cargo.toml index 23e4d55ca9c4c4bb9566748a87147612e11b4fce..6a7568a6267d5e97334b79acd4999b42065575ce 100644 --- a/primitives/core/Cargo.toml +++ b/primitives/core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-core" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -14,8 +14,8 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] derive_more = "0.99.2" -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../std" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../std" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } log = { version = "0.4.8", default-features = false } serde = { version = "1.0.101", optional = true, features = ["derive"] } byteorder = { version = "1.3.2", default-features = false } @@ -31,11 +31,12 @@ tiny-bip39 = { version = "0.7", optional = true } regex = { version = "1.3.1", optional = true } num-traits = { version = "0.2.8", default-features = false } zeroize = { version = "1.0.0", default-features = false } +secrecy = { version = "0.6.0", default-features = false } lazy_static = { version = "1.4.0", default-features = false, optional = true } parking_lot = { version = "0.10.0", optional = true } -sp-debug-derive = { version = "2.0.0-alpha.8", path = "../debug-derive" } -sp-externalities = { version = "0.8.0-alpha.8", optional = true, path = "../externalities" } -sp-storage = { version = "2.0.0-alpha.8", default-features = false, path = "../storage" } +sp-debug-derive = { version = "2.0.0-rc4", path = "../debug-derive" } +sp-externalities = { version = "0.8.0-rc4", optional = true, path = "../externalities" } +sp-storage = { version = "2.0.0-rc4", default-features = false, path = "../storage" } parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } futures = { version = "0.3.1", optional = true } @@ -50,15 +51,16 @@ twox-hash = { version = "1.5.0", default-features = false, optional = true } libsecp256k1 = { version = "0.3.2", default-features = false, features = ["hmac"], optional = true } merlin = { version = "2.0", default-features = false, optional = true } -sp-runtime-interface = { version = "2.0.0-alpha.8", default-features = false, path = "../runtime-interface" } +sp-runtime-interface = { version = "2.0.0-rc4", default-features = false, path = "../runtime-interface" } [dev-dependencies] -sp-serializer = { version = "2.0.0-alpha.8", path = "../serializer" } +sp-serializer = { version = "2.0.0-rc4", path = "../serializer" } pretty_assertions = "0.6.1" hex-literal = "0.2.1" rand = "0.7.2" criterion = "0.2.11" serde_json = "1.0" +rand_chacha = "0.2.2" [[bench]] name = "bench" @@ -105,6 +107,7 @@ std = [ "sp-storage/std", "sp-runtime-interface/std", "zeroize/alloc", + "secrecy/alloc", "futures", "futures/thread-pool", "libsecp256k1/std", diff --git a/primitives/core/src/crypto.rs b/primitives/core/src/crypto.rs index 73134dcbfa9da956abaf2d3509762cd60641257c..745f5776fe850c7095c77297ea9ab3678ac8450e 100644 --- a/primitives/core/src/crypto.rs +++ b/primitives/core/src/crypto.rs @@ -37,10 +37,16 @@ use regex::Regex; use base58::{FromBase58, ToBase58}; #[cfg(feature = "std")] use crate::hexdisplay::HexDisplay; -use zeroize::Zeroize; #[doc(hidden)] pub use sp_std::ops::Deref; use sp_runtime_interface::pass_by::PassByInner; +/// Trait to zeroize a memory buffer. +pub use zeroize::Zeroize; +/// Trait for accessing reference to `SecretString`. +pub use secrecy::ExposeSecret; +/// A store for sensitive data. +#[cfg(feature = "std")] +pub use secrecy::SecretString; /// The root phrase for our publicly known keys. pub const DEV_PHRASE: &str = "bottom drive obey lake curtain smoke basket hold race lonely fit walk"; @@ -79,51 +85,6 @@ impl> UncheckedInto for S { } } -/// A store for sensitive data. -/// -/// Calls `Zeroize::zeroize` upon `Drop`. -#[derive(Clone)] -pub struct Protected(T); - -impl AsRef for Protected { - fn as_ref(&self) -> &T { - &self.0 - } -} - -impl sp_std::ops::Deref for Protected { - type Target = T; - - fn deref(&self) -> &T { - &self.0 - } -} - -#[cfg(feature = "std")] -impl std::fmt::Debug for Protected { - fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(fmt, "") - } -} - -impl From for Protected { - fn from(t: T) -> Self { - Protected(t) - } -} - -impl Zeroize for Protected { - fn zeroize(&mut self) { - self.0.zeroize() - } -} - -impl Drop for Protected { - fn drop(&mut self) { - self.zeroize() - } -} - /// An error with the interpretation of a secret. #[derive(Debug, Clone, PartialEq, Eq)] #[cfg(feature = "full_crypto")] @@ -357,12 +318,20 @@ macro_rules! ss58_address_format { ( $( $identifier:tt => ($number:expr, $name:expr, $desc:tt) )* ) => ( /// A known address (sub)format/network ID for SS58. #[derive(Copy, Clone, PartialEq, Eq)] + #[cfg_attr(feature = "std", derive(Debug))] pub enum Ss58AddressFormat { $(#[doc = $desc] $identifier),*, /// Use a manually provided numeric value. Custom(u8), } + #[cfg(feature = "std")] + impl std::fmt::Display for Ss58AddressFormat { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{:?}", self) + } + } + static ALL_SS58_ADDRESS_FORMATS: [Ss58AddressFormat; 0 $(+ { let _ = $number; 1})*] = [ $(Ss58AddressFormat::$identifier),*, ]; @@ -462,8 +431,12 @@ ss58_address_format!( (16, "kulupu", "Kulupu mainnet, standard account (*25519).") DarwiniaAccount => (18, "darwinia", "Darwinia Chain mainnet, standard account (*25519).") + StafiAccount => + (20, "stafi", "Stafi mainnet, standard account (*25519).") RobonomicsAccount => (32, "robonomics", "Any Robonomics network standard account (*25519).") + DataHighwayAccount => + (33, "datahighway", "DataHighway mainnet, standard account (*25519).") CentrifugeAccount => (36, "centrifuge", "Centrifuge Chain mainnet, standard account (*25519).") SubstrateAccount => diff --git a/primitives/core/src/hasher.rs b/primitives/core/src/hasher.rs index 96a79bf5f65c01f86eefdd2a4e02f9076d549c5a..8ccaa4d90a78d7e697750f18f0b35fbcb8351396 100644 --- a/primitives/core/src/hasher.rs +++ b/primitives/core/src/hasher.rs @@ -36,3 +36,23 @@ pub mod blake2 { } } } + +pub mod keccak { + use hash_db::Hasher; + use hash256_std_hasher::Hash256StdHasher; + use crate::hash::H256; + + /// Concrete implementation of Hasher using Keccak 256-bit hashes + #[derive(Debug)] + pub struct KeccakHasher; + + impl Hasher for KeccakHasher { + type Out = H256; + type StdHasher = Hash256StdHasher; + const LENGTH: usize = 32; + + fn hash(x: &[u8]) -> Self::Out { + crate::hashing::keccak_256(x).into() + } + } +} diff --git a/primitives/core/src/lib.rs b/primitives/core/src/lib.rs index 91bdf6683db21618901f7be7fa8a3ba40d9c5c4c..1038c887e2174efc6303c427263c200f636f1c12 100644 --- a/primitives/core/src/lib.rs +++ b/primitives/core/src/lib.rs @@ -73,6 +73,8 @@ pub mod traits; pub mod testing; #[cfg(feature = "std")] pub mod tasks; +#[cfg(feature = "std")] +pub mod vrf; pub use self::hash::{H160, H256, H512, convert_hash}; pub use self::uint::{U256, U512}; @@ -83,6 +85,8 @@ pub use crypto::{DeriveJunction, Pair, Public}; pub use hash_db::Hasher; #[cfg(feature = "std")] pub use self::hasher::blake2::Blake2Hasher; +#[cfg(feature = "std")] +pub use self::hasher::keccak::KeccakHasher; pub use sp_storage as storage; @@ -91,9 +95,16 @@ pub use sp_std; /// Context for executing a call into the runtime. pub enum ExecutionContext { - /// Context for general importing (including own blocks). + /// Context used for general block import (including locally authored blocks). Importing, - /// Context used when syncing the blockchain. + /// Context used for importing blocks as part of an initial sync of the blockchain. + /// + /// We distinguish between major sync and import so that validators who are running + /// their initial sync (or catching up after some time offline) can use the faster + /// native runtime (since we can reasonably assume the network as a whole has already + /// come to a broad conensus on the block and it probably hasn't been crafted + /// specifically to attack this node), but when importing blocks at the head of the + /// chain in normal operation they can use the safer Wasm version. Syncing, /// Context used for block construction. BlockConstruction, diff --git a/primitives/core/src/offchain/mod.rs b/primitives/core/src/offchain/mod.rs index 1d77e10f59ca7fbca654c75f8f3489eec522d798..b2ff3552135ce935a2e8d3a79c7533baa5afa117 100644 --- a/primitives/core/src/offchain/mod.rs +++ b/primitives/core/src/offchain/mod.rs @@ -37,6 +37,9 @@ pub trait OffchainStorage: Clone + Send + Sync { /// Persist a value in storage under given key and prefix. fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]); + /// Clear a storage entry under given key and prefix. + fn remove(&mut self, prefix: &[u8], key: &[u8]); + /// Retrieve a value from storage under given key and prefix. fn get(&self, prefix: &[u8], key: &[u8]) -> Option>; @@ -219,7 +222,7 @@ pub struct Duration(u64); impl Duration { /// Create new duration representing given number of milliseconds. - pub fn from_millis(millis: u64) -> Self { + pub const fn from_millis(millis: u64) -> Self { Duration(millis) } @@ -346,9 +349,15 @@ pub trait Externalities: Send { /// Sets a value in the local storage. /// /// Note this storage is not part of the consensus, it's only accessible by - /// offchain worker tasks running on the same machine. It IS persisted between runs. + /// offchain worker tasks running on the same machine. It _is_ persisted between runs. fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]); + /// Removes a value in the local storage. + /// + /// Note this storage is not part of the consensus, it's only accessible by + /// offchain worker tasks running on the same machine. It _is_ persisted between runs. + fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]); + /// Sets a value in the local storage if it matches current value. /// /// Since multiple offchain workers may be running concurrently, to prevent @@ -357,7 +366,7 @@ pub trait Externalities: Send { /// Returns `true` if the value has been set, `false` otherwise. /// /// Note this storage is not part of the consensus, it's only accessible by - /// offchain worker tasks running on the same machine. It IS persisted between runs. + /// offchain worker tasks running on the same machine. It _is_ persisted between runs. fn local_storage_compare_and_set( &mut self, kind: StorageKind, @@ -370,7 +379,7 @@ pub trait Externalities: Send { /// /// If the value does not exist in the storage `None` will be returned. /// Note this storage is not part of the consensus, it's only accessible by - /// offchain worker tasks running on the same machine. It IS persisted between runs. + /// offchain worker tasks running on the same machine. It _is_ persisted between runs. fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option>; /// Initiates a http request given HTTP verb and the URL. @@ -513,6 +522,10 @@ impl Externalities for Box { (&mut **self).local_storage_set(kind, key, value) } + fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { + (&mut **self).local_storage_clear(kind, key) + } + fn local_storage_compare_and_set( &mut self, kind: StorageKind, @@ -618,6 +631,11 @@ impl Externalities for LimitedExternalities { self.externalities.local_storage_set(kind, key, value) } + fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { + self.check(Capability::OffchainWorkerDbWrite, "local_storage_clear"); + self.externalities.local_storage_clear(kind, key) + } + fn local_storage_compare_and_set( &mut self, kind: StorageKind, diff --git a/primitives/core/src/offchain/storage.rs b/primitives/core/src/offchain/storage.rs index 1826015b0d0cc15daf22ff12e673f7d57423948f..7d7c711ed95f046ccf33e1e3d2e8262c9d2bb06a 100644 --- a/primitives/core/src/offchain/storage.rs +++ b/primitives/core/src/offchain/storage.rs @@ -51,6 +51,11 @@ impl OffchainStorage for InMemOffchainStorage { self.storage.insert(key, value.to_vec()); } + fn remove(&mut self, prefix: &[u8], key: &[u8]) { + let key: Vec = prefix.iter().chain(key).cloned().collect(); + self.storage.remove(&key); + } + fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { let key: Vec = prefix.iter().chain(key).cloned().collect(); self.storage.get(&key).cloned() @@ -96,8 +101,9 @@ pub enum OffchainOverlayedChange { pub enum OffchainOverlayedChanges { /// Writing overlay changes to the offchain worker database is disabled by configuration. Disabled, - /// Overlay changes can be recorded using the inner collection of this variant. - Enabled(HashMap, OffchainOverlayedChange>), + /// Overlay changes can be recorded using the inner collection of this variant, + /// where the identifier is the tuple of `(prefix, key)`. + Enabled(HashMap<(Vec, Vec), OffchainOverlayedChange>), } impl Default for OffchainOverlayedChanges { @@ -135,23 +141,21 @@ impl OffchainOverlayedChanges { /// Remove a key and its associated value from the offchain database. pub fn remove(&mut self, prefix: &[u8], key: &[u8]) { if let Self::Enabled(ref mut storage) = self { - let key: Vec = prefix.iter().chain(key).cloned().collect(); - let _ = storage.insert(key, OffchainOverlayedChange::Remove); + let _ = storage.insert((prefix.to_vec(), key.to_vec()), OffchainOverlayedChange::Remove); } } /// Set the value associated with a key under a prefix to the value provided. pub fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { if let Self::Enabled(ref mut storage) = self { - let key = prefix.iter().chain(key).cloned().collect(); - let _ = storage.insert(key, OffchainOverlayedChange::SetValue(value.to_vec())); + let _ = storage.insert((prefix.to_vec(), key.to_vec()), OffchainOverlayedChange::SetValue(value.to_vec())); } } /// Obtain a associated value to the given key in storage with prefix. pub fn get(&self, prefix: &[u8], key: &[u8]) -> Option { if let Self::Enabled(ref storage) = self { - let key: Vec = prefix.iter().chain(key).cloned().collect(); + let key = (prefix.to_vec(), key.to_vec()); storage.get(&key).cloned() } else { None @@ -163,11 +167,11 @@ use std::collections::hash_map; /// Iterate by reference over the prepared offchain worker storage changes. pub struct OffchainOverlayedChangesIter<'i> { - inner: Option, OffchainOverlayedChange>>, + inner: Option, Vec), OffchainOverlayedChange>>, } impl<'i> Iterator for OffchainOverlayedChangesIter<'i> { - type Item = (&'i Vec, &'i OffchainOverlayedChange); + type Item = (&'i (Vec, Vec), &'i OffchainOverlayedChange); fn next(&mut self) -> Option { if let Some(ref mut iter) = self.inner { iter.next() @@ -192,11 +196,11 @@ impl<'i> OffchainOverlayedChangesIter<'i> { /// Iterate by value over the prepared offchain worker storage changes. pub struct OffchainOverlayedChangesIntoIter { - inner: Option,OffchainOverlayedChange>>, + inner: Option,Vec),OffchainOverlayedChange>>, } impl Iterator for OffchainOverlayedChangesIntoIter { - type Item = (Vec, OffchainOverlayedChange); + type Item = ((Vec, Vec), OffchainOverlayedChange); fn next(&mut self) -> Option { if let Some(ref mut iter) = self.inner { iter.next() @@ -220,11 +224,11 @@ impl OffchainOverlayedChangesIntoIter { /// Iterate over all items while draining them from the collection. pub struct OffchainOverlayedChangesDrain<'d> { - inner: Option,OffchainOverlayedChange>>, + inner: Option, Vec), OffchainOverlayedChange>>, } impl<'d> Iterator for OffchainOverlayedChangesDrain<'d> { - type Item = (Vec, OffchainOverlayedChange); + type Item = ((Vec, Vec), OffchainOverlayedChange); fn next(&mut self) -> Option { if let Some(ref mut iter) = self.inner { iter.next() @@ -281,9 +285,13 @@ mod test { ooc.set(STORAGE_PREFIX, b"ppp", b"rrr"); let mut iter = ooc.into_iter(); - let mut k = STORAGE_PREFIX.to_vec(); - k.extend_from_slice(&b"ppp"[..]); - assert_eq!(iter.next(), Some((k, OffchainOverlayedChange::SetValue(b"rrr".to_vec())))); + assert_eq!( + iter.next(), + Some( + ((STORAGE_PREFIX.to_vec(), b"ppp".to_vec()), + OffchainOverlayedChange::SetValue(b"rrr".to_vec())) + ) + ); assert_eq!(iter.next(), None); } } diff --git a/primitives/core/src/offchain/testing.rs b/primitives/core/src/offchain/testing.rs index 5e25e433a3cd5d5c03854ad5bbd89d664fa9654d..9145477722d78b1ffe86c5f55e076ba290b88712 100644 --- a/primitives/core/src/offchain/testing.rs +++ b/primitives/core/src/offchain/testing.rs @@ -21,12 +21,12 @@ //! the extra APIs. use std::{ - collections::BTreeMap, + collections::{BTreeMap, VecDeque}, sync::Arc, }; use crate::offchain::{ self, - storage::InMemOffchainStorage, + storage::{InMemOffchainStorage, OffchainOverlayedChange, OffchainOverlayedChanges}, HttpError, HttpRequestId as RequestId, HttpRequestStatus as RequestStatus, @@ -36,6 +36,7 @@ use crate::offchain::{ TransactionPool, OffchainStorage, }; + use parking_lot::RwLock; /// Pending request. @@ -61,6 +62,57 @@ pub struct PendingRequest { pub response_headers: Vec<(String, String)>, } +/// Sharable "persistent" offchain storage for test. +#[derive(Debug, Clone, Default)] +pub struct TestPersistentOffchainDB { + persistent: Arc>, +} + +impl TestPersistentOffchainDB { + /// Create a new and empty offchain storage db for persistent items + pub fn new() -> Self { + Self { + persistent: Arc::new(RwLock::new(InMemOffchainStorage::default())) + } + } + + /// Apply a set of off-chain changes directly to the test backend + pub fn apply_offchain_changes(&mut self, changes: &mut OffchainOverlayedChanges) { + let mut me = self.persistent.write(); + for ((_prefix, key), value_operation) in changes.drain() { + match value_operation { + OffchainOverlayedChange::SetValue(val) => me.set(b"", key.as_slice(), val.as_slice()), + OffchainOverlayedChange::Remove => me.remove(b"", key.as_slice()), + } + } + } +} + +impl OffchainStorage for TestPersistentOffchainDB { + fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { + self.persistent.write().set(prefix, key, value); + } + + fn remove(&mut self, prefix: &[u8], key: &[u8]) { + self.persistent.write().remove(prefix, key); + } + + fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { + self.persistent.read().get(prefix, key) + } + + fn compare_and_set( + &mut self, + prefix: &[u8], + key: &[u8], + old_value: Option<&[u8]>, + new_value: &[u8], + ) -> bool { + self.persistent.write().compare_and_set(prefix, key, old_value, new_value) + } +} + + /// Internal state of the externalities. /// /// This can be used in tests to respond or assert stuff about interactions. @@ -68,15 +120,16 @@ pub struct PendingRequest { pub struct OffchainState { /// A list of pending requests. pub requests: BTreeMap, - expected_requests: BTreeMap, + // Queue of requests that the test is expected to perform (in order). + expected_requests: VecDeque, /// Persistent local storage - pub persistent_storage: InMemOffchainStorage, + pub persistent_storage: TestPersistentOffchainDB, /// Local storage pub local_storage: InMemOffchainStorage, - /// Current timestamp (unix millis) - pub timestamp: u64, /// A supposedly random seed. pub seed: [u8; 32], + /// A timestamp simulating the current time. + pub timestamp: Timestamp, } impl OffchainState { @@ -104,8 +157,8 @@ impl OffchainState { } fn fulfill_expected(&mut self, id: u16) { - if let Some(mut req) = self.expected_requests.remove(&RequestId(id)) { - let response = req.response.take().expect("Response checked while added."); + if let Some(mut req) = self.expected_requests.pop_back() { + let response = req.response.take().expect("Response checked when added."); let headers = std::mem::take(&mut req.response_headers); self.fulfill_pending_request(id, req, response, headers); } @@ -117,11 +170,12 @@ impl OffchainState { /// before running the actual code that utilizes them (for instance before calling into runtime). /// Expected request has to be fulfilled before this struct is dropped, /// the `response` and `response_headers` fields will be used to return results to the callers. - pub fn expect_request(&mut self, id: u16, expected: PendingRequest) { + /// Requests are expected to be performed in the insertion order. + pub fn expect_request(&mut self, expected: PendingRequest) { if expected.response.is_none() { panic!("Expected request needs to have a response."); } - self.expected_requests.insert(RequestId(id), expected); + self.expected_requests.push_front(expected); } } @@ -145,6 +199,13 @@ impl TestOffchainExt { let state = ext.0.clone(); (ext, state) } + + /// Create new `TestOffchainExt` and a reference to the internal state. + pub fn with_offchain_db(offchain_db: TestPersistentOffchainDB) -> (Self, Arc>) { + let (ext, state) = Self::new(); + ext.0.write().persistent_storage = offchain_db; + (ext, state) + } } impl offchain::Externalities for TestOffchainExt { @@ -160,11 +221,11 @@ impl offchain::Externalities for TestOffchainExt { } fn timestamp(&mut self) -> Timestamp { - Timestamp::from_unix_millis(self.0.read().timestamp) + self.0.read().timestamp } - fn sleep_until(&mut self, _deadline: Timestamp) { - unimplemented!("not needed in tests so far") + fn sleep_until(&mut self, deadline: Timestamp) { + self.0.write().timestamp = deadline; } fn random_seed(&mut self) -> [u8; 32] { @@ -174,9 +235,17 @@ impl offchain::Externalities for TestOffchainExt { fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { let mut state = self.0.write(); match kind { - StorageKind::LOCAL => &mut state.local_storage, - StorageKind::PERSISTENT => &mut state.persistent_storage, - }.set(b"", key, value); + StorageKind::LOCAL => state.local_storage.set(b"", key, value), + StorageKind::PERSISTENT => state.persistent_storage.set(b"", key, value), + }; + } + + fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { + let mut state = self.0.write(); + match kind { + StorageKind::LOCAL => state.local_storage.remove(b"", key), + StorageKind::PERSISTENT => state.persistent_storage.remove(b"", key), + }; } fn local_storage_compare_and_set( @@ -188,17 +257,17 @@ impl offchain::Externalities for TestOffchainExt { ) -> bool { let mut state = self.0.write(); match kind { - StorageKind::LOCAL => &mut state.local_storage, - StorageKind::PERSISTENT => &mut state.persistent_storage, - }.compare_and_set(b"", key, old_value, new_value) + StorageKind::LOCAL => state.local_storage.compare_and_set(b"", key, old_value, new_value), + StorageKind::PERSISTENT => state.persistent_storage.compare_and_set(b"", key, old_value, new_value), + } } fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option> { let state = self.0.read(); match kind { - StorageKind::LOCAL => &state.local_storage, - StorageKind::PERSISTENT => &state.persistent_storage, - }.get(b"", key) + StorageKind::LOCAL => state.local_storage.get(b"", key), + StorageKind::PERSISTENT => state.persistent_storage.get(b"", key), + } } fn http_request_start(&mut self, method: &str, uri: &str, meta: &[u8]) -> Result { diff --git a/primitives/core/src/testing.rs b/primitives/core/src/testing.rs index e14eb6a7f37c6f6da25c94f5b750fd55c34d17e5..1d88e1fad5513d0cd417892d9ba3dc44d2e3a5b6 100644 --- a/primitives/core/src/testing.rs +++ b/primitives/core/src/testing.rs @@ -22,10 +22,12 @@ use crate::crypto::KeyTypeId; use crate::{ crypto::{Pair, Public, CryptoTypePublicPair}, ed25519, sr25519, ecdsa, - traits::BareCryptoStoreError + traits::Error, + vrf::{VRFTranscriptData, VRFSignature, make_transcript}, }; #[cfg(feature = "std")] use std::collections::HashSet; + /// Key type for generic Ed25519 key. pub const ED25519: KeyTypeId = KeyTypeId(*b"ed25"); /// Key type for generic Sr 25519 key. @@ -76,7 +78,7 @@ impl KeyStore { #[cfg(feature = "std")] impl crate::traits::BareCryptoStore for KeyStore { - fn keys(&self, id: KeyTypeId) -> Result, BareCryptoStoreError> { + fn keys(&self, id: KeyTypeId) -> Result, Error> { self.keys .get(&id) .map(|map| { @@ -106,11 +108,11 @@ impl crate::traits::BareCryptoStore for KeyStore { &mut self, id: KeyTypeId, seed: Option<&str>, - ) -> Result { + ) -> Result { match seed { Some(seed) => { let pair = sr25519::Pair::from_string(seed, None) - .map_err(|_| BareCryptoStoreError::ValidationError("Generates an `sr25519` pair.".to_owned()))?; + .map_err(|_| Error::ValidationError("Generates an `sr25519` pair.".to_owned()))?; self.keys.entry(id).or_default().insert(pair.public().to_raw_vec(), seed.into()); Ok(pair.public()) }, @@ -137,11 +139,11 @@ impl crate::traits::BareCryptoStore for KeyStore { &mut self, id: KeyTypeId, seed: Option<&str>, - ) -> Result { + ) -> Result { match seed { Some(seed) => { let pair = ed25519::Pair::from_string(seed, None) - .map_err(|_| BareCryptoStoreError::ValidationError("Generates an `ed25519` pair.".to_owned()))?; + .map_err(|_| Error::ValidationError("Generates an `ed25519` pair.".to_owned()))?; self.keys.entry(id).or_default().insert(pair.public().to_raw_vec(), seed.into()); Ok(pair.public()) }, @@ -168,11 +170,11 @@ impl crate::traits::BareCryptoStore for KeyStore { &mut self, id: KeyTypeId, seed: Option<&str>, - ) -> Result { + ) -> Result { match seed { Some(seed) => { let pair = ecdsa::Pair::from_string(seed, None) - .map_err(|_| BareCryptoStoreError::ValidationError("Generates an `ecdsa` pair.".to_owned()))?; + .map_err(|_| Error::ValidationError("Generates an `ecdsa` pair.".to_owned()))?; self.keys.entry(id).or_default().insert(pair.public().to_raw_vec(), seed.into()); Ok(pair.public()) }, @@ -201,7 +203,7 @@ impl crate::traits::BareCryptoStore for KeyStore { &self, id: KeyTypeId, keys: Vec, - ) -> std::result::Result, BareCryptoStoreError> { + ) -> std::result::Result, Error> { let provided_keys = keys.into_iter().collect::>(); let all_keys = self.keys(id)?.into_iter().collect::>(); @@ -213,31 +215,48 @@ impl crate::traits::BareCryptoStore for KeyStore { id: KeyTypeId, key: &CryptoTypePublicPair, msg: &[u8], - ) -> Result, BareCryptoStoreError> { + ) -> Result, Error> { use codec::Encode; match key.0 { ed25519::CRYPTO_ID => { let key_pair: ed25519::Pair = self .ed25519_key_pair(id, &ed25519::Public::from_slice(key.1.as_slice())) - .ok_or(BareCryptoStoreError::PairNotFound("ed25519".to_owned()))?; + .ok_or(Error::PairNotFound("ed25519".to_owned()))?; return Ok(key_pair.sign(msg).encode()); } sr25519::CRYPTO_ID => { let key_pair: sr25519::Pair = self .sr25519_key_pair(id, &sr25519::Public::from_slice(key.1.as_slice())) - .ok_or(BareCryptoStoreError::PairNotFound("sr25519".to_owned()))?; + .ok_or(Error::PairNotFound("sr25519".to_owned()))?; return Ok(key_pair.sign(msg).encode()); } ecdsa::CRYPTO_ID => { let key_pair: ecdsa::Pair = self .ecdsa_key_pair(id, &ecdsa::Public::from_slice(key.1.as_slice())) - .ok_or(BareCryptoStoreError::PairNotFound("ecdsa".to_owned()))?; + .ok_or(Error::PairNotFound("ecdsa".to_owned()))?; return Ok(key_pair.sign(msg).encode()); } - _ => Err(BareCryptoStoreError::KeyNotSupported(id)) + _ => Err(Error::KeyNotSupported(id)) } } + + fn sr25519_vrf_sign( + &self, + key_type: KeyTypeId, + public: &sr25519::Public, + transcript_data: VRFTranscriptData, + ) -> Result { + let transcript = make_transcript(transcript_data); + let pair = self.sr25519_key_pair(key_type, public) + .ok_or(Error::PairNotFound("Not found".to_owned()))?; + + let (inout, proof, _) = pair.as_ref().vrf_sign(transcript); + Ok(VRFSignature { + output: inout.to_output(), + proof, + }) + } } /// Macro for exporting functions from wasm in with the expected signature for using it with the @@ -358,10 +377,13 @@ impl SpawnBlockingExecutor { } #[cfg(feature = "std")] -impl crate::traits::SpawnBlocking for SpawnBlockingExecutor { +impl crate::traits::SpawnNamed for SpawnBlockingExecutor { fn spawn_blocking(&self, _: &'static str, future: futures::future::BoxFuture<'static, ()>) { self.0.spawn_ok(future); } + fn spawn(&self, _: &'static str, future: futures::future::BoxFuture<'static, ()>) { + self.0.spawn_ok(future); + } } #[cfg(test)] @@ -369,6 +391,7 @@ mod tests { use super::*; use crate::sr25519; use crate::testing::{ED25519, SR25519}; + use crate::vrf::VRFTranscriptValue; #[test] fn store_key_and_extract() { @@ -400,4 +423,42 @@ mod tests { assert!(public_keys.contains(&key_pair.public().into())); } + + #[test] + fn vrf_sign() { + let store = KeyStore::new(); + + let secret_uri = "//Alice"; + let key_pair = sr25519::Pair::from_string(secret_uri, None).expect("Generates key pair"); + + let transcript_data = VRFTranscriptData { + label: b"Test", + items: vec![ + ("one", VRFTranscriptValue::U64(1)), + ("two", VRFTranscriptValue::U64(2)), + ("three", VRFTranscriptValue::Bytes("test".as_bytes())), + ] + }; + + let result = store.read().sr25519_vrf_sign( + SR25519, + &key_pair.public(), + transcript_data.clone(), + ); + assert!(result.is_err()); + + store.write().insert_unknown( + SR25519, + secret_uri, + key_pair.public().as_ref(), + ).expect("Inserts unknown key"); + + let result = store.read().sr25519_vrf_sign( + SR25519, + &key_pair.public(), + transcript_data, + ); + + assert!(result.is_ok()); + } } diff --git a/primitives/core/src/traits.rs b/primitives/core/src/traits.rs index 880b34a1ed191f30c18748f1b77451234fc7386d..4481145818f2689ceac2836658fce24f7d9f99d9 100644 --- a/primitives/core/src/traits.rs +++ b/primitives/core/src/traits.rs @@ -19,9 +19,9 @@ use crate::{ crypto::{KeyTypeId, CryptoTypePublicPair}, + vrf::{VRFTranscriptData, VRFSignature}, ed25519, sr25519, ecdsa, }; - use std::{ borrow::Cow, fmt::{Debug, Display}, @@ -33,7 +33,7 @@ pub use sp_externalities::{Externalities, ExternalitiesExt}; /// BareCryptoStore error #[derive(Debug, derive_more::Display)] -pub enum BareCryptoStoreError { +pub enum Error { /// Public key type is not supported #[display(fmt="Key not supported: {:?}", _0)] KeyNotSupported(KeyTypeId), @@ -64,7 +64,7 @@ pub trait BareCryptoStore: Send + Sync { &mut self, id: KeyTypeId, seed: Option<&str>, - ) -> Result; + ) -> Result; /// Returns all ed25519 public keys for the given key type. fn ed25519_public_keys(&self, id: KeyTypeId) -> Vec; /// Generate a new ed25519 key pair for the given key type and an optional seed. @@ -76,7 +76,7 @@ pub trait BareCryptoStore: Send + Sync { &mut self, id: KeyTypeId, seed: Option<&str>, - ) -> Result; + ) -> Result; /// Returns all ecdsa public keys for the given key type. fn ecdsa_public_keys(&self, id: KeyTypeId) -> Vec; /// Generate a new ecdsa key pair for the given key type and an optional seed. @@ -88,7 +88,7 @@ pub trait BareCryptoStore: Send + Sync { &mut self, id: KeyTypeId, seed: Option<&str>, - ) -> Result; + ) -> Result; /// Insert a new key. This doesn't require any known of the crypto; but a public key must be /// manually provided. @@ -108,11 +108,11 @@ pub trait BareCryptoStore: Send + Sync { &self, id: KeyTypeId, keys: Vec - ) -> Result, BareCryptoStoreError>; + ) -> Result, Error>; /// List all supported keys /// /// Returns a set of public keys the signer supports. - fn keys(&self, id: KeyTypeId) -> Result, BareCryptoStoreError>; + fn keys(&self, id: KeyTypeId) -> Result, Error>; /// Checks if the private keys for the given public key and key type combinations exist. /// @@ -131,7 +131,7 @@ pub trait BareCryptoStore: Send + Sync { id: KeyTypeId, key: &CryptoTypePublicPair, msg: &[u8], - ) -> Result, BareCryptoStoreError>; + ) -> Result, Error>; /// Sign with any key /// @@ -144,7 +144,7 @@ pub trait BareCryptoStore: Send + Sync { id: KeyTypeId, keys: Vec, msg: &[u8] - ) -> Result<(CryptoTypePublicPair, Vec), BareCryptoStoreError> { + ) -> Result<(CryptoTypePublicPair, Vec), Error> { if keys.len() == 1 { return self.sign_with(id, &keys[0], msg).map(|s| (keys[0].clone(), s)); } else { @@ -154,7 +154,7 @@ pub trait BareCryptoStore: Send + Sync { } } } - Err(BareCryptoStoreError::KeyNotSupported(id)) + Err(Error::KeyNotSupported(id)) } /// Sign with all keys @@ -163,15 +163,36 @@ pub trait BareCryptoStore: Send + Sync { /// each key given that the key is supported. /// /// Returns a list of `Result`s each representing the SCALE encoded - /// signature of each key or a BareCryptoStoreError for non-supported keys. + /// signature of each key or a Error for non-supported keys. fn sign_with_all( &self, id: KeyTypeId, keys: Vec, msg: &[u8], - ) -> Result, BareCryptoStoreError>>, ()>{ + ) -> Result, Error>>, ()>{ Ok(keys.iter().map(|k| self.sign_with(id, k, msg)).collect()) } + + /// Generate VRF signature for given transcript data. + /// + /// Receives KeyTypeId and Public key to be able to map + /// them to a private key that exists in the keystore which + /// is, in turn, used for signing the provided transcript. + /// + /// Returns a result containing the signature data. + /// Namely, VRFOutput and VRFProof which are returned + /// inside the `VRFSignature` container struct. + /// + /// This function will return an error in the cases where + /// the public key and key type provided do not match a private + /// key in the keystore. Or, in the context of remote signing + /// an error could be a network one. + fn sr25519_vrf_sign( + &self, + key_type: KeyTypeId, + public: &sr25519::Public, + transcript_data: VRFTranscriptData, + ) -> Result; } /// A pointer to the key store. @@ -349,10 +370,14 @@ impl TaskExecutorExt { } } -/// Something that can spawn a blocking future. -pub trait SpawnBlocking { +/// Something that can spawn futures (blocking and non-blocking) with am assigned name. +pub trait SpawnNamed { /// Spawn the given blocking future. /// /// The given `name` is used to identify the future in tracing. fn spawn_blocking(&self, name: &'static str, future: futures::future::BoxFuture<'static, ()>); + /// Spawn the given non-blocking future. + /// + /// The given `name` is used to identify the future in tracing. + fn spawn(&self, name: &'static str, future: futures::future::BoxFuture<'static, ()>); } diff --git a/primitives/core/src/vrf.rs b/primitives/core/src/vrf.rs new file mode 100644 index 0000000000000000000000000000000000000000..d392587cb72e7acaec7139e81a6ca531bc2b3b67 --- /dev/null +++ b/primitives/core/src/vrf.rs @@ -0,0 +1,99 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! VRF-specifc data types and helpers + +use codec::Encode; +use merlin::Transcript; +use schnorrkel::vrf::{VRFOutput, VRFProof}; +/// An enum whose variants represent possible +/// accepted values to construct the VRF transcript +#[derive(Clone, Encode)] +pub enum VRFTranscriptValue<'a> { + /// Value is an array of bytes + Bytes(&'a [u8]), + /// Value is a u64 integer + U64(u64), +} +/// VRF Transcript data +#[derive(Clone, Encode)] +pub struct VRFTranscriptData<'a> { + /// The transcript's label + pub label: &'static [u8], + /// Additional data to be registered into the transcript + pub items: Vec<(&'static str, VRFTranscriptValue<'a>)>, +} +/// VRF signature data +pub struct VRFSignature { + /// The VRFOutput serialized + pub output: VRFOutput, + /// The calculated VRFProof + pub proof: VRFProof, +} + +/// Construct a `Transcript` object from data. +/// +/// Returns `merlin::Transcript` +pub fn make_transcript(data: VRFTranscriptData) -> Transcript { + let mut transcript = Transcript::new(data.label); + for (label, value) in data.items.into_iter() { + match value { + VRFTranscriptValue::Bytes(bytes) => { + transcript.append_message(label.as_bytes(), &bytes); + }, + VRFTranscriptValue::U64(val) => { + transcript.append_u64(label.as_bytes(), val); + } + } + } + transcript +} + + +#[cfg(test)] +mod tests { + use super::*; + use crate::vrf::VRFTranscriptValue; + use rand::RngCore; + use rand_chacha::{ + rand_core::SeedableRng, + ChaChaRng, + }; + + #[test] + fn transcript_creation_matches() { + let mut orig_transcript = Transcript::new(b"My label"); + orig_transcript.append_u64(b"one", 1); + orig_transcript.append_message(b"two", "test".as_bytes()); + + let new_transcript = make_transcript(VRFTranscriptData { + label: b"My label", + items: vec![ + ("one", VRFTranscriptValue::U64(1)), + ("two", VRFTranscriptValue::Bytes("test".as_bytes())), + ], + }); + let test = |t: Transcript| -> [u8; 16] { + let mut b = [0u8; 16]; + t.build_rng() + .finalize(&mut ChaChaRng::from_seed([0u8;32])) + .fill_bytes(&mut b); + b + }; + debug_assert!(test(orig_transcript) == test(new_transcript)); + } +} diff --git a/primitives/database/Cargo.toml b/primitives/database/Cargo.toml index b8ce17c8c6405fca0c46a2cc9a805583c766dca6..41ced29a57e7d9f2dedc45871c6a05657065603d 100644 --- a/primitives/database/Cargo.toml +++ b/primitives/database/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-database" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" diff --git a/primitives/database/src/lib.rs b/primitives/database/src/lib.rs index bc4c11f60a98d3a6e0a1a2b663bd6847aa82c1c4..1fb7b156661fcf8c3e3a514b7ba3cb58466c75a1 100644 --- a/primitives/database/src/lib.rs +++ b/primitives/database/src/lib.rs @@ -165,6 +165,12 @@ pub trait Database: Send + Sync { } } +impl std::fmt::Debug for dyn Database { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "Database") + } +} + /// Call `f` with the value previously stored against `key` and return the result, or `None` if /// `key` is not currently in the database. /// diff --git a/primitives/debug-derive/Cargo.toml b/primitives/debug-derive/Cargo.toml index d099f4917cd328a2b29e9a5725c9630a56dea549..fd63abcfa7a6cf7dda9bab20ddaa4adc780b7841 100644 --- a/primitives/debug-derive/Cargo.toml +++ b/primitives/debug-derive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-debug-derive" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" diff --git a/primitives/externalities/Cargo.toml b/primitives/externalities/Cargo.toml index 443885f94fdd8763162ad8859aa4fd63118ea945..65c59e41e4ab0a8683a376776dc1ea5a5a018e53 100644 --- a/primitives/externalities/Cargo.toml +++ b/primitives/externalities/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-externalities" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" license = "Apache-2.0" authors = ["Parity Technologies "] edition = "2018" @@ -13,7 +13,7 @@ documentation = "https://docs.rs/sp-externalities" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-storage = { version = "2.0.0-alpha.8", path = "../storage" } -sp-std = { version = "2.0.0-alpha.8", path = "../std" } +sp-storage = { version = "2.0.0-rc4", path = "../storage" } +sp-std = { version = "2.0.0-rc4", path = "../std" } environmental = { version = "1.1.1" } -codec = { package = "parity-scale-codec", version = "1.3.0" } +codec = { package = "parity-scale-codec", version = "1.3.1" } diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index cfb1d0878a4919ca5dd048e2910d56362410d129..8e141867195b743d99817fa464f930769af7fbf2 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -195,6 +195,29 @@ pub trait Externalities: ExtensionStore { /// The returned hash is defined by the `Block` and is SCALE encoded. fn storage_changes_root(&mut self, parent: &[u8]) -> Result>, ()>; + /// Start a new nested transaction. + /// + /// This allows to either commit or roll back all changes made after this call to the + /// top changes or the default child changes. For every transaction there cam be a + /// matching call to either `storage_rollback_transaction` or `storage_commit_transaction`. + /// Any transactions that are still open after returning from runtime are committed + /// automatically. + /// + /// Changes made without any open transaction are committed immediately. + fn storage_start_transaction(&mut self); + + /// Rollback the last transaction started by `storage_start_transaction`. + /// + /// Any changes made during that storage transaction are discarded. Returns an error when + /// no transaction is open that can be closed. + fn storage_rollback_transaction(&mut self) -> Result<(), ()>; + + /// Commit the last transaction started by `storage_start_transaction`. + /// + /// Any changes made during that storage transaction are committed. Returns an error when + /// no transaction is open that can be closed. + fn storage_commit_transaction(&mut self) -> Result<(), ()>; + /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! /// Benchmarking related functionality and shouldn't be used anywhere else! /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -210,6 +233,27 @@ pub trait Externalities: ExtensionStore { /// /// Commits all changes to the database and clears all caches. fn commit(&mut self); + + /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + /// Benchmarking related functionality and shouldn't be used anywhere else! + /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + /// + /// Gets the current read/write count for the benchmarking process. + fn read_write_count(&self) -> (u32, u32, u32, u32); + + /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + /// Benchmarking related functionality and shouldn't be used anywhere else! + /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + /// + /// Resets read/write count for the benchmarking process. + fn reset_read_write_count(&mut self); + + /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + /// Benchmarking related functionality and shouldn't be used anywhere else! + /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + /// + /// Adds new storage keys to the DB tracking whitelist. + fn set_whitelist(&mut self, new: Vec>); } /// Extension for the [`Externalities`] trait. diff --git a/primitives/finality-grandpa/Cargo.toml b/primitives/finality-grandpa/Cargo.toml index 54bbd3deb1ac010e170e165ea90fca6c2c18b969..7e77e1253cbb2749d1e410ee90ecdef538c20178 100644 --- a/primitives/finality-grandpa/Cargo.toml +++ b/primitives/finality-grandpa/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-finality-grandpa" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -14,15 +14,15 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.8", default-features = false, path = "../application-crypto" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +sp-application-crypto = { version = "2.0.0-rc4", default-features = false, path = "../application-crypto" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } grandpa = { package = "finality-grandpa", version = "0.12.3", default-features = false, features = ["derive-codec"] } log = { version = "0.4.8", optional = true } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-api = { version = "2.0.0-alpha.8", default-features = false, path = "../api" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../core" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../runtime" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../std" } +sp-api = { version = "2.0.0-rc4", default-features = false, path = "../api" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../core" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../runtime" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../std" } [features] default = ["std"] diff --git a/primitives/finality-grandpa/src/lib.rs b/primitives/finality-grandpa/src/lib.rs index 2e81c8cecbb708f952ae0dff7b5bd24c1a7e3995..f99880041c0c04d0b20df639ea5121b317571aec 100644 --- a/primitives/finality-grandpa/src/lib.rs +++ b/primitives/finality-grandpa/src/lib.rs @@ -29,6 +29,9 @@ use codec::{Encode, Decode, Input, Codec}; use sp_runtime::{ConsensusEngineId, RuntimeDebug, traits::NumberFor}; use sp_std::borrow::Cow; use sp_std::vec::Vec; +#[cfg(feature = "std")] +use sp_core::traits::BareCryptoStorePtr; +use sp_std::convert::TryInto; #[cfg(feature = "std")] use log::debug; @@ -254,7 +257,7 @@ impl Equivocation { /// Verifies the equivocation proof by making sure that both votes target /// different blocks and that its signatures are valid. -pub fn check_equivocation_proof(report: EquivocationProof) -> Result<(), ()> +pub fn check_equivocation_proof(report: EquivocationProof) -> bool where H: Clone + Encode + PartialEq, N: Clone + Encode + PartialEq, @@ -267,27 +270,27 @@ where if $equivocation.first.0.target_hash == $equivocation.second.0.target_hash && $equivocation.first.0.target_number == $equivocation.second.0.target_number { - return Err(()); + return false; } // check signatures on both votes are valid - check_message_signature( + let valid_first = check_message_signature( &$message($equivocation.first.0), &$equivocation.identity, &$equivocation.first.1, $equivocation.round_number, report.set_id, - )?; + ); - check_message_signature( + let valid_second = check_message_signature( &$message($equivocation.second.0), &$equivocation.identity, &$equivocation.second.1, $equivocation.round_number, report.set_id, - )?; + ); - return Ok(()); + return valid_first && valid_second; }; } @@ -329,7 +332,7 @@ pub fn check_message_signature( signature: &AuthoritySignature, round: RoundNumber, set_id: SetId, -) -> Result<(), ()> +) -> bool where H: Encode, N: Encode, @@ -348,7 +351,7 @@ pub fn check_message_signature_with_buffer( round: RoundNumber, set_id: SetId, buf: &mut Vec, -) -> Result<(), ()> +) -> bool where H: Encode, N: Encode, @@ -357,38 +360,44 @@ where localized_payload_with_buffer(round, set_id, message, buf); - if id.verify(&buf, signature) { - Ok(()) - } else { + let valid = id.verify(&buf, signature); + + if !valid { #[cfg(feature = "std")] debug!(target: "afg", "Bad signature on message from {:?}", id); - - Err(()) } + + valid } /// Localizes the message to the given set and round and signs the payload. #[cfg(feature = "std")] pub fn sign_message( + keystore: &BareCryptoStorePtr, message: grandpa::Message, - pair: &AuthorityPair, + public: AuthorityId, round: RoundNumber, set_id: SetId, -) -> grandpa::SignedMessage +) -> Option> where H: Encode, N: Encode, { - use sp_core::Pair; + use sp_core::crypto::Public; + use sp_application_crypto::AppKey; let encoded = localized_payload(round, set_id, &message); - let signature = pair.sign(&encoded[..]); + let signature = keystore.read() + .sign_with(AuthorityId::ID, &public.to_public_crypto_pair(), &encoded[..]) + .ok()? + .try_into() + .ok()?; - grandpa::SignedMessage { + Some(grandpa::SignedMessage { message, signature, - id: pair.public(), - } + id: public, + }) } /// WASM function call to check for pending changes. diff --git a/primitives/finality-tracker/Cargo.toml b/primitives/finality-tracker/Cargo.toml index cf8045d0d3eb886d6313448bc61100e5a50b6039..5cbd497becc865f9d176e34623eb591b1107e0f4 100644 --- a/primitives/finality-tracker/Cargo.toml +++ b/primitives/finality-tracker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-finality-tracker" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,9 +12,9 @@ description = "FRAME module that tracks the last finalized block, as perceived b targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/inherents" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/inherents" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } [features] default = ["std"] diff --git a/primitives/inherents/Cargo.toml b/primitives/inherents/Cargo.toml index 49180b6192dfdd8252e6ae71ae2c9ef94a41bd52..3532e08da1d6a177d241e6a731a2384e93e9278d 100644 --- a/primitives/inherents/Cargo.toml +++ b/primitives/inherents/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-inherents" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -15,9 +15,9 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] parking_lot = { version = "0.10.0", optional = true } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../std" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../core" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../std" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../core" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } derive_more = { version = "0.99.2", optional = true } [features] diff --git a/primitives/io/Cargo.toml b/primitives/io/Cargo.toml index df7f6ea1daa45f4972294e1b125f4f1e0a8e5e34..06df2cc5ed6c441980b8bb1eacff635d5220dad0 100644 --- a/primitives/io/Cargo.toml +++ b/primitives/io/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-io" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -14,16 +14,17 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } hash-db = { version = "0.15.2", default-features = false } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../core" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../std" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../core" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../std" } libsecp256k1 = { version = "0.3.4", optional = true } -sp-state-machine = { version = "0.8.0-alpha.8", optional = true, path = "../../primitives/state-machine" } -sp-wasm-interface = { version = "2.0.0-alpha.8", path = "../../primitives/wasm-interface", default-features = false } -sp-runtime-interface = { version = "2.0.0-alpha.8", default-features = false, path = "../runtime-interface" } -sp-trie = { version = "2.0.0-alpha.8", optional = true, path = "../../primitives/trie" } -sp-externalities = { version = "0.8.0-alpha.8", optional = true, path = "../externalities" } +sp-state-machine = { version = "0.8.0-rc4", optional = true, path = "../../primitives/state-machine" } +sp-wasm-interface = { version = "2.0.0-rc4", path = "../../primitives/wasm-interface", default-features = false } +sp-runtime-interface = { version = "2.0.0-rc4", default-features = false, path = "../runtime-interface" } +sp-trie = { version = "2.0.0-rc4", optional = true, path = "../../primitives/trie" } +sp-externalities = { version = "0.8.0-rc4", optional = true, path = "../externalities" } +sp-tracing = { version = "2.0.0-rc4", default-features = false, path = "../tracing" } log = { version = "0.4.8", optional = true } futures = { version = "0.3.1", features = ["thread-pool"], optional = true } parking_lot = { version = "0.10.0", optional = true } diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 687e01060f5a3dd6ae047255e0792e5ada6b70bd..c75c8e67cc29a9b1e2df86c3cad2c5eb2f81ad4d 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -155,6 +155,46 @@ pub trait Storage { fn next_key(&mut self, key: &[u8]) -> Option> { self.next_storage_key(&key) } + + /// Start a new nested transaction. + /// + /// This allows to either commit or roll back all changes that are made after this call. + /// For every transaction there must be a matching call to either `rollback_transaction` + /// or `commit_transaction`. This is also effective for all values manipulated using the + /// `DefaultChildStorage` API. + /// + /// # Warning + /// + /// This is a low level API that is potentially dangerous as it can easily result + /// in unbalanced transactions. For example, FRAME users should use high level storage + /// abstractions. + fn start_transaction(&mut self) { + self.storage_start_transaction(); + } + + /// Rollback the last transaction started by `start_transaction`. + /// + /// Any changes made during that transaction are discarded. + /// + /// # Panics + /// + /// Will panic if there is no open transaction. + fn rollback_transaction(&mut self) { + self.storage_rollback_transaction() + .expect("No open transaction that can be rolled back."); + } + + /// Commit the last transaction started by `start_transaction`. + /// + /// Any changes made during that transaction are committed. + /// + /// # Panics + /// + /// Will panic if there is no open transaction. + fn commit_transaction(&mut self) { + self.storage_commit_transaction() + .expect("No open transaction that can be committed."); + } } /// Interface for accessing the child storage for default child trie, @@ -216,7 +256,7 @@ pub trait DefaultChildStorage { /// Clear a child storage key. /// /// For the default child storage at `storage_key`, clear value at `key`. - fn clear ( + fn clear( &mut self, storage_key: &[u8], key: &[u8], @@ -300,6 +340,16 @@ pub trait Trie { fn blake2_256_ordered_root(input: Vec>) -> H256 { Layout::::ordered_trie_root(input) } + + /// A trie root formed from the iterated items. + fn keccak_256_root(input: Vec<(Vec, Vec)>) -> H256 { + Layout::::trie_root(input) + } + + /// A trie root formed from the enumerated items. + fn keccak_256_ordered_root(input: Vec>) -> H256 { + Layout::::ordered_trie_root(input) + } } /// Interface that provides miscellaneous functions for communicating between the runtime and the node. @@ -786,6 +836,16 @@ pub trait Offchain { .local_storage_set(kind, key, value) } + /// Remove a value from the local storage. + /// + /// Note this storage is not part of the consensus, it's only accessible by + /// offchain worker tasks running on the same machine. It IS persisted between runs. + fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { + self.extension::() + .expect("local_storage_clear can be called only in the offchain worker context") + .local_storage_clear(kind, key) + } + /// Sets a value in the local storage if it matches current value. /// /// Since multiple offchain workers may be running concurrently, to prevent @@ -945,6 +1005,55 @@ pub trait Logging { } } +#[cfg(feature = "std")] +sp_externalities::decl_extension! { + /// Extension to allow running traces in wasm via Proxy + pub struct TracingProxyExt(sp_tracing::proxy::TracingProxy); +} + +/// Interface that provides functions for profiling the runtime. +#[runtime_interface] +pub trait WasmTracing { + /// To create and enter a `tracing` span, using `sp_tracing::proxy` + /// Returns 0 value to indicate that no further traces should be attempted + fn enter_span(&mut self, target: &str, name: &str) -> u64 { + if sp_tracing::wasm_tracing_enabled() { + match self.extension::() { + Some(proxy) => return proxy.enter_span(target, name), + None => { + if self.register_extension(TracingProxyExt(sp_tracing::proxy::TracingProxy::new())).is_ok() { + if let Some(proxy) = self.extension::() { + return proxy.enter_span(target, name); + } + } else { + log::warn!( + target: "tracing", + "Unable to register extension: TracingProxyExt" + ); + } + } + } + } + log::debug!( + target: "tracing", + "Notify to runtime that tracing is disabled." + ); + 0 + } + + /// Exit a `tracing` span, using `sp_tracing::proxy` + fn exit_span(&mut self, id: u64) { + if let Some(proxy) = self.extension::() { + proxy.exit_span(id) + } else { + log::warn!( + target: "tracing", + "Unable to load extension: TracingProxyExt" + ); + } + } +} + /// Wasm-only interface that provides functions for interacting with the sandbox. #[runtime_interface(wasm_only)] pub trait Sandbox { @@ -1091,6 +1200,7 @@ pub type SubstrateHostFunctions = ( storage::HostFunctions, default_child_storage::HostFunctions, misc::HostFunctions, + wasm_tracing::HostFunctions, offchain::HostFunctions, crypto::HostFunctions, hashing::HostFunctions, diff --git a/primitives/keyring/Cargo.toml b/primitives/keyring/Cargo.toml index 18e814f3113f49320fb5321a351459aa05ed04f1..abd7f3d3d548574ed7c998ac932303f0c4aecf0d 100644 --- a/primitives/keyring/Cargo.toml +++ b/primitives/keyring/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-keyring" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -14,7 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-core = { version = "2.0.0-alpha.8", path = "../core" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../runtime" } +sp-core = { version = "2.0.0-rc4", path = "../core" } +sp-runtime = { version = "2.0.0-rc4", path = "../runtime" } lazy_static = "1.4.0" strum = { version = "0.16.0", features = ["derive"] } diff --git a/primitives/phragmen/Cargo.toml b/primitives/npos-elections/Cargo.toml similarity index 52% rename from primitives/phragmen/Cargo.toml rename to primitives/npos-elections/Cargo.toml index 071b81a06b44f425c42671ebd2fdfd00a6ff5dd7..0a55a3e8954eef3b604aa9033f6b9f48cc8ba333 100644 --- a/primitives/phragmen/Cargo.toml +++ b/primitives/npos-elections/Cargo.toml @@ -1,12 +1,12 @@ [package] -name = "sp-phragmen" -version = "2.0.0-alpha.8" +name = "sp-npos-elections" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" -description = "Phragmen primitives" +description = "NPoS election algorithm primitives" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] @@ -14,15 +14,14 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../std" } -sp-phragmen-compact = { version = "2.0.0-alpha.8", path = "./compact" } -sp-arithmetic = { version = "2.0.0-alpha.8", default-features = false, path = "../arithmetic" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../std" } +sp-npos-elections-compact = { version = "2.0.0-rc4", path = "./compact" } +sp-arithmetic = { version = "2.0.0-rc4", default-features = false, path = "../arithmetic" } [dev-dependencies] -substrate-test-utils = { version = "2.0.0-alpha.8", path = "../../test-utils" } +substrate-test-utils = { version = "2.0.0-rc4", path = "../../test-utils" } rand = "0.7.3" -sp-phragmen = { version = "2.0.0-alpha.8", path = "." } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } [features] default = ["std"] diff --git a/primitives/phragmen/benches/phragmen.rs b/primitives/npos-elections/benches/phragmen.rs similarity index 91% rename from primitives/phragmen/benches/phragmen.rs rename to primitives/npos-elections/benches/phragmen.rs index c01d9f400d69d2162bf92c1b416a0d0cd4d6f463..e2385665bf0655ef422db11aa96baf05ffa4ed4c 100644 --- a/primitives/phragmen/benches/phragmen.rs +++ b/primitives/npos-elections/benches/phragmen.rs @@ -25,10 +25,15 @@ extern crate test; use test::Bencher; use rand::{self, Rng}; -use sp_phragmen::{PhragmenResult, VoteWeight}; +use sp_npos_elections::{ElectionResult, VoteWeight}; use std::collections::BTreeMap; -use sp_runtime::{Perbill, traits::Zero}; +use sp_runtime::{Perbill, PerThing, traits::Zero}; +use sp_npos_elections::{ + balance_solution, assignment_ratio_to_staked, build_support_map, to_without_backing, VoteWeight, + ExtendedBalance, Assignment, StakedAssignment, IdentifierT, assignment_ratio_to_staked, + seq_phragmen, +}; // default params. Each will be scaled by the benchmarks individually. const VALIDATORS: u64 = 100; @@ -42,13 +47,7 @@ const PREFIX: AccountId = 1000_000; type AccountId = u64; mod bench_closure_and_slice { - use sp_phragmen::{ - VoteWeight, ExtendedBalance, Assignment, StakedAssignment, IdentifierT, - assignment_ratio_to_staked, - }; - use sp_runtime::{Perbill, PerThing}; - use rand::{self, Rng, RngCore}; - use test::Bencher; + use super::*; fn random_assignment() -> Assignment { let mut rng = rand::thread_rng(); @@ -60,8 +59,8 @@ mod bench_closure_and_slice { } /// Converts a vector of ratio assignments into ones with absolute budget value. - pub fn assignment_ratio_to_staked_slice( - ratio: Vec>, + pub fn assignment_ratio_to_staked_slice( + ratio: Vec>, stakes: &[VoteWeight], ) -> Vec> where @@ -135,7 +134,7 @@ fn do_phragmen( }); b.iter(|| { - let PhragmenResult { winners, assignments } = sp_phragmen::elect::( + let ElectionResult { winners, assignments } = seq_phragmen::( to_elect, Zero::zero(), candidates.clone(), @@ -146,14 +145,13 @@ fn do_phragmen( *stake_of_tree.get(who).unwrap() }; - // Do the benchmarking with equalize. + // Do the benchmarking with balancing. if eq_iters > 0 { - use sp_phragmen::{equalize, assignment_ratio_to_staked, build_support_map, to_without_backing}; let staked = assignment_ratio_to_staked(assignments, &stake_of); let winners = to_without_backing(winners); let mut support = build_support_map(winners.as_ref(), staked.as_ref()).0; - equalize( + balance_solution( staked.into_iter().map(|a| (a.clone(), stake_of(&a.who))).collect(), &mut support, eq_tolerance, diff --git a/primitives/phragmen/compact/Cargo.toml b/primitives/npos-elections/compact/Cargo.toml similarity index 80% rename from primitives/phragmen/compact/Cargo.toml rename to primitives/npos-elections/compact/Cargo.toml index d73af49679d45655f1e2665f771f700a9eae4457..61d1990a3a5bcdc0d33b6fd99d60088b5f3b1a85 100644 --- a/primitives/phragmen/compact/Cargo.toml +++ b/primitives/npos-elections/compact/Cargo.toml @@ -1,12 +1,12 @@ [package] -name = "sp-phragmen-compact" -version = "2.0.0-alpha.8" +name = "sp-npos-elections-compact" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" -description = "Phragmen Compact Solution" +description = "NPoS Compact Solution Type" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/phragmen/compact/src/assignment.rs b/primitives/npos-elections/compact/src/assignment.rs similarity index 96% rename from primitives/phragmen/compact/src/assignment.rs rename to primitives/npos-elections/compact/src/assignment.rs index a48cbd937913b78bf470ed69616323c60ee7e248..96c68ece92a19acb028d238962dd029f6067d0c4 100644 --- a/primitives/phragmen/compact/src/assignment.rs +++ b/primitives/npos-elections/compact/src/assignment.rs @@ -18,8 +18,8 @@ //! Code generation for the ratio assignment type. use crate::field_name_for; -use proc_macro2::{TokenStream as TokenStream2}; -use syn::{GenericArgument}; +use proc_macro2::TokenStream as TokenStream2; +use syn::GenericArgument; use quote::quote; fn from_impl(count: usize) -> TokenStream2 { @@ -57,7 +57,13 @@ fn from_impl(count: usize) -> TokenStream2 { let last = quote!(index_of_target(&distribution[#last_index].0).ok_or(_phragmen::Error::CompactInvalidIndex)?); quote!( - #c => compact.#field_name.push((index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?, [#inner], #last)), + #c => compact.#field_name.push( + ( + index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?, + [#inner], + #last, + ) + ), ) }).collect::(); diff --git a/primitives/phragmen/compact/src/lib.rs b/primitives/npos-elections/compact/src/lib.rs similarity index 77% rename from primitives/phragmen/compact/src/lib.rs rename to primitives/npos-elections/compact/src/lib.rs index cbd8596a5903ad810a292842fc5211ae69b8c673..1b88ff653108157f5845a3773dffe77922efabdd 100644 --- a/primitives/phragmen/compact/src/lib.rs +++ b/primitives/npos-elections/compact/src/lib.rs @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Proc macro for phragmen compact assignment. +//! Proc macro for a npos compact assignment. use proc_macro::TokenStream; use proc_macro2::{TokenStream as TokenStream2, Span, Ident}; @@ -29,7 +29,7 @@ mod staked; // prefix used for struct fields in compact. const PREFIX: &'static str = "votes"; -/// Generates a struct to store the phragmen assignments in a compact way. The struct can only store +/// Generates a struct to store the election assignments in a compact way. The struct can only store /// distributions up to the given input count. The given count must be greater than 2. /// /// ```ignore @@ -158,8 +158,25 @@ fn struct_def( ) }).collect::(); + + let len_impl = (1..=count).map(|c| { + let field_name = field_name_for(c); + quote!( + all_len = all_len.saturating_add(self.#field_name.len()); + ) + }).collect::(); + + let edge_count_impl = (1..count).map(|c| { + let field_name = field_name_for(c); + quote!( + all_edges = all_edges.saturating_add( + self.#field_name.len().saturating_mul(#c as usize) + ); + ) + }).collect::(); + Ok(quote! ( - /// A struct to encode a Phragmen assignment in a compact way. + /// A struct to encode a election assignment in a compact way. #[derive( Default, PartialEq, @@ -181,21 +198,45 @@ fn struct_def( { const LIMIT: usize = #count; } + + impl<#voter_type, #target_type, #weight_type> #ident<#voter_type, #target_type, #weight_type> { + /// Get the length of all the assignments that this type is encoding. This is basically + /// the same as the number of assignments, or the number of voters in total. + pub fn len(&self) -> usize { + let mut all_len = 0usize; + #len_impl + all_len + } + + /// Get the total count of edges. + pub fn edge_count(&self) -> usize { + let mut all_edges = 0usize; + #edge_count_impl + all_edges + } + + /// Get the average edge count. + pub fn average_edge_count(&self) -> usize { + self.edge_count().checked_div(self.len()).unwrap_or(0) + } + } )) } fn imports() -> Result { - let sp_phragmen_imports = match crate_name("sp-phragmen") { - Ok(sp_phragmen) => { - let ident = syn::Ident::new(&sp_phragmen, Span::call_site()); - quote!( extern crate #ident as _phragmen; ) + if std::env::var("CARGO_PKG_NAME").unwrap() == "sp-npos-elections" { + Ok(quote! { + use crate as _phragmen; + }) + } else { + match crate_name("sp-npos-elections") { + Ok(sp_npos_elections) => { + let ident = syn::Ident::new(&sp_npos_elections, Span::call_site()); + Ok(quote!( extern crate #ident as _phragmen; )) + }, + Err(e) => Err(syn::Error::new(Span::call_site(), &e)), } - Err(e) => return Err(syn::Error::new(Span::call_site(), &e)), - }; - - Ok(quote!( - #sp_phragmen_imports - )) + } } struct CompactSolutionDef { diff --git a/primitives/phragmen/compact/src/staked.rs b/primitives/npos-elections/compact/src/staked.rs similarity index 97% rename from primitives/phragmen/compact/src/staked.rs rename to primitives/npos-elections/compact/src/staked.rs index cb1675219580a478f6bbdbb500d5f9cb6e990853..e2680e18b6326964531737ce8235dae0bde65570 100644 --- a/primitives/phragmen/compact/src/staked.rs +++ b/primitives/npos-elections/compact/src/staked.rs @@ -57,7 +57,9 @@ fn from_impl(count: usize) -> TokenStream2 { let last = quote!(index_of_target(&distribution[#last_index].0).ok_or(_phragmen::Error::CompactInvalidIndex)?); quote!( - #c => compact.#field_name.push((index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?, [#inner], #last)), + #c => compact.#field_name.push( + (index_of_voter(&who).ok_or(_phragmen::Error::CompactInvalidIndex)?, [#inner], #last) + ), ) }).collect::(); diff --git a/primitives/phragmen/fuzzer/.gitignore b/primitives/npos-elections/fuzzer/.gitignore similarity index 100% rename from primitives/phragmen/fuzzer/.gitignore rename to primitives/npos-elections/fuzzer/.gitignore diff --git a/primitives/phragmen/fuzzer/Cargo.lock b/primitives/npos-elections/fuzzer/Cargo.lock similarity index 99% rename from primitives/phragmen/fuzzer/Cargo.lock rename to primitives/npos-elections/fuzzer/Cargo.lock index 3ef2a2732424a3ddd412192dc01ac0647c36ffe1..cd172421aeb28f8cddde2902801dfd322aaddd2c 100644 --- a/primitives/phragmen/fuzzer/Cargo.lock +++ b/primitives/npos-elections/fuzzer/Cargo.lock @@ -540,7 +540,7 @@ dependencies = [ [[package]] name = "memory-db" -version = "0.19.0" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "198831fe8722331a395bc199a5d08efbc197497ef354cb4c77b969c02ffc0fc4" dependencies = [ @@ -1234,20 +1234,20 @@ dependencies = [ ] [[package]] -name = "sp-phragmen" +name = "sp-npos-elections" version = "2.0.0-alpha.3" dependencies = [ "parity-scale-codec", "serde", "sp-core", - "sp-phragmen-compact", + "sp-npos-elections-compact", "sp-runtime", "sp-std", ] [[package]] -name = "sp-phragmen-compact" -version = "2.0.0-dev" +name = "sp-npos-elections-compact" +version = "2.0.0-rc3" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1256,12 +1256,12 @@ dependencies = [ ] [[package]] -name = "sp-phragmen-fuzzer" +name = "sp-npos-elections-fuzzer" version = "2.0.0" dependencies = [ "honggfuzz", "rand 0.7.3", - "sp-phragmen", + "sp-npos-elections", ] [[package]] diff --git a/primitives/phragmen/fuzzer/Cargo.toml b/primitives/npos-elections/fuzzer/Cargo.toml similarity index 60% rename from primitives/phragmen/fuzzer/Cargo.toml rename to primitives/npos-elections/fuzzer/Cargo.toml index 5a6b90925f5a871d1858c0f252175a483efd7983..b7c7dcab65427b8f4b84754022a3a15f348bf350 100644 --- a/primitives/phragmen/fuzzer/Cargo.toml +++ b/primitives/npos-elections/fuzzer/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "sp-phragmen-fuzzer" +name = "sp-npos-elections-fuzzer" version = "2.0.0-alpha.5" authors = ["Parity Technologies "] edition = "2018" @@ -7,16 +7,16 @@ license = "Apache-2.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Fuzzer for phragmén implementation." -documentation = "https://docs.rs/sp-phragmen-fuzzer" +documentation = "https://docs.rs/sp-npos-elections-fuzzer" publish = false [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-phragmen = { version = "2.0.0-alpha.8", path = ".." } -sp-std = { version = "2.0.0-alpha.8", path = "../../std" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../runtime" } +sp-npos-elections = { version = "2.0.0-rc4", path = ".." } +sp-std = { version = "2.0.0-rc4", path = "../../std" } +sp-runtime = { version = "2.0.0-rc4", path = "../../runtime" } honggfuzz = "0.5" rand = { version = "0.7.3", features = ["std", "small_rng"] } @@ -25,5 +25,5 @@ name = "reduce" path = "src/reduce.rs" [[bin]] -name = "equalize" -path = "src/equalize.rs" +name = "balance_solution" +path = "src/balance_solution.rs" diff --git a/primitives/phragmen/fuzzer/src/equalize.rs b/primitives/npos-elections/fuzzer/src/balance_solution.rs similarity index 85% rename from primitives/phragmen/fuzzer/src/equalize.rs rename to primitives/npos-elections/fuzzer/src/balance_solution.rs index 8ca417f269886e0ba8f9510d35dbb29c4ffbc421..e1bd3bd0a07dd8d3b149309246737be13d8749e6 100644 --- a/primitives/phragmen/fuzzer/src/equalize.rs +++ b/primitives/npos-elections/fuzzer/src/balance_solution.rs @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Fuzzing fro the equalize algorithm +//! Fuzzing fro the balance_solution algorithm //! //! It ensures that any solution which gets equalized will lead into a better or equally scored //! one. @@ -23,9 +23,9 @@ mod common; use common::to_range; use honggfuzz::fuzz; -use sp_phragmen::{ - equalize, assignment_ratio_to_staked, build_support_map, to_without_backing, elect, - PhragmenResult, VoteWeight, evaluate_support, is_score_better, +use sp_npos_elections::{ + balance_solution, assignment_ratio_to_staked, build_support_map, to_without_backing, seq_phragmen, + ElectionResult, VoteWeight, evaluate_support, is_score_better, }; use sp_std::collections::btree_map::BTreeMap; use sp_runtime::Perbill; @@ -39,7 +39,7 @@ fn generate_random_phragmen_result( to_elect: usize, edge_per_voter: u64, mut rng: impl RngCore, -) -> (PhragmenResult, BTreeMap) { +) -> (ElectionResult, BTreeMap) { let prefix = 100_000; // Note, it is important that stakes are always bigger than ed and let base_stake: u64 = 1_000_000_000; @@ -73,7 +73,7 @@ fn generate_random_phragmen_result( }); ( - elect::( + seq_phragmen::( to_elect, 0, candidates, @@ -86,7 +86,14 @@ fn generate_random_phragmen_result( fn main() { loop { fuzz!(|data: (usize, usize, usize, usize, usize, u64)| { - let (mut target_count, mut voter_count, mut iterations, mut edge_per_voter, mut to_elect, seed) = data; + let ( + mut target_count, + mut voter_count, + mut iterations, + mut edge_per_voter, + mut to_elect, + seed, + ) = data; let rng = rand::rngs::SmallRng::seed_from_u64(seed); target_count = to_range(target_count, 50, 2000); voter_count = to_range(voter_count, 50, 1000); @@ -95,7 +102,7 @@ fn main() { edge_per_voter = to_range(edge_per_voter, 1, target_count); println!("++ [{} / {} / {} / {}]", voter_count, target_count, to_elect, iterations); - let (PhragmenResult { winners, assignments }, stake_of_tree) = generate_random_phragmen_result( + let (ElectionResult { winners, assignments }, stake_of_tree) = generate_random_phragmen_result( voter_count as u64, target_count as u64, to_elect, @@ -117,7 +124,7 @@ fn main() { return; } - let i = equalize( + let i = balance_solution( &mut staked, &mut support, 10, @@ -131,7 +138,7 @@ fn main() { return; } - let enhance = is_score_better(initial_score, final_score); + let enhance = is_score_better(final_score, initial_score, Perbill::zero()); println!( "iter = {} // {:?} -> {:?} [{}]", @@ -140,6 +147,7 @@ fn main() { final_score, enhance, ); + // if more than one iteration has been done, or they must be equal. assert!(enhance || initial_score == final_score || i == 0) }); diff --git a/primitives/phragmen/fuzzer/src/common.rs b/primitives/npos-elections/fuzzer/src/common.rs similarity index 93% rename from primitives/phragmen/fuzzer/src/common.rs rename to primitives/npos-elections/fuzzer/src/common.rs index f1aab214de6ba96a4de688109ebda1360137071a..89fed14cfaeabb55a9658245170105e2304511e8 100644 --- a/primitives/phragmen/fuzzer/src/common.rs +++ b/primitives/npos-elections/fuzzer/src/common.rs @@ -19,8 +19,8 @@ /// converts x into the range [a, b] in a pseudo-fair way. pub fn to_range(x: usize, a: usize, b: usize) -> usize { - // does not work correctly if b < 2*a - assert!(b > 2 * a); + // does not work correctly if b < 2 * a + assert!(b >= 2 * a); let collapsed = x % b; if collapsed >= a { collapsed diff --git a/primitives/phragmen/fuzzer/src/reduce.rs b/primitives/npos-elections/fuzzer/src/reduce.rs similarity index 98% rename from primitives/phragmen/fuzzer/src/reduce.rs rename to primitives/npos-elections/fuzzer/src/reduce.rs index 7ac15dd5443b2d12ca5586277ba7d9f6fcab661d..d08a440a6291ff18190090d16f0c43f5ffa24707 100644 --- a/primitives/phragmen/fuzzer/src/reduce.rs +++ b/primitives/npos-elections/fuzzer/src/reduce.rs @@ -34,7 +34,7 @@ use honggfuzz::fuzz; mod common; use common::to_range; -use sp_phragmen::{StakedAssignment, ExtendedBalance, build_support_map, reduce}; +use sp_npos_elections::{StakedAssignment, ExtendedBalance, build_support_map, reduce}; use rand::{self, Rng, SeedableRng, RngCore}; type Balance = u128; diff --git a/primitives/phragmen/src/helpers.rs b/primitives/npos-elections/src/helpers.rs similarity index 55% rename from primitives/phragmen/src/helpers.rs rename to primitives/npos-elections/src/helpers.rs index 2674bc445de375bd771d619103c18bf86ae1b4e6..063eac70c57fd77325da73c99581532977f315ee 100644 --- a/primitives/phragmen/src/helpers.rs +++ b/primitives/npos-elections/src/helpers.rs @@ -15,39 +15,74 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Helper methods for phragmen. +//! Helper methods for npos-elections. -use crate::{Assignment, ExtendedBalance, VoteWeight, IdentifierT, StakedAssignment, WithApprovalOf}; -use sp_arithmetic::PerThing; +use crate::{Assignment, ExtendedBalance, VoteWeight, IdentifierT, StakedAssignment, WithApprovalOf, Error}; +use sp_arithmetic::{PerThing, InnerOf}; use sp_std::prelude::*; /// Converts a vector of ratio assignments into ones with absolute budget value. -pub fn assignment_ratio_to_staked( - ratio: Vec>, +/// +/// Note that this will NOT attempt at normalizing the result. +pub fn assignment_ratio_to_staked( + ratio: Vec>, stake_of: FS, ) -> Vec> where for<'r> FS: Fn(&'r A) -> VoteWeight, - T: sp_std::ops::Mul, - ExtendedBalance: From<::Inner>, + P: sp_std::ops::Mul, + ExtendedBalance: From>, { ratio .into_iter() .map(|a| { let stake = stake_of(&a.who); - a.into_staked(stake.into(), true) + a.into_staked(stake.into()) }) .collect() } +/// Same as [`assignment_ratio_to_staked`] and try and do normalization. +pub fn assignment_ratio_to_staked_normalized( + ratio: Vec>, + stake_of: FS, +) -> Result>, Error> +where + for<'r> FS: Fn(&'r A) -> VoteWeight, + P: sp_std::ops::Mul, + ExtendedBalance: From>, +{ + let mut staked = assignment_ratio_to_staked(ratio, &stake_of); + staked.iter_mut().map(|a| + a.try_normalize(stake_of(&a.who).into()).map_err(|err| Error::ArithmeticError(err)) + ).collect::>()?; + Ok(staked) +} + /// Converts a vector of staked assignments into ones with ratio values. -pub fn assignment_staked_to_ratio( +/// +/// Note that this will NOT attempt at normalizing the result. +pub fn assignment_staked_to_ratio( + staked: Vec>, +) -> Vec> +where + ExtendedBalance: From>, +{ + staked.into_iter().map(|a| a.into_assignment()).collect() +} + +/// Same as [`assignment_staked_to_ratio`] and try and do normalization. +pub fn assignment_staked_to_ratio_normalized( staked: Vec>, -) -> Vec> +) -> Result>, Error> where - ExtendedBalance: From<::Inner>, + ExtendedBalance: From>, { - staked.into_iter().map(|a| a.into_assignment(true)).collect() + let mut ratio = staked.into_iter().map(|a| a.into_assignment()).collect::>(); + ratio.iter_mut().map(|a| + a.try_normalize().map_err(|err| Error::ArithmeticError(err)) + ).collect::>()?; + Ok(ratio) } /// consumes a vector of winners with backing stake to just winners. diff --git a/primitives/phragmen/src/lib.rs b/primitives/npos-elections/src/lib.rs similarity index 81% rename from primitives/phragmen/src/lib.rs rename to primitives/npos-elections/src/lib.rs index 03bbb279d8f987d92a37bd130c5c6113faa039df..592ed3b7173504ff8da810dd1c32289912f73c9f 100644 --- a/primitives/phragmen/src/lib.rs +++ b/primitives/npos-elections/src/lib.rs @@ -15,28 +15,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Rust implementation of the Phragmén election algorithm. This is used in several pallets to -//! optimally distribute the weight of a set of voters among an elected set of candidates. In the -//! context of staking this is mapped to validators and nominators. +//! A set of election algorithms to be used with a substrate runtime, typically within the staking +//! sub-system. Notable implementation include //! -//! The algorithm has two phases: -//! - Sequential phragmen: performed in [`elect`] function which is first pass of the distribution -//! The results are not optimal but the execution time is less. -//! - Equalize post-processing: tries to further distribute the weight fairly among candidates. -//! Incurs more execution time. +//! - [`seq_phragmen`]: Implements the Phragmén Sequential Method. An un-ranked, relatively fast +//! election method that ensures PJR, but does not provide a constant factor approximation of the +//! maximin problem. +//! - [`balance_solution`]: Implements the star balancing algorithm. This iterative process can +//! increase a solutions score, as described in [`evaluate_support`]. //! -//! The main objective of the assignments done by phragmen is to maximize the minimum backed -//! candidate in the elected set. -//! -//! Reference implementation: https://github.com/w3f/consensus -//! Further details: -//! https://research.web3.foundation/en/latest/polkadot/NPoS/4.%20Sequential%20Phragm%C3%A9n%E2%80%99s%20method/ +//! More information can be found at: https://arxiv.org/abs/2004.12990 #![cfg_attr(not(feature = "std"), no_std)] use sp_std::{prelude::*, collections::btree_map::BTreeMap, fmt::Debug, cmp::Ordering, convert::TryFrom}; use sp_arithmetic::{ - PerThing, Rational128, + PerThing, Rational128, ThresholdOrd, InnerOf, Normalizable, helpers_128bit::multiply_by_rational, traits::{Zero, Saturating, Bounded, SaturatedConversion}, }; @@ -67,7 +61,7 @@ pub use codec; pub use sp_arithmetic; // re-export the compact solution type. -pub use sp_phragmen_compact::generate_compact_solution_type; +pub use sp_npos_elections_compact::generate_compact_solution_type; /// A trait to limit the number of votes per voter. The generated compact type will implement this. pub trait VotingLimit { @@ -90,6 +84,8 @@ pub enum Error { CompactTargetOverflow, /// One of the index functions returned none. CompactInvalidIndex, + /// An error occurred in some arithmetic operation. + ArithmeticError(&'static str), } /// A type which is used in the API of this crate as a numeric weight of a vote, most often the @@ -100,7 +96,7 @@ pub type VoteWeight = u64; pub type ExtendedBalance = u128; /// The score of an assignment. This can be computed from the support map via [`evaluate_support`]. -pub type PhragmenScore = [ExtendedBalance; 3]; +pub type ElectionScore = [ExtendedBalance; 3]; /// A winner, with their respective approval stake. pub type WithApprovalOf = (A, ExtendedBalance); @@ -110,7 +106,7 @@ pub type WithApprovalOf = (A, ExtendedBalance); /// bigger than u64::max_value() is needed. For maximum accuracy we simply use u128; const DEN: u128 = u128::max_value(); -/// A candidate entity for phragmen election. +/// A candidate entity for the election. #[derive(Clone, Default, Debug)] struct Candidate { /// Identifier. @@ -147,9 +143,9 @@ struct Edge { candidate_index: usize, } -/// Final result of the phragmen election. +/// Final result of the election. #[derive(Debug)] -pub struct PhragmenResult { +pub struct ElectionResult { /// Just winners zipped with their approval stake. Note that the approval stake is merely the /// sub of their received stake and could be used for very basic sorting and approval voting. pub winners: Vec>, @@ -161,16 +157,16 @@ pub struct PhragmenResult { /// A voter's stake assignment among a set of targets, represented as ratios. #[derive(Debug, Clone, Default)] #[cfg_attr(feature = "std", derive(PartialEq, Eq, Encode, Decode))] -pub struct Assignment { +pub struct Assignment { /// Voter's identifier. pub who: AccountId, /// The distribution of the voter's stake. - pub distribution: Vec<(AccountId, T)>, + pub distribution: Vec<(AccountId, P)>, } -impl Assignment +impl Assignment where - ExtendedBalance: From<::Inner>, + ExtendedBalance: From>, { /// Convert from a ratio assignment into one with absolute values aka. [`StakedAssignment`]. /// @@ -179,50 +175,49 @@ where /// distribution's sum is exactly equal to the total budget, by adding or subtracting the /// remainder from the last distribution. /// - /// If an edge ratio is [`Bounded::max_value()`], it is dropped. This edge can never mean + /// If an edge ratio is [`Bounded::min_value()`], it is dropped. This edge can never mean /// anything useful. - pub fn into_staked(self, stake: ExtendedBalance, fill: bool) -> StakedAssignment + pub fn into_staked(self, stake: ExtendedBalance) -> StakedAssignment where - T: sp_std::ops::Mul, + P: sp_std::ops::Mul, { - let mut sum: ExtendedBalance = Bounded::min_value(); - let mut distribution = self - .distribution + let distribution = self.distribution .into_iter() .filter_map(|(target, p)| { // if this ratio is zero, then skip it. - if p == Bounded::min_value() { + if p.is_zero() { None } else { // NOTE: this mul impl will always round to the nearest number, so we might both // overflow and underflow. let distribution_stake = p * stake; - // defensive only. We assume that balance cannot exceed extended balance. - sum = sum.saturating_add(distribution_stake); Some((target, distribution_stake)) } }) .collect::>(); - if fill { - // NOTE: we can do this better. - // https://revs.runtime-revolution.com/getting-100-with-rounded-percentages-273ffa70252b - if let Some(leftover) = stake.checked_sub(sum) { - if let Some(last) = distribution.last_mut() { - last.1 = last.1.saturating_add(leftover); - } - } else if let Some(excess) = sum.checked_sub(stake) { - if let Some(last) = distribution.last_mut() { - last.1 = last.1.saturating_sub(excess); - } - } - } - StakedAssignment { who: self.who, distribution, } } + + /// Try and normalize this assignment. + /// + /// If `Ok(())` is returned, then the assignment MUST have been successfully normalized to 100%. + pub fn try_normalize(&mut self) -> Result<(), &'static str> { + self.distribution + .iter() + .map(|(_, p)| *p) + .collect::>() + .normalize(P::one()) + .map(|normalized_ratios| + self.distribution + .iter_mut() + .zip(normalized_ratios) + .for_each(|((_, old), corrected)| { *old = corrected; }) + ) + } } /// A voter's stake assignment among a set of targets, represented as absolute values in the scale @@ -249,42 +244,23 @@ impl StakedAssignment { /// /// If an edge stake is so small that it cannot be represented in `T`, it is ignored. This edge /// can never be re-created and does not mean anything useful anymore. - pub fn into_assignment(self, fill: bool) -> Assignment + pub fn into_assignment(self) -> Assignment where - ExtendedBalance: From<::Inner>, + ExtendedBalance: From>, + AccountId: IdentifierT, { - let accuracy: u128 = T::ACCURACY.saturated_into(); - let mut sum: u128 = Zero::zero(); - let stake = self.distribution.iter().map(|x| x.1).sum(); - let mut distribution = self - .distribution + let stake = self.total(); + let distribution = self.distribution .into_iter() .filter_map(|(target, w)| { - let per_thing = T::from_rational_approximation(w, stake); + let per_thing = P::from_rational_approximation(w, stake); if per_thing == Bounded::min_value() { None } else { - sum += per_thing.clone().deconstruct().saturated_into(); Some((target, per_thing)) } }) - .collect::>(); - - if fill { - if let Some(leftover) = accuracy.checked_sub(sum) { - if let Some(last) = distribution.last_mut() { - last.1 = last.1.saturating_add( - T::from_parts(leftover.saturated_into()) - ); - } - } else if let Some(excess) = sum.checked_sub(accuracy) { - if let Some(last) = distribution.last_mut() { - last.1 = last.1.saturating_sub( - T::from_parts(excess.saturated_into()) - ); - } - } - } + .collect::>(); Assignment { who: self.who, @@ -292,16 +268,40 @@ impl StakedAssignment { } } + /// Try and normalize this assignment. + /// + /// If `Ok(())` is returned, then the assignment MUST have been successfully normalized to + /// `stake`. + /// + /// NOTE: current implementation of `.normalize` is almost safe to `expect()` upon. The only + /// error case is when the input cannot fit in `T`, or the sum of input cannot fit in `T`. + /// Sadly, both of these are dependent upon the implementation of `VoteLimit`, i.e. the limit + /// of edges per voter which is enforced from upstream. Hence, at this crate, we prefer + /// returning a result and a use the name prefix `try_`. + pub fn try_normalize(&mut self, stake: ExtendedBalance) -> Result<(), &'static str> { + self.distribution + .iter() + .map(|(_, ref weight)| *weight) + .collect::>() + .normalize(stake) + .map(|normalized_weights| + self.distribution + .iter_mut() + .zip(normalized_weights.into_iter()) + .for_each(|((_, weight), corrected)| { *weight = corrected; }) + ) + } + /// Get the total stake of this assignment (aka voter budget). pub fn total(&self) -> ExtendedBalance { self.distribution.iter().fold(Zero::zero(), |a, b| a.saturating_add(b.1)) } } -/// A structure to demonstrate the phragmen result from the perspective of the candidate, i.e. how +/// A structure to demonstrate the election result from the perspective of the candidate, i.e. how /// much support each candidate is receiving. /// -/// This complements the [`PhragmenResult`] and is needed to run the equalize post-processing. +/// This complements the [`ElectionResult`] and is needed to run the balancing post-processing. /// /// This, at the current version, resembles the `Exposure` defined in the Staking pallet, yet /// they do not necessarily have to be the same. @@ -332,12 +332,12 @@ pub type SupportMap = BTreeMap>; /// responsibility of the caller to make sure only those candidates who have a sensible economic /// value are passed in. From the perspective of this function, a candidate can easily be among the /// winner with no backing stake. -pub fn elect( +pub fn seq_phragmen( candidate_count: usize, minimum_candidate_count: usize, initial_candidates: Vec, initial_voters: Vec<(AccountId, VoteWeight, Vec)>, -) -> Option> where +) -> Option> where AccountId: Default + Ord + Clone, R: PerThing, { @@ -388,7 +388,6 @@ pub fn elect( // we have already checked that we have more candidates than minimum_candidate_count. - // run phragmen. let to_elect = candidate_count.min(candidates.len()); elected_candidates = Vec::with_capacity(candidate_count); assigned = Vec::with_capacity(candidate_count); @@ -524,13 +523,13 @@ pub fn elect( } } - Some(PhragmenResult { + Some(ElectionResult { winners: elected_candidates, assignments: assigned, }) } -/// Build the support map from the given phragmen result. It maps a flat structure like +/// Build the support map from the given election result. It maps a flat structure like /// /// ```nocompile /// assignments: vec![ @@ -588,7 +587,7 @@ pub fn build_support_map( (supports, errors) } -/// Evaluate a phragmen result, given the support map. The returned tuple contains: +/// Evaluate a support map. The returned tuple contains: /// /// - Minimum support. This value must be **maximized**. /// - Sum of all supports. This value must be **maximized**. @@ -597,7 +596,7 @@ pub fn build_support_map( /// `O(E)` where `E` is the total number of edges. pub fn evaluate_support( support: &SupportMap, -) -> PhragmenScore { +) -> ElectionScore { let mut min_support = ExtendedBalance::max_value(); let mut sum: ExtendedBalance = Zero::zero(); // NOTE: The third element might saturate but fine for now since this will run on-chain and need @@ -614,38 +613,51 @@ pub fn evaluate_support( [min_support, sum, sum_squared] } -/// Compares two sets of phragmen scores based on desirability and returns true if `that` is -/// better than `this`. +/// Compares two sets of election scores based on desirability and returns true if `this` is +/// better than `that`. /// -/// Evaluation is done in a lexicographic manner. +/// Evaluation is done in a lexicographic manner, and if each element of `this` is `that * epsilon` +/// greater or less than `that`. /// /// Note that the third component should be minimized. -pub fn is_score_better(this: PhragmenScore, that: PhragmenScore) -> bool { - match that +pub fn is_score_better(this: ElectionScore, that: ElectionScore, epsilon: P) -> bool + where ExtendedBalance: From> +{ + match this .iter() .enumerate() - .map(|(i, e)| e.cmp(&this[i])) - .collect::>() + .map(|(i, e)| ( + e.ge(&that[i]), + e.tcmp(&that[i], epsilon.mul_ceil(that[i])), + )) + .collect::>() .as_slice() { - [Ordering::Greater, _, _] => true, - [Ordering::Equal, Ordering::Greater, _] => true, - [Ordering::Equal, Ordering::Equal, Ordering::Less] => true, + // epsilon better in the score[0], accept. + [(_, Ordering::Greater), _, _] => true, + + // less than epsilon better in score[0], but more than epsilon better in the second. + [(true, Ordering::Equal), (_, Ordering::Greater), _] => true, + + // less than epsilon better in score[0, 1], but more than epsilon better in the third + [(true, Ordering::Equal), (true, Ordering::Equal), (_, Ordering::Less)] => true, + + // anything else is not a good score. _ => false, } } -/// Performs equalize post-processing to the output of the election algorithm. This happens in +/// Performs balancing post-processing to the output of the election algorithm. This happens in /// rounds. The number of rounds and the maximum diff-per-round tolerance can be tuned through input /// parameters. /// /// Returns the number of iterations that were preformed. /// -/// - `assignments`: exactly the same is the output of phragmen. +/// - `assignments`: exactly the same as the output of [`seq_phragmen`]. /// - `supports`: mutable reference to s `SupportMap`. This parameter is updated. /// - `tolerance`: maximum difference that can occur before an early quite happens. /// - `iterations`: maximum number of iterations that will be processed. -pub fn equalize( +pub fn balance_solution( assignments: &mut Vec>, supports: &mut SupportMap, tolerance: ExtendedBalance, @@ -659,7 +671,7 @@ pub fn equalize( for assignment in assignments.iter_mut() { let voter_budget = assignment.total(); let StakedAssignment { who, distribution } = assignment; - let diff = do_equalize( + let diff = do_balancing( who, voter_budget, distribution, @@ -676,9 +688,9 @@ pub fn equalize( } } -/// actually perform equalize. same interface is `equalize`. Just called in loops with a check for +/// actually perform balancing. same interface is `balance_solution`. Just called in loops with a check for /// maximum difference. -fn do_equalize( +fn do_balancing( voter: &AccountId, budget: ExtendedBalance, elected_edges: &mut Vec<(AccountId, ExtendedBalance)>, diff --git a/primitives/phragmen/src/mock.rs b/primitives/npos-elections/src/mock.rs similarity index 98% rename from primitives/phragmen/src/mock.rs rename to primitives/npos-elections/src/mock.rs index fb801125158bc1dc0f6a19d965cc91eb20b0f561..b9c2396b08bb00e39d1851d1d03b2ac83f7fb357 100644 --- a/primitives/phragmen/src/mock.rs +++ b/primitives/npos-elections/src/mock.rs @@ -15,11 +15,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Mock file for phragmen. +//! Mock file for npos-elections. #![cfg(test)] -use crate::{elect, PhragmenResult, Assignment, VoteWeight, ExtendedBalance}; +use crate::{seq_phragmen, ElectionResult, Assignment, VoteWeight, ExtendedBalance}; use sp_arithmetic::{PerThing, traits::{SaturatedConversion, Zero, One}}; use sp_std::collections::btree_map::BTreeMap; use sp_runtime::assert_eq_error_rate; @@ -326,7 +326,7 @@ pub(crate) fn run_and_compare( min_to_elect: usize, ) { // run fixed point code. - let PhragmenResult { winners, assignments } = elect::<_, Output>( + let ElectionResult { winners, assignments } = seq_phragmen::<_, Output>( to_elect, min_to_elect, candidates.clone(), diff --git a/primitives/phragmen/src/node.rs b/primitives/npos-elections/src/node.rs similarity index 100% rename from primitives/phragmen/src/node.rs rename to primitives/npos-elections/src/node.rs diff --git a/primitives/phragmen/src/reduce.rs b/primitives/npos-elections/src/reduce.rs similarity index 99% rename from primitives/phragmen/src/reduce.rs rename to primitives/npos-elections/src/reduce.rs index 2878aa78c45d92c8cfce53117727021e94f2e91e..d0b4afe73dff9e1483dd803cec7a969a078e52d4 100644 --- a/primitives/phragmen/src/reduce.rs +++ b/primitives/npos-elections/src/reduce.rs @@ -640,8 +640,8 @@ fn reduce_all(assignments: &mut Vec>) -> u32 num_changed } -/// Reduce the given [`Vec>`]. This removes redundant edges from without changing the -/// overall backing of any of the elected candidates. +/// Reduce the given [`Vec>`]. This removes redundant edges from +/// without changing the overall backing of any of the elected candidates. /// /// Returns the number of edges removed. /// diff --git a/primitives/phragmen/src/tests.rs b/primitives/npos-elections/src/tests.rs similarity index 75% rename from primitives/phragmen/src/tests.rs rename to primitives/npos-elections/src/tests.rs index c7b43e8a3ca9449b6015140a99ff813518686743..80c742117d979168cfc825e22ffcccaaa31259de 100644 --- a/primitives/phragmen/src/tests.rs +++ b/primitives/npos-elections/src/tests.rs @@ -15,14 +15,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Tests for phragmen. - -#![cfg(test)] +//! Tests for npos-elections. use crate::mock::*; use crate::{ - elect, equalize, build_support_map, is_score_better, helpers::*, - Support, StakedAssignment, Assignment, PhragmenResult, ExtendedBalance, + seq_phragmen, balance_solution, build_support_map, is_score_better, helpers::*, + Support, StakedAssignment, Assignment, ElectionResult, ExtendedBalance, }; use substrate_test_utils::assert_eq_uvec; use sp_arithmetic::{Perbill, Permill, Percent, PerU16}; @@ -83,7 +81,7 @@ fn phragmen_poc_works() { ]; let stake_of = create_stake_of(&[(10, 10), (20, 20), (30, 30)]); - let PhragmenResult { winners, assignments } = elect::<_, Perbill>( + let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( 2, 2, candidates, @@ -146,7 +144,7 @@ fn phragmen_poc_works() { Support:: { total: 35, voters: vec![(20, 20), (30, 15)] }, ); - equalize( + balance_solution( &mut staked, &mut support_map, 0, @@ -240,11 +238,14 @@ fn phragmen_accuracy_on_large_scale_only_validators() { (5, (u64::max_value() - 2).into()), ]); - let PhragmenResult { winners, assignments } = elect::<_, Perbill>( + let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( 2, 2, candidates.clone(), - auto_generate_self_voters(&candidates).iter().map(|(ref v, ref vs)| (v.clone(), stake_of(v), vs.clone())).collect::>(), + auto_generate_self_voters(&candidates) + .iter() + .map(|(ref v, ref vs)| (v.clone(), stake_of(v), vs.clone())) + .collect::>(), ).unwrap(); assert_eq_uvec!(winners, vec![(1, 18446744073709551614u128), (5, 18446744073709551613u128)]); @@ -270,7 +271,7 @@ fn phragmen_accuracy_on_large_scale_validators_and_nominators() { (14, u64::max_value().into()), ]); - let PhragmenResult { winners, assignments } = elect::<_, Perbill>( + let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( 2, 2, candidates, @@ -313,7 +314,7 @@ fn phragmen_accuracy_on_small_scale_self_vote() { (30, 1), ]); - let PhragmenResult { winners, assignments: _ } = elect::<_, Perbill>( + let ElectionResult { winners, assignments: _ } = seq_phragmen::<_, Perbill>( 3, 3, candidates, @@ -343,7 +344,7 @@ fn phragmen_accuracy_on_small_scale_no_self_vote() { (3, 1), ]); - let PhragmenResult { winners, assignments: _ } = elect::<_, Perbill>( + let ElectionResult { winners, assignments: _ } = seq_phragmen::<_, Perbill>( 3, 3, candidates, @@ -376,7 +377,7 @@ fn phragmen_large_scale_test() { (50, 990000000000000000), ]); - let PhragmenResult { winners, assignments } = elect::<_, Perbill>( + let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( 2, 2, candidates, @@ -402,7 +403,7 @@ fn phragmen_large_scale_test_2() { (50, nom_budget.into()), ]); - let PhragmenResult { winners, assignments } = elect::<_, Perbill>( + let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( 2, 2, candidates, @@ -478,7 +479,7 @@ fn elect_has_no_entry_barrier() { (2, 10), ]); - let PhragmenResult { winners, assignments: _ } = elect::<_, Perbill>( + let ElectionResult { winners, assignments: _ } = seq_phragmen::<_, Perbill>( 3, 3, candidates, @@ -505,7 +506,7 @@ fn minimum_to_elect_is_respected() { (2, 10), ]); - let maybe_result = elect::<_, Perbill>( + let maybe_result = seq_phragmen::<_, Perbill>( 10, 10, candidates, @@ -531,7 +532,7 @@ fn self_votes_should_be_kept() { (1, 8), ]); - let result = elect::<_, Perbill>( + let result = seq_phragmen::<_, Perbill>( 2, 2, candidates, @@ -570,7 +571,7 @@ fn self_votes_should_be_kept() { &Support { total: 24u128, voters: vec![(20u64, 20u128), (1u64, 4u128)] }, ); - equalize( + balance_solution( &mut staked_assignments, &mut supports, 0, @@ -587,67 +588,286 @@ fn self_votes_should_be_kept() { ); } -#[test] -fn assignment_convert_works() { - let staked = StakedAssignment { - who: 1 as AccountId, - distribution: vec![ - (20, 100 as ExtendedBalance), - (30, 25), - ], - }; +mod assignment_convert_normalize { + use super::*; + #[test] + fn assignment_convert_works() { + let staked = StakedAssignment { + who: 1 as AccountId, + distribution: vec![ + (20, 100 as ExtendedBalance), + (30, 25), + ], + }; - let assignment = staked.clone().into_assignment(true); - assert_eq!( - assignment, - Assignment { + let assignment = staked.clone().into_assignment(); + assert_eq!( + assignment, + Assignment { + who: 1, + distribution: vec![ + (20, Perbill::from_percent(80)), + (30, Perbill::from_percent(20)), + ] + } + ); + + assert_eq!( + assignment.into_staked(125), + staked, + ); + } + + #[test] + fn assignment_convert_will_not_normalize() { + assert_eq!( + Assignment { + who: 1, + distribution: vec![ + (2, Perbill::from_percent(33)), + (3, Perbill::from_percent(66)), + ] + }.into_staked(100), + StakedAssignment { + who: 1, + distribution: vec![ + (2, 33), + (3, 66), + // sum is not 100! + ], + }, + ); + + assert_eq!( + StakedAssignment { + who: 1, + distribution: vec![ + (2, 333_333_333_333_333), + (3, 333_333_333_333_333), + (4, 666_666_666_666_333), + ], + }.into_assignment(), + Assignment { + who: 1, + distribution: vec![ + (2, Perbill::from_parts(250000000)), + (3, Perbill::from_parts(250000000)), + (4, Perbill::from_parts(499999999)), + // sum is not 100%! + ] + }, + ) + } + + #[test] + fn assignment_can_normalize() { + let mut a = Assignment { who: 1, distribution: vec![ - (20, Perbill::from_percent(80)), - (30, Perbill::from_percent(20)), + (2, Perbill::from_parts(330000000)), + (3, Perbill::from_parts(660000000)), + // sum is not 100%! ] - } - ); + }; + a.try_normalize().unwrap(); + assert_eq!( + a, + Assignment { + who: 1, + distribution: vec![ + (2, Perbill::from_parts(340000000)), + (3, Perbill::from_parts(660000000)), + ] + }, + ); + } - assert_eq!( - assignment.into_staked(125, true), - staked, - ); + #[test] + fn staked_assignment_can_normalize() { + let mut a = StakedAssignment { + who: 1, + distribution: vec![ + (2, 33), + (3, 66), + ] + }; + a.try_normalize(100).unwrap(); + assert_eq!( + a, + StakedAssignment { + who: 1, + distribution: vec![ + (2, 34), + (3, 66), + ] + }, + ); + } } -#[test] -fn score_comparison_is_lexicographical() { - // only better in the fist parameter, worse in the other two ✅ - assert_eq!( - is_score_better([10, 20, 30], [12, 10, 35]), - true, - ); +mod score { + use super::*; + #[test] + fn score_comparison_is_lexicographical_no_epsilon() { + let epsilon = Perbill::zero(); + // only better in the fist parameter, worse in the other two ✅ + assert_eq!( + is_score_better([12, 10, 35], [10, 20, 30], epsilon), + true, + ); - // worse in the first, better in the other two ❌ - assert_eq!( - is_score_better([10, 20, 30], [9, 30, 10]), - false, - ); + // worse in the first, better in the other two ❌ + assert_eq!( + is_score_better([9, 30, 10], [10, 20, 30], epsilon), + false, + ); - // equal in the first, the second one dictates. - assert_eq!( - is_score_better([10, 20, 30], [10, 25, 40]), - true, - ); + // equal in the first, the second one dictates. + assert_eq!( + is_score_better([10, 25, 40], [10, 20, 30], epsilon), + true, + ); - // equal in the first two, the last one dictates. - assert_eq!( - is_score_better([10, 20, 30], [10, 20, 40]), - false, - ); + // equal in the first two, the last one dictates. + assert_eq!( + is_score_better([10, 20, 40], [10, 20, 30], epsilon), + false, + ); + } + + #[test] + fn score_comparison_with_epsilon() { + let epsilon = Perbill::from_percent(1); + + { + // no more than 1 percent (10) better in the first param. + assert_eq!( + is_score_better([1009, 5000, 100000], [1000, 5000, 100000], epsilon), + false, + ); + + // now equal, still not better. + assert_eq!( + is_score_better([1010, 5000, 100000], [1000, 5000, 100000], epsilon), + false, + ); + + // now it is. + assert_eq!( + is_score_better([1011, 5000, 100000], [1000, 5000, 100000], epsilon), + true, + ); + } + + { + // First score score is epsilon better, but first score is no longer `ge`. Then this is + // still not a good solution. + assert_eq!( + is_score_better([999, 6000, 100000], [1000, 5000, 100000], epsilon), + false, + ); + } + + { + // first score is equal or better, but not epsilon. Then second one is the determinant. + assert_eq!( + is_score_better([1005, 5000, 100000], [1000, 5000, 100000], epsilon), + false, + ); + + assert_eq!( + is_score_better([1005, 5050, 100000], [1000, 5000, 100000], epsilon), + false, + ); + + assert_eq!( + is_score_better([1005, 5051, 100000], [1000, 5000, 100000], epsilon), + true, + ); + } + + { + // first score and second are equal or less than epsilon more, third is determinant. + assert_eq!( + is_score_better([1005, 5025, 100000], [1000, 5000, 100000], epsilon), + false, + ); + + assert_eq!( + is_score_better([1005, 5025, 99_000], [1000, 5000, 100000], epsilon), + false, + ); + + assert_eq!( + is_score_better([1005, 5025, 98_999], [1000, 5000, 100000], epsilon), + true, + ); + } + } + + #[test] + fn score_comparison_large_value() { + // some random value taken from eras in kusama. + let initial = [12488167277027543u128, 5559266368032409496, 118749283262079244270992278287436446]; + // this claim is 0.04090% better in the third component. It should be accepted as better if + // epsilon is smaller than 5/10_0000 + let claim = [12488167277027543u128, 5559266368032409496, 118700736389524721358337889258988054]; + + assert_eq!( + is_score_better( + claim.clone(), + initial.clone(), + Perbill::from_rational_approximation(1u32, 10_000), + ), + true, + ); + + assert_eq!( + is_score_better( + claim.clone(), + initial.clone(), + Perbill::from_rational_approximation(2u32, 10_000), + ), + true, + ); + + assert_eq!( + is_score_better( + claim.clone(), + initial.clone(), + Perbill::from_rational_approximation(3u32, 10_000), + ), + true, + ); + + assert_eq!( + is_score_better( + claim.clone(), + initial.clone(), + Perbill::from_rational_approximation(4u32, 10_000), + ), + true, + ); + + assert_eq!( + is_score_better( + claim.clone(), + initial.clone(), + Perbill::from_rational_approximation(5u32, 10_000), + ), + false, + ); + } } mod compact { use codec::{Decode, Encode}; - use crate::{generate_compact_solution_type, VoteWeight}; - use super::{AccountId}; - // these need to come from the same dev-dependency `sp-phragmen`, not from the crate. - use sp_phragmen::{Assignment, StakedAssignment, Error as PhragmenError, ExtendedBalance}; + use super::AccountId; + // these need to come from the same dev-dependency `sp-npos-elections`, not from the crate. + use crate::{ + generate_compact_solution_type, VoteWeight, Assignment, StakedAssignment, + Error as PhragmenError, ExtendedBalance, + }; use sp_std::{convert::{TryInto, TryFrom}, fmt::Debug}; use sp_arithmetic::Percent; @@ -672,6 +892,8 @@ mod compact { compact, Decode::decode(&mut &encoded[..]).unwrap(), ); + assert_eq!(compact.len(), 4); + assert_eq!(compact.edge_count(), 2 + 4); } fn basic_ratio_test_with() where @@ -747,6 +969,13 @@ mod compact { target_index, ).unwrap(); + // basically number of assignments that it is encoding. + assert_eq!(compacted.len(), assignments.len()); + assert_eq!( + compacted.edge_count(), + assignments.iter().fold(0, |a, b| a + b.distribution.len()), + ); + assert_eq!( compacted, TestCompact { @@ -844,6 +1073,11 @@ mod compact { voter_index, target_index, ).unwrap(); + assert_eq!(compacted.len(), assignments.len()); + assert_eq!( + compacted.edge_count(), + assignments.iter().fold(0, |a, b| a + b.distribution.len()), + ); assert_eq!( compacted, diff --git a/primitives/offchain/Cargo.toml b/primitives/offchain/Cargo.toml index ad2f1ee453b3022f273cc499526ba6e40ab41c51..44eb1bc0e10ce42206f9520fe82bc6bd6343a854 100644 --- a/primitives/offchain/Cargo.toml +++ b/primitives/offchain/Cargo.toml @@ -1,7 +1,7 @@ [package] description = "Substrate offchain workers primitives" name = "sp-offchain" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" license = "Apache-2.0" authors = ["Parity Technologies "] edition = "2018" @@ -12,12 +12,12 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../core" } -sp-api = { version = "2.0.0-alpha.8", default-features = false, path = "../api" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../runtime" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../core" } +sp-api = { version = "2.0.0-rc4", default-features = false, path = "../api" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../runtime" } [dev-dependencies] -sp-state-machine = { version = "0.8.0-alpha.8", default-features = false, path = "../state-machine" } +sp-state-machine = { version = "0.8.0-rc4", default-features = false, path = "../state-machine" } [features] default = ["std"] diff --git a/primitives/panic-handler/Cargo.toml b/primitives/panic-handler/Cargo.toml index 161214d801ef1785783204de85a33cfe5852f8bc..f350d317a031331aceb940a57c77a6ce661e7d5d 100644 --- a/primitives/panic-handler/Cargo.toml +++ b/primitives/panic-handler/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-panic-handler" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" diff --git a/primitives/rpc/Cargo.toml b/primitives/rpc/Cargo.toml index ee64741b602559148bda5f4c4f753a5ec5043447..86809803b4730195ae754775f5a0eed45ec091dd 100644 --- a/primitives/rpc/Cargo.toml +++ b/primitives/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-rpc" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,7 +13,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", features = ["derive"] } -sp-core = { version = "2.0.0-alpha.8", path = "../core" } +sp-core = { version = "2.0.0-rc4", path = "../core" } [dev-dependencies] serde_json = "1.0.41" diff --git a/primitives/rpc/src/number.rs b/primitives/rpc/src/number.rs index 63aa643fb6fca6e323842afe94ade5d3cffc9881..3d7e74753526c9a756cdd1a9b9f4d9f0d78a1b6d 100644 --- a/primitives/rpc/src/number.rs +++ b/primitives/rpc/src/number.rs @@ -15,65 +15,79 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Chain RPC Block number type. +//! A number type that can be serialized both as a number or a string that encodes a number in a +//! string. -use serde::{Serialize, Deserialize}; use std::{convert::TryFrom, fmt::Debug}; +use serde::{Serialize, Deserialize}; use sp_core::U256; -/// RPC Block number type +/// A number type that can be serialized both as a number or a string that encodes a number in a +/// string. +/// +/// We allow two representations of the block number as input. Either we deserialize to the type +/// that is specified in the block type or we attempt to parse given hex value. /// -/// We allow two representations of the block number as input. -/// Either we deserialize to the type that is specified in the block type -/// or we attempt to parse given hex value. -/// We do that for consistency with the returned type, default generic header -/// serializes block number as hex to avoid overflows in JavaScript. -#[derive(Serialize, Deserialize, Debug, PartialEq)] +/// The primary motivation for having this type is to avoid overflows when using big integers in +/// JavaScript (which we consider as an important RPC API consumer). +#[derive(Copy, Clone, Serialize, Deserialize, Debug, PartialEq)] #[serde(untagged)] -pub enum NumberOrHex { - /// The original header number type of block. - Number(Number), - /// Hex representation of the block number. +pub enum NumberOrHex { + /// The number represented directly. + Number(u64), + /// Hex representation of the number. Hex(U256), } -impl + From + Debug + PartialOrd> NumberOrHex { - /// Attempts to convert into concrete block number. - /// - /// Fails in case hex number is too big. - pub fn to_number(self) -> Result { - let num = match self { - NumberOrHex::Number(n) => n, - NumberOrHex::Hex(h) => { - let l = h.low_u64(); - if U256::from(l) != h { - return Err(format!("`{}` does not fit into u64 type; unsupported for now.", h)) - } else { - Number::try_from(l) - .map_err(|_| format!("`{}` does not fit into block number type.", h))? - } - }, - }; - // FIXME <2329>: Database seems to limit the block number to u32 for no reason - if num > Number::from(u32::max_value()) { - return Err(format!("`{:?}` > u32::max_value(), the max block number is u32.", num)) +impl NumberOrHex { + /// Converts this number into an U256. + pub fn into_u256(self) -> U256 { + match self { + NumberOrHex::Number(n) => n.into(), + NumberOrHex::Hex(h) => h, } - Ok(num) } } -impl From for NumberOrHex { +impl From for NumberOrHex { fn from(n: u64) -> Self { NumberOrHex::Number(n) } } -impl From for NumberOrHex { +impl From for NumberOrHex { fn from(n: U256) -> Self { NumberOrHex::Hex(n) } } +/// An error type that signals an out-of-range conversion attempt. +pub struct TryFromIntError(pub(crate) ()); + +impl TryFrom for u32 { + type Error = TryFromIntError; + fn try_from(num_or_hex: NumberOrHex) -> Result { + let num_or_hex = num_or_hex.into_u256(); + if num_or_hex > U256::from(u32::max_value()) { + return Err(TryFromIntError(())); + } else { + Ok(num_or_hex.as_u32()) + } + } +} + +impl TryFrom for u64 { + type Error = TryFromIntError; + fn try_from(num_or_hex: NumberOrHex) -> Result { + let num_or_hex = num_or_hex.into_u256(); + if num_or_hex > U256::from(u64::max_value()) { + return Err(TryFromIntError(())); + } else { + Ok(num_or_hex.as_u64()) + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -81,10 +95,11 @@ mod tests { #[test] fn should_serialize_and_deserialize() { - assert_deser(r#""0x1234""#, NumberOrHex::::Hex(0x1234.into())); - assert_deser(r#""0x0""#, NumberOrHex::::Hex(0.into())); - assert_deser(r#"5"#, NumberOrHex::Number(5_u64)); - assert_deser(r#"10000"#, NumberOrHex::Number(10000_u32)); - assert_deser(r#"0"#, NumberOrHex::Number(0_u16)); + assert_deser(r#""0x1234""#, NumberOrHex::Hex(0x1234.into())); + assert_deser(r#""0x0""#, NumberOrHex::Hex(0.into())); + assert_deser(r#"5"#, NumberOrHex::Number(5)); + assert_deser(r#"10000"#, NumberOrHex::Number(10000)); + assert_deser(r#"0"#, NumberOrHex::Number(0)); + assert_deser(r#"1000000000000"#, NumberOrHex::Number(1000000000000)); } } diff --git a/primitives/runtime-interface/Cargo.toml b/primitives/runtime-interface/Cargo.toml index 75bed6ad7b276a349e1b0cffd64aee0167ad5ade..dc37c186293d5319e4abd01bf65c80fbd5927ecf 100644 --- a/primitives/runtime-interface/Cargo.toml +++ b/primitives/runtime-interface/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-runtime-interface" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,20 +13,20 @@ documentation = "https://docs.rs/sp-runtime-interface/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-wasm-interface = { version = "2.0.0-alpha.8", path = "../wasm-interface", default-features = false } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../std" } -sp-tracing = { version = "2.0.0-alpha.8", default-features = false, path = "../tracing" } -sp-runtime-interface-proc-macro = { version = "2.0.0-alpha.8", path = "proc-macro" } -sp-externalities = { version = "0.8.0-alpha.8", optional = true, path = "../externalities" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } +sp-wasm-interface = { version = "2.0.0-rc4", path = "../wasm-interface", default-features = false } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../std" } +sp-tracing = { version = "2.0.0-rc4", default-features = false, path = "../tracing" } +sp-runtime-interface-proc-macro = { version = "2.0.0-rc4", path = "proc-macro" } +sp-externalities = { version = "0.8.0-rc4", optional = true, path = "../externalities" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } static_assertions = "1.0.0" primitive-types = { version = "0.7.0", default-features = false } [dev-dependencies] -sp-runtime-interface-test-wasm = { version = "2.0.0-alpha.8", path = "test-wasm" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../primitives/state-machine" } -sp-core = { version = "2.0.0-alpha.8", path = "../core" } -sp-io = { version = "2.0.0-alpha.8", path = "../io" } +sp-runtime-interface-test-wasm = { version = "2.0.0-rc4", path = "test-wasm" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../primitives/state-machine" } +sp-core = { version = "2.0.0-rc4", path = "../core" } +sp-io = { version = "2.0.0-rc4", path = "../io" } rustversion = "1.0.0" trybuild = "1.0.23" diff --git a/primitives/runtime-interface/proc-macro/Cargo.toml b/primitives/runtime-interface/proc-macro/Cargo.toml index d8f39ba645603aa9baffdc0053116012eacf06bb..dfb3840a08e87d4b1041ef388ffce2ca0693c74d 100644 --- a/primitives/runtime-interface/proc-macro/Cargo.toml +++ b/primitives/runtime-interface/proc-macro/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-runtime-interface-proc-macro" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" diff --git a/primitives/runtime-interface/src/impls.rs b/primitives/runtime-interface/src/impls.rs index 217316c3dd79c9f5060d07f2d636b6f11b013492..259d3517f001dcd415b452c09273baa2d2346589 100644 --- a/primitives/runtime-interface/src/impls.rs +++ b/primitives/runtime-interface/src/impls.rs @@ -365,6 +365,10 @@ impl PassBy for Option { type PassBy = Codec; } +impl PassBy for (u32, u32, u32, u32) { + type PassBy = Codec; +} + /// Implement `PassBy` with `Inner` for the given fixed sized hash types. macro_rules! for_primitive_types { { $( $hash:ident $n:expr ),* $(,)? } => { diff --git a/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml b/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml index 461010f11c44b984a0f26c9a4137c219382672fe..9ad22599ad08eef5aa962b51bf0cd2dad2596dbd 100644 --- a/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml +++ b/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-runtime-interface-test-wasm-deprecated" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" @@ -13,10 +13,10 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-runtime-interface = { version = "2.0.0-alpha.8", default-features = false, path = "../" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../io" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../core" } +sp-runtime-interface = { version = "2.0.0-rc4", default-features = false, path = "../" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../io" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../core" } [build-dependencies] wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } diff --git a/primitives/runtime-interface/test-wasm-deprecated/build.rs b/primitives/runtime-interface/test-wasm-deprecated/build.rs index cd5db582f385c8e0310a9dcbad07857df566539d..a4f323566006cf7c003cec2e413e1d23ecffead1 100644 --- a/primitives/runtime-interface/test-wasm-deprecated/build.rs +++ b/primitives/runtime-interface/test-wasm-deprecated/build.rs @@ -20,7 +20,7 @@ use wasm_builder_runner::WasmBuilder; fn main() { WasmBuilder::new() .with_current_project() - .with_wasm_builder_from_crates_or_path("1.0.9", "../../../utils/wasm-builder") + .with_wasm_builder_from_crates_or_path("1.0.11", "../../../utils/wasm-builder") .export_heap_base() .import_memory() .build() diff --git a/primitives/runtime-interface/test-wasm/Cargo.toml b/primitives/runtime-interface/test-wasm/Cargo.toml index 4d9dd1b0b4cf2386960f7e1130209685876e253f..7973f152bcb0c8c48b14362f9b90dec2676cc5ed 100644 --- a/primitives/runtime-interface/test-wasm/Cargo.toml +++ b/primitives/runtime-interface/test-wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-runtime-interface-test-wasm" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" @@ -13,10 +13,10 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-runtime-interface = { version = "2.0.0-alpha.8", default-features = false, path = "../" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../io" } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../core" } +sp-runtime-interface = { version = "2.0.0-rc4", default-features = false, path = "../" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../io" } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../core" } [build-dependencies] wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } diff --git a/primitives/runtime-interface/test-wasm/build.rs b/primitives/runtime-interface/test-wasm/build.rs index cd5db582f385c8e0310a9dcbad07857df566539d..a4f323566006cf7c003cec2e413e1d23ecffead1 100644 --- a/primitives/runtime-interface/test-wasm/build.rs +++ b/primitives/runtime-interface/test-wasm/build.rs @@ -20,7 +20,7 @@ use wasm_builder_runner::WasmBuilder; fn main() { WasmBuilder::new() .with_current_project() - .with_wasm_builder_from_crates_or_path("1.0.9", "../../../utils/wasm-builder") + .with_wasm_builder_from_crates_or_path("1.0.11", "../../../utils/wasm-builder") .export_heap_base() .import_memory() .build() diff --git a/primitives/runtime-interface/test/Cargo.toml b/primitives/runtime-interface/test/Cargo.toml index 6a964dce47c1adf48ec166ae3b13f7a6a797bd09..bdbe7ff902ee9760b1d04ebc3bbce0c232ed4d46 100644 --- a/primitives/runtime-interface/test/Cargo.toml +++ b/primitives/runtime-interface/test/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-runtime-interface-test" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,12 +12,12 @@ repository = "https://github.com/paritytech/substrate/" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-runtime-interface = { version = "2.0.0-alpha.8", path = "../" } -sc-executor = { version = "0.8.0-alpha.8", path = "../../../client/executor" } -sp-runtime-interface-test-wasm = { version = "2.0.0-alpha.8", path = "../test-wasm" } -sp-runtime-interface-test-wasm-deprecated = { version = "2.0.0-alpha.8", path = "../test-wasm-deprecated" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../../primitives/state-machine" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../runtime" } -sp-core = { version = "2.0.0-alpha.8", path = "../../core" } -sp-io = { version = "2.0.0-alpha.8", path = "../../io" } +sp-runtime-interface = { version = "2.0.0-rc4", path = "../" } +sc-executor = { version = "0.8.0-rc4", path = "../../../client/executor" } +sp-runtime-interface-test-wasm = { version = "2.0.0-rc4", path = "../test-wasm" } +sp-runtime-interface-test-wasm-deprecated = { version = "2.0.0-rc4", path = "../test-wasm-deprecated" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../../primitives/state-machine" } +sp-runtime = { version = "2.0.0-rc4", path = "../../runtime" } +sp-core = { version = "2.0.0-rc4", path = "../../core" } +sp-io = { version = "2.0.0-rc4", path = "../../io" } tracing = "0.1.13" diff --git a/primitives/runtime-interface/test/src/lib.rs b/primitives/runtime-interface/test/src/lib.rs index 06bc4e8ed8d46951f8e84c0291849062902f933e..109caab6062f8e7a6b5d49069366528b236c8c36 100644 --- a/primitives/runtime-interface/test/src/lib.rs +++ b/primitives/runtime-interface/test/src/lib.rs @@ -55,7 +55,6 @@ fn call_wasm_method_with_result( &mut ext_ext, sp_core::traits::MissingHostFunctions::Disallow, ).map_err(|e| format!("Failed to execute `{}`: {}", method, e))?; - Ok(ext) } diff --git a/primitives/runtime/Cargo.toml b/primitives/runtime/Cargo.toml index dad1691fcf6bf7a0d39ce940f458ebba525ccf68..9bc972646f93e2eee47b06cc1be9e9fe63569950 100644 --- a/primitives/runtime/Cargo.toml +++ b/primitives/runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-runtime" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -15,24 +15,25 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../core" } -sp-application-crypto = { version = "2.0.0-alpha.8", default-features = false, path = "../application-crypto" } -sp-arithmetic = { version = "2.0.0-alpha.8", default-features = false, path = "../arithmetic" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../std" } -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../io" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../core" } +sp-application-crypto = { version = "2.0.0-rc4", default-features = false, path = "../application-crypto" } +sp-arithmetic = { version = "2.0.0-rc4", default-features = false, path = "../arithmetic" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../std" } +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../io" } log = { version = "0.4.8", optional = true } paste = "0.1.6" rand = { version = "0.7.2", optional = true } impl-trait-for-tuples = "0.1.3" -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../inherents" } +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../inherents" } parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } hash256-std-hasher = { version = "0.15.2", default-features = false } +either = { version = "1.5", default-features = false } [dev-dependencies] serde_json = "1.0.41" rand = "0.7.2" -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../primitives/state-machine" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../primitives/state-machine" } [features] bench = [] @@ -51,4 +52,5 @@ std = [ "sp-inherents/std", "parity-util-mem/std", "hash256-std-hasher/std", + "either/use_std", ] diff --git a/primitives/runtime/src/curve.rs b/primitives/runtime/src/curve.rs index be47b566e93c0c27e91826abb4f2af1a5af604bc..27eb89a76947e039b396870ae3a3e2dc772f6b57 100644 --- a/primitives/runtime/src/curve.rs +++ b/primitives/runtime/src/curve.rs @@ -17,7 +17,7 @@ //! Provides some utilities to define a piecewise linear function. -use crate::{Perbill, traits::{AtLeast32Bit, SaturatedConversion}}; +use crate::{Perbill, traits::{AtLeast32BitUnsigned, SaturatedConversion}}; use core::ops::Sub; /// Piecewise Linear function in [0, 1] -> [0, 1]. @@ -36,7 +36,7 @@ fn abs_sub + Clone>(a: N, b: N) -> N where { impl<'a> PiecewiseLinear<'a> { /// Compute `f(n/d)*d` with `n <= d`. This is useful to avoid loss of precision. pub fn calculate_for_fraction_times_denominator(&self, n: N, d: N) -> N where - N: AtLeast32Bit + Clone + N: AtLeast32BitUnsigned + Clone { let n = n.min(d.clone()); @@ -80,7 +80,7 @@ impl<'a> PiecewiseLinear<'a> { // This is guaranteed not to overflow on whatever values nor lose precision. // `q` must be superior to zero. fn multiply_by_rational_saturating(value: N, p: u32, q: u32) -> N - where N: AtLeast32Bit + Clone + where N: AtLeast32BitUnsigned + Clone { let q = q.max(1); diff --git a/primitives/runtime/src/generic/checked_extrinsic.rs b/primitives/runtime/src/generic/checked_extrinsic.rs index 5e4150cf2b8dc4a37e92d7be5db501732d9ae468..f355308a59f9704d7f7343dcd904d8458ad63184 100644 --- a/primitives/runtime/src/generic/checked_extrinsic.rs +++ b/primitives/runtime/src/generic/checked_extrinsic.rs @@ -19,7 +19,8 @@ //! stage. use crate::traits::{ - self, Member, MaybeDisplay, SignedExtension, Dispatchable, DispatchInfoOf, ValidateUnsigned, + self, Member, MaybeDisplay, SignedExtension, Dispatchable, DispatchInfoOf, PostDispatchInfoOf, + ValidateUnsigned, }; use crate::transaction_validity::{TransactionValidity, TransactionSource}; @@ -67,7 +68,7 @@ where self, info: &DispatchInfoOf, len: usize, - ) -> crate::ApplyExtrinsicResult { + ) -> crate::ApplyExtrinsicResultWithInfo> { let (maybe_who, pre) = if let Some((id, extra)) = self.signed { let pre = Extra::pre_dispatch(extra, &id, &self.function, info, len)?; (Some(id), pre) @@ -81,8 +82,7 @@ where Ok(info) => info, Err(err) => err.post_info, }; - let res = res.map(|_| ()).map_err(|e| e.error); - Extra::post_dispatch(pre, info, &post_info, len, &res)?; + Extra::post_dispatch(pre, info, &post_info, len, &res.map(|_| ()).map_err(|e| e.error))?; Ok(res) } } diff --git a/primitives/runtime/src/generic/header.rs b/primitives/runtime/src/generic/header.rs index 24cceef2cdc331ca81831457e8cd757e85c616d7..e6c800e5787ff2e7e0f7efb77efffacbc2be9c7e 100644 --- a/primitives/runtime/src/generic/header.rs +++ b/primitives/runtime/src/generic/header.rs @@ -21,7 +21,7 @@ use serde::{Deserialize, Serialize}; use crate::codec::{Decode, Encode, Codec, Input, Output, HasCompact, EncodeAsRef, Error}; use crate::traits::{ - self, Member, AtLeast32Bit, SimpleBitOps, Hash as HashT, + self, Member, AtLeast32BitUnsigned, SimpleBitOps, Hash as HashT, MaybeSerializeDeserialize, MaybeSerialize, MaybeDisplay, MaybeMallocSizeOf, }; @@ -123,7 +123,7 @@ impl codec::EncodeLike for Header where impl traits::Header for Header where Number: Member + MaybeSerializeDeserialize + Debug + sp_std::hash::Hash + MaybeDisplay + - AtLeast32Bit + Codec + Copy + Into + TryFrom + sp_std::str::FromStr + + AtLeast32BitUnsigned + Codec + Copy + Into + TryFrom + sp_std::str::FromStr + MaybeMallocSizeOf, Hash: HashT, Hash::Output: Default + sp_std::hash::Hash + Copy + Member + Ord + @@ -171,7 +171,8 @@ impl traits::Header for Header where } impl Header where - Number: Member + sp_std::hash::Hash + Copy + MaybeDisplay + AtLeast32Bit + Codec + Into + TryFrom, + Number: Member + sp_std::hash::Hash + Copy + MaybeDisplay + AtLeast32BitUnsigned + Codec + + Into + TryFrom, Hash: HashT, Hash::Output: Default + sp_std::hash::Hash + Copy + Member + MaybeDisplay + SimpleBitOps + Codec, { diff --git a/primitives/runtime/src/generic/unchecked_extrinsic.rs b/primitives/runtime/src/generic/unchecked_extrinsic.rs index 41ff2609fc8ccc1939dca2a23445052927a61a3a..d16d404ddfd0e9207771b606d224d54df7288998 100644 --- a/primitives/runtime/src/generic/unchecked_extrinsic.rs +++ b/primitives/runtime/src/generic/unchecked_extrinsic.rs @@ -27,6 +27,7 @@ use crate::{ }, generic::CheckedExtrinsic, transaction_validity::{TransactionValidityError, InvalidTransaction}, + OpaqueExtrinsic, }; const TRANSACTION_VERSION: u8 = 4; @@ -316,6 +317,23 @@ where } } +impl From> + for OpaqueExtrinsic +where + Address: Encode, + Signature: Encode, + Call: Encode, + Extra: SignedExtension, +{ + fn from(extrinsic: UncheckedExtrinsic) -> Self { + OpaqueExtrinsic::from_bytes(extrinsic.encode().as_slice()) + .expect( + "both OpaqueExtrinsic and UncheckedExtrinsic have encoding that is compatible with \ + raw Vec encoding; qed" + ) + } +} + #[cfg(test)] mod tests { use super::*; @@ -424,4 +442,13 @@ mod tests { let as_vec: Vec = Decode::decode(&mut encoded.as_slice()).unwrap(); assert_eq!(as_vec.encode(), encoded); } + + #[test] + fn conversion_to_opaque() { + let ux = Ex::new_unsigned(vec![0u8; 0]); + let encoded = ux.encode(); + let opaque: OpaqueExtrinsic = ux.into(); + let opaque_encoded = opaque.encode(); + assert_eq!(opaque_encoded, encoded); + } } diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 361da3ee5749a7035773aad8e96dbbe86690f563..b27cb0c63351cb8bbab7069b03b68fabde998a3f 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -71,8 +71,8 @@ pub use sp_core::RuntimeDebug; /// Re-export top-level arithmetic stuff. pub use sp_arithmetic::{ - Perquintill, Perbill, Permill, Percent, PerU16, Rational128, Fixed64, Fixed128, - PerThing, traits::SaturatedConversion, + PerThing, traits::SaturatedConversion, Perquintill, Perbill, Permill, Percent, PerU16, InnerOf, + Rational128, FixedI64, FixedI128, FixedU128, FixedPointNumber, FixedPointOperand, }; /// Re-export 128 bit helpers. pub use sp_arithmetic::helpers_128bit; @@ -81,6 +81,8 @@ pub use sp_arithmetic::biguint; pub use random_number_generator::RandomNumberGenerator; +pub use either::Either; + /// An abstraction over justification for a block's validity under a consensus algorithm. /// /// Essentially a finality proof. The exact formulation will vary between consensus @@ -536,6 +538,10 @@ pub type DispatchOutcome = Result<(), DispatchError>; /// - The extrinsic supplied a bad signature. This transaction won't become valid ever. pub type ApplyExtrinsicResult = Result; +/// Same as `ApplyExtrinsicResult` but augmented with `PostDispatchInfo` on success. +pub type ApplyExtrinsicResultWithInfo = + Result, transaction_validity::TransactionValidityError>; + /// Verify a signature on an encoded value in a lazy manner. This can be /// an optimization if the signature scheme has an "unsigned" escape hash. pub fn verify_encoded_lazy( @@ -708,7 +714,14 @@ macro_rules! assert_eq_error_rate { /// Simple blob to hold an extrinsic without committing to its format and ensure it is serialized /// correctly. #[derive(PartialEq, Eq, Clone, Default, Encode, Decode)] -pub struct OpaqueExtrinsic(pub Vec); +pub struct OpaqueExtrinsic(Vec); + +impl OpaqueExtrinsic { + /// Convert an encoded extrinsic to an `OpaqueExtrinsic`. + pub fn from_bytes(mut bytes: &[u8]) -> Result { + OpaqueExtrinsic::decode(&mut bytes) + } +} #[cfg(feature = "std")] impl parity_util_mem::MallocSizeOf for OpaqueExtrinsic { diff --git a/primitives/runtime/src/offchain/mod.rs b/primitives/runtime/src/offchain/mod.rs index 427b54468f48df5284f4c193cf16b3ca8f697df4..fe5844ce3004b62286968cf829e4b71793515446 100644 --- a/primitives/runtime/src/offchain/mod.rs +++ b/primitives/runtime/src/offchain/mod.rs @@ -19,5 +19,6 @@ pub mod http; pub mod storage; +pub mod storage_lock; pub use sp_core::offchain::*; diff --git a/primitives/runtime/src/offchain/storage.rs b/primitives/runtime/src/offchain/storage.rs index f8dcd73fa2bc609bffcfbd3cf6ba308d0a38b69b..2f62d400c0b950b326f70edae145de259eb16e7d 100644 --- a/primitives/runtime/src/offchain/storage.rs +++ b/primitives/runtime/src/offchain/storage.rs @@ -50,6 +50,11 @@ impl<'a> StorageValueRef<'a> { }) } + /// Remove the associated value from the storage. + pub fn clear(&mut self) { + sp_io::offchain::local_storage_clear(self.kind, self.key) + } + /// Retrieve & decode the value from storage. /// /// Note that if you want to do some checks based on the value @@ -67,7 +72,8 @@ impl<'a> StorageValueRef<'a> { /// Function `f` should return a new value that we should attempt to write to storage. /// This function returns: /// 1. `Ok(Ok(T))` in case the value has been successfully set. - /// 2. `Ok(Err(T))` in case the value was returned, but it couldn't have been set. + /// 2. `Ok(Err(T))` in case the value was calculated by the passed closure `f`, + /// but it could not be stored. /// 3. `Err(_)` in case `f` returns an error. pub fn mutate(&self, f: F) -> Result, E> where T: codec::Codec, diff --git a/primitives/runtime/src/offchain/storage_lock.rs b/primitives/runtime/src/offchain/storage_lock.rs new file mode 100644 index 0000000000000000000000000000000000000000..9d4e671db6ea18ac90a1c8f90194ccf8c4d611dd --- /dev/null +++ b/primitives/runtime/src/offchain/storage_lock.rs @@ -0,0 +1,593 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! # Off-chain Storage Lock +//! +//! A storage-based lock with a defined expiry time. +//! +//! The lock is using Local Storage and allows synchronizing access to critical +//! section of your code for concurrently running Off-chain Workers. Usage of +//! `PERSISTENT` variant of the storage persists the lock value across a full node +//! restart or re-orgs. +//! +//! A use case for the lock is to make sure that a particular section of the +//! code is only run by one Off-chain Worker at a time. This may include +//! performing a side-effect (i.e. an HTTP call) or alteration of single or +//! multiple Local Storage entries. +//! +//! One use case would be collective updates of multiple data items or append / +//! remove of i.e. sets, vectors which are stored in the off-chain storage DB. +//! +//! ## Example: +//! +//! ```rust +//! # use codec::{Decode, Encode, Codec}; +//! // in your off-chain worker code +//! use sp_runtime::offchain::{ +//! storage::StorageValueRef, +//! storage_lock::{StorageLock, Time}, +//! }; +//! +//! fn append_to_in_storage_vec<'a, T>(key: &'a [u8], _: T) where T: Codec { +//! // `access::lock` defines the storage entry which is used for +//! // persisting the lock in the underlying database. +//! // The entry name _must_ be unique and can be interpreted as a +//! // unique mutex instance reference tag. +//! let mut lock = StorageLock::

= <

::Block as BlockT>::Hash; pub type TransactionFor

= <

::Block as BlockT>::Extrinsic; /// Type of transactions event stream for a pool. pub type TransactionStatusStreamFor

= TransactionStatusStream, BlockHash

>; +/// Transaction type for a local pool. +pub type LocalTransactionFor

= <

::Block as BlockT>::Extrinsic; /// Typical future type used in transaction pool api. pub type PoolFuture = std::pin::Pin> + Send>>; @@ -162,7 +164,7 @@ pub trait InPoolTransaction { /// Get priority of the transaction. fn priority(&self) -> &TransactionPriority; /// Get longevity of the transaction. - fn longevity(&self) ->&TransactionLongevity; + fn longevity(&self) -> &TransactionLongevity; /// Get transaction dependencies. fn requires(&self) -> &[TransactionTag]; /// Get tags that transaction provides. @@ -251,12 +253,14 @@ pub enum ChainEvent { NewBlock { /// Is this the new best block. is_new_best: bool, - /// Id of the just imported block. - id: BlockId, + /// Hash of the block. + hash: B::Hash, /// Header of the just imported block header: B::Header, - /// List of retracted blocks ordered by block number. - retracted: Vec, + /// Tree route from old best to new best parent that was calculated on import. + /// + /// If `None`, no re-org happened on import. + tree_route: Option>>, }, /// An existing block has been finalized. Finalized { @@ -271,6 +275,28 @@ pub trait MaintainedTransactionPool: TransactionPool { fn maintain(&self, event: ChainEvent) -> Pin + Send>>; } +/// Transaction pool interface for submitting local transactions that exposes a +/// blocking interface for submission. +pub trait LocalTransactionPool: Send + Sync { + /// Block type. + type Block: BlockT; + /// Transaction hash type. + type Hash: Hash + Eq + Member + Serialize; + /// Error type. + type Error: From + crate::error::IntoPoolError; + + /// Submits the given local unverified transaction to the pool blocking the + /// current thread for any necessary pre-verification. + /// NOTE: It MUST NOT be used for transactions that originate from the + /// network or RPC, since the validation is performed with + /// `TransactionSource::Local`. + fn submit_local( + &self, + at: &BlockId, + xt: LocalTransactionFor, + ) -> Result; +} + /// An abstraction for transaction pool. /// /// This trait is used by offchain calls to be able to submit transactions. @@ -289,7 +315,7 @@ pub trait OffchainSubmitTransaction: Send + Sync { ) -> Result<(), ()>; } -impl OffchainSubmitTransaction for TPool { +impl OffchainSubmitTransaction for TPool { fn submit_at( &self, at: &BlockId, @@ -301,15 +327,14 @@ impl OffchainSubmitTransaction for TPool { extrinsic ); - let result = futures::executor::block_on(self.submit_one( - &at, TransactionSource::Local, extrinsic, - )); + let result = self.submit_local(&at, extrinsic); - result.map(|_| ()) - .map_err(|e| log::warn!( + result.map(|_| ()).map_err(|e| { + log::warn!( target: "txpool", "(offchain call) Error submitting a transaction to the pool: {:?}", e - )) + ) + }) } } diff --git a/primitives/trie/Cargo.toml b/primitives/trie/Cargo.toml index ca459f30572f38c4e46e0f7d91cf8a590daaea5d..1ebc974bfb19f8d984ae4116be4fb03b3930abf0 100644 --- a/primitives/trie/Cargo.toml +++ b/primitives/trie/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-trie" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] description = "Patricia trie stuff using a parity-scale-codec node format" repository = "https://github.com/paritytech/substrate/" @@ -17,20 +17,20 @@ name = "bench" harness = false [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../std" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../std" } hash-db = { version = "0.15.2", default-features = false } -trie-db = { version = "0.20.1", default-features = false } +trie-db = { version = "0.21.0", default-features = false } trie-root = { version = "0.16.0", default-features = false } -memory-db = { version = "0.20.0", default-features = false } -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../core" } +memory-db = { version = "0.21.0", default-features = false } +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../core" } [dev-dependencies] -trie-bench = "0.21.0" +trie-bench = "0.22.0" trie-standardmap = "0.15.2" criterion = "0.2.11" hex-literal = "0.2.1" -sp-runtime = { version = "2.0.0-alpha.8", path = "../runtime" } +sp-runtime = { version = "2.0.0-rc4", path = "../runtime" } [features] default = ["std"] diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index f328b71750ba42a2252b19f71375f2db10d34076..7d1879a4f9f749a4bd3521cb72cc36641c7082ff 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -24,9 +24,7 @@ mod node_codec; mod storage_proof; mod trie_stream; -use sp_std::boxed::Box; -use sp_std::marker::PhantomData; -use sp_std::vec::Vec; +use sp_std::{boxed::Box, marker::PhantomData, vec::Vec, borrow::Borrow}; use hash_db::{Hasher, Prefix}; use trie_db::proof::{generate_proof, verify_proof}; pub use trie_db::proof::VerifyError; @@ -53,6 +51,7 @@ pub struct Layout(sp_std::marker::PhantomData); impl TrieLayout for Layout { const USE_EXTENSION: bool = false; + const ALLOW_EMPTY: bool = true; type Hash = H; type Codec = NodeCodec; } @@ -162,23 +161,24 @@ pub fn verify_trie_proof<'a, L: TrieConfiguration, I, K, V>( } /// Determine a trie root given a hash DB and delta values. -pub fn delta_trie_root( +pub fn delta_trie_root( db: &mut DB, mut root: TrieHash, delta: I ) -> Result, Box>> where - I: IntoIterator)>, - A: AsRef<[u8]> + Ord, - B: AsRef<[u8]>, + I: IntoIterator, + A: Borrow<[u8]>, + B: Borrow>, + V: Borrow<[u8]>, DB: hash_db::HashDB, { { let mut trie = TrieDBMut::::from_existing(&mut *db, &mut root)?; for (key, change) in delta { - match change { - Some(val) => trie.insert(key.as_ref(), val.as_ref())?, - None => trie.remove(key.as_ref())?, + match change.borrow() { + Some(val) => trie.insert(key.borrow(), val.borrow())?, + None => trie.remove(key.borrow())?, }; } } @@ -230,16 +230,17 @@ pub fn child_trie_root( /// Determine a child trie root given a hash DB and delta values. H is the default hasher, /// but a generic implementation may ignore this type parameter and use other hashers. -pub fn child_delta_trie_root( +pub fn child_delta_trie_root( keyspace: &[u8], db: &mut DB, root_data: RD, delta: I, ) -> Result<::Out, Box>> where - I: IntoIterator)>, - A: AsRef<[u8]> + Ord, - B: AsRef<[u8]>, + I: IntoIterator, + A: Borrow<[u8]>, + B: Borrow>, + V: Borrow<[u8]>, RD: AsRef<[u8]>, DB: hash_db::HashDB { @@ -252,9 +253,9 @@ pub fn child_delta_trie_root( let mut trie = TrieDBMut::::from_existing(&mut db, &mut root)?; for (key, change) in delta { - match change { - Some(val) => trie.insert(key.as_ref(), val.as_ref())?, - None => trie.remove(key.as_ref())?, + match change.borrow() { + Some(val) => trie.insert(key.borrow(), val.borrow())?, + None => trie.remove(key.borrow())?, }; } } diff --git a/primitives/utils/Cargo.toml b/primitives/utils/Cargo.toml index d7efdf494204b00d5ee5ae1c143934be95dff328..96c7825515ec54374705711503fcfcc075cd8645 100644 --- a/primitives/utils/Cargo.toml +++ b/primitives/utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-utils" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,6 +13,7 @@ futures = "0.3.4" futures-core = "0.3.4" lazy_static = "1.4.0" prometheus = "0.8.0" +futures-timer = "3.0.2" [features] default = ["metered"] diff --git a/primitives/utils/src/lib.rs b/primitives/utils/src/lib.rs index 644e94651d69dbf6112d8f7ce0727179ad43688a..77bcd096561b45befcee7f80a0611108ce756830 100644 --- a/primitives/utils/src/lib.rs +++ b/primitives/utils/src/lib.rs @@ -18,4 +18,5 @@ //! Utilities Primitives for Substrate pub mod metrics; -pub mod mpsc; \ No newline at end of file +pub mod mpsc; +pub mod status_sinks; diff --git a/primitives/utils/src/metrics.rs b/primitives/utils/src/metrics.rs index b991ce016b115b103fe2ff59de009549f8c8922c..a66589b5927fe2fa6386f5f2024e0e342f665825 100644 --- a/primitives/utils/src/metrics.rs +++ b/primitives/utils/src/metrics.rs @@ -23,8 +23,8 @@ use prometheus::{ core::{ AtomicU64, GenericGauge, GenericCounter }, }; -#[cfg(features = "metered")] -use prometheus::{core::GenericGaugeVec, Opts}; +#[cfg(feature = "metered")] +use prometheus::{core::GenericCounterVec, Opts}; lazy_static! { @@ -37,9 +37,9 @@ lazy_static! { ).expect("Creating of statics doesn't fail. qed"); } -#[cfg(features = "metered")] +#[cfg(feature = "metered")] lazy_static! { - pub static ref UNBOUNDED_CHANNELS_COUNTER : GenericGaugeVec = GenericGaugeVec::new( + pub static ref UNBOUNDED_CHANNELS_COUNTER : GenericCounterVec = GenericCounterVec::new( Opts::new("unbounded_channel_len", "Items in each mpsc::unbounded instance"), &["entity", "action"] // 'name of channel, send|received|dropped ).expect("Creating of statics doesn't fail. qed"); @@ -52,7 +52,7 @@ pub fn register_globals(registry: &Registry) -> Result<(), PrometheusError> { registry.register(Box::new(TOKIO_THREADS_ALIVE.clone()))?; registry.register(Box::new(TOKIO_THREADS_TOTAL.clone()))?; - #[cfg(features = "metered")] + #[cfg(feature = "metered")] registry.register(Box::new(UNBOUNDED_CHANNELS_COUNTER.clone()))?; Ok(()) diff --git a/primitives/utils/src/mpsc.rs b/primitives/utils/src/mpsc.rs index 827195388f9336f89cd115a74674452c8cb6ccba..70baa006bdcdc0d86b99aec71859361806f9044c 100644 --- a/primitives/utils/src/mpsc.rs +++ b/primitives/utils/src/mpsc.rs @@ -17,7 +17,7 @@ //! Features to meter unbounded channels -#[cfg(not(features = "metered"))] +#[cfg(not(feature = "metered"))] mod inner { // just aliased, non performance implications use futures::channel::mpsc::{self, UnboundedReceiver, UnboundedSender}; @@ -31,22 +31,29 @@ mod inner { } -#[cfg(features = "metered")] +#[cfg(feature = "metered")] mod inner { //tracing implementation use futures::channel::mpsc::{self, UnboundedReceiver, UnboundedSender, TryRecvError, TrySendError, SendError }; - use futures::{sink::Sink, task::{Poll, Context}, stream::Stream}; + use futures::{sink::Sink, task::{Poll, Context}, stream::{Stream, FusedStream}}; use std::pin::Pin; use crate::metrics::UNBOUNDED_CHANNELS_COUNTER; /// Wrapper Type around `UnboundedSender` that increases the global /// measure when a message is added - #[derive(Debug, Clone)] + #[derive(Debug)] pub struct TracingUnboundedSender(&'static str, UnboundedSender); + // Strangely, deriving `Clone` requires that `T` is also `Clone`. + impl Clone for TracingUnboundedSender { + fn clone(&self) -> Self { + Self(self.0, self.1.clone()) + } + } + /// Wrapper Type around `UnboundedReceiver` that decreases the global /// measure when a message is polled #[derive(Debug)] @@ -88,7 +95,7 @@ mod inner { /// Proxy function to mpsc::UnboundedSender pub fn unbounded_send(&self, msg: T) -> Result<(), TrySendError> { self.1.unbounded_send(msg).map(|s|{ - UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.0, &"send"]).incr(); + UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.0, &"send"]).inc(); s }) } @@ -104,13 +111,19 @@ mod inner { fn consume(&mut self) { // consume all items, make sure to reflect the updated count let mut count = 0; - while let Ok(Some(..)) = self.try_next() { - count += 1; - } + loop { + if self.1.is_terminated() { + break; + } + match self.try_next() { + Ok(Some(..)) => count += 1, + _ => break + } + } // and discount the messages if count > 0 { - UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.0, &"dropped"]).incr_by(count); + UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.0, &"dropped"]).inc_by(count); } } @@ -127,7 +140,7 @@ mod inner { pub fn try_next(&mut self) -> Result, TryRecvError> { self.1.try_next().map(|s| { if s.is_some() { - UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.0, &"received"]).incr(); + UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.0, &"received"]).inc(); } s }) @@ -153,7 +166,7 @@ mod inner { match Pin::new(&mut s.1).poll_next(cx) { Poll::Ready(msg) => { if msg.is_some() { - UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.0, "received"]).incr(); + UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[s.0, "received"]).inc(); } Poll::Ready(msg) } @@ -164,6 +177,11 @@ mod inner { } } + impl FusedStream for TracingUnboundedReceiver { + fn is_terminated(&self) -> bool { + self.1.is_terminated() + } + } impl Sink for TracingUnboundedSender { type Error = SendError; diff --git a/client/service/src/status_sinks.rs b/primitives/utils/src/status_sinks.rs similarity index 96% rename from client/service/src/status_sinks.rs rename to primitives/utils/src/status_sinks.rs index 4b1dce52f9a319ee02c2033f0214fffd6fe56a86..47bccebb960b4966371cc116245a0377e6de754c 100644 --- a/client/service/src/status_sinks.rs +++ b/primitives/utils/src/status_sinks.rs @@ -19,7 +19,7 @@ use std::time::Duration; use std::pin::Pin; use std::task::{Poll, Context}; use futures_timer::Delay; -use sp_utils::mpsc::TracingUnboundedSender; +use crate::mpsc::TracingUnboundedSender; /// Holds a list of `UnboundedSender`s, each associated with a certain time period. Every time the /// period elapses, we push an element on the sender. @@ -109,7 +109,7 @@ impl futures::Future for YieldAfter { mod tests { use super::StatusSinks; use futures::prelude::*; - use futures::channel::mpsc; + use crate::mpsc::tracing_unbounded; use std::time::Duration; use std::task::Poll; @@ -120,7 +120,7 @@ mod tests { let mut status_sinks = StatusSinks::new(); - let (tx, rx) = mpsc::unbounded(); + let (tx, rx) = tracing_unbounded("status_sink_test"); status_sinks.push(Duration::from_millis(100), tx); let mut val_order = 5; diff --git a/primitives/version/Cargo.toml b/primitives/version/Cargo.toml index cf23c8f5958a3ee41d817cc89c6cf8b84b5dcb40..181b793bd57afeb8f9279fc6652302c7b6bb9e65 100644 --- a/primitives/version/Cargo.toml +++ b/primitives/version/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-version" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -16,9 +16,9 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] impl-serde = { version = "0.2.3", optional = true } serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../std" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../runtime" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../std" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../runtime" } [features] default = ["std"] diff --git a/primitives/wasm-interface/Cargo.toml b/primitives/wasm-interface/Cargo.toml index b3f468a83a949ef3dcad3655f24c011b23f10fd9..8b32cde969c67e0932ce88e0b0fc96d865967550 100644 --- a/primitives/wasm-interface/Cargo.toml +++ b/primitives/wasm-interface/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-wasm-interface" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -15,8 +15,8 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] wasmi = { version = "0.6.2", optional = true } impl-trait-for-tuples = "0.1.2" -sp-std = { version = "2.0.0-alpha.8", path = "../std", default-features = false } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +sp-std = { version = "2.0.0-rc4", path = "../std", default-features = false } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } [features] default = [ "std" ] diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index 48261b4eeb9af77ee99fc23645afcecc6aaa53d5..f67f1560c1578a646828f12fdd2bf3c20f976fe9 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-test-utils" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" diff --git a/test-utils/client/Cargo.toml b/test-utils/client/Cargo.toml index 5b71bc0051486a70bbb0575bafcab955c73a8236..a9d8590f021fa9a20f5f8333f8b6e492b8638d72 100644 --- a/test-utils/client/Cargo.toml +++ b/test-utils/client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-test-client" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,17 +12,18 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sc-client-api = { version = "2.0.0-alpha.8", path = "../../client/api" } -sc-client-db = { version = "0.8.0-alpha.8", features = ["test-helpers"], path = "../../client/db" } -sp-consensus = { version = "0.8.0-alpha.8", path = "../../primitives/consensus/common" } -sc-executor = { version = "0.8.0-alpha.8", path = "../../client/executor" } -sc-consensus = { version = "0.8.0-alpha.8", path = "../../client/consensus/common" } -sc-service = { version = "0.8.0-alpha.8", default-features = false, features = ["test-helpers"], path = "../../client/service" } +sc-client-api = { version = "2.0.0-rc4", path = "../../client/api" } +sc-light = { version = "2.0.0-rc4", path = "../../client/light" } +sc-client-db = { version = "0.8.0-rc4", features = ["test-helpers"], path = "../../client/db" } +sp-consensus = { version = "0.8.0-rc4", path = "../../primitives/consensus/common" } +sc-executor = { version = "0.8.0-rc4", path = "../../client/executor" } +sc-consensus = { version = "0.8.0-rc4", path = "../../client/consensus/common" } +sc-service = { version = "0.8.0-rc4", default-features = false, features = ["test-helpers"], path = "../../client/service" } futures = "0.3.4" hash-db = "0.15.2" -sp-keyring = { version = "2.0.0-alpha.8", path = "../../primitives/keyring" } -codec = { package = "parity-scale-codec", version = "1.3.0" } -sp-core = { version = "2.0.0-alpha.8", path = "../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../primitives/runtime" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../primitives/blockchain" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../primitives/state-machine" } +sp-keyring = { version = "2.0.0-rc4", path = "../../primitives/keyring" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +sp-core = { version = "2.0.0-rc4", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-rc4", path = "../../primitives/runtime" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../primitives/blockchain" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../primitives/state-machine" } diff --git a/test-utils/client/src/client_ext.rs b/test-utils/client/src/client_ext.rs index 706a7b6e95a85a00083e792c06bba47333c99e7c..a74bd3258ef0f97782fbba780e8c01d26124f480 100644 --- a/test-utils/client/src/client_ext.rs +++ b/test-utils/client/src/client_ext.rs @@ -19,6 +19,7 @@ use sc_service::client::Client; use sc_client_api::backend::Finalizer; +use sc_client_api::client::BlockBackend; use sp_consensus::{ BlockImportParams, BlockImport, BlockOrigin, Error as ConsensusError, ForkChoiceStrategy, diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index ffd93970f41dad59a3f12ceced0d0534baa2049e..2ab9e4066ddd1e230e54e874571b06d504be6614 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -46,7 +46,7 @@ use sp_runtime::traits::{Block as BlockT, BlakeTwo256}; use sc_service::client::{LocalCallExecutor, ClientConfig}; /// Test client light database backend. -pub type LightBackend = client::light::backend::Backend< +pub type LightBackend = sc_light::Backend< sc_client_db::light::LightStorage, BlakeTwo256, >; diff --git a/test-utils/runtime/Cargo.toml b/test-utils/runtime/Cargo.toml index 2a7cb9a234ce3463e5d72a66b336fe0d3e63a8e3..71987da150476146271fa1890a504c36a709ad19 100644 --- a/test-utils/runtime/Cargo.toml +++ b/test-utils/runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-test-runtime" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" @@ -13,35 +13,35 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/application-crypto" } -sp-consensus-aura = { version = "0.8.0-alpha.8", default-features = false, path = "../../primitives/consensus/aura" } -sp-consensus-babe = { version = "0.8.0-alpha.8", default-features = false, path = "../../primitives/consensus/babe" } -sp-block-builder = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/block-builder" } -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -frame-executive = { version = "2.0.0-alpha.8", default-features = false, path = "../../frame/executive" } -sp-inherents = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/inherents" } -sp-keyring = { version = "2.0.0-alpha.8", optional = true, path = "../../primitives/keyring" } -memory-db = { version = "0.20.0", default-features = false } -sp-offchain = { path = "../../primitives/offchain", default-features = false, version = "2.0.0-alpha.8"} -sp-core = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/std" } -sp-runtime-interface = { path = "../../primitives/runtime-interface", default-features = false, version = "2.0.0-alpha.8"} -sp-io = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/io" } -frame-support = { version = "2.0.0-alpha.8", default-features = false, path = "../../frame/support" } -sp-version = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/version" } -sp-session = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/session" } -sp-api = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/api" } -sp-runtime = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/runtime" } -pallet-babe = { version = "2.0.0-alpha.8", default-features = false, path = "../../frame/babe" } -frame-system = { version = "2.0.0-alpha.8", default-features = false, path = "../../frame/system" } -frame-system-rpc-runtime-api = { version = "2.0.0-alpha.8", default-features = false, path = "../../frame/system/rpc/runtime-api" } -pallet-timestamp = { version = "2.0.0-alpha.8", default-features = false, path = "../../frame/timestamp" } -sp-finality-grandpa = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/finality-grandpa" } -sp-trie = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/trie" } -sp-transaction-pool = { version = "2.0.0-alpha.8", default-features = false, path = "../../primitives/transaction-pool" } -trie-db = { version = "0.20.1", default-features = false } +sp-application-crypto = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/application-crypto" } +sp-consensus-aura = { version = "0.8.0-rc4", default-features = false, path = "../../primitives/consensus/aura" } +sp-consensus-babe = { version = "0.8.0-rc4", default-features = false, path = "../../primitives/consensus/babe" } +sp-block-builder = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/block-builder" } +codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false, features = ["derive"] } +frame-executive = { version = "2.0.0-rc4", default-features = false, path = "../../frame/executive" } +sp-inherents = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/inherents" } +sp-keyring = { version = "2.0.0-rc4", optional = true, path = "../../primitives/keyring" } +memory-db = { version = "0.21.0", default-features = false } +sp-offchain = { path = "../../primitives/offchain", default-features = false, version = "2.0.0-rc4"} +sp-core = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/std" } +sp-runtime-interface = { path = "../../primitives/runtime-interface", default-features = false, version = "2.0.0-rc4"} +sp-io = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/io" } +frame-support = { version = "2.0.0-rc4", default-features = false, path = "../../frame/support" } +sp-version = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/version" } +sp-session = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/session" } +sp-api = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/api" } +sp-runtime = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/runtime" } +pallet-babe = { version = "2.0.0-rc4", default-features = false, path = "../../frame/babe" } +frame-system = { version = "2.0.0-rc4", default-features = false, path = "../../frame/system" } +frame-system-rpc-runtime-api = { version = "2.0.0-rc4", default-features = false, path = "../../frame/system/rpc/runtime-api" } +pallet-timestamp = { version = "2.0.0-rc4", default-features = false, path = "../../frame/timestamp" } +sp-finality-grandpa = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/finality-grandpa" } +sp-trie = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/trie" } +sp-transaction-pool = { version = "2.0.0-rc4", default-features = false, path = "../../primitives/transaction-pool" } +trie-db = { version = "0.21.0", default-features = false } parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } -sc-service = { version = "0.8.0-alpha.8", default-features = false, optional = true, features = ["test-helpers"], path = "../../client/service" } +sc-service = { version = "0.8.0-rc4", default-features = false, optional = true, features = ["test-helpers"], path = "../../client/service" } # 3rd party cfg-if = "0.1.10" @@ -49,10 +49,10 @@ log = { version = "0.4.8", optional = true } serde = { version = "1.0.101", optional = true, features = ["derive"] } [dev-dependencies] -sc-block-builder = { version = "0.8.0-alpha.8", path = "../../client/block-builder" } -sc-executor = { version = "0.8.0-alpha.8", path = "../../client/executor" } -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "./client" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../primitives/state-machine" } +sc-block-builder = { version = "0.8.0-rc4", path = "../../client/block-builder" } +sc-executor = { version = "0.8.0-rc4", path = "../../client/executor" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "./client" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../primitives/state-machine" } [build-dependencies] wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../utils/wasm-builder-runner" } diff --git a/test-utils/runtime/build.rs b/test-utils/runtime/build.rs index 7d30dacfc837ed32cdb8896fc919c64fc1243749..69ff73a3ffc849168e51092a6391815ea0aa856f 100644 --- a/test-utils/runtime/build.rs +++ b/test-utils/runtime/build.rs @@ -20,7 +20,7 @@ use wasm_builder_runner::WasmBuilder; fn main() { WasmBuilder::new() .with_current_project() - .with_wasm_builder_from_crates_or_path("1.0.9", "../../utils/wasm-builder") + .with_wasm_builder_from_crates_or_path("1.0.11", "../../utils/wasm-builder") .export_heap_base() // Note that we set the stack-size to 1MB explicitly even though it is set // to this value by default. This is because some of our tests (`restoration_of_globals`) diff --git a/test-utils/runtime/client/Cargo.toml b/test-utils/runtime/client/Cargo.toml index 87e088464972630fad33e847d24514392aec3f8a..09f2c3f152abb98b606f334e8444ff3f8c21f084 100644 --- a/test-utils/runtime/client/Cargo.toml +++ b/test-utils/runtime/client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-test-runtime-client" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,16 +12,17 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-consensus = { version = "0.8.0-alpha.8", path = "../../../primitives/consensus/common" } -sc-block-builder = { version = "0.8.0-alpha.8", path = "../../../client/block-builder" } -substrate-test-client = { version = "2.0.0-alpha.8", path = "../../client" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -substrate-test-runtime = { version = "2.0.0-alpha.8", path = "../../runtime" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.8", path = "../../../primitives/api" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } -codec = { package = "parity-scale-codec", version = "1.3.0" } -sc-client-api = { version = "2.0.0-alpha.8", path = "../../../client/api" } -sc-consensus = { version = "0.8.0-alpha.8", path = "../../../client/consensus/common" } -sc-service = { version = "0.8.0-alpha.8", default-features = false, path = "../../../client/service" } +sc-light = { version = "2.0.0-rc4", path = "../../../client/light" } +sp-consensus = { version = "0.8.0-rc4", path = "../../../primitives/consensus/common" } +sc-block-builder = { version = "0.8.0-rc4", path = "../../../client/block-builder" } +substrate-test-client = { version = "2.0.0-rc4", path = "../../client" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +substrate-test-runtime = { version = "2.0.0-rc4", path = "../../runtime" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-rc4", path = "../../../primitives/api" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +sc-client-api = { version = "2.0.0-rc4", path = "../../../client/api" } +sc-consensus = { version = "0.8.0-rc4", path = "../../../client/consensus/common" } +sc-service = { version = "0.8.0-rc4", default-features = false, path = "../../../client/service" } futures = "0.3.4" diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index 5b0eafb4a3d643e9a1b9dbe31ce88a56cfeb8392..4e9034fb4d493935c3ca47321681e6a94b1bf4c8 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -35,12 +35,9 @@ use sp_core::{sr25519, ChangesTrieConfiguration}; use sp_core::storage::{ChildInfo, Storage, StorageChild}; use substrate_test_runtime::genesismap::{GenesisConfig, additional_storage_with_genesis}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Hash as HashT, NumberFor, HashFor}; -use sc_service::client::{ - light::fetcher::{ - Fetcher, - RemoteHeaderRequest, RemoteReadRequest, RemoteReadChildRequest, - RemoteCallRequest, RemoteChangesRequest, RemoteBodyRequest, - }, +use sc_client_api::light::{ + RemoteCallRequest, RemoteChangesRequest, RemoteBodyRequest, + Fetcher, RemoteHeaderRequest, RemoteReadRequest, RemoteReadChildRequest, }; /// A prelude to import in tests. @@ -78,10 +75,10 @@ pub type Executor = client::LocalCallExecutor< pub type LightBackend = substrate_test_client::LightBackend; /// Test client light executor. -pub type LightExecutor = client::light::call_executor::GenesisCallExecutor< +pub type LightExecutor = sc_light::GenesisCallExecutor< LightBackend, client::LocalCallExecutor< - client::light::backend::Backend< + sc_light::Backend< sc_client_db::light::LightStorage, HashFor >, @@ -350,7 +347,7 @@ pub fn new_light() -> ( ) { let storage = sc_client_db::light::LightStorage::new_test(); - let blockchain = Arc::new(client::light::blockchain::Blockchain::new(storage)); + let blockchain = Arc::new(sc_light::Blockchain::new(storage)); let backend = Arc::new(LightBackend::new(blockchain.clone())); let executor = new_native_executor(); let local_call_executor = client::LocalCallExecutor::new(backend.clone(), executor, sp_core::tasks::executor(), Default::default()); diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 5ac5535a874786c24c6e79ac8498ce7b81147f87..06054c1240f3bb9e59d81bc1d320345d626f7fcc 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -184,7 +184,7 @@ impl ExtrinsicT for Extrinsic { } impl sp_runtime::traits::Dispatchable for Extrinsic { - type Origin = (); + type Origin = Origin; type Trait = (); type Info = (); type PostInfo = (); @@ -194,10 +194,20 @@ impl sp_runtime::traits::Dispatchable for Extrinsic { } impl Extrinsic { + /// Convert `&self` into `&Transfer`. + /// + /// Panics if this is no `Transfer` extrinsic. pub fn transfer(&self) -> &Transfer { + self.try_transfer().expect("cannot convert to transfer ref") + } + + /// Try to convert `&self` into `&Transfer`. + /// + /// Returns `None` if this is no `Transfer` extrinsic. + pub fn try_transfer(&self) -> Option<&Transfer> { match self { - Extrinsic::Transfer { ref transfer, .. } => transfer, - _ => panic!("cannot convert to transfer ref"), + Extrinsic::Transfer { ref transfer, .. } => Some(transfer), + _ => None, } } } @@ -208,6 +218,8 @@ pub type AccountSignature = sr25519::Signature; pub type AccountId = ::Signer; /// A simple hash type for all our hashing. pub type Hash = H256; +/// The hashing algorithm used. +pub type Hashing = BlakeTwo256; /// The block number type used in this runtime. pub type BlockNumber = u64; /// Index of a transaction. @@ -219,7 +231,7 @@ pub type Digest = sp_runtime::generic::Digest; /// A test block. pub type Block = sp_runtime::generic::Block; /// A test block's header. -pub type Header = sp_runtime::generic::Header; +pub type Header = sp_runtime::generic::Header; /// Run whatever tests we have. pub fn run_tests(mut input: &[u8]) -> Vec { @@ -311,6 +323,9 @@ cfg_if! { fn test_ecdsa_crypto() -> (ecdsa::AppSignature, ecdsa::AppPublic); /// Run various tests against storage. fn test_storage(); + /// Test that ensures that we can call a function that takes multiple + /// arguments. + fn test_multiple_arguments(data: Vec, other: Vec, num: u32); } } } else { @@ -357,6 +372,9 @@ cfg_if! { fn test_ecdsa_crypto() -> (ecdsa::AppSignature, ecdsa::AppPublic); /// Run various tests against storage. fn test_storage(); + /// Test that ensures that we can call a function that takes multiple + /// arguments. + fn test_multiple_arguments(data: Vec, other: Vec, num: u32); } } } @@ -399,12 +417,13 @@ parameter_types! { } impl frame_system::Trait for Runtime { + type BaseCallFilter = (); type Origin = Origin; type Call = Extrinsic; type Index = u64; type BlockNumber = u64; type Hash = H256; - type Hashing = BlakeTwo256; + type Hashing = Hashing; type AccountId = u64; type Lookup = IdentityLookup; type Header = Header; @@ -414,6 +433,7 @@ impl frame_system::Trait for Runtime { type DbWeight = (); type BlockExecutionWeight = (); type ExtrinsicBaseWeight = (); + type MaximumExtrinsicWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -465,7 +485,7 @@ fn code_using_trie() -> u64 { let mut root = sp_std::default::Default::default(); let _ = { let v = &pairs; - let mut t = TrieDBMut::::new(&mut mdb, &mut root); + let mut t = TrieDBMut::::new(&mut mdb, &mut root); for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; @@ -476,7 +496,7 @@ fn code_using_trie() -> u64 { t }; - if let Ok(trie) = TrieDB::::new(&mdb, &root) { + if let Ok(trie) = TrieDB::::new(&mdb, &root) { if let Ok(iter) = trie.iter() { let mut iter_pairs = Vec::new(); for pair in iter { @@ -637,6 +657,11 @@ cfg_if! { test_read_storage(); test_read_child_storage(); } + + fn test_multiple_arguments(data: Vec, other: Vec, num: u32) { + assert_eq!(&data[..], &other[..]); + assert_eq!(data.len(), num as usize); + } } impl sp_consensus_aura::AuraApi for Runtime { @@ -858,6 +883,11 @@ cfg_if! { test_read_storage(); test_read_child_storage(); } + + fn test_multiple_arguments(data: Vec, other: Vec, num: u32) { + assert_eq!(&data[..], &other[..]); + assert_eq!(data.len(), num as usize); + } } impl sp_consensus_aura::AuraApi for Runtime { diff --git a/test-utils/runtime/transaction-pool/Cargo.toml b/test-utils/runtime/transaction-pool/Cargo.toml index 4f1c816d090cda0a8a769df2fe6e7eee62f0c0ee..f29ae2b7bf46753606dbd6e7945307f13732fd7f 100644 --- a/test-utils/runtime/transaction-pool/Cargo.toml +++ b/test-utils/runtime/transaction-pool/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-test-runtime-transaction-pool" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,12 +12,12 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../client" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../client" } parking_lot = "0.10.0" -codec = { package = "parity-scale-codec", version = "1.3.0" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-transaction-pool = { version = "2.0.0-alpha.8", path = "../../../primitives/transaction-pool" } -sc-transaction-graph = { version = "2.0.0-alpha.8", path = "../../../client/transaction-pool/graph" } +codec = { package = "parity-scale-codec", version = "1.3.1" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-transaction-pool = { version = "2.0.0-rc4", path = "../../../primitives/transaction-pool" } +sc-transaction-graph = { version = "2.0.0-rc4", path = "../../../client/transaction-pool/graph" } futures = { version = "0.3.1", features = ["compat"] } derive_more = "0.99.2" diff --git a/test-utils/runtime/transaction-pool/src/lib.rs b/test-utils/runtime/transaction-pool/src/lib.rs index c7778a51da1054664836cfb661a916ae5b6dece7..17cecd394ab916eaf16a2d5e5cfb0fffb745189a 100644 --- a/test-utils/runtime/transaction-pool/src/lib.rs +++ b/test-utils/runtime/transaction-pool/src/lib.rs @@ -23,17 +23,18 @@ use codec::Encode; use parking_lot::RwLock; use sp_runtime::{ generic::{self, BlockId}, - traits::{BlakeTwo256, Hash as HashT}, + traits::{BlakeTwo256, Hash as HashT, Block as _, Header as _}, transaction_validity::{ TransactionValidity, ValidTransaction, TransactionValidityError, InvalidTransaction, TransactionSource, }, }; -use std::collections::{HashSet, HashMap}; +use std::collections::{HashSet, HashMap, BTreeMap}; use substrate_test_runtime_client::{ runtime::{Index, AccountId, Block, BlockNumber, Extrinsic, Hash, Header, Transfer}, AccountKeyring::{self, *}, }; +use sp_blockchain::CachedHeaderMetadata; /// Error type used by [`TestApi`]. #[derive(Debug, derive_more::From, derive_more::Display)] @@ -53,9 +54,8 @@ impl std::error::Error for Error { #[derive(Default)] pub struct ChainState { - pub block_by_number: HashMap>, - pub block_by_hash: HashMap>, - pub header_by_number: HashMap, + pub block_by_number: BTreeMap>, + pub block_by_hash: HashMap, pub nonces: HashMap, pub invalid_hashes: HashSet, } @@ -70,11 +70,7 @@ pub struct TestApi { impl TestApi { /// Test Api with Alice nonce set initially. pub fn with_alice_nonce(nonce: u64) -> Self { - let api = TestApi { - valid_modifier: RwLock::new(Box::new(|_| {})), - chain: Default::default(), - validation_requests: RwLock::new(Default::default()), - }; + let api = Self::empty(); api.chain.write().nonces.insert(Alice.into(), nonce); @@ -89,6 +85,9 @@ impl TestApi { validation_requests: RwLock::new(Default::default()), }; + // Push genesis block + api.push_block(0, Vec::new()); + api } @@ -97,46 +96,61 @@ impl TestApi { *self.valid_modifier.write() = modifier; } - /// Push block as a part of canonical chain under given number. + /// Push block under given number. + /// + /// If multiple blocks exists with the same block number, the first inserted block will be + /// interpreted as part of the canonical chain. pub fn push_block(&self, block_number: BlockNumber, xts: Vec) -> Header { - let mut chain = self.chain.write(); - chain.block_by_number.insert(block_number, xts.clone()); - let header = Header { - number: block_number, - digest: Default::default(), - extrinsics_root: Default::default(), - parent_hash: block_number + let parent_hash = { + let chain = self.chain.read(); + block_number .checked_sub(1) .and_then(|num| { - chain.header_by_number.get(&num) - .cloned().map(|h| h.hash()) - }).unwrap_or_default(), - state_root: Default::default(), + chain.block_by_number + .get(&num) + .map(|blocks| { + blocks[0].header.hash() + }) + }).unwrap_or_default() }; - chain.block_by_hash.insert(header.hash(), xts); - chain.header_by_number.insert(block_number, header.clone()); - header + + self.push_block_with_parent(parent_hash, xts) } - /// Push a block without a number. + /// Push a block using the given `parent`. /// - /// As a part of non-canonical chain. - pub fn push_fork_block(&self, block_hash: Hash, xts: Vec) { + /// Panics if `parent` does not exists. + pub fn push_block_with_parent( + &self, + parent: Hash, + xts: Vec, + ) -> Header { let mut chain = self.chain.write(); - chain.block_by_hash.insert(block_hash, xts); - } - pub fn push_fork_block_with_parent(&self, parent: Hash, xts: Vec) -> Header { - let mut chain = self.chain.write(); - let blocknum = chain.block_by_number.keys().max().expect("block_by_number shouldn't be empty"); + // `Hash::default()` is the genesis parent hash + let block_number = if parent == Hash::default() { + 0 + } else { + *chain.block_by_hash + .get(&parent) + .expect("`parent` exists") + .header() + .number() + 1 + }; + let header = Header { - number: *blocknum, + number: block_number, digest: Default::default(), - extrinsics_root: Default::default(), + extrinsics_root: Hash::random(), parent_hash: parent, state_root: Default::default(), }; - chain.block_by_hash.insert(header.hash(), xts); + + let hash = header.hash(); + let block = Block::new(header.clone(), xts); + chain.block_by_hash.insert(hash, block.clone()); + chain.block_by_number.entry(block_number).or_default().push(block); + header } @@ -170,11 +184,19 @@ impl TestApi { let mut chain = self.chain.write(); chain.nonces.entry(account).and_modify(|n| *n += 1).or_insert(1); } + + /// Calculate a tree route between the two given blocks. + pub fn tree_route( + &self, + from: Hash, + to: Hash, + ) -> Result, Error> { + sp_blockchain::tree_route(self, from, to) + } } impl sc_transaction_graph::ChainApi for TestApi { type Block = Block; - type Hash = Hash; type Error = Error; type ValidationFuture = futures::future::Ready>; type BodyFuture = futures::future::Ready>, Error>>; @@ -187,13 +209,19 @@ impl sc_transaction_graph::ChainApi for TestApi { ) -> Self::ValidationFuture { self.validation_requests.write().push(uxt.clone()); - let chain_nonce = self.chain.read().nonces.get(&uxt.transfer().from).cloned().unwrap_or(0); - let requires = if chain_nonce == uxt.transfer().nonce { - vec![] + let (requires, provides) = if let Some(transfer) = uxt.try_transfer() { + let chain_nonce = self.chain.read().nonces.get(&transfer.from).cloned().unwrap_or(0); + let requires = if chain_nonce == transfer.nonce { + vec![] + } else { + vec![vec![chain_nonce as u8]] + }; + let provides = vec![vec![transfer.nonce as u8]]; + + (requires, provides) } else { - vec![vec![chain_nonce as u8]] + (Vec::new(), vec![uxt.encode()]) }; - let provides = vec![vec![uxt.transfer().nonce as u8]]; if self.chain.read().invalid_hashes.contains(&self.hash_and_length(&uxt).0) { return futures::future::ready(Ok( @@ -218,7 +246,14 @@ impl sc_transaction_graph::ChainApi for TestApi { &self, at: &BlockId, ) -> Result>, Error> { - Ok(Some(number_of(at))) + Ok(match at { + generic::BlockId::Hash(x) => self.chain + .read() + .block_by_hash + .get(x) + .map(|b| *b.header.number()), + generic::BlockId::Number(num) => Some(*num), + }) } fn block_id_to_hash( @@ -227,34 +262,60 @@ impl sc_transaction_graph::ChainApi for TestApi { ) -> Result>, Error> { Ok(match at { generic::BlockId::Hash(x) => Some(x.clone()), - generic::BlockId::Number(num) => { - self.chain.read() - .header_by_number.get(num) - .map(|h| h.hash()) - .or_else(|| Some(Default::default())) - }, + generic::BlockId::Number(num) => self.chain + .read() + .block_by_number + .get(num) + .map(|blocks| blocks[0].header().hash()), }) } fn hash_and_length( &self, ex: &sc_transaction_graph::ExtrinsicFor, - ) -> (Self::Hash, usize) { + ) -> (Hash, usize) { Self::hash_and_length_inner(ex) } fn block_body(&self, id: &BlockId) -> Self::BodyFuture { futures::future::ready(Ok(match id { - BlockId::Number(num) => self.chain.read().block_by_number.get(num).cloned(), - BlockId::Hash(hash) => self.chain.read().block_by_hash.get(hash).cloned(), + BlockId::Number(num) => self.chain + .read() + .block_by_number + .get(num) + .map(|b| b[0].extrinsics().to_vec()), + BlockId::Hash(hash) => self.chain + .read() + .block_by_hash + .get(hash) + .map(|b| b.extrinsics().to_vec()), })) } } -fn number_of(at: &BlockId) -> u64 { - match at { - generic::BlockId::Number(n) => *n as u64, - _ => 0, +impl sp_blockchain::HeaderMetadata for TestApi { + type Error = Error; + + fn header_metadata( + &self, + hash: Hash, + ) -> Result, Self::Error> { + let chain = self.chain.read(); + let block = chain.block_by_hash.get(&hash).expect("Hash exists"); + + Ok(block.header().into()) + } + + fn insert_header_metadata( + &self, + _: Hash, + _: CachedHeaderMetadata, + ) { + unimplemented!("Not implemented for tests") + } + + fn remove_header_metadata(&self, _: Hash) { + unimplemented!("Not implemented for tests") } } diff --git a/test-utils/src/lib.rs b/test-utils/src/lib.rs index e600ab9fce92697fd18d4c0125d80c966472ad13..8163460df7427fb3bb7b7de3e187b487eee21378 100644 --- a/test-utils/src/lib.rs +++ b/test-utils/src/lib.rs @@ -38,7 +38,7 @@ /// ``` #[macro_export] macro_rules! assert_eq_uvec { - ( $x:expr, $y:expr ) => { + ( $x:expr, $y:expr $(,)? ) => { $crate::__assert_eq_uvec!($x, $y); $crate::__assert_eq_uvec!($y, $x); } diff --git a/utils/browser/Cargo.toml b/utils/browser/Cargo.toml index 35e798a392c1d7712a9cf91a5778e47621ec10c8..ed02e8e2fa64eab81bc9bec34ae5c91a9a9ec66e 100644 --- a/utils/browser/Cargo.toml +++ b/utils/browser/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-browser-utils" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" authors = ["Parity Technologies "] description = "Utilities for creating a browser light-client." edition = "2018" @@ -15,18 +15,18 @@ targets = ["x86_64-unknown-linux-gnu"] futures = { version = "0.3", features = ["compat"] } futures01 = { package = "futures", version = "0.1.29" } log = "0.4.8" -libp2p-wasm-ext = { version = "0.18.0", features = ["websocket"] } +libp2p-wasm-ext = { version = "0.19.0", features = ["websocket"] } console_error_panic_hook = "0.1.6" console_log = "0.1.2" js-sys = "0.3.34" wasm-bindgen = "0.2.57" wasm-bindgen-futures = "0.4.7" kvdb-web = "0.6" -sp-database = { version = "2.0.0-alpha.8", path = "../../primitives/database" } -sc-informant = { version = "0.8.0-alpha.8", path = "../../client/informant" } -sc-service = { version = "0.8.0-alpha.8", path = "../../client/service", default-features = false } -sc-network = { path = "../../client/network", version = "0.8.0-alpha.8"} -sc-chain-spec = { path = "../../client/chain-spec", version = "2.0.0-alpha.8"} +sp-database = { version = "2.0.0-rc4", path = "../../primitives/database" } +sc-informant = { version = "0.8.0-rc4", path = "../../client/informant" } +sc-service = { version = "0.8.0-rc4", path = "../../client/service", default-features = false } +sc-network = { path = "../../client/network", version = "0.8.0-rc4"} +sc-chain-spec = { path = "../../client/chain-spec", version = "2.0.0-rc4"} # Imported just for the `no_cc` feature clear_on_drop = { version = "0.2.3", features = ["no_cc"] } diff --git a/utils/browser/src/lib.rs b/utils/browser/src/lib.rs index 408ba24cfed229e0c86474cdfe5a3c0370512052..c8034d9466fe78d37967681c8af4345f19d1a307 100644 --- a/utils/browser/src/lib.rs +++ b/utils/browser/src/lib.rs @@ -17,17 +17,17 @@ use futures01::sync::mpsc as mpsc01; use log::{debug, info}; -use std::sync::Arc; use sc_network::config::TransportConfig; use sc_service::{ - AbstractService, RpcSession, Role, Configuration, + RpcSession, Role, Configuration, TaskManager, RpcHandlers, config::{DatabaseConfig, KeystoreConfig, NetworkConfiguration}, GenericChainSpec, RuntimeGenesis }; use wasm_bindgen::prelude::*; -use futures::{prelude::*, channel::{oneshot, mpsc}, future::{poll_fn, ok}, compat::*}; -use std::task::Poll; -use std::pin::Pin; +use futures::{ + prelude::*, channel::{oneshot, mpsc}, compat::*, future::{ready, ok, select} +}; +use std::{sync::Arc, pin::Pin}; use sc_chain_spec::Extension; use libp2p_wasm_ext::{ExtTransport, ffi}; @@ -64,7 +64,7 @@ where network, telemetry_endpoints: chain_spec.telemetry_endpoints().clone(), chain_spec: Box::new(chain_spec), - task_executor: Arc::new(move |fut, _| wasm_bindgen_futures::spawn_local(fut)), + task_executor: (|fut, _| wasm_bindgen_futures::spawn_local(fut)).into(), telemetry_external_transport: Some(transport), role: Role::Light, database: { @@ -86,6 +86,7 @@ where pruning: Default::default(), rpc_cors: Default::default(), rpc_http: Default::default(), + rpc_ipc: Default::default(), rpc_ws: Default::default(), rpc_ws_max_connections: Default::default(), rpc_methods: Default::default(), @@ -97,6 +98,11 @@ where wasm_method: Default::default(), max_runtime_instances: 8, announce_block: true, + base_path: None, + informant_output_format: sc_informant::OutputFormat { + enable_color: false, + prefix: String::new(), + }, }; Ok(config) @@ -115,36 +121,25 @@ struct RpcMessage { } /// Create a Client object that connects to a service. -pub fn start_client(mut service: impl AbstractService) -> Client { - // Spawn informant - wasm_bindgen_futures::spawn_local( - sc_informant::build(&service, sc_informant::OutputFormat::Plain).map(drop) - ); - +pub fn start_client(mut task_manager: TaskManager, rpc_handlers: Arc) -> Client { // We dispatch a background task responsible for processing the service. // // The main action performed by the code below consists in polling the service with // `service.poll()`. // The rest consists in handling RPC requests. - let (rpc_send_tx, mut rpc_send_rx) = mpsc::unbounded::(); - wasm_bindgen_futures::spawn_local(poll_fn(move |cx| { - loop { - match Pin::new(&mut rpc_send_rx).poll_next(cx) { - Poll::Ready(Some(message)) => { - let fut = service - .rpc_query(&message.session, &message.rpc_json) - .boxed(); - let _ = message.send_back.send(fut); - }, - Poll::Pending => break, - Poll::Ready(None) => return Poll::Ready(()), - } - } - - Pin::new(&mut service) - .poll(cx) - .map(drop) - })); + let (rpc_send_tx, rpc_send_rx) = mpsc::unbounded::(); + wasm_bindgen_futures::spawn_local( + select( + rpc_send_rx.for_each(move |message| { + let fut = rpc_handlers.rpc_query(&message.session, &message.rpc_json); + let _ = message.send_back.send(fut); + ready(()) + }), + Box::pin(async move { + let _ = task_manager.future().await; + }), + ).map(drop) + ); Client { rpc_send_tx, diff --git a/utils/build-script-utils/Cargo.toml b/utils/build-script-utils/Cargo.toml index 6c3f17be2bf126e43fdf68b7590bb591410acf22..9eada7bf820a03fc413a6b5ac6b5af4ee4c687dc 100644 --- a/utils/build-script-utils/Cargo.toml +++ b/utils/build-script-utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-build-script-utils" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" diff --git a/utils/fork-tree/Cargo.toml b/utils/fork-tree/Cargo.toml index d663774bd7ed143350f3c893ef35dfb3c8e09de4..a1aaea70b1f54aa3aa9ff882e99a97c5c1ae989b 100644 --- a/utils/fork-tree/Cargo.toml +++ b/utils/fork-tree/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fork-tree" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -13,4 +13,4 @@ documentation = "https://docs.rs/fork-tree" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "1.3.1", features = ["derive"] } diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index 71bd64a0a6b9e470e5645c4888aede4e40152043..003b4d9c05b3e2ea03f4b010d91af6324ca98adc 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-benchmarking-cli" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,18 +12,18 @@ description = "CLI for benchmarking FRAME" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -frame-benchmarking = { version = "2.0.0-alpha.8", path = "../../../frame/benchmarking" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../primitives/core" } -sc-service = { version = "0.8.0-alpha.8", default-features = false, path = "../../../client/service" } -sc-cli = { version = "0.8.0-alpha.8", path = "../../../client/cli" } -sc-client-db = { version = "0.8.0-alpha.8", path = "../../../client/db" } -sc-executor = { version = "0.8.0-alpha.8", path = "../../../client/executor" } -sp-externalities = { version = "0.8.0-alpha.8", path = "../../../primitives/externalities" } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../primitives/runtime" } -sp-state-machine = { version = "0.8.0-alpha.8", path = "../../../primitives/state-machine" } +frame-benchmarking = { version = "2.0.0-rc4", path = "../../../frame/benchmarking" } +sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } +sc-service = { version = "0.8.0-rc4", default-features = false, path = "../../../client/service" } +sc-cli = { version = "0.8.0-rc4", path = "../../../client/cli" } +sc-client-db = { version = "0.8.0-rc4", path = "../../../client/db" } +sc-executor = { version = "0.8.0-rc4", path = "../../../client/executor" } +sp-externalities = { version = "0.8.0-rc4", path = "../../../primitives/externalities" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../primitives/runtime" } +sp-state-machine = { version = "0.8.0-rc4", path = "../../../primitives/state-machine" } structopt = "0.3.8" -codec = { version = "1.3.0", package = "parity-scale-codec" } +codec = { version = "1.3.1", package = "parity-scale-codec" } [features] default = ["db"] -db = ["sc-client-db/kvdb-rocksdb", "sc-client-db/parity-db"] +db = ["sc-client-db/with-kvdb-rocksdb", "sc-client-db/with-parity-db"] diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index ed63ec5e5d350a9b04b245a7d14e23ec5e821851..7f55672885d463dfa13b262208d280ad1300c27e 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -17,7 +17,7 @@ use crate::BenchmarkCmd; use codec::{Decode, Encode}; -use frame_benchmarking::{Analysis, BenchmarkBatch}; +use frame_benchmarking::{Analysis, BenchmarkBatch, BenchmarkSelector}; use sc_cli::{SharedParams, CliConfiguration, ExecutionStrategy, Result}; use sc_client_db::BenchmarkingState; use sc_executor::NativeExecutor; @@ -27,7 +27,12 @@ use sc_service::{Configuration, NativeExecutionDispatch}; use sp_runtime::{ traits::{Block as BlockT, Header as HeaderT, NumberFor}, }; -use sp_core::{tasks, testing::KeyStore, traits::KeystoreExt}; +use sp_core::{ + tasks, + testing::KeyStore, + traits::KeystoreExt, + offchain::{OffchainExt, testing::TestOffchainExt}, +}; use std::fmt::Debug; impl BenchmarkCmd { @@ -56,6 +61,8 @@ impl BenchmarkCmd { let mut extensions = Extensions::default(); extensions.register(KeystoreExt(KeyStore::new())); + let (offchain, _) = TestOffchainExt::new(); + extensions.register(OffchainExt::new(offchain)); let result = StateMachine::<_, _, NumberFor, _>::new( &state, @@ -100,15 +107,22 @@ impl BenchmarkCmd { if self.raw_data { // Print the table header - batch.results[0].0.iter().for_each(|param| print!("{:?},", param.0)); + batch.results[0].components.iter().for_each(|param| print!("{:?},", param.0)); - print!("extrinsic_time,storage_root_time\n"); + print!("extrinsic_time,storage_root_time,reads,repeat_reads,writes,repeat_writes\n"); // Print the values batch.results.iter().for_each(|result| { - let parameters = &result.0; + let parameters = &result.components; parameters.iter().for_each(|param| print!("{:?},", param.1)); // Print extrinsic time and storage root time - print!("{:?},{:?}\n", result.1, result.2); + print!("{:?},{:?},{:?},{:?},{:?},{:?}\n", + result.extrinsic_time, + result.storage_root_time, + result.reads, + result.repeat_reads, + result.writes, + result.repeat_writes, + ); }); println!(); @@ -116,13 +130,27 @@ impl BenchmarkCmd { // Conduct analysis. if !self.no_median_slopes { - if let Some(analysis) = Analysis::median_slopes(&batch.results) { - println!("Median Slopes Analysis\n========\n{}", analysis); + println!("Median Slopes Analysis\n========"); + if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::ExtrinsicTime) { + println!("-- Extrinsic Time --\n{}", analysis); + } + if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Reads) { + println!("Reads = {:?}", analysis); + } + if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Writes) { + println!("Writes = {:?}", analysis); } } if !self.no_min_squares { - if let Some(analysis) = Analysis::min_squares_iqr(&batch.results) { - println!("Min Squares Analysis\n========\n{}", analysis); + println!("Min Squares Analysis\n========"); + if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime) { + println!("-- Extrinsic Time --\n{}", analysis); + } + if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads) { + println!("Reads = {:?}", analysis); + } + if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes) { + println!("Writes = {:?}", analysis); } } }, diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 7704f032b87fa9388d8902daa053a2ddae586a74..149b971577faba53a850052a962c5598a06defd0 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -21,7 +21,7 @@ use sc_cli::{ExecutionStrategy, WasmExecutionMethod}; use std::fmt::Debug; /// The `benchmark` command used to benchmark FRAME Pallets. -#[derive(Debug, structopt::StructOpt, Clone)] +#[derive(Debug, structopt::StructOpt)] pub struct BenchmarkCmd { /// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`). #[structopt(short, long)] diff --git a/utils/frame/rpc/support/Cargo.toml b/utils/frame/rpc/support/Cargo.toml index 1dbb8b09a87b989596eed64da1e0ccabdc55a6fa..ec4d06c93c9cca1d1cb8f3a55268fbd2ea9480b8 100644 --- a/utils/frame/rpc/support/Cargo.toml +++ b/utils/frame/rpc/support/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-frame-rpc-support" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies ", "Andrew Dirksen "] edition = "2018" license = "Apache-2.0" @@ -13,14 +13,14 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] futures = { version = "0.3.0", features = ["compat"] } -jsonrpc-client-transports = { version = "14.0.5", default-features = false, features = ["http"] } -jsonrpc-core = "14" -codec = { package = "parity-scale-codec", version = "1" } +jsonrpc-client-transports = { version = "14.2.0", default-features = false, features = ["http"] } +jsonrpc-core = "14.2.0" +codec = { package = "parity-scale-codec", version = "1.3.1" } serde = "1" -frame-support = { version = "2.0.0-alpha.8", path = "../../../../frame/support" } -sp-storage = { version = "2.0.0-alpha.8", path = "../../../../primitives/storage" } -sc-rpc-api = { version = "0.8.0-alpha.8", path = "../../../../client/rpc-api" } +frame-support = { version = "2.0.0-rc4", path = "../../../../frame/support" } +sp-storage = { version = "2.0.0-rc4", path = "../../../../primitives/storage" } +sc-rpc-api = { version = "0.8.0-rc4", path = "../../../../client/rpc-api" } [dev-dependencies] -frame-system = { version = "2.0.0-alpha.8", path = "../../../../frame/system" } +frame-system = { version = "2.0.0-rc4", path = "../../../../frame/system" } tokio = "0.2" diff --git a/utils/frame/rpc/system/Cargo.toml b/utils/frame/rpc/system/Cargo.toml index 991878cef0c1105fe61b1d352e04398015e5631c..1d655bcca3470bdfe2af43135bf51e1ba03a8bfe 100644 --- a/utils/frame/rpc/system/Cargo.toml +++ b/utils/frame/rpc/system/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-frame-rpc-system" -version = "2.0.0-alpha.8" +version = "2.0.0-rc4" authors = ["Parity Technologies "] edition = "2018" license = "Apache-2.0" @@ -12,22 +12,24 @@ description = "FRAME's system exposed over Substrate RPC" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sc-client-api = { version = "2.0.0-alpha.8", path = "../../../../client/api" } -codec = { package = "parity-scale-codec", version = "1.3.0" } +sc-client-api = { version = "2.0.0-rc4", path = "../../../../client/api" } +codec = { package = "parity-scale-codec", version = "1.3.1" } futures = { version = "0.3.4", features = ["compat"] } -jsonrpc-core = "14.0.3" -jsonrpc-core-client = "14.0.5" -jsonrpc-derive = "14.0.3" +jsonrpc-core = "14.2.0" +jsonrpc-core-client = "14.2.0" +jsonrpc-derive = "14.2.1" log = "0.4.8" serde = { version = "1.0.101", features = ["derive"] } -sp-runtime = { version = "2.0.0-alpha.8", path = "../../../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.8", path = "../../../../primitives/api" } -frame-system-rpc-runtime-api = { version = "2.0.0-alpha.8", path = "../../../../frame/system/rpc/runtime-api" } -sp-core = { version = "2.0.0-alpha.8", path = "../../../../primitives/core" } -sp-blockchain = { version = "2.0.0-alpha.8", path = "../../../../primitives/blockchain" } -sp-transaction-pool = { version = "2.0.0-alpha.8", path = "../../../../primitives/transaction-pool" } +sp-runtime = { version = "2.0.0-rc4", path = "../../../../primitives/runtime" } +sp-api = { version = "2.0.0-rc4", path = "../../../../primitives/api" } +frame-system-rpc-runtime-api = { version = "2.0.0-rc4", path = "../../../../frame/system/rpc/runtime-api" } +sp-core = { version = "2.0.0-rc4", path = "../../../../primitives/core" } +sp-blockchain = { version = "2.0.0-rc4", path = "../../../../primitives/blockchain" } +sp-transaction-pool = { version = "2.0.0-rc4", path = "../../../../primitives/transaction-pool" } +sp-block-builder = { version = "2.0.0-rc4", path = "../../../../primitives/block-builder" } +sc-rpc-api = { version = "0.8.0-rc4", path = "../../../../client/rpc-api" } [dev-dependencies] -substrate-test-runtime-client = { version = "2.0.0-alpha.8", path = "../../../../test-utils/runtime/client" } +substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../../../test-utils/runtime/client" } env_logger = "0.7.0" -sc-transaction-pool = { version = "2.0.0-alpha.8", path = "../../../../client/transaction-pool" } +sc-transaction-pool = { version = "2.0.0-rc4", path = "../../../../client/transaction-pool" } diff --git a/utils/frame/rpc/system/src/lib.rs b/utils/frame/rpc/system/src/lib.rs index 415f9541b60bdabe3747f983449837f93059745f..6927f05b4f05b14867bfe3a9a7c70fd732eca7ce 100644 --- a/utils/frame/rpc/system/src/lib.rs +++ b/utils/frame/rpc/system/src/lib.rs @@ -22,8 +22,8 @@ use std::sync::Arc; use codec::{self, Codec, Decode, Encode}; use sc_client_api::light::{future_header, RemoteBlockchain, Fetcher, RemoteCallRequest}; use jsonrpc_core::{ - Error, ErrorCode, - futures::future::{result, Future}, + Error as RpcError, ErrorCode, + futures::future::{self as rpc_future,result, Future}, }; use jsonrpc_derive::rpc; use futures::future::{ready, TryFutureExt}; @@ -35,18 +35,20 @@ use sp_runtime::{ generic::BlockId, traits, }; -use sp_core::hexdisplay::HexDisplay; +use sp_core::{hexdisplay::HexDisplay, Bytes}; use sp_transaction_pool::{TransactionPool, InPoolTransaction}; +use sp_block_builder::BlockBuilder; +use sc_rpc_api::DenyUnsafe; pub use frame_system_rpc_runtime_api::AccountNonceApi; pub use self::gen_client::Client as SystemClient; /// Future that resolves to account nonce. -pub type FutureResult = Box + Send>; +pub type FutureResult = Box + Send>; /// System RPC methods. #[rpc] -pub trait SystemApi { +pub trait SystemApi { /// Returns the next valid index (aka nonce) for given account. /// /// This method takes into consideration all pending transactions @@ -54,34 +56,57 @@ pub trait SystemApi { /// it fallbacks to query the index from the runtime (aka. state nonce). #[rpc(name = "system_accountNextIndex", alias("account_nextIndex"))] fn nonce(&self, account: AccountId) -> FutureResult; + + /// Dry run an extrinsic at a given block. Return SCALE encoded ApplyExtrinsicResult. + #[rpc(name = "system_dryRun", alias("system_dryRunAt"))] + fn dry_run(&self, extrinsic: Bytes, at: Option) -> FutureResult; +} + +/// Error type of this RPC api. +pub enum Error { + /// The transaction was not decodable. + DecodeError, + /// The call to runtime failed. + RuntimeError, } -const RUNTIME_ERROR: i64 = 1; +impl From for i64 { + fn from(e: Error) -> i64 { + match e { + Error::RuntimeError => 1, + Error::DecodeError => 2, + } + } +} /// An implementation of System-specific RPC methods on full client. pub struct FullSystem { client: Arc, pool: Arc

, + deny_unsafe: DenyUnsafe, _marker: std::marker::PhantomData, } impl FullSystem { /// Create new `FullSystem` given client and transaction pool. - pub fn new(client: Arc, pool: Arc

) -> Self { + pub fn new(client: Arc, pool: Arc

, deny_unsafe: DenyUnsafe,) -> Self { FullSystem { client, pool, + deny_unsafe, _marker: Default::default(), } } } -impl SystemApi for FullSystem +impl SystemApi<::Hash, AccountId, Index> + for FullSystem where C: sp_api::ProvideRuntimeApi, C: HeaderBackend, C: Send + Sync + 'static, C::Api: AccountNonceApi, + C::Api: BlockBuilder, P: TransactionPool + 'static, Block: traits::Block, AccountId: Clone + std::fmt::Display + Codec, @@ -93,8 +118,8 @@ where let best = self.client.info().best_hash; let at = BlockId::hash(best); - let nonce = api.account_nonce(&at, account.clone()).map_err(|e| Error { - code: ErrorCode::ServerError(RUNTIME_ERROR), + let nonce = api.account_nonce(&at, account.clone()).map_err(|e| RpcError { + code: ErrorCode::ServerError(Error::RuntimeError.into()), message: "Unable to query nonce.".into(), data: Some(format!("{:?}", e).into()), })?; @@ -104,6 +129,38 @@ where Box::new(result(get_nonce())) } + + fn dry_run(&self, extrinsic: Bytes, at: Option<::Hash>) -> FutureResult { + if let Err(err) = self.deny_unsafe.check_if_safe() { + return Box::new(rpc_future::err(err.into())); + } + + let dry_run = || { + let api = self.client.runtime_api(); + let at = BlockId::::hash(at.unwrap_or_else(|| + // If the block hash is not supplied assume the best block. + self.client.info().best_hash + )); + + let uxt: ::Extrinsic = Decode::decode(&mut &*extrinsic).map_err(|e| RpcError { + code: ErrorCode::ServerError(Error::DecodeError.into()), + message: "Unable to dry run extrinsic.".into(), + data: Some(format!("{:?}", e).into()), + })?; + + let result = api.apply_extrinsic(&at, uxt) + .map_err(|e| RpcError { + code: ErrorCode::ServerError(Error::RuntimeError.into()), + message: "Unable to dry run extrinsic.".into(), + data: Some(format!("{:?}", e).into()), + })?; + + Ok(Encode::encode(&result).into()) + }; + + + Box::new(result(dry_run())) + } } /// An implementation of System-specific RPC methods on light client. @@ -131,7 +188,8 @@ impl LightSystem { } } -impl SystemApi for LightSystem +impl SystemApi<::Hash, AccountId, Index> + for LightSystem where P: TransactionPool + 'static, C: HeaderBackend, @@ -165,8 +223,8 @@ where ).compat(); let future_nonce = future_nonce.and_then(|nonce| Decode::decode(&mut &nonce[..]) .map_err(|e| ClientError::CallResultDecode("Cannot decode account nonce", e))); - let future_nonce = future_nonce.map_err(|e| Error { - code: ErrorCode::ServerError(RUNTIME_ERROR), + let future_nonce = future_nonce.map_err(|e| RpcError { + code: ErrorCode::ServerError(Error::RuntimeError.into()), message: "Unable to query nonce.".into(), data: Some(format!("{:?}", e).into()), }); @@ -176,6 +234,14 @@ where Box::new(future_nonce) } + + fn dry_run(&self, _extrinsic: Bytes, _at: Option<::Hash>) -> FutureResult { + Box::new(result(Err(RpcError { + code: ErrorCode::MethodNotFound, + message: "Unable to dry run extrinsic.".into(), + data: None, + }))) + } } /// Adjust account nonce from state, so that tx with the nonce will be @@ -222,16 +288,15 @@ mod tests { use super::*; use futures::executor::block_on; - use substrate_test_runtime_client::{ - runtime::Transfer, - AccountKeyring, - }; + use substrate_test_runtime_client::{runtime::Transfer, AccountKeyring}; use sc_transaction_pool::{BasicPool, FullChainApi}; + use sp_runtime::{ApplyExtrinsicResult, transaction_validity::{TransactionValidityError, InvalidTransaction}}; #[test] fn should_return_next_nonce_for_some_account() { - // given let _ = env_logger::try_init(); + + // given let client = Arc::new(substrate_test_runtime_client::new()); let pool = Arc::new( BasicPool::new( @@ -257,7 +322,7 @@ mod tests { let ext1 = new_transaction(1); block_on(pool.submit_one(&BlockId::number(0), source, ext1)).unwrap(); - let accounts = FullSystem::new(client, pool); + let accounts = FullSystem::new(client, pool, DenyUnsafe::Yes); // when let nonce = accounts.nonce(AccountKeyring::Alice.into()); @@ -265,4 +330,91 @@ mod tests { // then assert_eq!(nonce.wait().unwrap(), 2); } + + #[test] + fn dry_run_should_deny_unsafe() { + let _ = env_logger::try_init(); + + // given + let client = Arc::new(substrate_test_runtime_client::new()); + let pool = Arc::new( + BasicPool::new( + Default::default(), + Arc::new(FullChainApi::new(client.clone())), + None, + ).0 + ); + + let accounts = FullSystem::new(client, pool, DenyUnsafe::Yes); + + // when + let res = accounts.dry_run(vec![].into(), None); + + // then + assert_eq!(res.wait(), Err(RpcError::method_not_found())); + } + + #[test] + fn dry_run_should_work() { + let _ = env_logger::try_init(); + + // given + let client = Arc::new(substrate_test_runtime_client::new()); + let pool = Arc::new( + BasicPool::new( + Default::default(), + Arc::new(FullChainApi::new(client.clone())), + None, + ).0 + ); + + let accounts = FullSystem::new(client, pool, DenyUnsafe::No); + + let tx = Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Bob.into(), + amount: 5, + nonce: 0, + }.into_signed_tx(); + + // when + let res = accounts.dry_run(tx.encode().into(), None); + + // then + let bytes = res.wait().unwrap().0; + let apply_res: ApplyExtrinsicResult = Decode::decode(&mut bytes.as_slice()).unwrap(); + assert_eq!(apply_res, Ok(Ok(()))); + } + + #[test] + fn dry_run_should_indicate_error() { + let _ = env_logger::try_init(); + + // given + let client = Arc::new(substrate_test_runtime_client::new()); + let pool = Arc::new( + BasicPool::new( + Default::default(), + Arc::new(FullChainApi::new(client.clone())), + None, + ).0 + ); + + let accounts = FullSystem::new(client, pool, DenyUnsafe::No); + + let tx = Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Bob.into(), + amount: 5, + nonce: 100, + }.into_signed_tx(); + + // when + let res = accounts.dry_run(tx.encode().into(), None); + + // then + let bytes = res.wait().unwrap().0; + let apply_res: ApplyExtrinsicResult = Decode::decode(&mut bytes.as_slice()).unwrap(); + assert_eq!(apply_res, Err(TransactionValidityError::Invalid(InvalidTransaction::Stale))); + } } diff --git a/utils/prometheus/Cargo.toml b/utils/prometheus/Cargo.toml index c425a7c87947790ec6e6bc11f2ead6bc8c913cc0..322935a8847b4a8b047c258f0d846800af1d28b9 100644 --- a/utils/prometheus/Cargo.toml +++ b/utils/prometheus/Cargo.toml @@ -1,7 +1,7 @@ [package] description = "Endpoint to expose Prometheus metrics" name = "substrate-prometheus-endpoint" -version = "0.8.0-alpha.8" +version = "0.8.0-rc4" license = "Apache-2.0" authors = ["Parity Technologies "] edition = "2018" @@ -18,6 +18,7 @@ futures-util = { version = "0.3.1", default-features = false, features = ["io"] derive_more = "0.99" [target.'cfg(not(target_os = "unknown"))'.dependencies] -async-std = { version = "1.0.1", features = ["unstable"] } +# async-std is temporarily pinned to <1.6 because version 1.6.0 is buggy +async-std = { version = "1.0.1, <1.6", features = ["unstable"] } hyper = { version = "0.13.1", default-features = false, features = ["stream"] } tokio = "0.2" diff --git a/utils/wasm-builder/Cargo.toml b/utils/wasm-builder/Cargo.toml index 2b7a632b559b6599236cc2f6097745a8c27389f2..e46db432689e06938e81ffa7ce20b264c6d44fc1 100644 --- a/utils/wasm-builder/Cargo.toml +++ b/utils/wasm-builder/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-wasm-builder" -version = "1.0.10" +version = "1.0.11" authors = ["Parity Technologies "] description = "Utility for building WASM binaries" edition = "2018" @@ -14,10 +14,10 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] build-helper = "0.1.1" -cargo_metadata = "0.9.0" +cargo_metadata = "0.10.0" tempfile = "3.1.0" toml = "0.5.4" -walkdir = "2.2.9" +walkdir = "2.3.1" fs2 = "0.4.3" wasm-gc-api = "0.1.11" atty = "0.2.13" diff --git a/utils/wasm-builder/README.md b/utils/wasm-builder/README.md index 5f4ca615d5065dca62e1d9c7d0463975c52acb2c..b72e7e16d4ff4110597e4f44119d571842b79e3e 100644 --- a/utils/wasm-builder/README.md +++ b/utils/wasm-builder/README.md @@ -1,11 +1,11 @@ -# WASM builder is a utility for building a project as a WASM binary +# Wasm builder is a utility for building a project as a Wasm binary -The WASM builder is a tool that integrates the process of building the WASM binary of your project into the main +The Wasm builder is a tool that integrates the process of building the WASM binary of your project into the main `cargo` build process. ## Project setup -A project that should be compiled as a WASM binary needs to: +A project that should be compiled as a Wasm binary needs to: 1. Add a `build.rs` file. 2. Add `substrate-wasm-builder` as dependency into `build-dependencies`. @@ -31,12 +31,20 @@ As the final step, you need to add the following to your project: include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); ``` -This will include the generated WASM binary as two constants `WASM_BINARY` and `WASM_BINARY_BLOATY`. -The former is a compact WASM binary and the latter is not compacted. +This will include the generated Wasm binary as two constants `WASM_BINARY` and `WASM_BINARY_BLOATY`. +The former is a compact Wasm binary and the latter is not compacted. + +### Feature + +Wasm builder supports to enable cargo features while building the Wasm binary. By default it will +enable all features in the wasm build that are enabled for the native build except the +`default` and `std` features. Besides that, wasm builder supports the special `runtime-wasm` +feature. This `runtime-wasm` feature will be enabled by the wasm builder when it compiles the +Wasm binary. If this feature is not present, it will not be enabled. ## Environment variables -By using environment variables, you can configure which WASM binaries are built and how: +By using environment variables, you can configure which Wasm binaries are built and how: - `SKIP_WASM_BUILD` - Skips building any wasm binary. This is useful when only native should be recompiled. - `BUILD_DUMMY_WASM_BINARY` - Builds dummy wasm binaries. These dummy binaries are empty and useful @@ -44,7 +52,7 @@ By using environment variables, you can configure which WASM binaries are built - `WASM_BUILD_TYPE` - Sets the build type for building wasm binaries. Supported values are `release` or `debug`. By default the build type is equal to the build type used by the main build. - `TRIGGER_WASM_BUILD` - Can be set to trigger a wasm build. On subsequent calls the value of the variable - needs to change. As WASM builder instructs `cargo` to watch for file changes + needs to change. As wasm builder instructs `cargo` to watch for file changes this environment variable should only be required in certain circumstances. - `WASM_BUILD_RUSTFLAGS` - Extend `RUSTFLAGS` given to `cargo build` while building the wasm binary. - `WASM_BUILD_NO_COLOR` - Disable color output of the wasm build. @@ -59,7 +67,7 @@ be `NODE_RUNTIME`. ## Prerequisites: -WASM builder requires the following prerequisites for building the WASM binary: +Wasm builder requires the following prerequisites for building the Wasm binary: - rust nightly + `wasm32-unknown-unknown` toolchain @@ -67,4 +75,4 @@ If a specific rust nightly is installed with `rustup`, it is important that the as well. For example if installing the rust nightly from 20.02.2020 using `rustup install nightly-2020-02-20`, the wasm target needs to be installed as well `rustup target add wasm32-unknown-unknown --toolchain nightly-2020-02-20`. -License: GPL-3.0 +License: Apache-2.0 diff --git a/utils/wasm-builder/src/lib.rs b/utils/wasm-builder/src/lib.rs index 9f9e77275a9962c752be4bdaf68989977f6adaf0..95b75c5867f6097d4d8eb51703713228650000c9 100644 --- a/utils/wasm-builder/src/lib.rs +++ b/utils/wasm-builder/src/lib.rs @@ -15,14 +15,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! # WASM builder is a utility for building a project as a WASM binary +//! # Wasm builder is a utility for building a project as a Wasm binary //! -//! The WASM builder is a tool that integrates the process of building the WASM binary of your project into the main +//! The Wasm builder is a tool that integrates the process of building the WASM binary of your project into the main //! `cargo` build process. //! //! ## Project setup //! -//! A project that should be compiled as a WASM binary needs to: +//! A project that should be compiled as a Wasm binary needs to: //! //! 1. Add a `build.rs` file. //! 2. Add `substrate-wasm-builder` as dependency into `build-dependencies`. @@ -48,12 +48,20 @@ //! include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); //! ``` //! -//! This will include the generated WASM binary as two constants `WASM_BINARY` and `WASM_BINARY_BLOATY`. -//! The former is a compact WASM binary and the latter is not compacted. +//! This will include the generated Wasm binary as two constants `WASM_BINARY` and `WASM_BINARY_BLOATY`. +//! The former is a compact Wasm binary and the latter is not compacted. +//! +//! ### Feature +//! +//! Wasm builder supports to enable cargo features while building the Wasm binary. By default it will +//! enable all features in the wasm build that are enabled for the native build except the +//! `default` and `std` features. Besides that, wasm builder supports the special `runtime-wasm` +//! feature. This `runtime-wasm` feature will be enabled by the wasm builder when it compiles the +//! Wasm binary. If this feature is not present, it will not be enabled. //! //! ## Environment variables //! -//! By using environment variables, you can configure which WASM binaries are built and how: +//! By using environment variables, you can configure which Wasm binaries are built and how: //! //! - `SKIP_WASM_BUILD` - Skips building any wasm binary. This is useful when only native should be recompiled. //! - `BUILD_DUMMY_WASM_BINARY` - Builds dummy wasm binaries. These dummy binaries are empty and useful @@ -61,7 +69,7 @@ //! - `WASM_BUILD_TYPE` - Sets the build type for building wasm binaries. Supported values are `release` or `debug`. //! By default the build type is equal to the build type used by the main build. //! - `TRIGGER_WASM_BUILD` - Can be set to trigger a wasm build. On subsequent calls the value of the variable -//! needs to change. As WASM builder instructs `cargo` to watch for file changes +//! needs to change. As wasm builder instructs `cargo` to watch for file changes //! this environment variable should only be required in certain circumstances. //! - `WASM_BUILD_RUSTFLAGS` - Extend `RUSTFLAGS` given to `cargo build` while building the wasm binary. //! - `WASM_BUILD_NO_COLOR` - Disable color output of the wasm build. @@ -76,7 +84,7 @@ //! //! ## Prerequisites: //! -//! WASM builder requires the following prerequisites for building the WASM binary: +//! Wasm builder requires the following prerequisites for building the Wasm binary: //! //! - rust nightly + `wasm32-unknown-unknown` toolchain //! @@ -185,6 +193,17 @@ fn write_file_if_changed(file: PathBuf, content: String) { } } +/// Copy `src` to `dst` if the `dst` does not exist or is different. +fn copy_file_if_changed(src: PathBuf, dst: PathBuf) { + let src_file = fs::read_to_string(&src).ok(); + let dst_file = fs::read_to_string(&dst).ok(); + + if src_file != dst_file { + fs::copy(&src, &dst) + .expect(&format!("Copying `{}` to `{}` can not fail; qed", src.display(), dst.display())); + } +} + /// Get a cargo command that compiles with nightly fn get_nightly_cargo() -> CargoCommand { let env_cargo = CargoCommand::new( diff --git a/utils/wasm-builder/src/wasm_project.rs b/utils/wasm-builder/src/wasm_project.rs index d1c926c90410f2472ccce09b97c5c8138923e7b1..7df3524e8aaf8a9da430ba5962ec70fb4e3d4da5 100644 --- a/utils/wasm-builder/src/wasm_project.rs +++ b/utils/wasm-builder/src/wasm_project.rs @@ -26,7 +26,7 @@ use toml::value::Table; use build_helper::rerun_if_changed; -use cargo_metadata::MetadataCommand; +use cargo_metadata::{MetadataCommand, Metadata}; use walkdir::WalkDir; @@ -302,10 +302,23 @@ fn create_wasm_workspace_project(wasm_workspace: &Path, workspace_root_path: &Pa wasm_workspace_toml.insert("patch".into(), patch.into()); } - fs::write( + write_file_if_changed( wasm_workspace.join("Cargo.toml"), toml::to_string_pretty(&wasm_workspace_toml).expect("Wasm workspace toml is valid; qed"), - ).expect("WASM workspace `Cargo.toml` writing can not fail; qed"); + ); +} + +/// Find a package by the given `manifest_path` in the metadata. +/// +/// Panics if the package could not be found. +fn find_package_by_manifest_path<'a>( + manifest_path: &Path, + crate_metadata: &'a cargo_metadata::Metadata, +) -> &'a cargo_metadata::Package { + crate_metadata.packages + .iter() + .find(|p| p.manifest_path == manifest_path) + .expect("Wasm project exists in its own metadata; qed") } /// Get a list of enabled features for the project. @@ -313,10 +326,7 @@ fn project_enabled_features( cargo_manifest: &Path, crate_metadata: &cargo_metadata::Metadata, ) -> Vec { - let package = crate_metadata.packages - .iter() - .find(|p| p.manifest_path == cargo_manifest) - .expect("Wasm project exists in its own metadata; qed"); + let package = find_package_by_manifest_path(cargo_manifest, crate_metadata); let mut enabled_features = package.features.keys() .filter(|f| { @@ -338,23 +348,34 @@ fn project_enabled_features( enabled_features } +/// Returns if the project has the `runtime-wasm` feature +fn has_runtime_wasm_feature_declared( + cargo_manifest: &Path, + crate_metadata: &cargo_metadata::Metadata, +) -> bool { + let package = find_package_by_manifest_path(cargo_manifest, crate_metadata); + + package.features.keys().any(|k| k == "runtime-wasm") +} + /// Create the project used to build the wasm binary. /// /// # Returns /// The path to the created project. -fn create_project( - cargo_manifest: &Path, - wasm_workspace: &Path, - crate_metadata: &cargo_metadata::Metadata, -) -> PathBuf { +fn create_project(cargo_manifest: &Path, wasm_workspace: &Path, crate_metadata: &Metadata) -> PathBuf { let crate_name = get_crate_name(cargo_manifest); let crate_path = cargo_manifest.parent().expect("Parent path exists; qed"); let wasm_binary = get_wasm_binary_name(cargo_manifest); let project_folder = wasm_workspace.join(&crate_name); - fs::create_dir_all(project_folder.join("src")).expect("Wasm project dir create can not fail; qed"); + fs::create_dir_all(project_folder.join("src")) + .expect("Wasm project dir create can not fail; qed"); - let enabled_features = project_enabled_features(&cargo_manifest, &crate_metadata); + let mut enabled_features = project_enabled_features(&cargo_manifest, &crate_metadata); + + if has_runtime_wasm_feature_declared(cargo_manifest, crate_metadata) { + enabled_features.push("runtime-wasm".into()); + } write_file_if_changed( project_folder.join("Cargo.toml"), @@ -386,8 +407,7 @@ fn create_project( if let Some(crate_lock_file) = find_cargo_lock(cargo_manifest) { // Use the `Cargo.lock` of the main project. - fs::copy(crate_lock_file, wasm_workspace.join("Cargo.lock")) - .expect("Copying the `Cargo.lock` can not fail; qed"); + crate::copy_file_if_changed(crate_lock_file, wasm_workspace.join("Cargo.lock")); } project_folder @@ -519,22 +539,33 @@ fn generate_rerun_if_changed_instructions( .exec() .expect("`cargo metadata` can not fail!"); - // Start with the dependencies of the crate we want to compile for wasm. - let mut dependencies = metadata.packages + let package = metadata.packages .iter() .find(|p| p.manifest_path == cargo_manifest) - .expect("The crate package is contained in its own metadata; qed") - .dependencies - .iter() - .collect::>(); + .expect("The crate package is contained in its own metadata; qed"); + + // Start with the dependencies of the crate we want to compile for wasm. + let mut dependencies = package.dependencies.iter().collect::>(); // Collect all packages by follow the dependencies of all packages we find. let mut packages = HashSet::new(); + packages.insert(DeduplicatePackage::from(package)); + while let Some(dependency) = dependencies.pop() { + let path_or_git_dep = dependency.source + .as_ref() + .map(|s| s.starts_with("git+")) + .unwrap_or(true); + let package = metadata.packages .iter() .filter(|p| !p.manifest_path.starts_with(wasm_workspace)) - .find(|p| dependency.req.matches(&p.version) && dependency.name == p.name); + .find(|p| { + // Check that the name matches and that the version matches or this is + // a git or path dep. A git or path dependency can only occur once, so we don't + // need to check the version. + (path_or_git_dep || dependency.req.matches(&p.version)) && dependency.name == p.name + }); if let Some(package) = package { if packages.insert(DeduplicatePackage::from(package)) { @@ -544,21 +575,7 @@ fn generate_rerun_if_changed_instructions( } // Make sure that if any file/folder of a dependency change, we need to rerun the `build.rs` - packages.into_iter() - .filter(|p| !p.manifest_path.starts_with(wasm_workspace)) - .for_each(|package| { - let mut manifest_path = package.manifest_path.clone(); - if manifest_path.ends_with("Cargo.toml") { - manifest_path.pop(); - } - - rerun_if_changed(&manifest_path); - - WalkDir::new(manifest_path) - .into_iter() - .filter_map(|p| p.ok()) - .for_each(|p| rerun_if_changed(p.path())); - }); + packages.iter().for_each(package_rerun_if_changed); // Register our env variables println!("cargo:rerun-if-env-changed={}", crate::SKIP_BUILD_ENV); @@ -568,8 +585,32 @@ fn generate_rerun_if_changed_instructions( println!("cargo:rerun-if-env-changed={}", crate::WASM_BUILD_TOOLCHAIN); } -/// Copy the WASM binary to the target directory set in `WASM_TARGET_DIRECTORY` environment variable. -/// If the variable is not set, this is a no-op. +/// Track files and paths related to the given package to rerun `build.rs` on any relevant change. +fn package_rerun_if_changed(package: &DeduplicatePackage) { + let mut manifest_path = package.manifest_path.clone(); + if manifest_path.ends_with("Cargo.toml") { + manifest_path.pop(); + } + + WalkDir::new(&manifest_path) + .into_iter() + .filter_entry(|p| { + // Ignore this entry if it is a directory that contains a `Cargo.toml` that is not the + // `Cargo.toml` related to the current package. This is done to ignore sub-crates of a crate. + // If such a sub-crate is a dependency, it will be processed independently anyway. + p.path() == manifest_path + || !p.path().is_dir() + || !p.path().join("Cargo.toml").exists() + }) + .filter_map(|p| p.ok().map(|p| p.into_path())) + .filter(|p| { + p.is_dir() || p.extension().map(|e| e == "rs" || e == "toml").unwrap_or_default() + }) + .for_each(|p| rerun_if_changed(p)); +} + +/// Copy the WASM binary to the target directory set in `WASM_TARGET_DIRECTORY` environment +/// variable. If the variable is not set, this is a no-op. fn copy_wasm_to_target_directory(cargo_manifest: &Path, wasm_binary: &WasmBinary) { let target_dir = match env::var(crate::WASM_TARGET_DIRECTORY) { Ok(path) => PathBuf::from(path),