diff --git a/.editorconfig b/.editorconfig index 47fde53b690b5b86037ede8cbf0337db8a472d7d..2b40ec32fac3e935b4f85e70bf339224d5e8f8b0 100644 --- a/.editorconfig +++ b/.editorconfig @@ -14,3 +14,9 @@ indent_style=space indent_size=2 tab_width=8 end_of_line=lf + +[*.sh] +indent_style=space +indent_size=2 +tab_width=8 +end_of_line=lf diff --git a/.github/ISSUE_TEMPLATE/release.md b/.github/ISSUE_TEMPLATE/release.md new file mode 100644 index 0000000000000000000000000000000000000000..6067dbf12fa70ab43a3065e59e7e68375b721049 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/release.md @@ -0,0 +1,9 @@ +--- +title: Release failure for {{ ref }} +--- + +Pipeline for release {{ ref }} failed. Please investigate. + +If the pipeline has failed before pushing to crates.io, delete the release tag +and fix the release as necessary, retagging after complete. If the pipeline has +failed after pushing to crates.io, create a new tag incrementing the version. diff --git a/.github/workflows/check-gitlab-pipeline.yml b/.github/workflows/check-gitlab-pipeline.yml new file mode 100644 index 0000000000000000000000000000000000000000..c87f17c2f732e447cacf297cfe41e7b266a4bc99 --- /dev/null +++ b/.github/workflows/check-gitlab-pipeline.yml @@ -0,0 +1,30 @@ +# A github action to track the status of the gitlab pipeline for tagged +# releases, and cancel the release/create a new issue if it fails + +name: Monitor gitlab pipeline status + +on: + push: + tags: + - v* + - ci-release-* + +jobs: + monitor: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Monitor pipeline + run: env; ./.maintain/github/check_gitlab_pipeline.sh + id: monitor_pipeline + env: + TAGGER: ${{ github.event.pusher.name }} + - name: Create Issue + if: failure() + uses: JasonEtco/create-an-issue@v2 + with: + filename: .github/ISSUE_TEMPLATE/release.md + assignees: ${{ github.event.pusher.name }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/trigger-review-pipeline.yml b/.github/workflows/trigger-review-pipeline.yml new file mode 100644 index 0000000000000000000000000000000000000000..af54ec4358b438b7cd53513c33e661fef833aa94 --- /dev/null +++ b/.github/workflows/trigger-review-pipeline.yml @@ -0,0 +1,20 @@ +name: Trigger pipeline for review + +on: + pull_request: + types: [ready_for_review] + +jobs: + trigger: + runs-on: ubuntu-latest + + steps: + - name: Trigger pipeline + run: | + curl -X POST \ + -F token="$TOKEN" \ + -F ref="$REF" \ + https://gitlab.parity.io/api/v4/projects/145/trigger/pipeline + env: + REF: ${{ github.event.number }} + TOKEN: ${{ secrets.GITLAB_TRIGGER_TOKEN }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 226695043f7fa93756ed70ca0a9530a36a4dde15..509bd8f07e4adf4667ce0babe36fae876c3b34de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,14 +32,17 @@ stages: variables: GIT_STRATEGY: fetch GIT_DEPTH: 100 - CARGO_HOME: "/ci-cache/${CI_PROJECT_NAME}/cargo/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}" + SCCACHE_DIR: "/ci-cache/${CI_PROJECT_NAME}/sccache" CARGO_INCREMENTAL: 0 CI_SERVER_NAME: "GitLab CI" DOCKER_OS: "debian:stretch" ARCH: "x86_64" # FIXME set to release CARGO_UNLEASH_INSTALL_PARAMS: "--version 1.0.0-alpha.10" - CARGO_UNLEASH_PKG_DEF: "--skip node node-* pallet-template pallet-example pallet-example-* subkey chain-spec-builder sp-arithmetic-fuzzer" + CARGO_UNLEASH_PKG_DEF: "--skip node node-* pallet-template pallet-example pallet-example-* subkey chain-spec-builder" + CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: "wasm-bindgen-test-runner" + WASM_BINDGEN_TEST_TIMEOUT: 120 + CHROMEDRIVER_ARGS: "--log-level=INFO --whitelisted-ips=127.0.0.1" .collect-artifacts: &collect-artifacts @@ -193,15 +196,15 @@ test-linux-stable: &test-linux - WASM_BUILD_NO_COLOR=1 time cargo test --all --release --verbose --locked |& tee output.log - sccache -s - after_script: - - echo "___Collecting warnings for check_warnings job___" + - echo "____Test job successful, checking for warnings____" - awk '/^warning:/,/^$/ { print }' output.log > ${CI_COMMIT_SHORT_SHA}_warnings.log - artifacts: - name: $CI_COMMIT_SHORT_SHA - expire_in: 3 days - paths: - - ${CI_COMMIT_SHORT_SHA}_warnings.log - + - if [ -s ${CI_COMMIT_SHORT_SHA}_warnings.log ]; then + cat ${CI_COMMIT_SHORT_SHA}_warnings.log; + exit 1; + else + echo "___No warnings___"; + fi + test-dependency-rules: stage: test <<: *docker-env @@ -284,7 +287,7 @@ test-runtime-benchmarks: - $DEPLOY_TAG script: - cd bin/node/cli - - BUILD_DUMMY_WASM_BINARY=1 time cargo check --verbose --features runtime-benchmarks + - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --features runtime-benchmarks - sccache -s test-linux-stable-int: @@ -310,7 +313,6 @@ test-linux-stable-int: paths: - ${CI_COMMIT_SHORT_SHA}_int_failure.log - check-web-wasm: stage: test <<: *docker-env @@ -322,7 +324,6 @@ check-web-wasm: - time cargo build --target=wasm32-unknown-unknown -p sp-io - time cargo build --target=wasm32-unknown-unknown -p sp-runtime - time cargo build --target=wasm32-unknown-unknown -p sp-std - - time cargo build --target=wasm32-unknown-unknown -p sc-client - time cargo build --target=wasm32-unknown-unknown -p sc-consensus-aura - time cargo build --target=wasm32-unknown-unknown -p sc-consensus-babe - time cargo build --target=wasm32-unknown-unknown -p sp-consensus @@ -360,6 +361,15 @@ cargo-check-macos: #### stage: build +test-browser-node: + stage: build + <<: *docker-env + needs: + - job: check-web-wasm + artifacts: false + script: + - cargo +nightly test --target wasm32-unknown-unknown -p node-browser-testing -Z features=itarget + build-linux-substrate: &build-binary stage: build <<: *collect-artifacts @@ -430,27 +440,6 @@ build-rust-doc-release: - echo "" > ./crate-docs/index.html - sccache -s - -check_warnings: - stage: build - <<: *docker-env - except: - variables: - - $DEPLOY_TAG - variables: - GIT_STRATEGY: none - needs: - - job: test-linux-stable - artifacts: true - script: - - if [ -s ${CI_COMMIT_SHORT_SHA}_warnings.log ]; then - cat ${CI_COMMIT_SHORT_SHA}_warnings.log; - exit 1; - else - echo "___No warnings___"; - fi - - check-polkadot-companion-status: stage: post-build-test image: parity/tools:latest @@ -582,46 +571,6 @@ publish-s3-doc: - aws s3 ls s3://${BUCKET}/${PREFIX}/ --human-readable --summarize - -publish-gh-doc: - stage: publish - image: parity/tools:latest - allow_failure: true - dependencies: - - build-rust-doc-release - cache: {} - <<: *build-only - <<: *kubernetes-build - variables: - GIT_STRATEGY: none - GITHUB_API: "https://api.github.com" - script: - - test -r ./crate-docs/index.html || ( - echo "./crate-docs/index.html not present, build:rust:doc:release job not complete"; - exit 1 - ) - - test "${GITHUB_USER}" -a "${GITHUB_EMAIL}" -a "${GITHUB_TOKEN}" || ( - echo "environment variables for github insufficient"; - exit 1 - ) - - | - cat > ${HOME}/.gitconfig <&1 | sed -r "s|(${GITHUB_USER}):[a-f0-9]+@|\1:REDACTED@|g" - after_script: - - rm -vrf ${HOME}/.gitconfig - publish-draft-release: stage: publish image: parity/tools:latest diff --git a/.maintain/github/check_gitlab_pipeline.sh b/.maintain/github/check_gitlab_pipeline.sh new file mode 100755 index 0000000000000000000000000000000000000000..4e02dfdb2a428cda4a4a6dc224d69b383693e4ce --- /dev/null +++ b/.maintain/github/check_gitlab_pipeline.sh @@ -0,0 +1,37 @@ +#!/bin/bash +SUBSTRATE_API_BASEURL="https://gitlab.parity.io/api/v4/projects/145" + +TAG_NAME=$(echo "$GITHUB_REF" | sed -E 's_refs/tags/(.*)_\1_') +PIPELINE_ID=$(curl -s $SUBSTRATE_API_BASEURL/pipelines | jq -r "map(select(.ref==\"$TAG_NAME\")) | .[0] | .id") +if [ "$PIPELINE_ID" == "null" ]; then + echo "[!] Pipeline for $TAG_NAME not found. Exiting." + exit 1 +fi + +echo "[+] Pipeline path: https://gitlab.parity.io/parity/substrate/pipelines/$PIPELINE_ID" + +# 130 minute job max +for (( c=0; c < 180; c++ )); do + out=$(curl -s "$SUBSTRATE_API_BASEURL/pipelines/$PIPELINE_ID" | jq -r .status) + case $out in + "success") + echo "[+] Pipeline $PIPELINE_ID for $TAG_NAME succeeded!" + exit 0 + ;; + "failed") + echo "[!] Pipeline $PIPELINE_ID for $TAG_NAME failed. Cannot proceed. Check job output on gitlab!" + exit 1 + ;; + "cancelled") + echo "[!] Pipeline $PIPELINE_ID for $TAG_NAME was cancelled. Cannot proceed!" + exit 1 + ;; + "running") + echo "[-] Pipeline $PIPELINE_ID for $TAG_NAME still in progress..." + esac + sleep 60 +done +# If we reach here, we timed out after 30 minutes +echo "[!] Pipeline $PIPELINE_ID for $TAG_NAME timed out! Cannot proceed" +echo "::set-output name=pipeline_status::timedout" +exit 1 diff --git a/.maintain/gitlab/check_polkadot_companion_status.sh b/.maintain/gitlab/check_polkadot_companion_status.sh index b54f457dc56830ef1ad2892809025ff13cf4f38f..5387e68f25cbba284433b8946f20f6eacfc76b9f 100755 --- a/.maintain/gitlab/check_polkadot_companion_status.sh +++ b/.maintain/gitlab/check_polkadot_companion_status.sh @@ -87,7 +87,8 @@ fi curl -H "${github_header}" -sS -o companion_pr_reviews.json \ ${github_api_polkadot_pull_url}/${pr_companion}/reviews -if [ "$(jq -r -e '.[].state' < companion_pr_reviews.json | uniq)" != "APPROVED" ] +if [ -n "$(jq -r -e '.[].state | select(. == "CHANGES_REQUESTED")' < companion_pr_reviews.json)" ] && \ + [ -z "$(jq -r -e '.[].state | select(. == "APPROVED")' < companion_pr_reviews.json)" ] then boldprint "polkadot pr #${pr_companion} not APPROVED" exit 1 diff --git a/.maintain/gitlab/skip_if_draft.sh b/.maintain/gitlab/skip_if_draft.sh new file mode 100755 index 0000000000000000000000000000000000000000..a234a6c18e21c8e415c840f6bba1cf988d2a744c --- /dev/null +++ b/.maintain/gitlab/skip_if_draft.sh @@ -0,0 +1,14 @@ +#!/bin/sh +url="https://api.github.com/repos/paritytech/substrate/pulls/${CI_COMMIT_REF_NAME}" +echo "[+] API URL: $url" + +draft_state=$(curl "$url" | jq -r .draft) +echo "[+] Draft state: $draft_state" + +if [ "$draft_state" = 'true' ]; then + echo "[!] PR is currently a draft, stopping pipeline" + exit 1 +else + echo "[+] PR is not a draft. Proceeding with CI pipeline" + exit 0 +fi diff --git a/.maintain/monitoring/grafana-dashboards/README_dashboard.md b/.maintain/monitoring/grafana-dashboards/README_dashboard.md new file mode 100644 index 0000000000000000000000000000000000000000..37bebc6f8eaae0e9430a612d7bc9676ac666e648 --- /dev/null +++ b/.maintain/monitoring/grafana-dashboards/README_dashboard.md @@ -0,0 +1,14 @@ +## Substrate Dashboard + +Shared templated Grafana dashboards. + +To import the dashboards follow the [Grafana +documentation](https://grafana.com/docs/grafana/latest/reference/export_import/). +You can see an example setup [here](../../../.maintain/sentry-node). + +#### Required labels on Prometheus metrics + +- `instance` referring to a single scrape target (see [Prometheus docs for + details](https://prometheus.io/docs/concepts/jobs_instances/)). + +- `network` referring to the Blockchain network e.g. Kusama. diff --git a/.maintain/monitoring/grafana-dashboards/substrate-dashboard.json b/.maintain/monitoring/grafana-dashboards/substrate-dashboard.json new file mode 100644 index 0000000000000000000000000000000000000000..629b22617b22a970eb1f439179f62711d1bac9b4 --- /dev/null +++ b/.maintain/monitoring/grafana-dashboards/substrate-dashboard.json @@ -0,0 +1,1650 @@ +{ + "annotations": { + "list": [ + { + "$$hashKey": "object:15", + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1586424254170, + "links": [ + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "With love from ColmenaLabs", + "tooltip": "", + "type": "link", + "url": "https://colmenalabs.org" + }, + { + "icon": "external link", + "tags": [], + "targetBlank": true, + "title": "Polkastats.io", + "tooltip": "", + "type": "link", + "url": "https://polkastats.io" + } + ], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate([[metric_namespace]]_block_height{status=\"finalized\",instance=\"[[instance]]\",network=\"[[network]]\"}[10m])/rate([[metric_namespace]]_block_height{status=\"finalized\",instance=\"[[instance]]\",network=\"[[network]]\"}[1m])", + "intervalFactor": 1, + "legendFormat": "rate[10m] / rate[1m]", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Relative Block Production Speed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 0 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "[[metric_namespace]]_sub_libp2p_peers_count{instance=\"[[instance]]\",network=\"[[network]]\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Peers count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "6.4.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "scalar([[metric_namespace]]_block_height{status=\"best\",instance=\"[[instance]]\",network=\"[[network]]\"})-scalar([[metric_namespace]]_block_height{status=\"finalized\",instance=\"[[instance]]\",network=\"[[network]]\"})", + "intervalFactor": 2, + "legendFormat": "[[hostname]]", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Diff -> ( Best Block - Finalized )", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 0 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate([[metric_namespace]]_block_height{status=\"finalized\",instance=\"[[instance]]\",network=\"[[network]]\"}[10m])*60", + "intervalFactor": 10, + "legendFormat": "{{instance}} Blocks / minute", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Block rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 10, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase([[metric_namespace]]_block_height{instance=\"[[instance]]\",network=\"[[network]]\",status=~\"finalized|sync_target\"}[1m])", + "intervalFactor": 5, + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blocks Av per min", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 6 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "[[metric_namespace]]_block_height{instance=\"[[instance]]\",network=\"[[network]]\",status=~\"finalized|sync_target\"}", + "legendFormat": "{{instance}} {{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Block Finalized", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 6 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "[[metric_namespace]]_block_height{status=\"best\",instance=\"[[instance]]\",network=\"[[network]]\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Block height", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 6 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "data": "", + "expr": "[[metric_namespace]]_ready_transactions_number{instance=\"[[instance]]\",network=\"[[network]]\"}", + "hide": false, + "legendFormat": "{{instance}}", + "refId": "A", + "target": "txcount", + "type": "timeseries" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TXs Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "[[metric_namespace]]_sync_extra_finality_proofs_active{instance=\"[[instance]]\",network=\"[[network]]\"}", + "legendFormat": "{{instance}} active", + "refId": "A" + }, + { + "expr": "[[metric_namespace]]_sync_extra_finality_proofs_failed{instance=\"[[instance]]\",network=\"[[network]]\"}", + "legendFormat": "{{instance}} failed", + "refId": "B" + }, + { + "expr": "[[metric_namespace]]_sync_extra_finality_proofs_importing{instance=\"[[instance]]\",network=\"[[network]]\"}", + "legendFormat": "{{instance}} importing", + "refId": "C" + }, + { + "expr": "[[metric_namespace]]_sync_extra_finality_proofs_pending{instance=\"[[instance]]\",network=\"[[network]]\"}", + "legendFormat": "{{instance}} pending", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sync Proof", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 12 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "[[metric_namespace]]_sync_extra_justifications_active{instance=\"[[instance]]\",network=\"[[network]]\"}", + "legendFormat": "{{instance}} active", + "refId": "A" + }, + { + "expr": "[[metric_namespace]]_sync_extra_justifications_failed{instance=\"[[instance]]\",network=\"[[network]]\"}", + "legendFormat": "{{instance}} failed", + "refId": "B" + }, + { + "expr": "[[metric_namespace]]_sync_extra_justifications_importing{instance=\"[[instance]]\",network=\"[[network]]\"}", + "legendFormat": "{{instance}} importing", + "refId": "C" + }, + { + "expr": "[[metric_namespace]]_sync_extra_justifications_pending{instance=\"[[instance]]\",network=\"[[network]]\"}", + "legendFormat": "{{instance}} pending", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sync justifications", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 12 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "[[metric_namespace]]_sub_libp2p_connections{instance=\"[[instance]]\",network=\"[[network]]\"}", + "hide": false, + "legendFormat": "{{instance}} connections", + "refId": "A" + }, + { + "expr": "[[metric_namespace]]_sub_libp2p_is_major_syncing{instance=\"[[instance]]\",network=\"[[network]]\"}", + "hide": false, + "legendFormat": "{{instance}} syncing", + "refId": "B" + }, + { + "expr": "[[metric_namespace]]_sub_libp2p_kbuckets_num_nodes{instance=\"[[instance]]\",network=\"[[network]]\"}", + "hide": false, + "legendFormat": "{{instance}} num_nodes", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "sub_libp2p", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 12 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "[[metric_namespace]]_sub_libp2p_notifications_total{instance=\"[[instance]]\",network=\"[[network]]\",protocol=\"FRNK\",direction=\"in\"}", + "hide": false, + "legendFormat": "{{instance}} FRNK in", + "refId": "A" + }, + { + "expr": "[[metric_namespace]]_sub_libp2p_notifications_total{instance=\"[[instance]]\",network=\"[[network]]\",protocol=\"FRNK\",direction=\"out\"}", + "hide": false, + "legendFormat": "{{instance}} FRNK out", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "libp2p_notifications", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "[[metric_namespace]]_cpu_usage_percentage{instance=\"[[instance]]\",network=\"[[network]]\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU usage %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 18 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "[[metric_namespace]]_memory_usage_bytes{instance=\"[[instance]]\",network=\"[[network]]\"}", + "legendFormat": "{{instance}} Mem bytes", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "[[metric_namespace]]_sub_libp2p_network_per_sec_bytes", + "hide": false, + "legendFormat": "{{instance}}", + "refId": "A" + }, + { + "expr": "[[metric_namespace]]_sub_libp2p_notifications_total", + "hide": true, + "legendFormat": "{{instance}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "libp2p_network_per_sec_bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 18 + }, + "hiddenSeries": false, + "id": 29, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "6.5.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "[[metric_namespace]]_sub_libp2p_notifications_total{instance=\"[[instance]]\",network=\"[[network]]\",protocol=\"dot1\",direction=\"in\"}", + "hide": false, + "legendFormat": "{{instance}} dot1 in", + "refId": "B" + }, + { + "expr": "[[metric_namespace]]_sub_libp2p_notifications_total{instance=\"[[instance]]\",network=\"[[network]]\",protocol=\"dot2\",direction=\"in\"}", + "hide": false, + "legendFormat": "{{instance}} dot2 in", + "refId": "C" + }, + { + "expr": "[[metric_namespace]]_sub_libp2p_notifications_total{instance=\"[[instance]]\",network=\"[[network]]\",protocol=\"dot2\",direction=\"out\"}", + "hide": false, + "legendFormat": "{{instance}} dot2 out", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "libp2p_notifications", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": "substrate", + "value": "substrate" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "metric_namespace", + "options": [ + { + "selected": true, + "text": "substrate", + "value": "substrate" + }, + { + "selected": false, + "text": "polkadot", + "value": "polkadot" + } + ], + "query": "substrate, polkadot", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "dev", + "value": "dev" + }, + "datasource": "Prometheus", + "definition": "label_values(network)", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": false, + "name": "network", + "options": [], + "query": "label_values(network)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "validator-a:9615", + "value": "validator-a:9615" + }, + "datasource": "Prometheus", + "definition": "label_values(instance)", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Substrate Dashboard", + "uid": "ColmenaLabs", + "variables": { + "list": [] + }, + "version": 2 +} diff --git a/.maintain/sentry-node/docker-compose.yml b/.maintain/sentry-node/docker-compose.yml index db835b057969642c9da77a4bde57c3178745bbd3..376538dde5786e15d1524dc8f75072aa4c1cb59d 100644 --- a/.maintain/sentry-node/docker-compose.yml +++ b/.maintain/sentry-node/docker-compose.yml @@ -19,6 +19,9 @@ # - validator-a: localhost:9944 # - validator-b: localhost:9945 # - sentry-a: localhost:9946 +# - grafana: localhost:3001 +# - prometheus: localhost:9090 + version: "3.7" services: @@ -79,13 +82,12 @@ services: - "--chain=local" - "--port" - "30333" - - "--charlie" - - "--bootnodes" + - "--sentry" - "/dns4/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR" - - "--bootnodes" - - "/dns4/validator-b/tcp/30333/p2p/QmSVnNf9HwVMT1Y4cK1P6aoJcEZjmoTXpjKBmAABLMnZEk" - "--reserved-nodes" - "/dns4/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR" + - "--bootnodes" + - "/dns4/validator-b/tcp/30333/p2p/QmSVnNf9HwVMT1Y4cK1P6aoJcEZjmoTXpjKBmAABLMnZEk" - "--no-telemetry" - "--rpc-cors" - "all" @@ -94,7 +96,7 @@ services: - "--unsafe-rpc-external" - "--log" - "sub-authority-discovery=trace" - - "--sentry" + - "--prometheus-external" validator-b: image: parity/substrate @@ -127,12 +129,43 @@ services: - "--unsafe-rpc-external" - "--log" - "sub-authority-discovery=trace" + - "--prometheus-external" ui: image: polkadot-js/apps ports: - "3000:80" + prometheus: + image: prom/prometheus + networks: + - network-a + - internet + ports: + - "9090:9090" + links: + - validator-a:validator-a + - sentry-a:sentry-a + - validator-b:validator-b + volumes: + - ./prometheus/:/etc/prometheus/ + restart: always + + grafana: + image: grafana/grafana + user: "104" + depends_on: + - prometheus + networks: + - network-a + - internet + ports: + - 3001:3000 + volumes: + - ./grafana/provisioning/:/etc/grafana/provisioning + - ../monitoring/grafana-dashboards/:/etc/grafana/provisioning/dashboard-definitions + restart: always + networks: network-a: internet: diff --git a/.maintain/sentry-node/grafana/provisioning/dashboards/dashboards.yml b/.maintain/sentry-node/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000000000000000000000000000000000000..ad9164fd8ea01023a2b5736fdea9bfc0259f1eed --- /dev/null +++ b/.maintain/sentry-node/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: +- name: 'Prometheus' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: false + options: + path: /etc/grafana/provisioning/dashboard-definitions diff --git a/.maintain/sentry-node/grafana/provisioning/datasources/datasource.yml b/.maintain/sentry-node/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000000000000000000000000000000000000..c02bb38b3d378112480b0772855e79b0f02cb02f --- /dev/null +++ b/.maintain/sentry-node/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,50 @@ +# config file version +apiVersion: 1 + +# list of datasources that should be deleted from the database +deleteDatasources: + - name: Prometheus + orgId: 1 + +# list of datasources to insert/update depending +# whats available in the database +datasources: + # name of the datasource. Required +- name: Prometheus + # datasource type. Required + type: prometheus + # access mode. direct or proxy. Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://prometheus:9090 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: false + # basic auth username, if used + basicAuthUser: + # basic auth password, if used + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: true + # fields that will be converted to json and stored in json_data + jsonData: + graphiteVersion: "1.1" + tlsAuth: false + tlsAuthWithCACert: false + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: true diff --git a/.maintain/sentry-node/prometheus/prometheus.yml b/.maintain/sentry-node/prometheus/prometheus.yml new file mode 100644 index 0000000000000000000000000000000000000000..831b84ba0b7018948ae235619eb600b51e75c75d --- /dev/null +++ b/.maintain/sentry-node/prometheus/prometheus.yml @@ -0,0 +1,15 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'substrate_validator-a' + static_configs: + - targets: ['validator-a:9615'] + labels: + network: dev + - targets: ['sentry-a:9615'] + labels: + network: dev + - targets: ['validator-b:9615'] + labels: + network: dev diff --git a/Cargo.lock b/Cargo.lock index 6038064ab0b44b05a917e72950700c33f8d9d56a..6269e0336025ee9148b7d670fdda833970378a0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,12 +70,11 @@ dependencies = [ [[package]] name = "alga" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658f9468113d34781f6ca9d014d174c74b73de870f1e0e3ad32079bbab253b19" +checksum = "4f823d037a7ec6ea2197046bafd4ae150e6bc36f9ca347404f46a46823fa84f2" dependencies = [ "approx", - "libm", "num-complex", "num-traits 0.2.11", ] @@ -100,9 +99,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7825f6833612eb2414095684fcf6c635becf3ce97fe48cf6421321e93bfbd53c" +checksum = "d9a60d744a80c30fcb657dfe2c1b22bcb3e814c1a1e3674f32bf5820b570fbff" [[package]] name = "app_dirs" @@ -127,15 +126,15 @@ dependencies = [ [[package]] name = "arbitrary" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75153c95fdedd7db9732dfbfc3702324a1627eec91ba56e37cd0ac78314ab2ed" +checksum = "1148c9b25d393a07c4cc3ef5dd30f82a40a1c261018c4a670611ed8e76cad3ea" [[package]] name = "arc-swap" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7b8a9123b8027467bce0099fe556c628a53c8d83df0507084c31e9ba2e39aff" +checksum = "b585a98a234c46fc563103e9278c9391fde1f4e6850334da895d27edb9580f62" [[package]] name = "arrayref" @@ -174,7 +173,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d0864d84b8e07b145449be9a8537db86bf9de5ce03b913214694643b4743502" dependencies = [ "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -234,14 +233,14 @@ dependencies = [ [[package]] name = "async-tls" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce6977f57fa68da77ffe5542950d47e9c23d65f5bc7cb0a9f8700996913eec7" +checksum = "95fd83426b89b034bf4e9ceb9c533c2f2386b813fd3dcae0a425ec6f1837d78a" dependencies = [ "futures 0.3.4", - "rustls 0.16.0", + "rustls", "webpki", - "webpki-roots 0.17.0", + "webpki-roots 0.19.0", ] [[package]] @@ -269,9 +268,9 @@ checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" [[package]] name = "backtrace" -version = "0.3.45" +version = "0.3.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad235dabf00f36301792cfe82499880ba54c6486be094d1047b02bacb67c14e8" +checksum = "b1e692897359247cc6bb902933361652380af0f1b7651ae5c5013407f30e109e" dependencies = [ "backtrace-sys", "cfg-if", @@ -281,9 +280,9 @@ dependencies = [ [[package]] name = "backtrace-sys" -version = "0.1.34" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca797db0057bae1a7aa2eef3283a874695455cecf08a43bfb8507ee0ebc1ed69" +checksum = "78848718ee1255a2485d1309ad9cdecfc2e7d0362dd11c6829364c6b35ae1bc7" dependencies = [ "cc", "libc", @@ -295,15 +294,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5024ee8015f02155eee35c711107ddd9a9bf3cb689cf2a9089c97e79b6e1ae83" -[[package]] -name = "base64" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" -dependencies = [ - "byteorder 1.3.4", -] - [[package]] name = "base64" version = "0.11.0" @@ -399,6 +389,17 @@ dependencies = [ "constant_time_eq", ] +[[package]] +name = "blake2s_simd" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab9e07352b829279624ceb7c64adb4f585dacdb81d35cafae81139ccd617cf44" +dependencies = [ + "arrayref", + "arrayvec 0.5.1", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.7.3" @@ -439,21 +440,21 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "slab", ] [[package]] name = "bs58" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b170cd256a3f9fa6b9edae3e44a7dfdfc77e8124dbc3e2612d75f9c3e2396dae" +checksum = "476e9cd489f9e121e02ffa6014a8ef220ecb15c05ed23fc34cca13925dc283fb" [[package]] name = "bstr" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "502ae1441a0a5adb8fbd38a5955a6416b9493e92b465de5e4a9bde6a539c2c48" +checksum = "2889e6d50f394968c8bf4240dc3f2a7eb4680844d27308f798229ac9d4725f41" dependencies = [ "lazy_static", "memchr", @@ -546,9 +547,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" +checksum = "9c9384ca4b90c0ea47e19a5c996d6643a3e73dedf9b89c65efb67587e34da1bb" dependencies = [ "jobserver", ] @@ -579,11 +580,12 @@ dependencies = [ [[package]] name = "chain-spec-builder" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "ansi_term 0.12.1", "node-cli", "rand 0.7.3", + "sc-chain-spec", "sc-keystore", "sp-core", "structopt", @@ -604,9 +606,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "0.29.2" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92986241798376849e1a007827041fed9bb36195822c2049d18e174420e0534" +checksum = "fe6837df1d5cba2397b835c8530f51723267e16abbf83892e9e5af4f0e5dd10a" dependencies = [ "glob 0.3.0", "libc", @@ -740,7 +742,7 @@ dependencies = [ "gimli", "log", "serde", - "smallvec 1.2.0", + "smallvec 1.3.0", "target-lexicon", "thiserror", ] @@ -778,7 +780,7 @@ checksum = "518344698fa6c976d853319218415fdfb4f1bc6b42d0b2e2df652e55dff1f778" dependencies = [ "cranelift-codegen", "log", - "smallvec 1.2.0", + "smallvec 1.3.0", "target-lexicon", ] @@ -1001,7 +1003,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47c5e5ac752e18207b12e16b10631ae5f7f68f8805f335f9b817ead83d9ffce1" dependencies = [ "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -1045,13 +1047,13 @@ checksum = "11c0346158a19b3627234e15596f5e465c360fcdb97d817bcb255e0510f5a788" [[package]] name = "derive_more" -version = "0.99.3" +version = "0.99.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a806e96c59a76a5ba6e18735b6cf833344671e61e7863f2edb5c518ea2cac95c" +checksum = "e2323f3f47db9a0e77ce7a300605d8d2098597fc451ed1a97bb1f6411bb550a7" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -1103,9 +1105,9 @@ dependencies = [ [[package]] name = "doc-comment" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "807e5847c39ad6a11eac66de492ed1406f76a260eb8656e8740cad9eabc69c27" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "ed25519-dalek" @@ -1138,22 +1140,22 @@ dependencies = [ [[package]] name = "enumflags2" -version = "0.6.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33121c8782ba948ba332dab29311b026a8716dc65a1599e5b88f392d38496af8" +checksum = "83c8d82922337cd23a15f88b70d8e4ef5f11da38dd7cdb55e84dd5de99695da0" dependencies = [ "enumflags2_derive", ] [[package]] name = "enumflags2_derive" -version = "0.6.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecf634c5213044b8d54a46dd282cf5dd1f86bb5cb53e92c409cb4680a7fb9894" +checksum = "946ee94e3dbf58fdd324f9ce245c7b238d46a66f00e86a020b71996349e46cce" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -1190,18 +1192,18 @@ checksum = "516aa8d7a71cb00a1c4146f0798549b93d083d4f189b3ced8f3de6b8f11ee6c4" [[package]] name = "erased-serde" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7d80305c9bd8cd78e3c753eb9fb110f83621e5211f1a3afffcc812b104daf9" +checksum = "d88b6d1705e16a4d62e05ea61cc0496c2bd190f4fa8e5c1f11ce747be6bcf3d1" dependencies = [ "serde", ] [[package]] name = "errno" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a071601ed01b988f896ab14b95e67335d1eeb50190932a1320f7fe3cadc84e" +checksum = "b480f641ccf0faf324e20c1d3e53d81b7484c698b42ea677f6907ae4db195371" dependencies = [ "errno-dragonfly", "libc", @@ -1218,33 +1220,6 @@ dependencies = [ "libc", ] -[[package]] -name = "ethbloom" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e7abcddbdd5db30aeed4deb586adc4824e6c247e2f7238d1187f752893f096b" -dependencies = [ - "crunchy", - "fixed-hash", - "impl-rlp", - "impl-serde 0.3.0", - "tiny-keccak 2.0.1", -] - -[[package]] -name = "ethereum-types" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "964c23cdee0ca07d5be2a628b46d5c11a2134ce554a8c16d8dbc2db647e4fd4d" -dependencies = [ - "ethbloom", - "fixed-hash", - "impl-rlp", - "impl-serde 0.3.0", - "primitive-types", - "uint", -] - [[package]] name = "evm" version = "0.16.1" @@ -1334,7 +1309,7 @@ checksum = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", "synstructure", ] @@ -1404,9 +1379,9 @@ checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" [[package]] name = "flate2" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd6d6f4752952feb71363cffc9ebac9411b75b87c6ab6058c40c8900cf43c0f" +checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42" dependencies = [ "cfg-if", "crc32fast", @@ -1423,19 +1398,20 @@ checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" [[package]] name = "fork-tree" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "parity-scale-codec", ] [[package]] name = "frame-benchmarking" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", "linregress", "parity-scale-codec", + "paste", "sp-api", "sp-io", "sp-runtime", @@ -1445,12 +1421,11 @@ dependencies = [ [[package]] name = "frame-benchmarking-cli" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-benchmarking", "parity-scale-codec", "sc-cli", - "sc-client", "sc-client-db", "sc-executor", "sc-service", @@ -1463,7 +1438,7 @@ dependencies = [ [[package]] name = "frame-executive" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -1477,12 +1452,13 @@ dependencies = [ "sp-io", "sp-runtime", "sp-std", + "sp-tracing", "sp-version", ] [[package]] name = "frame-metadata" -version = "11.0.0-alpha.5" +version = "11.0.0-dev" dependencies = [ "parity-scale-codec", "serde", @@ -1492,7 +1468,7 @@ dependencies = [ [[package]] name = "frame-support" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "bitmask", "frame-metadata", @@ -1512,37 +1488,37 @@ dependencies = [ "sp-runtime", "sp-state-machine", "sp-std", - "tracing", + "sp-tracing", ] [[package]] name = "frame-support-procedural" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support-procedural-tools", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] name = "frame-support-procedural-tools" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support-procedural-tools-derive", "proc-macro-crate", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] name = "frame-support-procedural-tools-derive" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -1552,6 +1528,7 @@ dependencies = [ "frame-support", "parity-scale-codec", "pretty_assertions", + "rustversion", "serde", "sp-core", "sp-inherents", @@ -1563,7 +1540,7 @@ dependencies = [ [[package]] name = "frame-system" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "criterion 0.2.11", "frame-support", @@ -1581,7 +1558,7 @@ dependencies = [ [[package]] name = "frame-system-rpc-runtime-api" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "parity-scale-codec", "sp-api", @@ -1742,7 +1719,7 @@ dependencies = [ "proc-macro-hack", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -1791,7 +1768,6 @@ dependencies = [ "proc-macro-hack", "proc-macro-nested", "slab", - "tokio-io", ] [[package]] @@ -1877,7 +1853,7 @@ dependencies = [ "byteorder 1.3.4", "fallible-iterator", "indexmap", - "smallvec 1.2.0", + "smallvec 1.3.0", "stable_deref_trait", ] @@ -1895,9 +1871,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "globset" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "925aa2cac82d8834e2b2a4415b6f6879757fb5c0928fc445ae76461a12eed8f2" +checksum = "7ad1da430bd7281dde2576f44c84cc3f0f7b475e7202cd503042dff01a8c8120" dependencies = [ "aho-corasick", "bstr", @@ -1950,20 +1926,20 @@ dependencies = [ [[package]] name = "h2" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5c295d1c0c68e4e42003d75f908f5e16a1edd1cbe0b0d02e4dc2006a384f47" +checksum = "377038bf3c89d18d6ca1431e7a5027194fbd724ca10592b9487ede5e8e144f42" dependencies = [ "bytes 0.5.4", "fnv", "futures-core", "futures-sink", "futures-util", - "http 0.2.0", + "http 0.2.1", "indexmap", "log", "slab", - "tokio 0.2.13", + "tokio 0.2.18", "tokio-util", ] @@ -2003,9 +1979,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.8" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1010591b26bbfe835e9faeabeb11866061cc7dcebffd56ad7d0942d0e61aefd8" +checksum = "8a0d737e0f947a1864e93d33fdef4af8445a00d1ed8dc0c8ddb73139ea6abf15" dependencies = [ "libc", ] @@ -2080,9 +2056,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b708cc7f06493459026f53b9a61a7a121a5d1ec6238dee58ea4941132b30156b" +checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" dependencies = [ "bytes 0.5.4", "fnv", @@ -2108,7 +2084,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" dependencies = [ "bytes 0.5.4", - "http 0.2.0", + "http 0.2.1", ] [[package]] @@ -2158,16 +2134,16 @@ dependencies = [ [[package]] name = "hyper" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7b15203263d1faa615f9337d79c1d37959439dc46c2b4faab33286fadc2a1c5" +checksum = "96816e1d921eca64d208a85aab4f7798455a8e34229ee5a88c935bdee1b78b14" dependencies = [ "bytes 0.5.4", "futures-channel", "futures-core", "futures-util", - "h2 0.2.2", - "http 0.2.0", + "h2 0.2.4", + "http 0.2.1", "http-body 0.3.1", "httparse", "itoa", @@ -2175,7 +2151,7 @@ dependencies = [ "net2", "pin-project", "time", - "tokio 0.2.13", + "tokio 0.2.18", "tower-service", "want 0.3.0", ] @@ -2189,11 +2165,11 @@ dependencies = [ "bytes 0.5.4", "ct-logs", "futures-util", - "hyper 0.13.3", + "hyper 0.13.5", "log", - "rustls 0.17.0", + "rustls", "rustls-native-certs", - "tokio 0.2.13", + "tokio 0.2.18", "tokio-rustls", "webpki", ] @@ -2264,7 +2240,7 @@ checksum = "7ef5550a42e3740a0e71f909d4c861056a284060af885ae7aa6242820f920d9d" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -2290,9 +2266,9 @@ checksum = "141340095b15ed7491bd3d4ced9d20cebfb826174b6bb03386381f62b01e3d77" [[package]] name = "intervalier" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14200459dc2319eb13708aed1c1efb8307e0e0e801e7282476939492e1492631" +checksum = "64fa110ec7b8f493f416eed552740d10e7030ad5f63b2308f82c9608ec2df275" dependencies = [ "futures 0.3.4", "futures-timer 2.0.2", @@ -2307,11 +2283,17 @@ dependencies = [ "libc", ] +[[package]] +name = "ip_network" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee15951c035f79eddbef745611ec962f63f4558f1dadf98ab723cc603487c6f" + [[package]] name = "ipnet" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a859057dc563d1388c1e816f98a1892629075fc046ed06e845b883bb8b2916fb" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" [[package]] name = "itertools" @@ -2339,18 +2321,18 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cb931d43e71f560c81badb0191596562bafad2be06a3f9025b845c847c60df5" +checksum = "6a27d435371a2fa5b6d2b028a74bbdb1234f308da363226a2854ca3ff8ba7055" dependencies = [ "wasm-bindgen", ] [[package]] name = "jsonrpc-client-transports" -version = "14.0.5" +version = "14.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a9ae166c4d1f702d297cd76d4b55758ace80272ffc6dbb139fdc1bf810de40b" +checksum = "2307a7e78cf969759e390a8a2151ea12e783849a45bb00aa871b468ba58ea79e" dependencies = [ "failure", "futures 0.1.29", @@ -2365,9 +2347,9 @@ dependencies = [ [[package]] name = "jsonrpc-core" -version = "14.0.5" +version = "14.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe3b688648f1ef5d5072229e2d672ecb92cbff7d1c79bcf3fd5898f3f3df0970" +checksum = "25525f6002338fb4debb5167a89a0b47f727a5a48418417545ad3429758b7fec" dependencies = [ "futures 0.1.29", "log", @@ -2378,9 +2360,9 @@ dependencies = [ [[package]] name = "jsonrpc-core-client" -version = "14.0.5" +version = "14.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080dc110be17701097df238fad3c816d4a478a1899dfbcf8ec8957dd40ec7304" +checksum = "87f9382e831a6d630c658df103aac3f971da096deb57c136ea2b760d3b4e3f9f" dependencies = [ "jsonrpc-client-transports", ] @@ -2394,41 +2376,41 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] name = "jsonrpc-http-server" -version = "14.0.6" +version = "14.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816d63997ea45d3634608edbef83ddb35e661f7c0b27b5b72f237e321f0e9807" +checksum = "d52860f0549694aa4abb12766856f56952ab46d3fb9f0815131b2db3d9cc2f29" dependencies = [ "hyper 0.12.35", "jsonrpc-core", "jsonrpc-server-utils", "log", "net2", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "unicase", ] [[package]] name = "jsonrpc-pubsub" -version = "14.0.6" +version = "14.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b31c9b90731276fdd24d896f31bb10aecf2e5151733364ae81123186643d939" +checksum = "c4ca5e391d6c6a2261d4adca029f427fe63ea546ad6cef2957c654c08495ec16" dependencies = [ "jsonrpc-core", "log", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "serde", ] [[package]] name = "jsonrpc-server-utils" -version = "14.0.5" +version = "14.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b7635e618a0edbbe0d2a2bbbc69874277c49383fcf6c3c0414491cfb517d22" +checksum = "1f06add502b48351e05dd95814835327fb115e4e9f834ca42fd522d3b769d4d2" dependencies = [ "bytes 0.4.12", "globset", @@ -2442,14 +2424,14 @@ dependencies = [ [[package]] name = "jsonrpc-ws-server" -version = "14.0.6" +version = "14.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94e5773b2ae66e0e02c80775ce6bbba6f15d5bb47c14ec36a36fcf94f8df851" +checksum = "017a7dd5083d9ed62c5e1dd3e317975c33c3115dac5447f4480fe05a8c354754" dependencies = [ "jsonrpc-core", "jsonrpc-server-utils", "log", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "slab", "ws", ] @@ -2497,7 +2479,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cad096c6849b2ef027fabe35c4aed356d0e3d3f586d0a8361e5e17f1e50a7ce5" dependencies = [ "parity-util-mem", - "smallvec 1.2.0", + "smallvec 1.3.0", ] [[package]] @@ -2508,7 +2490,7 @@ checksum = "4aa954d12cfac958822dfd77aab34f3eec71f103b918c4ab79ab59a36ee594ea" dependencies = [ "kvdb", "parity-util-mem", - "parking_lot 0.10.0", + "parking_lot 0.10.2", ] [[package]] @@ -2524,10 +2506,10 @@ dependencies = [ "num_cpus", "owning_ref", "parity-util-mem", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "regex", "rocksdb", - "smallvec 1.2.0", + "smallvec 1.3.0", ] [[package]] @@ -2567,9 +2549,9 @@ checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" [[package]] name = "libc" -version = "0.2.67" +version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb147597cdf94ed43ab7a9038716637d2d1bf2bc571da995d0028dec06bd3018" +checksum = "99e85c08494b21a9054e7fe1374a732aeadaff3980b6990b94bfd3a70f690005" [[package]] name = "libflate" @@ -2595,15 +2577,15 @@ dependencies = [ [[package]] name = "libm" -version = "0.1.4" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a" +checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" [[package]] name = "libp2p" -version = "0.16.2" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bba17ee9cac4bb89de5812159877d9b4f0a993bf41697a5a875940cd1eb71f24" +checksum = "32ea742c86405b659c358223a8f0f9f5a9eb27bb6083894c6340959b05269662" dependencies = [ "bytes 0.5.4", "futures 0.3.4", @@ -2629,33 +2611,34 @@ dependencies = [ "libp2p-wasm-ext", "libp2p-websocket", "libp2p-yamux", - "parity-multiaddr", - "parity-multihash", - "parking_lot 0.10.0", + "multihash", + "parity-multiaddr 0.8.0", + "parking_lot 0.10.2", "pin-project", - "smallvec 1.2.0", + "smallvec 1.3.0", "wasm-timer", ] [[package]] name = "libp2p-core" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b874594c4b29de1a29f27871feba8e6cd13aa54a8a1e8f8c7cf3dfac5ca287c" +checksum = "a1d2c17158c4dca984a77a5927aac6f0862d7f50c013470a415f93be498b5739" dependencies = [ "asn1_der", "bs58", "ed25519-dalek", + "either", "fnv", "futures 0.3.4", "futures-timer 3.0.2", "lazy_static", "libsecp256k1", "log", + "multihash", "multistream-select", - "parity-multiaddr", - "parity-multihash", - "parking_lot 0.10.0", + "parity-multiaddr 0.8.0", + "parking_lot 0.10.2", "pin-project", "prost", "prost-build", @@ -2663,7 +2646,7 @@ dependencies = [ "ring", "rw-stream-sink", "sha2", - "smallvec 1.2.0", + "smallvec 1.3.0", "thiserror", "unsigned-varint", "void", @@ -2672,19 +2655,19 @@ dependencies = [ [[package]] name = "libp2p-core-derive" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d472e9d522f588805c77801de10b957be84e10f019ca5f869fa1825b15ea9b" +checksum = "329127858e4728db5ab60c33d5ae352a999325fdf190ed022ec7d3a4685ae2e6" dependencies = [ "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] name = "libp2p-deflate" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e25004d4d9837b44b22c5f1a69be1724a5168fef6cff1716b5176a972c3aa62" +checksum = "4ad32b006ea922da8cc66e537cf2df4b0fad8ebaa467d2a8c63d7784ac252ec6" dependencies = [ "flate2", "futures 0.3.4", @@ -2693,9 +2676,9 @@ dependencies = [ [[package]] name = "libp2p-dns" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b99e552f9939b606eb4b59f7f64d9b01e3f96752f47e350fc3c5fc646ed3f649" +checksum = "c0d0993481203d68e5ce2f787d033fb0cac6b850659ed6c784612db678977c71" dependencies = [ "futures 0.3.4", "libp2p-core", @@ -2704,9 +2687,9 @@ dependencies = [ [[package]] name = "libp2p-floodsub" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d3234f12e44f9a50351a9807b97fe7de11eb9ae4482370392ba10da6dc90722" +checksum = "3673153ca967c179d745fadf047d069355d6669ecf7f261b450fbaebf1bffd3d" dependencies = [ "cuckoofilter", "fnv", @@ -2716,16 +2699,16 @@ dependencies = [ "prost", "prost-build", "rand 0.7.3", - "smallvec 1.2.0", + "smallvec 1.3.0", ] [[package]] name = "libp2p-gossipsub" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d46cb3e0841bd951cbf4feae56cdc081e6347836a644fb260c3ec554149b4006" +checksum = "3f7f3f79f060864db0317cc47641b7d35276dee52a0ffa91553fbd0c153863a3" dependencies = [ - "base64 0.11.0", + "base64", "byteorder 1.3.4", "bytes 0.5.4", "fnv", @@ -2739,16 +2722,16 @@ dependencies = [ "prost-build", "rand 0.7.3", "sha2", - "smallvec 1.2.0", + "smallvec 1.3.0", "unsigned-varint", "wasm-timer", ] [[package]] name = "libp2p-identify" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfeb935a9bd41263e4f3a24b988e9f4a044f3ae89ac284e83c17fe2f84e0d66b" +checksum = "a38ca3eb807789e26f41c82ca7cd2b3843c66c5587b8b5f709a2f421f3061414" dependencies = [ "futures 0.3.4", "libp2p-core", @@ -2756,15 +2739,15 @@ dependencies = [ "log", "prost", "prost-build", - "smallvec 1.2.0", + "smallvec 1.3.0", "wasm-timer", ] [[package]] name = "libp2p-kad" -version = "0.16.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464dc8412978d40f0286be72ed9ab5e0e1386a4a06e7f174526739b5c3c1f041" +checksum = "a92cda1fb8149ea64d092a2b99d2bd7a2c309eee38ea322d02e4480bd6ee1759" dependencies = [ "arrayvec 0.5.1", "bytes 0.5.4", @@ -2775,12 +2758,12 @@ dependencies = [ "libp2p-core", "libp2p-swarm", "log", - "parity-multihash", + "multihash", "prost", "prost-build", "rand 0.7.3", "sha2", - "smallvec 1.2.0", + "smallvec 1.3.0", "uint", "unsigned-varint", "void", @@ -2789,9 +2772,9 @@ dependencies = [ [[package]] name = "libp2p-mdns" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "881fcfb360c2822db9f0e6bb6f89529621556ed9a8b038313414eda5107334de" +checksum = "41e908d2aaf8ff0ec6ad1f02fe1844fd777fb0b03a68a226423630750ab99471" dependencies = [ "async-std", "data-encoding", @@ -2804,16 +2787,16 @@ dependencies = [ "log", "net2", "rand 0.7.3", - "smallvec 1.2.0", + "smallvec 1.3.0", "void", "wasm-timer", ] [[package]] name = "libp2p-mplex" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8507b37ad0eed275efcde67a023c3d85af6c80768b193845b9288e848e1af95" +checksum = "0832882b06619b2e81d74e71447753ea3c068164a0bca67847d272e856a04a02" dependencies = [ "bytes 0.5.4", "fnv", @@ -2821,15 +2804,15 @@ dependencies = [ "futures_codec", "libp2p-core", "log", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "unsigned-varint", ] [[package]] name = "libp2p-noise" -version = "0.16.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15a8a3d71f898beb6f854c8aae27aa1d198e0d1f2e49412261c2d90ef39675a" +checksum = "918e94a649e1139c24ee9f1f8c1f2adaba6d157b9471af787f2d9beac8c29c77" dependencies = [ "curve25519-dalek", "futures 0.3.4", @@ -2848,9 +2831,9 @@ dependencies = [ [[package]] name = "libp2p-ping" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d22f2f228b3a828dca1cb8aa9fa331e0bc9c36510cb2c1916956e20dc85e8c" +checksum = "f9bfbf87eebb492d040f9899c5c81c9738730465ac5e78d9b7a7d086d0f07230" dependencies = [ "futures 0.3.4", "libp2p-core", @@ -2863,9 +2846,9 @@ dependencies = [ [[package]] name = "libp2p-plaintext" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56126a204d7b3382bac163143ff4125a14570b3ba76ba979103d1ae1abed1923" +checksum = "fabb00553a49bf6d4a8ce362f6eefac410227a14d03c3acffbb8cc3f022ea019" dependencies = [ "bytes 0.5.4", "futures 0.3.4", @@ -2881,9 +2864,9 @@ dependencies = [ [[package]] name = "libp2p-pnet" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b916938a8868f75180aeeffcc6a516a922d165e8fa2a90b57bad989d1ccbb57a" +checksum = "9f81b8b37ff529e1f51c20c396dac657def2108da174c1d27e57e72c9fe2d411" dependencies = [ "futures 0.3.4", "log", @@ -2895,9 +2878,9 @@ dependencies = [ [[package]] name = "libp2p-secio" -version = "0.16.1" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1219e9ecb4945d7331a05f5ffe96a1f6e28051bfa1223d4c60353c251de0354e" +checksum = "a7a0509a7e47245259954fef58b85b81bf4d29ae33a4365e38d718a866698774" dependencies = [ "aes-ctr", "ctr", @@ -2925,23 +2908,24 @@ dependencies = [ [[package]] name = "libp2p-swarm" -version = "0.16.1" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "275471e7c0e88ae004660866cd54f603bd8bd1f4caef541a27f50dd8640c4d4c" +checksum = "44ab289ae44cc691da0a6fe96aefa43f26c86c6c7813998e203f6d80f1860f18" dependencies = [ "futures 0.3.4", "libp2p-core", "log", - "smallvec 1.2.0", + "rand 0.7.3", + "smallvec 1.3.0", "void", "wasm-timer", ] [[package]] name = "libp2p-tcp" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9e80ad4e3535345f3d666554ce347d3100453775611c05c60786bf9a1747a10" +checksum = "b37ea44823d3ed223e4605da94b50177bc520f05ae2452286700549a32d81669" dependencies = [ "async-std", "futures 0.3.4", @@ -2954,9 +2938,9 @@ dependencies = [ [[package]] name = "libp2p-uds" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d329564a43da9d0e055a5b938633c4a8ceab1f59cec133fbc4647917c07341" +checksum = "281c18ea2faacb9c8a6ff76c4405df5918d9a263770e3847bf03f099abadc010" dependencies = [ "async-std", "futures 0.3.4", @@ -2966,9 +2950,9 @@ dependencies = [ [[package]] name = "libp2p-wasm-ext" -version = "0.16.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "923581c055bc4b8c5f42d4ce5ef43e52fe5216f1ea4bc26476cb8a966ce6220b" +checksum = "e3ac7dbde0f88cad191dcdfd073b8bae28d01823e8ca313f117b6ecb914160c3" dependencies = [ "futures 0.3.4", "js-sys", @@ -2980,9 +2964,9 @@ dependencies = [ [[package]] name = "libp2p-websocket" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5351ca9eea122081c1c0f9323164d2918cac29b5a6bfe5054d4ba8ec9447cf42" +checksum = "6874c9069ce93d899df9dc7b29f129c706b2a0fdc048f11d878935352b580190" dependencies = [ "async-tls", "bytes 0.5.4", @@ -2991,7 +2975,7 @@ dependencies = [ "libp2p-core", "log", "quicksink", - "rustls 0.16.0", + "rustls", "rw-stream-sink", "soketto", "url 2.1.1", @@ -3001,13 +2985,13 @@ dependencies = [ [[package]] name = "libp2p-yamux" -version = "0.16.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dac30de24ccde0e67f363d71a125c587bbe6589503f664947e9b084b68a34f1" +checksum = "02f91aea50f6571e0bc6c058dc0e9b270afd41ec28dd94e9e4bf607e78b9ab87" dependencies = [ "futures 0.3.4", "libp2p-core", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "thiserror", "yamux", ] @@ -3089,9 +3073,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" dependencies = [ "scopeguard", ] @@ -3162,18 +3146,18 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9" +checksum = "b4fc2c02a7e374099d4ee95a193111f72d2110197fe200272371758f6c3643d8" dependencies = [ - "rustc_version", + "autocfg 1.0.0", ] [[package]] name = "memory-db" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f58381b20ebe2c578e75dececd9da411414903415349548ccc46aac3209cdfbc" +checksum = "be512cb2ccb4ecbdca937fdd4a62ea5f09f8e7195466a85e4632b3d5bcce82e6" dependencies = [ "ahash", "hash-db", @@ -3268,23 +3252,38 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0debeb9fcf88823ea64d64e4a815ab1643f33127d995978e099942ce38f25238" +[[package]] +name = "multihash" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47fbc227f7e2b1cb701f95404579ecb2668abbdd3c7ef7a6cbb3cc0d3b236869" +dependencies = [ + "blake2b_simd", + "blake2s_simd", + "digest", + "sha-1", + "sha2", + "sha3", + "unsigned-varint", +] + [[package]] name = "multimap" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97fbd5d00e0e37bfb10f433af8f5aaf631e739368dc9fc28286ca81ca4948dc" +checksum = "d8883adfde9756c1d30b0f519c9b8c502a94b41ac62f696453c37c7fc0a958ce" [[package]] name = "multistream-select" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f938ffe420493e77c8b6cbcc3f282283f68fc889c5dcbc8e51668d5f3a01ad94" +checksum = "74cdcf7cfb3402881e15a1f95116cb033d69b33c83d481e1234777f5ef0c3d2c" dependencies = [ "bytes 0.5.4", - "futures 0.1.29", + "futures 0.3.4", "log", - "smallvec 1.2.0", - "tokio-io", + "pin-project", + "smallvec 1.3.0", "unsigned-varint", ] @@ -3354,22 +3353,51 @@ dependencies = [ [[package]] name = "node-bench" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ + "derive_more", + "fs_extra", + "hash-db", + "hex", + "kvdb", + "kvdb-rocksdb", + "lazy_static", "log", "node-primitives", "node-testing", + "rand 0.7.3", "sc-cli", "sc-client-api", "serde", "serde_json", + "sp-core", "sp-runtime", + "sp-state-machine", + "sp-trie", "structopt", + "tempfile", +] + +[[package]] +name = "node-browser-testing" +version = "2.0.0" +dependencies = [ + "futures 0.3.4", + "futures-timer 3.0.2", + "jsonrpc-core", + "libp2p", + "node-cli", + "sc-rpc-api", + "serde", + "serde_json", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test", ] [[package]] name = "node-cli" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "assert_cmd", "frame-benchmarking-cli", @@ -3395,14 +3423,16 @@ dependencies = [ "pallet-timestamp", "pallet-transaction-payment", "parity-scale-codec", + "platforms", "rand 0.7.3", + "regex", "sc-authority-discovery", "sc-basic-authorship", "sc-chain-spec", "sc-cli", - "sc-client", "sc-client-api", "sc-client-db", + "sc-consensus", "sc-consensus-babe", "sc-consensus-epochs", "sc-finality-grandpa", @@ -3434,14 +3464,13 @@ dependencies = [ "substrate-build-script-utils", "tempfile", "tracing", - "vergen", "wasm-bindgen", "wasm-bindgen-futures", ] [[package]] name = "node-executor" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "criterion 0.3.1", "frame-benchmarking", @@ -3475,7 +3504,7 @@ dependencies = [ [[package]] name = "node-inspect" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "derive_more", "log", @@ -3491,7 +3520,7 @@ dependencies = [ [[package]] name = "node-primitives" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "pretty_assertions", "sp-core", @@ -3501,14 +3530,14 @@ dependencies = [ [[package]] name = "node-rpc" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "jsonrpc-core", "node-primitives", "node-runtime", "pallet-contracts-rpc", "pallet-transaction-payment-rpc", - "sc-client", + "sc-client-api", "sc-consensus-babe", "sc-consensus-babe-rpc", "sc-consensus-epochs", @@ -3524,7 +3553,7 @@ dependencies = [ [[package]] name = "node-rpc-client" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "env_logger 0.7.1", "futures 0.1.29", @@ -3537,7 +3566,7 @@ dependencies = [ [[package]] name = "node-runtime" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-benchmarking", "frame-executive", @@ -3563,6 +3592,7 @@ dependencies = [ "pallet-indices", "pallet-membership", "pallet-offences", + "pallet-offences-benchmarking", "pallet-randomness-collective-flip", "pallet-recovery", "pallet-scheduler", @@ -3600,15 +3630,15 @@ dependencies = [ [[package]] name = "node-template" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "futures 0.3.4", "log", "node-template-runtime", "sc-basic-authorship", "sc-cli", - "sc-client", "sc-client-api", + "sc-consensus", "sc-consensus-aura", "sc-executor", "sc-finality-grandpa", @@ -3624,12 +3654,11 @@ dependencies = [ "sp-transaction-pool", "structopt", "substrate-build-script-utils", - "vergen", ] [[package]] name = "node-template-runtime" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-executive", "frame-support", @@ -3661,12 +3690,13 @@ dependencies = [ [[package]] name = "node-testing" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "criterion 0.3.1", "frame-support", "frame-system", "fs_extra", + "futures 0.3.4", "log", "node-executor", "node-primitives", @@ -3684,7 +3714,6 @@ dependencies = [ "parity-scale-codec", "sc-block-builder", "sc-cli", - "sc-client", "sc-client-api", "sc-client-db", "sc-executor", @@ -3707,13 +3736,12 @@ dependencies = [ [[package]] name = "node-transaction-factory" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "log", "parity-scale-codec", "sc-block-builder", "sc-cli", - "sc-client", "sc-client-api", "sc-service", "sp-api", @@ -3788,9 +3816,9 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da4dc79f9e6c81bef96148c8f6b8e72ad4541caa4a24373e900a36da07de03a3" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" dependencies = [ "autocfg 1.0.0", "num-bigint", @@ -3814,13 +3842,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" dependencies = [ "autocfg 1.0.0", + "libm", ] [[package]] name = "num_cpus" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" dependencies = [ "hermit-abi", "libc", @@ -3897,7 +3926,7 @@ dependencies = [ [[package]] name = "pallet-assets" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -3911,7 +3940,7 @@ dependencies = [ [[package]] name = "pallet-aura" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -3919,7 +3948,7 @@ dependencies = [ "pallet-session", "pallet-timestamp", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "serde", "sp-application-crypto", "sp-consensus-aura", @@ -3933,7 +3962,7 @@ dependencies = [ [[package]] name = "pallet-authority-discovery" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -3951,7 +3980,7 @@ dependencies = [ [[package]] name = "pallet-authorship" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -3967,7 +3996,7 @@ dependencies = [ [[package]] name = "pallet-babe" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -3988,7 +4017,7 @@ dependencies = [ [[package]] name = "pallet-balances" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-benchmarking", "frame-support", @@ -4004,7 +4033,7 @@ dependencies = [ [[package]] name = "pallet-benchmark" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-benchmarking", "frame-support", @@ -4018,7 +4047,7 @@ dependencies = [ [[package]] name = "pallet-collective" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-benchmarking", "frame-support", @@ -4035,7 +4064,7 @@ dependencies = [ [[package]] name = "pallet-contracts" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "assert_matches", "frame-support", @@ -4045,6 +4074,7 @@ dependencies = [ "pallet-contracts-primitives", "pallet-randomness-collective-flip", "pallet-timestamp", + "pallet-transaction-payment", "parity-scale-codec", "parity-wasm 0.41.0", "pwasm-utils", @@ -4060,7 +4090,7 @@ dependencies = [ [[package]] name = "pallet-contracts-primitives" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "parity-scale-codec", "sp-runtime", @@ -4069,7 +4099,7 @@ dependencies = [ [[package]] name = "pallet-contracts-rpc" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -4088,7 +4118,7 @@ dependencies = [ [[package]] name = "pallet-contracts-rpc-runtime-api" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "pallet-contracts-primitives", "parity-scale-codec", @@ -4099,7 +4129,7 @@ dependencies = [ [[package]] name = "pallet-democracy" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-benchmarking", "frame-support", @@ -4118,7 +4148,7 @@ dependencies = [ [[package]] name = "pallet-elections" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4134,7 +4164,7 @@ dependencies = [ [[package]] name = "pallet-elections-phragmen" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4153,7 +4183,7 @@ dependencies = [ [[package]] name = "pallet-evm" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "evm", "frame-support", @@ -4173,7 +4203,7 @@ dependencies = [ [[package]] name = "pallet-example" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-benchmarking", "frame-support", @@ -4189,7 +4219,7 @@ dependencies = [ [[package]] name = "pallet-example-offchain-worker" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4204,7 +4234,7 @@ dependencies = [ [[package]] name = "pallet-finality-tracker" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4221,7 +4251,7 @@ dependencies = [ [[package]] name = "pallet-generic-asset" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4235,7 +4265,7 @@ dependencies = [ [[package]] name = "pallet-grandpa" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4253,7 +4283,7 @@ dependencies = [ [[package]] name = "pallet-identity" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "enumflags2", "frame-benchmarking", @@ -4270,7 +4300,7 @@ dependencies = [ [[package]] name = "pallet-im-online" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-benchmarking", "frame-support", @@ -4289,7 +4319,7 @@ dependencies = [ [[package]] name = "pallet-indices" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4305,7 +4335,7 @@ dependencies = [ [[package]] name = "pallet-membership" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4319,7 +4349,7 @@ dependencies = [ [[package]] name = "pallet-nicks" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4334,7 +4364,7 @@ dependencies = [ [[package]] name = "pallet-offences" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4348,9 +4378,27 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-offences-benchmarking" +version = "2.0.0-dev" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "pallet-im-online", + "pallet-offences", + "pallet-session", + "pallet-staking", + "parity-scale-codec", + "sp-io", + "sp-runtime", + "sp-staking", + "sp-std", +] + [[package]] name = "pallet-randomness-collective-flip" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4364,7 +4412,7 @@ dependencies = [ [[package]] name = "pallet-recovery" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "enumflags2", "frame-support", @@ -4380,7 +4428,7 @@ dependencies = [ [[package]] name = "pallet-scheduler" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-benchmarking", "frame-support", @@ -4395,7 +4443,7 @@ dependencies = [ [[package]] name = "pallet-scored-pool" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4410,7 +4458,7 @@ dependencies = [ [[package]] name = "pallet-session" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4430,19 +4478,27 @@ dependencies = [ [[package]] name = "pallet-session-benchmarking" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-benchmarking", + "frame-support", "frame-system", + "pallet-balances", "pallet-session", "pallet-staking", + "pallet-staking-reward-curve", + "pallet-timestamp", + "parity-scale-codec", + "serde", + "sp-core", + "sp-io", "sp-runtime", "sp-std", ] [[package]] name = "pallet-society" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4458,7 +4514,7 @@ dependencies = [ [[package]] name = "pallet-staking" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "env_logger 0.7.1", "frame-benchmarking", @@ -4472,7 +4528,7 @@ dependencies = [ "pallet-staking-reward-curve", "pallet-timestamp", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "rand 0.7.3", "rand_chacha 0.2.2", "serde", @@ -4488,20 +4544,41 @@ dependencies = [ "substrate-test-utils", ] +[[package]] +name = "pallet-staking-fuzz" +version = "0.0.0" +dependencies = [ + "frame-support", + "frame-system", + "honggfuzz", + "pallet-balances", + "pallet-indices", + "pallet-session", + "pallet-staking", + "pallet-staking-reward-curve", + "pallet-timestamp", + "parity-scale-codec", + "sp-core", + "sp-io", + "sp-phragmen", + "sp-runtime", + "sp-std", +] + [[package]] name = "pallet-staking-reward-curve" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "proc-macro-crate", "proc-macro2", "quote 1.0.3", "sp-runtime", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] name = "pallet-sudo" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4515,7 +4592,7 @@ dependencies = [ [[package]] name = "pallet-template" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4527,7 +4604,7 @@ dependencies = [ [[package]] name = "pallet-timestamp" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-benchmarking", "frame-support", @@ -4545,7 +4622,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -4556,11 +4633,12 @@ dependencies = [ "sp-io", "sp-runtime", "sp-std", + "sp-storage", ] [[package]] name = "pallet-transaction-payment-rpc" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -4577,7 +4655,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc-runtime-api" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "parity-scale-codec", @@ -4590,7 +4668,7 @@ dependencies = [ [[package]] name = "pallet-treasury" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-benchmarking", "frame-support", @@ -4606,7 +4684,7 @@ dependencies = [ [[package]] name = "pallet-utility" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-benchmarking", "frame-support", @@ -4622,7 +4700,7 @@ dependencies = [ [[package]] name = "pallet-vesting" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "enumflags2", "frame-benchmarking", @@ -4639,6 +4717,20 @@ dependencies = [ "sp-storage", ] +[[package]] +name = "parity-db" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d595e372d119261593297debbe4193811a4dc811d2a1ccbb8caaa6666ad7ab" +dependencies = [ + "blake2-rfc", + "crc32fast", + "libc", + "log", + "memmap", + "parking_lot 0.10.2", +] + [[package]] name = "parity-multiaddr" version = "0.7.3" @@ -4657,6 +4749,24 @@ dependencies = [ "url 2.1.1", ] +[[package]] +name = "parity-multiaddr" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4db35e222f783ef4e6661873f6c165c4eb7b65e0c408349818517d5705c2d7d3" +dependencies = [ + "arrayref", + "bs58", + "byteorder 1.3.4", + "data-encoding", + "multihash", + "percent-encoding 2.1.0", + "serde", + "static_assertions", + "unsigned-varint", + "url 2.1.1", +] + [[package]] name = "parity-multihash" version = "0.2.3" @@ -4694,7 +4804,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -4705,19 +4815,16 @@ checksum = "aa9777aa91b8ad9dd5aaa04a9b6bcb02c7f1deb952fca5a66034d5e63afc5c6f" [[package]] name = "parity-util-mem" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e42755f26e5ea21a6a819d9e63cbd70713e9867a2b767ec2cc65ca7659532c5" +checksum = "2c6e2583649a3ca84894d1d71da249abcfda54d5aca24733d72ca10d0f02361c" dependencies = [ "cfg-if", - "ethereum-types", - "hashbrown", "impl-trait-for-tuples", - "lru", "parity-util-mem-derive", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "primitive-types", - "smallvec 1.2.0", + "smallvec 1.3.0", "winapi 0.3.8", ] @@ -4728,7 +4835,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f557c32c6d268a07c921471619c0295f5efad3a0e76d4f97a05c091a51d110b2" dependencies = [ "proc-macro2", - "syn 1.0.16", + "syn 1.0.17", "synstructure", ] @@ -4760,12 +4867,12 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc" +checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" dependencies = [ "lock_api", - "parking_lot_core 0.7.0", + "parking_lot_core 0.7.2", ] [[package]] @@ -4785,23 +4892,23 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7582838484df45743c8434fbff785e8edf260c28748353d44bc0da32e0ceabf1" +checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" dependencies = [ "cfg-if", "cloudabi", "libc", "redox_syscall", - "smallvec 1.2.0", + "smallvec 1.3.0", "winapi 0.3.8", ] [[package]] name = "paste" -version = "0.1.7" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e1afe738d71b1ebab5f1207c055054015427dbfc7bbe9ee1266894156ec046" +checksum = "ab4fb1930692d1b6a9cfabdde3d06ea0a7d186518e2f4d67660d8970e2fa647a" dependencies = [ "paste-impl", "proc-macro-hack", @@ -4809,14 +4916,14 @@ dependencies = [ [[package]] name = "paste-impl" -version = "0.1.7" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d4dc4a7f6f743211c5aab239640a65091535d97d43d92a52bca435a640892bb" +checksum = "a62486e111e571b1e93b710b61e8f493c0013be39629b714cb166bdb06aa5a8a" dependencies = [ "proc-macro-hack", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -4865,22 +4972,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7804a463a8d9572f13453c516a5faea534a2403d7ced2f0c7e100eeff072772c" +checksum = "6f6a7f5eee6292c559c793430c55c00aea9d3b3d1905e855806ca4d7253426a2" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385322a45f2ecf3410c68d2a549a4a2685e8051d0f278e39743ff4e451cb9b3f" +checksum = "8988430ce790d8682672117bc06dda364c0be32d3abd738234f19f3240bad99a" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -4907,6 +5014,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "platforms" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feb3b2b1033b8a60b4da6ee470325f887758c95d5320f52f9ce0df055a55940e" + [[package]] name = "plotters" version = "0.2.12" @@ -4987,52 +5100,47 @@ dependencies = [ [[package]] name = "proc-macro-error" -version = "0.4.11" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7959c6467d962050d639361f7703b2051c43036d03493c36f01d440fdd3138a" +checksum = "98e9e4b82e0ef281812565ea4751049f1bdcdfccda7d3f459f2e138a40c08678" dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", "version_check", ] [[package]] name = "proc-macro-error-attr" -version = "0.4.11" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4002d9f55991d5e019fb940a90e1a95eb80c24e77cb2462dd4dc869604d543a" +checksum = "4f5444ead4e9935abd7f27dc51f7e852a0569ac888096d5ec2499470794e2e53" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", "syn-mid", "version_check", ] [[package]] name = "proc-macro-hack" -version = "0.5.11" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" -dependencies = [ - "proc-macro2", - "quote 1.0.3", - "syn 1.0.16", -] +checksum = "0d659fe7c6d27f25e9d80a1a094c223f5246f6a6596453e09d7229bf42750b63" [[package]] name = "proc-macro-nested" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" +checksum = "8e946095f9d3ed29ec38de908c22f95d9ac008e424c7bcae54c75a79c527c694" [[package]] name = "proc-macro2" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c09721c6781493a2a492a96b5a5bf19b65917fe6728884e7c44dd0c60ca3435" +checksum = "df246d292ff63439fea9bc8c0a270bed0e390d5ebd4db4ba15aba81111b5abe3" dependencies = [ "unicode-xid 0.2.0", ] @@ -5104,7 +5212,7 @@ dependencies = [ "itertools", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -5119,9 +5227,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.10.2" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a5325d019a4d837d3abde0a836920f959e33d350f77b5f1e289e061e774942" +checksum = "8e86d370532557ae7573551a1ec8235a0f8d6cb276c7c9e6aa490b511c447485" [[package]] name = "pwasm-utils" @@ -5154,9 +5262,9 @@ dependencies = [ [[package]] name = "quicksink" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8461ef7445f61fd72d8dcd0629ce724b9131b3c2eb36e83a5d3d4161c127530" +checksum = "77de3c815e5a160b1539c6592796801df2043ae35e123b46d73380cfa57af858" dependencies = [ "futures-core", "futures-sink", @@ -5234,7 +5342,7 @@ dependencies = [ "rand_isaac", "rand_jitter", "rand_os", - "rand_pcg", + "rand_pcg 0.1.2", "rand_xorshift", "winapi 0.3.8", ] @@ -5250,6 +5358,7 @@ dependencies = [ "rand_chacha 0.2.2", "rand_core 0.5.1", "rand_hc 0.2.0", + "rand_pcg 0.2.1", ] [[package]] @@ -5359,6 +5468,15 @@ dependencies = [ "rand_core 0.4.2", ] +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "rand_xorshift" version = "0.1.1" @@ -5445,11 +5563,31 @@ dependencies = [ "rust-argon2", ] +[[package]] +name = "ref-cast" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a214c7875e1b63fc1618db7c80efc0954f6156c9ff07699fd9039e255accdd1" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602eb59cda66fcb9aec25841fb76bc01d2b34282dcdd705028da297db6f3eec8" +dependencies = [ + "proc-macro2", + "quote 1.0.3", + "syn 1.0.17", +] + [[package]] name = "regex" -version = "1.3.4" +version = "1.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322cf97724bea3ee221b78fe25ac9c46114ebb51747ad5babd51a2fc6a8235a8" +checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" dependencies = [ "aho-corasick", "memchr", @@ -5468,9 +5606,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.16" +version = "0.6.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1132f845907680735a84409c3bebc64d1364a5683ffbce899550cd09d5eaefc1" +checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" [[package]] name = "region" @@ -5495,9 +5633,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.16.11" +version = "0.16.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "741ba1704ae21999c00942f9f5944f801e977f54302af346b596287599ad1862" +checksum = "1ba5a8ec64ee89a76c98c549af81ff14813df09c3e6dc4766c3856da48597a0c" dependencies = [ "cc", "lazy_static", @@ -5516,9 +5654,9 @@ checksum = "cabe4fa914dec5870285fa7f71f602645da47c486e68486d2b4ceb4a343e90ac" [[package]] name = "rlp" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a44d5ae8afcb238af8b75640907edc6c931efcfab2c854e81ed35fa080f84cd" +checksum = "4a7d3f9bed94764eac15b8f14af59fac420c236adaff743b7bcc88e265cb4345" dependencies = [ "rustc-hex", ] @@ -5549,7 +5687,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bc8af4bda8e1ff4932523b94d3dd20ee30a87232323eda55903ffd71d2fb017" dependencies = [ - "base64 0.11.0", + "base64", "blake2b_simd", "constant_time_eq", "crossbeam-utils", @@ -5582,26 +5720,13 @@ dependencies = [ "semver 0.9.0", ] -[[package]] -name = "rustls" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b25a18b1bf7387f0145e7f8324e700805aade3842dd3db2e74e4cdeb4677c09e" -dependencies = [ - "base64 0.10.1", - "log", - "ring", - "sct", - "webpki", -] - [[package]] name = "rustls" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1" dependencies = [ - "base64 0.11.0", + "base64", "log", "ring", "sct", @@ -5615,7 +5740,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75ffeb84a6bd9d014713119542ce415db3a3e4748f0bfce1e1416cd224a23a5" dependencies = [ "openssl-probe", - "rustls 0.17.0", + "rustls", "schannel", "security-framework", ] @@ -5628,7 +5753,7 @@ checksum = "b3bba175698996010c4f6dce5e7f173b6eb781fce25d2cfc45e27091ce0b79f6" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -5644,9 +5769,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8" +checksum = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76" [[package]] name = "safe-mix" @@ -5688,7 +5813,7 @@ dependencies = [ [[package]] name = "sc-authority-discovery" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "bytes 0.5.4", "derive_more", @@ -5718,13 +5843,13 @@ dependencies = [ [[package]] name = "sc-basic-authorship" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "futures 0.3.4", "futures-timer 3.0.2", "log", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "sc-block-builder", "sc-client-api", "sc-telemetry", @@ -5742,7 +5867,7 @@ dependencies = [ [[package]] name = "sc-block-builder" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "parity-scale-codec", "sc-client-api", @@ -5759,7 +5884,7 @@ dependencies = [ [[package]] name = "sc-chain-spec" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "impl-trait-for-tuples", "sc-chain-spec-derive", @@ -5767,23 +5892,24 @@ dependencies = [ "sc-telemetry", "serde", "serde_json", + "sp-chain-spec", "sp-core", "sp-runtime", ] [[package]] name = "sc-chain-spec-derive" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "proc-macro-crate", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] name = "sc-cli" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "ansi_term 0.12.1", "app_dirs", @@ -5815,55 +5941,17 @@ dependencies = [ "sp-runtime", "sp-state-machine", "sp-utils", + "sp-version", "structopt", "substrate-prometheus-endpoint", "tempfile", "time", - "tokio 0.2.13", -] - -[[package]] -name = "sc-client" -version = "0.8.0-alpha.5" -dependencies = [ - "derive_more", - "env_logger 0.7.1", - "fnv", - "futures 0.3.4", - "hash-db", - "hex-literal", - "kvdb", - "kvdb-memorydb", - "log", - "parity-scale-codec", - "parking_lot 0.10.0", - "sc-block-builder", - "sc-client-api", - "sc-executor", - "sc-telemetry", - "sp-api", - "sp-blockchain", - "sp-consensus", - "sp-core", - "sp-externalities", - "sp-inherents", - "sp-keyring", - "sp-panic-handler", - "sp-runtime", - "sp-state-machine", - "sp-std", - "sp-trie", - "sp-utils", - "sp-version", - "substrate-prometheus-endpoint", - "substrate-test-runtime-client", - "tempfile", - "tracing", + "tokio 0.2.18", ] [[package]] name = "sc-client-api" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "derive_more", "fnv", @@ -5871,16 +5959,18 @@ dependencies = [ "hash-db", "hex-literal", "kvdb", + "kvdb-memorydb", "lazy_static", "log", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "sc-executor", "sc-telemetry", "sp-api", "sp-blockchain", "sp-consensus", "sp-core", + "sp-database", "sp-externalities", "sp-inherents", "sp-keyring", @@ -5893,12 +5983,15 @@ dependencies = [ "sp-trie", "sp-utils", "sp-version", + "substrate-prometheus-endpoint", + "substrate-test-runtime", ] [[package]] name = "sc-client-db" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ + "blake2-rfc", "env_logger 0.7.1", "hash-db", "kvdb", @@ -5906,18 +5999,18 @@ dependencies = [ "kvdb-rocksdb", "linked-hash-map", "log", + "parity-db", "parity-scale-codec", "parity-util-mem", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "quickcheck", - "rand 0.7.3", - "sc-client", "sc-client-api", "sc-executor", "sc-state-db", "sp-blockchain", "sp-consensus", "sp-core", + "sp-database", "sp-keyring", "sp-runtime", "sp-state-machine", @@ -5927,9 +6020,19 @@ dependencies = [ "tempfile", ] +[[package]] +name = "sc-consensus" +version = "0.8.0-dev" +dependencies = [ + "sc-client-api", + "sp-blockchain", + "sp-consensus", + "sp-runtime", +] + [[package]] name = "sc-consensus-aura" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "derive_more", "env_logger 0.7.1", @@ -5937,9 +6040,8 @@ dependencies = [ "futures-timer 3.0.2", "log", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "sc-block-builder", - "sc-client", "sc-client-api", "sc-consensus-slots", "sc-executor", @@ -5967,7 +6069,7 @@ dependencies = [ [[package]] name = "sc-consensus-babe" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "derive_more", "env_logger 0.7.1", @@ -5980,11 +6082,10 @@ dependencies = [ "num-rational", "num-traits 0.2.11", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "pdqselect", "rand 0.7.3", "sc-block-builder", - "sc-client", "sc-client-api", "sc-consensus-epochs", "sc-consensus-slots", @@ -6017,7 +6118,7 @@ dependencies = [ [[package]] name = "sc-consensus-babe-rpc" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "derive_more", "futures 0.3.4", @@ -6042,11 +6143,11 @@ dependencies = [ [[package]] name = "sc-consensus-epochs" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "fork-tree", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "sc-client-api", "sp-blockchain", "sp-runtime", @@ -6054,7 +6155,7 @@ dependencies = [ [[package]] name = "sc-consensus-manual-seal" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "assert_matches", "derive_more", @@ -6064,9 +6165,8 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "log", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "sc-basic-authorship", - "sc-client", "sc-client-api", "sc-transaction-pool", "serde", @@ -6078,12 +6178,12 @@ dependencies = [ "substrate-test-runtime-client", "substrate-test-runtime-transaction-pool", "tempfile", - "tokio 0.2.13", + "tokio 0.2.18", ] [[package]] name = "sc-consensus-pow" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "derive_more", "futures 0.3.4", @@ -6103,13 +6203,13 @@ dependencies = [ [[package]] name = "sc-consensus-slots" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "futures 0.3.4", "futures-timer 3.0.2", "log", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "sc-client-api", "sc-telemetry", "sp-api", @@ -6124,7 +6224,7 @@ dependencies = [ [[package]] name = "sc-consensus-uncles" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "log", "sc-client-api", @@ -6137,7 +6237,7 @@ dependencies = [ [[package]] name = "sc-executor" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "assert_matches", "derive_more", @@ -6147,11 +6247,12 @@ dependencies = [ "log", "parity-scale-codec", "parity-wasm 0.41.0", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "sc-executor-common", "sc-executor-wasmi", "sc-executor-wasmtime", "sc-runtime-test", + "sp-api", "sp-core", "sp-externalities", "sp-io", @@ -6171,11 +6272,12 @@ dependencies = [ [[package]] name = "sc-executor-common" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "derive_more", "log", "parity-scale-codec", + "parity-wasm 0.41.0", "sp-allocator", "sp-core", "sp-runtime-interface", @@ -6186,11 +6288,10 @@ dependencies = [ [[package]] name = "sc-executor-wasmi" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "log", "parity-scale-codec", - "parity-wasm 0.41.0", "sc-executor-common", "sp-allocator", "sp-core", @@ -6201,9 +6302,11 @@ dependencies = [ [[package]] name = "sc-executor-wasmtime" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "assert_matches", + "cranelift-codegen", + "cranelift-wasm", "log", "parity-scale-codec", "parity-wasm 0.41.0", @@ -6214,11 +6317,13 @@ dependencies = [ "sp-runtime-interface", "sp-wasm-interface", "substrate-wasmtime", + "substrate-wasmtime-runtime", + "wasmtime-environ", ] [[package]] name = "sc-finality-grandpa" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "assert_matches", "env_logger 0.7.1", @@ -6228,12 +6333,12 @@ dependencies = [ "futures-timer 3.0.2", "log", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "pin-project", "rand 0.7.3", "sc-block-builder", - "sc-client", "sc-client-api", + "sc-consensus", "sc-keystore", "sc-network", "sc-network-gossip", @@ -6256,12 +6361,12 @@ dependencies = [ "substrate-prometheus-endpoint", "substrate-test-runtime-client", "tempfile", - "tokio 0.2.13", + "tokio 0.2.18", ] [[package]] name = "sc-informant" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "ansi_term 0.12.1", "futures 0.3.4", @@ -6277,11 +6382,11 @@ dependencies = [ [[package]] name = "sc-keystore" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "derive_more", "hex", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "rand 0.7.3", "serde_json", "sp-application-crypto", @@ -6292,7 +6397,7 @@ dependencies = [ [[package]] name = "sc-network" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "assert_matches", "async-std", @@ -6308,6 +6413,7 @@ dependencies = [ "futures-timer 3.0.2", "futures_codec", "hex", + "ip_network", "libp2p", "linked-hash-map", "linked_hash_set", @@ -6315,14 +6421,13 @@ dependencies = [ "lru", "nohash-hasher", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "pin-project", "prost", "prost-build", "quickcheck", "rand 0.7.3", "sc-block-builder", - "sc-client", "sc-client-api", "sc-peerset", "serde", @@ -6352,8 +6457,9 @@ dependencies = [ [[package]] name = "sc-network-gossip" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ + "async-std", "futures 0.3.4", "futures-timer 3.0.2", "libp2p", @@ -6375,12 +6481,13 @@ dependencies = [ "futures-timer 3.0.2", "libp2p", "log", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "rand 0.7.3", "sc-block-builder", - "sc-client", "sc-client-api", + "sc-consensus", "sc-network", + "sc-service", "sp-blockchain", "sp-consensus", "sp-consensus-babe", @@ -6393,7 +6500,7 @@ dependencies = [ [[package]] name = "sc-offchain" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "bytes 0.5.4", "env_logger 0.7.1", @@ -6401,12 +6508,12 @@ dependencies = [ "fnv", "futures 0.3.4", "futures-timer 3.0.2", - "hyper 0.13.3", + "hyper 0.13.5", "hyper-rustls", "log", "num_cpus", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "rand 0.7.3", "sc-client-api", "sc-client-db", @@ -6421,12 +6528,12 @@ dependencies = [ "sp-utils", "substrate-test-runtime-client", "threadpool", - "tokio 0.2.13", + "tokio 0.2.18", ] [[package]] name = "sc-peerset" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "futures 0.3.4", "libp2p", @@ -6439,7 +6546,7 @@ dependencies = [ [[package]] name = "sc-rpc" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "assert_matches", "futures 0.1.29", @@ -6449,9 +6556,8 @@ dependencies = [ "jsonrpc-pubsub", "log", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "sc-block-builder", - "sc-client", "sc-client-api", "sc-executor", "sc-keystore", @@ -6461,6 +6567,7 @@ dependencies = [ "serde_json", "sp-api", "sp-blockchain", + "sp-chain-spec", "sp-core", "sp-io", "sp-offchain", @@ -6477,7 +6584,7 @@ dependencies = [ [[package]] name = "sc-rpc-api" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "derive_more", "futures 0.3.4", @@ -6487,9 +6594,10 @@ dependencies = [ "jsonrpc-pubsub", "log", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "serde", "serde_json", + "sp-chain-spec", "sp-core", "sp-rpc", "sp-runtime", @@ -6499,7 +6607,7 @@ dependencies = [ [[package]] name = "sc-rpc-server" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "jsonrpc-core", "jsonrpc-http-server", @@ -6526,24 +6634,26 @@ dependencies = [ [[package]] name = "sc-service" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "derive_more", "exit-future", "futures 0.1.29", "futures 0.3.4", - "futures-diagnose", "futures-timer 3.0.2", + "hash-db", "lazy_static", "log", "netstat2", - "parity-multiaddr", + "parity-multiaddr 0.7.3", "parity-scale-codec", "parity-util-mem", - "parking_lot 0.10.0", + "parking_lot 0.10.2", + "pin-project", "procfs", + "rand 0.7.3", + "sc-block-builder", "sc-chain-spec", - "sc-client", "sc-client-api", "sc-client-db", "sc-executor", @@ -6561,20 +6671,24 @@ dependencies = [ "slog", "sp-api", "sp-application-crypto", + "sp-block-builder", "sp-blockchain", "sp-consensus", "sp-consensus-babe", "sp-core", + "sp-externalities", "sp-finality-grandpa", "sp-io", "sp-runtime", "sp-session", + "sp-state-machine", "sp-transaction-pool", + "sp-trie", "sp-utils", + "sp-version", "substrate-prometheus-endpoint", "substrate-test-runtime-client", "sysinfo", - "target_info", "tracing", "wasm-timer", ] @@ -6587,42 +6701,57 @@ dependencies = [ "fdlimit", "futures 0.1.29", "futures 0.3.4", + "hex-literal", "log", - "sc-client", + "parity-scale-codec", + "parking_lot 0.10.2", + "sc-block-builder", + "sc-client-api", + "sc-client-db", + "sc-executor", "sc-network", "sc-service", + "sp-api", + "sp-blockchain", "sp-consensus", "sp-core", + "sp-externalities", + "sp-panic-handler", "sp-runtime", + "sp-state-machine", + "sp-storage", "sp-transaction-pool", + "sp-trie", + "substrate-test-runtime", + "substrate-test-runtime-client", "tempfile", "tokio 0.1.22", ] [[package]] name = "sc-state-db" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "env_logger 0.7.1", "log", "parity-scale-codec", "parity-util-mem", "parity-util-mem-derive", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "sc-client-api", "sp-core", ] [[package]] name = "sc-telemetry" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "bytes 0.5.4", "futures 0.3.4", "futures-timer 3.0.2", "libp2p", "log", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "pin-project", "rand 0.7.3", "serde", @@ -6636,11 +6765,11 @@ dependencies = [ [[package]] name = "sc-tracing" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "erased-serde", "log", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "sc-telemetry", "serde", "serde_json", @@ -6651,7 +6780,7 @@ dependencies = [ [[package]] name = "sc-transaction-graph" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "assert_matches", "criterion 0.3.1", @@ -6661,7 +6790,7 @@ dependencies = [ "log", "parity-scale-codec", "parity-util-mem", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "serde", "sp-blockchain", "sp-core", @@ -6674,7 +6803,7 @@ dependencies = [ [[package]] name = "sc-transaction-pool" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "assert_matches", "derive_more", @@ -6685,7 +6814,7 @@ dependencies = [ "log", "parity-scale-codec", "parity-util-mem", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "sc-client-api", "sc-transaction-graph", "sp-api", @@ -6693,8 +6822,10 @@ dependencies = [ "sp-core", "sp-keyring", "sp-runtime", + "sp-tracing", "sp-transaction-pool", "sp-utils", + "substrate-prometheus-endpoint", "substrate-test-runtime-client", "substrate-test-runtime-transaction-pool", "wasm-timer", @@ -6702,9 +6833,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507a9e6e8ffe0a4e0ebb9a10293e62fdf7657c06f1b8bb07a8fcf697d2abf295" +checksum = "039c25b130bd8c1321ee2d7de7fde2659fa9c2744e4bb29711cfc852ea53cd19" dependencies = [ "lazy_static", "winapi 0.3.8", @@ -6757,7 +6888,7 @@ checksum = "f8584eea9b9ff42825b46faf46a8c24d2cff13ec152fa2a50df788b87c07ee28" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -6772,21 +6903,22 @@ dependencies = [ [[package]] name = "security-framework" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97bbedbe81904398b6ebb054b3e912f99d55807125790f3198ac990d98def5b0" +checksum = "572dfa3a0785509e7a44b5b4bebcf94d41ba34e9ed9eb9df722545c3b3c4144a" dependencies = [ "bitflags", "core-foundation", "core-foundation-sys", + "libc", "security-framework-sys", ] [[package]] name = "security-framework-sys" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06fd2f23e31ef68dd2328cc383bd493142e46107a3a0e24f7d734e3f3b80fe4c" +checksum = "8ddb15a5fec93b7021b8a9e96009c5d8d51c15673569f7c0f6b7204e5b7b404f" dependencies = [ "core-foundation-sys", "libc", @@ -6837,29 +6969,29 @@ checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" [[package]] name = "serde" -version = "1.0.105" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e707fbbf255b8fc8c3b99abb91e7257a622caeb20a9818cbadbeeede4e0932ff" +checksum = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.105" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac5d00fc561ba2724df6758a17de23df5914f20e41cb00f94d5b7ae42fffaff8" +checksum = "9e549e3abf4fb8621bd1609f11dfc9f5e50320802273b12f3811a67e6716ea6c" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] name = "serde_json" -version = "1.0.48" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9371ade75d4c2d6cb154141b9752cf3781ec9c05e0e5cf35060e1e70ee7b9c25" +checksum = "da07b57ee2623368351e9a0488bb0b261322a15a6e0ae53e243cbdc0f4208da9" dependencies = [ "itoa", "ryu", @@ -6982,7 +7114,7 @@ checksum = "a945ec7f7ce853e89ffa36be1e27dce9a43e82ff9093bf3461c30d5da74ed11b" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -6996,9 +7128,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" +checksum = "05720e22615919e4734f6a99ceae50d00226c3c5aca406e102ebc33298214e0a" [[package]] name = "snow" @@ -7024,23 +7156,23 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c9dab3f95c9ebdf3a88268c19af668f637a3c5039c2c56ff2d40b1b2d64a25b" dependencies = [ - "base64 0.11.0", + "base64", "bytes 0.5.4", "flate2", "futures 0.3.4", - "http 0.2.0", + "http 0.2.1", "httparse", "log", "rand 0.7.3", "sha1", - "smallvec 1.2.0", + "smallvec 1.3.0", "static_assertions", "thiserror", ] [[package]] name = "sp-allocator" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "derive_more", "log", @@ -7051,7 +7183,7 @@ dependencies = [ [[package]] name = "sp-api" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "hash-db", "parity-scale-codec", @@ -7066,13 +7198,13 @@ dependencies = [ [[package]] name = "sp-api-proc-macro" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "blake2-rfc", "proc-macro-crate", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -7096,7 +7228,7 @@ dependencies = [ [[package]] name = "sp-application-crypto" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "parity-scale-codec", "serde", @@ -7118,7 +7250,7 @@ dependencies = [ [[package]] name = "sp-arithmetic" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "criterion 0.3.1", "integer-sqrt", @@ -7127,13 +7259,14 @@ dependencies = [ "primitive-types", "rand 0.7.3", "serde", + "serde_json", "sp-debug-derive", "sp-std", ] [[package]] name = "sp-arithmetic-fuzzer" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "honggfuzz", "num-bigint", @@ -7144,7 +7277,7 @@ dependencies = [ [[package]] name = "sp-authority-discovery" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "parity-scale-codec", "sp-api", @@ -7155,7 +7288,7 @@ dependencies = [ [[package]] name = "sp-authorship" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "parity-scale-codec", "sp-inherents", @@ -7165,7 +7298,7 @@ dependencies = [ [[package]] name = "sp-block-builder" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "parity-scale-codec", "sp-api", @@ -7176,22 +7309,30 @@ dependencies = [ [[package]] name = "sp-blockchain" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "derive_more", "log", "lru", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "sp-block-builder", "sp-consensus", "sp-runtime", "sp-state-machine", ] +[[package]] +name = "sp-chain-spec" +version = "2.0.0-dev" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "sp-consensus" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "derive_more", "futures 0.3.4", @@ -7200,7 +7341,7 @@ dependencies = [ "libp2p", "log", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "serde", "sp-core", "sp-inherents", @@ -7214,7 +7355,7 @@ dependencies = [ [[package]] name = "sp-consensus-aura" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "parity-scale-codec", "sp-api", @@ -7227,7 +7368,7 @@ dependencies = [ [[package]] name = "sp-consensus-babe" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "parity-scale-codec", "sp-api", @@ -7242,7 +7383,7 @@ dependencies = [ [[package]] name = "sp-consensus-pow" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "parity-scale-codec", "sp-api", @@ -7253,7 +7394,7 @@ dependencies = [ [[package]] name = "sp-consensus-vrf" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "parity-scale-codec", "schnorrkel", @@ -7264,7 +7405,7 @@ dependencies = [ [[package]] name = "sp-core" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "base58", "blake2-rfc", @@ -7280,10 +7421,11 @@ dependencies = [ "lazy_static", "libsecp256k1", "log", + "merlin", "num-traits 0.2.11", "parity-scale-codec", "parity-util-mem", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "pretty_assertions", "primitive-types", "rand 0.7.3", @@ -7300,33 +7442,42 @@ dependencies = [ "sp-storage", "substrate-bip39", "tiny-bip39", - "tiny-keccak 2.0.1", + "tiny-keccak 2.0.2", "twox-hash", "wasmi", "zeroize", ] +[[package]] +name = "sp-database" +version = "2.0.0-dev" +dependencies = [ + "kvdb", + "parking_lot 0.10.2", +] + [[package]] name = "sp-debug-derive" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] name = "sp-externalities" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "environmental", + "parity-scale-codec", "sp-std", "sp-storage", ] [[package]] name = "sp-finality-grandpa" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "parity-scale-codec", "serde", @@ -7338,7 +7489,7 @@ dependencies = [ [[package]] name = "sp-finality-tracker" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "parity-scale-codec", "sp-inherents", @@ -7347,23 +7498,25 @@ dependencies = [ [[package]] name = "sp-inherents" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "derive_more", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "sp-core", "sp-std", ] [[package]] name = "sp-io" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ + "futures 0.3.4", "hash-db", "libsecp256k1", "log", "parity-scale-codec", + "parking_lot 0.10.2", "sp-core", "sp-externalities", "sp-runtime-interface", @@ -7375,7 +7528,7 @@ dependencies = [ [[package]] name = "sp-keyring" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "lazy_static", "sp-core", @@ -7385,15 +7538,17 @@ dependencies = [ [[package]] name = "sp-offchain" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "sp-api", + "sp-core", "sp-runtime", + "sp-state-machine", ] [[package]] name = "sp-panic-handler" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "backtrace", "log", @@ -7401,11 +7556,12 @@ dependencies = [ [[package]] name = "sp-phragmen" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "parity-scale-codec", "rand 0.7.3", "serde", + "sp-arithmetic", "sp-phragmen", "sp-phragmen-compact", "sp-runtime", @@ -7420,12 +7576,23 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] -name = "sp-rpc" +name = "sp-phragmen-fuzzer" version = "2.0.0-alpha.5" +dependencies = [ + "honggfuzz", + "rand 0.7.3", + "sp-phragmen", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "sp-rpc" +version = "2.0.0-dev" dependencies = [ "serde", "serde_json", @@ -7434,7 +7601,7 @@ dependencies = [ [[package]] name = "sp-runtime" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "hash256-std-hasher", "impl-trait-for-tuples", @@ -7450,12 +7617,13 @@ dependencies = [ "sp-core", "sp-inherents", "sp-io", + "sp-state-machine", "sp-std", ] [[package]] name = "sp-runtime-interface" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "parity-scale-codec", "primitive-types", @@ -7467,6 +7635,7 @@ dependencies = [ "sp-runtime-interface-test-wasm", "sp-state-machine", "sp-std", + "sp-tracing", "sp-wasm-interface", "static_assertions", "trybuild", @@ -7474,13 +7643,13 @@ dependencies = [ [[package]] name = "sp-runtime-interface-proc-macro" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "Inflector", "proc-macro-crate", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -7488,12 +7657,14 @@ name = "sp-runtime-interface-test" version = "2.0.0-dev" dependencies = [ "sc-executor", + "sp-core", "sp-io", "sp-runtime", "sp-runtime-interface", "sp-runtime-interface-test-wasm", "sp-runtime-interface-test-wasm-deprecated", "sp-state-machine", + "tracing", ] [[package]] @@ -7520,7 +7691,7 @@ dependencies = [ [[package]] name = "sp-sandbox" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "assert_matches", "parity-scale-codec", @@ -7534,7 +7705,7 @@ dependencies = [ [[package]] name = "sp-serializer" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "serde", "serde_json", @@ -7542,7 +7713,7 @@ dependencies = [ [[package]] name = "sp-session" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "sp-api", "sp-core", @@ -7552,7 +7723,7 @@ dependencies = [ [[package]] name = "sp-staking" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "parity-scale-codec", "sp-runtime", @@ -7561,14 +7732,14 @@ dependencies = [ [[package]] name = "sp-state-machine" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "hash-db", "hex-literal", "log", "num-traits 0.2.11", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "rand 0.7.3", "sp-core", "sp-externalities", @@ -7581,13 +7752,14 @@ dependencies = [ [[package]] name = "sp-std" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" [[package]] name = "sp-storage" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "impl-serde 0.2.3", + "ref-cast", "serde", "sp-debug-derive", "sp-std", @@ -7607,7 +7779,7 @@ dependencies = [ [[package]] name = "sp-timestamp" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -7618,9 +7790,16 @@ dependencies = [ "wasm-timer", ] +[[package]] +name = "sp-tracing" +version = "2.0.0-dev" +dependencies = [ + "tracing", +] + [[package]] name = "sp-transaction-pool" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "derive_more", "futures 0.3.4", @@ -7634,7 +7813,7 @@ dependencies = [ [[package]] name = "sp-trie" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "criterion 0.2.11", "hash-db", @@ -7652,7 +7831,7 @@ dependencies = [ [[package]] name = "sp-utils" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "futures 0.3.4", "futures-core", @@ -7662,7 +7841,7 @@ dependencies = [ [[package]] name = "sp-version" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "impl-serde 0.2.3", "parity-scale-codec", @@ -7673,7 +7852,7 @@ dependencies = [ [[package]] name = "sp-wasm-interface" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -7743,9 +7922,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "structopt" -version = "0.3.11" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fe43617218c0805c6eb37160119dc3c548110a67786da7218d1c6555212f073" +checksum = "863246aaf5ddd0d6928dfeb1a9ca65f505599e4e1b399935ef7e75107516b4ef" dependencies = [ "clap", "lazy_static", @@ -7754,15 +7933,15 @@ dependencies = [ [[package]] name = "structopt-derive" -version = "0.4.4" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6e79c80e0f4efd86ca960218d4e056249be189ff1c42824dcd9a7f51a56f0bd" +checksum = "d239ca4b13aee7a2142e6795cbd69e457665ff8037aed33b3effdc430d2f927a" dependencies = [ "heck", "proc-macro-error", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -7783,12 +7962,12 @@ dependencies = [ "heck", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] name = "subkey" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "clap", "derive_more", @@ -7829,7 +8008,7 @@ dependencies = [ [[package]] name = "substrate-browser-utils" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "chrono", "clear_on_drop", @@ -7848,17 +8027,21 @@ dependencies = [ "sc-informant", "sc-network", "sc-service", + "sp-database", "wasm-bindgen", "wasm-bindgen-futures", ] [[package]] name = "substrate-build-script-utils" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" +dependencies = [ + "platforms", +] [[package]] name = "substrate-frame-rpc-support" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "frame-support", "frame-system", @@ -7869,12 +8052,12 @@ dependencies = [ "sc-rpc-api", "serde", "sp-storage", - "tokio 0.2.13", + "tokio 0.2.18", ] [[package]] name = "substrate-frame-rpc-system" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" dependencies = [ "env_logger 0.7.1", "frame-system-rpc-runtime-api", @@ -7884,7 +8067,7 @@ dependencies = [ "jsonrpc-derive", "log", "parity-scale-codec", - "sc-client", + "sc-client-api", "sc-transaction-pool", "serde", "sp-api", @@ -7897,15 +8080,15 @@ dependencies = [ [[package]] name = "substrate-prometheus-endpoint" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" dependencies = [ "async-std", "derive_more", "futures-util", - "hyper 0.13.3", + "hyper 0.13.5", "log", "prometheus", - "tokio 0.2.13", + "tokio 0.2.18", ] [[package]] @@ -7915,10 +8098,11 @@ dependencies = [ "futures 0.3.4", "hash-db", "parity-scale-codec", - "sc-client", "sc-client-api", "sc-client-db", + "sc-consensus", "sc-executor", + "sc-service", "sp-blockchain", "sp-consensus", "sp-core", @@ -7943,8 +8127,8 @@ dependencies = [ "parity-scale-codec", "parity-util-mem", "sc-block-builder", - "sc-client", "sc-executor", + "sc-service", "serde", "sp-api", "sp-application-crypto", @@ -7976,10 +8160,12 @@ dependencies = [ "futures 0.3.4", "parity-scale-codec", "sc-block-builder", - "sc-client", "sc-client-api", + "sc-consensus", + "sc-service", "sp-api", "sp-blockchain", + "sp-consensus", "sp-core", "sp-runtime", "substrate-test-client", @@ -7993,7 +8179,7 @@ dependencies = [ "derive_more", "futures 0.3.4", "parity-scale-codec", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "sc-transaction-graph", "sp-blockchain", "sp-runtime", @@ -8003,7 +8189,7 @@ dependencies = [ [[package]] name = "substrate-test-utils" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" [[package]] name = "substrate-wasm-builder" @@ -8118,9 +8304,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "123bd9499cfb380418d509322d7a6d52e5315f064fe4b3ad18a53d6b92c07859" +checksum = "0df0eb663f387145cab623dea85b09c2c5b4b0aef44e945d928e682fce71bb03" dependencies = [ "proc-macro2", "quote 1.0.3", @@ -8135,7 +8321,7 @@ checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -8155,15 +8341,15 @@ checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", "unicode-xid 0.2.0", ] [[package]] name = "sysinfo" -version = "0.12.0" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ccb41798287e8e299a701b5560d886d6ca2c3e7115e9ea2cb68c123aec339b7" +checksum = "1cac193374347e7c263c5f547524f36ff8ec6702d56c8799c8331d26dffe8c1e" dependencies = [ "cfg-if", "doc-comment", @@ -8186,12 +8372,6 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab0e7238dcc7b40a7be719a25365910f6807bd864f4cce6b2e6b873658e2b19d" -[[package]] -name = "target_info" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c63f48baada5c52e65a29eef93ab4f8982681b67f9e8d29c7b05abcfec2b9ffe" - [[package]] name = "tempfile" version = "3.1.0" @@ -8224,7 +8404,7 @@ dependencies = [ "lazy_static", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", "version_check", ] @@ -8239,22 +8419,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.11" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee14bf8e6767ab4c687c9e8bc003879e042a96fd67a3ba5934eadb6536bef4db" +checksum = "54b3d3d2ff68104100ab257bb6bb0cb26c901abe4bd4ba15961f3bf867924012" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.11" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7b51e1fbc44b5a0840be594fbc0f960be09050f2617e61e6aa43bef97cd3ef4" +checksum = "ca972988113b7715266f91250ddb98070d033c62a011fa0fcc57434a649310dd" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -8277,20 +8457,19 @@ dependencies = [ [[package]] name = "time" -version = "0.1.42" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" dependencies = [ "libc", - "redox_syscall", "winapi 0.3.8", ] [[package]] name = "tiny-bip39" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6848cd8f566953ce1e8faeba12ee23cbdbb0437754792cd857d44628b5685e3" +checksum = "b0165e045cc2ae1660270ca65e1676dbaab60feb0f91b10f7d0665e9b47e31f2" dependencies = [ "failure", "hmac", @@ -8313,9 +8492,9 @@ dependencies = [ [[package]] name = "tiny-keccak" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2953ca5148619bc99695c1274cb54c5275bbb913c6adad87e72eaf8db9787f69" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" dependencies = [ "crunchy", ] @@ -8356,12 +8535,13 @@ dependencies = [ [[package]] name = "tokio" -version = "0.2.13" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa5e81d6bc4e67fe889d5783bd2a128ab2e0cfa487e0be16b6a8d177b101616" +checksum = "34ef16d072d2b6dc8b4a56c70f5c5ced1a37752116f8e7c1e80c659aa7cb6713" dependencies = [ "bytes 0.5.4", "fnv", + "futures-core", "iovec", "lazy_static", "libc", @@ -8459,7 +8639,7 @@ checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -8488,8 +8668,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4adb8b3e5f86b707f1b54e7c15b6de52617a823608ccda98a15d3a24222f265a" dependencies = [ "futures-core", - "rustls 0.17.0", - "tokio 0.2.13", + "rustls", + "tokio 0.2.18", "webpki", ] @@ -8592,16 +8772,16 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "571da51182ec208780505a32528fc5512a8fe1443ab960b3f2f3ef093cd16930" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" dependencies = [ "bytes 0.5.4", "futures-core", "futures-sink", "log", "pin-project-lite", - "tokio 0.2.13", + "tokio 0.2.18", ] [[package]] @@ -8637,7 +8817,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fbad39da2f9af1cae3016339ad7f2c7a9e870f12e8fd04c4fd7ef35b30c0d2b" dependencies = [ "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", ] [[package]] @@ -8657,9 +8837,9 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f105ed33e42b534284b691e804e909c42a8898afcf22896a32255c05a1a50488" +checksum = "c48b309cdda1abbdada28424bdc46f8b85362b3e66d6786d91223e83874429c7" dependencies = [ "criterion 0.2.11", "hash-db", @@ -8673,15 +8853,15 @@ dependencies = [ [[package]] name = "trie-db" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de9222c50cc325855621271157c973da27a0dcd26fa06f8edf81020bd2333df0" +checksum = "bcc309f34008563989045a4c4dbcc5770467f3a3785ee80a9b5cc0d83362475f" dependencies = [ "hash-db", "hashbrown", "log", "rustc-hex", - "smallvec 1.2.0", + "smallvec 1.3.0", ] [[package]] @@ -8711,9 +8891,9 @@ checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" [[package]] name = "trybuild" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24b4e093c5ed1a60b22557090120aa14f90ca801549c0949d775ea07c1407720" +checksum = "459186ab1afd6d93bd23c2269125f4f7694f8771fe0e64434b4bdc212b94034d" dependencies = [ "glob 0.3.0", "lazy_static", @@ -8745,9 +8925,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.11.2" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" [[package]] name = "uint" @@ -8785,7 +8965,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" dependencies = [ - "smallvec 1.2.0", + "smallvec 1.3.0", ] [[package]] @@ -8814,9 +8994,9 @@ checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" [[package]] name = "unsigned-varint" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38e01ad4b98f042e166c1bf9a13f9873a99d79eaa171ce7ca81e6dd0f895d8a" +checksum = "f67332660eb59a6f1eb24ff1220c9e8d01738a8503c6002e30bcfe4bd9f2b4a9" dependencies = [ "bytes 0.5.4", "futures-io", @@ -8870,16 +9050,6 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" -[[package]] -name = "vergen" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ce50d8996df1f85af15f2cd8d33daae6e479575123ef4314a51a70a230739cb" -dependencies = [ - "bitflags", - "chrono", -] - [[package]] name = "version_check" version = "0.9.1" @@ -8906,9 +9076,9 @@ dependencies = [ [[package]] name = "wabt-sys" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af5d153dc96aad7dc13ab90835b892c69867948112d95299e522d370c4e13a08" +checksum = "23d7043ebb3e5d96fad7a8d3ca22ee9880748ff8c3e18092cfb2a49d3b8f9084" dependencies = [ "cc", "cmake", @@ -8964,34 +9134,36 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasm-bindgen" -version = "0.2.59" +version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3557c397ab5a8e347d434782bcd31fc1483d927a6826804cec05cc792ee2519d" +checksum = "2cc57ce05287f8376e998cbddfb4c8cb43b84a7ec55cf4551d7c00eef317a47f" dependencies = [ "cfg-if", + "serde", + "serde_json", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.59" +version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0da9c9a19850d3af6df1cb9574970b566d617ecfaf36eb0b706b6f3ef9bd2f8" +checksum = "d967d37bf6c16cca2973ca3af071d0a2523392e4a594548155d89a678f4237cd" dependencies = [ "bumpalo", "lazy_static", "log", "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "457414a91863c0ec00090dba537f88ab955d93ca6555862c29b6d860990b8a8a" +checksum = "7add542ea1ac7fdaa9dc25e031a6af33b7d63376292bd24140c637d00d1c312a" dependencies = [ "cfg-if", "js-sys", @@ -9001,9 +9173,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.59" +version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6fde1d36e75a714b5fe0cffbb78978f222ea6baebb726af13c78869fdb4205" +checksum = "8bd151b63e1ea881bb742cd20e1d6127cef28399558f3b5d415289bc41eee3a4" dependencies = [ "quote 1.0.3", "wasm-bindgen-macro-support", @@ -9011,22 +9183,46 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.59" +version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25bda4168030a6412ea8a047e27238cadf56f0e53516e1e83fec0a8b7c786f6d" +checksum = "d68a5b36eef1be7868f668632863292e37739656a80fc4b9acec7b0bd35a4931" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.59" +version = "0.2.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf76fe7d25ac79748a37538b7daeed1c7a6867c92d3245c12c6222e4a20d639" + +[[package]] +name = "wasm-bindgen-test" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "648da3460c6d2aa04b715a936329e2e311180efe650b2127d6267f4193ccac14" +dependencies = [ + "console_error_panic_hook", + "js-sys", + "scoped-tls", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc9f36ad51f25b0219a3d4d13b90eb44cd075dff8b6280cca015775d7acaddd8" +checksum = "cf2f86cd78a2aa7b1fb4bb6ed854eccb7f9263089c79542dca1576a1518a8467" +dependencies = [ + "proc-macro2", + "quote 1.0.3", +] [[package]] name = "wasm-gc-api" @@ -9108,7 +9304,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80f3dea0e60c076dd0da27fa10c821323903c9554c617ed32eaab8e7a7e36c89" dependencies = [ "anyhow", - "base64 0.11.0", + "base64", "bincode", "cranelift-codegen", "cranelift-entity", @@ -9148,27 +9344,27 @@ dependencies = [ [[package]] name = "wast" -version = "10.0.0" +version = "14.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4efb62ecebf5cc9dbf2954309a20d816289c6550c0597a138b9e811cefc05007" +checksum = "47b11c94c63d5365a76ea287f8e6e5b6050233fae4b2423aea2a1e126a385e17" dependencies = [ "leb128", ] [[package]] name = "wat" -version = "1.0.11" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdea5e25273cc3a62f3ae3a1a4c7d7996625875b50c0b4475fee6698c2b069c" +checksum = "03db18bc33cff3859c296efbefdcc00763a644539feeadca3415a1cee8a2835d" dependencies = [ "wast", ] [[package]] name = "web-sys" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721c6263e2c66fd44501cc5efbfa2b7dfa775d13e4ea38c46299646ed1f9c70a" +checksum = "2d6f51648d8c56c366144378a33290049eafdd784071077f6fe37dae64c1c4cb" dependencies = [ "js-sys", "wasm-bindgen", @@ -9186,18 +9382,18 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.17.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a262ae37dd9d60f60dd473d1158f9fbebf110ba7b6a5051c8160460f6043718b" +checksum = "91cd5736df7f12a964a5067a12c62fa38e1bd8080aff1f80bc29be7c80d19ab4" dependencies = [ "webpki", ] [[package]] name = "webpki-roots" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91cd5736df7f12a964a5067a12c62fa38e1bd8080aff1f80bc29be7c80d19ab4" +checksum = "f8eff4b7516a57307f9349c64bf34caa34b940b66fed4b2fb3136cb7386e5739" dependencies = [ "webpki", ] @@ -9241,9 +9437,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ccfbf554c6ad11084fb7517daca16cfdcaccbdadba4fc336f032a8b12c2ad80" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ "winapi 0.3.8", ] @@ -9308,7 +9504,7 @@ dependencies = [ "futures 0.3.4", "log", "nohash-hasher", - "parking_lot 0.10.0", + "parking_lot 0.10.2", "rand 0.7.3", "static_assertions", ] @@ -9330,7 +9526,7 @@ checksum = "de251eec69fc7c1bc3923403d18ececb929380e016afe103da75f396704f8ca2" dependencies = [ "proc-macro2", "quote 1.0.3", - "syn 1.0.16", + "syn 1.0.17", "synstructure", ] diff --git a/Cargo.toml b/Cargo.toml index b5829073853dc21cdf305b39de6564421cb63505..35d396741f19c43161ba2973aa546b4e737582c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "bin/node-template/runtime", "bin/node-template/pallets/template", "bin/node/bench", + "bin/node/browser-testing", "bin/node/cli", "bin/node/executor", "bin/node/primitives", @@ -14,7 +15,6 @@ members = [ "bin/node/transaction-factory", "bin/utils/subkey", "bin/utils/chain-spec-builder", - "client", "client/api", "client/authority-discovery", "client/basic-authorship", @@ -25,6 +25,7 @@ members = [ "client/consensus/aura", "client/consensus/babe", "client/consensus/babe/rpc", + "client/consensus/common", "client/consensus/manual-seal", "client/consensus/pow", "client/consensus/uncles", @@ -94,6 +95,7 @@ members = [ "frame/society", "frame/staking", "frame/staking/reward-curve", + "frame/staking/fuzzer", "frame/sudo", "frame/support", "frame/support/procedural", @@ -122,6 +124,8 @@ members = [ "primitives/consensus/pow", "primitives/consensus/vrf", "primitives/core", + "primitives/chain-spec", + "primitives/database", "primitives/debug-derive", "primitives/storage", "primitives/externalities", @@ -132,6 +136,7 @@ members = [ "primitives/offchain", "primitives/panic-handler", "primitives/phragmen", + "primitives/phragmen/fuzzer", "primitives/phragmen/compact", "primitives/rpc", "primitives/runtime-interface", @@ -156,6 +161,7 @@ members = [ "primitives/timestamp", "primitives/test-primitives", "primitives/transaction-pool", + "primitives/tracing", "primitives/trie", "primitives/utils", "primitives/wasm-interface", @@ -175,3 +181,4 @@ members = [ [profile.release] # Substrate runtime requires unwinding. panic = "unwind" + diff --git a/Process.toml b/Process.toml new file mode 100644 index 0000000000000000000000000000000000000000..ecaf5c7120ff5cb1b5cc0f923b51e7fda4df9fca --- /dev/null +++ b/Process.toml @@ -0,0 +1,24 @@ +[Networking] +owner = "tomaka" +whitelist = [] +matrix_room_id = "!vUADSGcyXmxhKLeDsW:matrix.parity.io" + +[Client] +owner = "gnunicorn" +whitelist = [] +matrix_room_id = "!aenJixaHcSKbJOWxYk:matrix.parity.io" + +[Runtime] +owner = "gavofyork" +whitelist = [] +matrix_room_id = "!yBKstWVBkwzUkPslsp:matrix.parity.io" + +[Consensus] +owner = "andresilva" +whitelist = [] +matrix_room_id = "!XdNWDTfVNFVixljKZU:matrix.parity.io" + +[Smart Contracts] +owner = "pepyakin" +whitelist = [] +matrix_room_id = "!yBKstWVBkwzUkPslsp:matrix.parity.io" diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index b312d311118026092cc11b338b5c47dcf4428fc3..53b6da219f9a8d2d5673721341137ffcf4f8d616 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -1,13 +1,17 @@ [package] name = "node-template" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Anonymous"] +description = "Substrate Node template" edition = "2018" license = "Unlicense" build = "build.rs" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [[bin]] name = "node-template" @@ -16,29 +20,25 @@ futures = "0.3.4" log = "0.4.8" structopt = "0.3.8" -sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sc-executor = { version = "0.8.0-alpha.5", path = "../../../client/executor" } -sc-service = { version = "0.8.0-alpha.5", path = "../../../client/service" } -sp-inherents = { version = "2.0.0-alpha.5", path = "../../../primitives/inherents" } -sc-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../client/transaction-pool" } -sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../primitives/transaction-pool" } -sc-network = { version = "0.8.0-alpha.5", path = "../../../client/network" } -sc-consensus-aura = { version = "0.8.0-alpha.5", path = "../../../client/consensus/aura" } -sp-consensus-aura = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/aura" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } -sc-finality-grandpa = { version = "0.8.0-alpha.5", path = "../../../client/finality-grandpa" } -sp-finality-grandpa = { version = "2.0.0-alpha.5", path = "../../../primitives/finality-grandpa" } -sc-client = { version = "0.8.0-alpha.5", path = "../../../client/" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sc-basic-authorship = { path = "../../../client/basic-authorship", version = "0.8.0-alpha.5"} +sc-cli = { version = "0.8.0-dev", path = "../../../client/cli" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sc-executor = { version = "0.8.0-dev", path = "../../../client/executor" } +sc-service = { version = "0.8.0-dev", default-features = false, path = "../../../client/service" } +sp-inherents = { version = "2.0.0-dev", path = "../../../primitives/inherents" } +sc-transaction-pool = { version = "2.0.0-dev", path = "../../../client/transaction-pool" } +sp-transaction-pool = { version = "2.0.0-dev", path = "../../../primitives/transaction-pool" } +sc-network = { version = "0.8.0-dev", path = "../../../client/network" } +sc-consensus-aura = { version = "0.8.0-dev", path = "../../../client/consensus/aura" } +sp-consensus-aura = { version = "0.8.0-dev", path = "../../../primitives/consensus/aura" } +sp-consensus = { version = "0.8.0-dev", path = "../../../primitives/consensus/common" } +sc-consensus = { version = "0.8.0-dev", path = "../../../client/consensus/common" } +sc-finality-grandpa = { version = "0.8.0-dev", path = "../../../client/finality-grandpa" } +sp-finality-grandpa = { version = "2.0.0-dev", path = "../../../primitives/finality-grandpa" } +sc-client-api = { version = "2.0.0-dev", path = "../../../client/api" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sc-basic-authorship = { path = "../../../client/basic-authorship", version = "0.8.0-dev"} -node-template-runtime = { version = "2.0.0-alpha.5", path = "../runtime" } +node-template-runtime = { version = "2.0.0-dev", path = "../runtime" } [build-dependencies] -vergen = "3.0.4" -build-script-utils = { version = "2.0.0-alpha.5", package = "substrate-build-script-utils", path = "../../../utils/build-script-utils" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +substrate-build-script-utils = { version = "2.0.0-dev", path = "../../../utils/build-script-utils" } diff --git a/bin/node-template/node/build.rs b/bin/node-template/node/build.rs index 222cbb409285b40e7204cd609d444854dd4082aa..e3bfe3116bf28dba1872f7d0b64c2ee0c9c71c3c 100644 --- a/bin/node-template/node/build.rs +++ b/bin/node-template/node/build.rs @@ -1,9 +1,7 @@ -use vergen::{ConstantsFlags, generate_cargo_keys}; - -const ERROR_MSG: &str = "Failed to generate metadata files"; +use substrate_build_script_utils::{generate_cargo_keys, rerun_if_git_head_changed}; fn main() { - generate_cargo_keys(ConstantsFlags::SHA_SHORT).expect(ERROR_MSG); + generate_cargo_keys(); - build_script_utils::rerun_if_git_head_changed(); + rerun_if_git_head_changed(); } diff --git a/bin/node-template/node/src/chain_spec.rs b/bin/node-template/node/src/chain_spec.rs index b57000fed7b642bd5d08503feb2e6dcd3b6f354d..fb53edd9a1a06b44fd2626feda427b79b0c08a5f 100644 --- a/bin/node-template/node/src/chain_spec.rs +++ b/bin/node-template/node/src/chain_spec.rs @@ -3,10 +3,10 @@ use node_template_runtime::{ AccountId, AuraConfig, BalancesConfig, GenesisConfig, GrandpaConfig, SudoConfig, SystemConfig, WASM_BINARY, Signature }; -use sp_consensus_aura::sr25519::{AuthorityId as AuraId}; -use sp_finality_grandpa::{AuthorityId as GrandpaId}; -use sc_service; +use sp_consensus_aura::sr25519::AuthorityId as AuraId; +use sp_finality_grandpa::AuthorityId as GrandpaId; use sp_runtime::traits::{Verify, IdentifyAccount}; +use sc_service::ChainType; // Note this is the URL for the telemetry server //const STAGING_TELEMETRY_URL: &str = "wss://telemetry.polkadot.io/submit/"; @@ -14,17 +14,6 @@ use sp_runtime::traits::{Verify, IdentifyAccount}; /// Specialized `ChainSpec`. This is a specialization of the general Substrate ChainSpec type. pub type ChainSpec = sc_service::GenericChainSpec; -/// The chain specification option. This is expected to come in from the CLI and -/// is little more than one of a number of alternatives which can easily be converted -/// from a string (`--chain=...`) into a `ChainSpec`. -#[derive(Clone, Debug)] -pub enum Alternative { - /// Whatever the current runtime is, with just Alice as an auth. - Development, - /// Whatever the current runtime is, with simple Alice/Bob auths. - LocalTestnet, -} - /// Helper function to generate a crypto pair from seed pub fn get_from_seed(seed: &str) -> ::Public { TPublic::Pair::from_string(&format!("//{}", seed), None) @@ -42,80 +31,72 @@ pub fn get_account_id_from_seed(seed: &str) -> AccountId where } /// Helper function to generate an authority key for Aura -pub fn get_authority_keys_from_seed(s: &str) -> (AuraId, GrandpaId) { +pub fn authority_keys_from_seed(s: &str) -> (AuraId, GrandpaId) { ( get_from_seed::(s), get_from_seed::(s), ) } -impl Alternative { - /// Get an actual chain config from one of the alternatives. - pub(crate) fn load(self) -> Result { - Ok(match self { - Alternative::Development => ChainSpec::from_genesis( - "Development", - "dev", - || testnet_genesis( - vec![ - get_authority_keys_from_seed("Alice"), - ], - get_account_id_from_seed::("Alice"), - vec![ - get_account_id_from_seed::("Alice"), - get_account_id_from_seed::("Bob"), - get_account_id_from_seed::("Alice//stash"), - get_account_id_from_seed::("Bob//stash"), - ], - true, - ), - vec![], - None, - None, - None, - None - ), - Alternative::LocalTestnet => ChainSpec::from_genesis( - "Local Testnet", - "local_testnet", - || testnet_genesis( - vec![ - get_authority_keys_from_seed("Alice"), - get_authority_keys_from_seed("Bob"), - ], - get_account_id_from_seed::("Alice"), - vec![ - get_account_id_from_seed::("Alice"), - get_account_id_from_seed::("Bob"), - get_account_id_from_seed::("Charlie"), - get_account_id_from_seed::("Dave"), - get_account_id_from_seed::("Eve"), - get_account_id_from_seed::("Ferdie"), - get_account_id_from_seed::("Alice//stash"), - get_account_id_from_seed::("Bob//stash"), - get_account_id_from_seed::("Charlie//stash"), - get_account_id_from_seed::("Dave//stash"), - get_account_id_from_seed::("Eve//stash"), - get_account_id_from_seed::("Ferdie//stash"), - ], - true, - ), - vec![], - None, - None, - None, - None - ), - }) - } +pub fn development_config() -> ChainSpec { + ChainSpec::from_genesis( + "Development", + "dev", + ChainType::Development, + || testnet_genesis( + vec![ + authority_keys_from_seed("Alice"), + ], + get_account_id_from_seed::("Alice"), + vec![ + get_account_id_from_seed::("Alice"), + get_account_id_from_seed::("Bob"), + get_account_id_from_seed::("Alice//stash"), + get_account_id_from_seed::("Bob//stash"), + ], + true, + ), + vec![], + None, + None, + None, + None, + ) +} - pub(crate) fn from(s: &str) -> Option { - match s { - "dev" => Some(Alternative::Development), - "" | "local" => Some(Alternative::LocalTestnet), - _ => None, - } - } +pub fn local_testnet_config() -> ChainSpec { + ChainSpec::from_genesis( + "Local Testnet", + "local_testnet", + ChainType::Local, + || testnet_genesis( + vec![ + authority_keys_from_seed("Alice"), + authority_keys_from_seed("Bob"), + ], + get_account_id_from_seed::("Alice"), + vec![ + get_account_id_from_seed::("Alice"), + get_account_id_from_seed::("Bob"), + get_account_id_from_seed::("Charlie"), + get_account_id_from_seed::("Dave"), + get_account_id_from_seed::("Eve"), + get_account_id_from_seed::("Ferdie"), + get_account_id_from_seed::("Alice//stash"), + get_account_id_from_seed::("Bob//stash"), + get_account_id_from_seed::("Charlie//stash"), + get_account_id_from_seed::("Dave//stash"), + get_account_id_from_seed::("Eve//stash"), + get_account_id_from_seed::("Ferdie//stash"), + ], + true, + ), + vec![], + None, + None, + None, + None, + ) } fn testnet_genesis(initial_authorities: Vec<(AuraId, GrandpaId)>, @@ -141,10 +122,3 @@ fn testnet_genesis(initial_authorities: Vec<(AuraId, GrandpaId)>, }), } } - -pub fn load_spec(id: &str) -> Result, String> { - Ok(match Alternative::from(id) { - Some(spec) => Box::new(spec.load()?), - None => Box::new(ChainSpec::from_json_file(std::path::PathBuf::from(id))?), - }) -} diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index 0f4c301dbff5b7defe0565bf71e85b21a6418867..baac33e9aca83311fe76a378afe04966a0337c6c 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -14,36 +14,67 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use sp_consensus_aura::sr25519::{AuthorityPair as AuraPair}; -use sc_cli::VersionInfo; -use crate::service; use crate::chain_spec; use crate::cli::Cli; +use crate::service; +use sc_cli::SubstrateCli; -/// Parse and run command line arguments -pub fn run(version: VersionInfo) -> sc_cli::Result<()> { - let opt = sc_cli::from_args::(&version); +impl SubstrateCli for Cli { + fn impl_name() -> &'static str { + "Substrate Node" + } + + fn impl_version() -> &'static str { + env!("SUBSTRATE_CLI_IMPL_VERSION") + } - let mut config = sc_service::Configuration::from_version(&version); + fn description() -> &'static str { + env!("CARGO_PKG_DESCRIPTION") + } + + fn author() -> &'static str { + env!("CARGO_PKG_AUTHORS") + } + + fn support_url() -> &'static str { + "support.anonymous.an" + } - match opt.subcommand { + fn copyright_start_year() -> i32 { + 2017 + } + + fn executable_name() -> &'static str { + env!("CARGO_PKG_NAME") + } + + fn load_spec(&self, id: &str) -> Result, String> { + Ok(match id { + "dev" => Box::new(chain_spec::development_config()), + "" | "local" => Box::new(chain_spec::local_testnet_config()), + path => Box::new(chain_spec::ChainSpec::from_json_file( + std::path::PathBuf::from(path), + )?), + }) + } +} + +/// Parse and run command line arguments +pub fn run() -> sc_cli::Result<()> { + let cli = Cli::from_args(); + + match &cli.subcommand { Some(subcommand) => { - subcommand.init(&version)?; - subcommand.update_config(&mut config, chain_spec::load_spec, &version)?; - subcommand.run( - config, - |config: _| Ok(new_full_start!(config).0), - ) - }, + let runner = cli.create_runner(subcommand)?; + runner.run_subcommand(subcommand, |config| Ok(new_full_start!(config).0)) + } None => { - opt.run.init(&version)?; - opt.run.update_config(&mut config, chain_spec::load_spec, &version)?; - opt.run.run( - config, + let runner = cli.create_runner(&cli.run)?; + runner.run_node( service::new_light, service::new_full, - &version, + node_template_runtime::VERSION ) - }, + } } } diff --git a/bin/node-template/node/src/main.rs b/bin/node-template/node/src/main.rs index 91b2c257e0cd733f7e87d00e02857101810a2fb0..369e6932a030811b542ae8de9f26e9324f22e069 100644 --- a/bin/node-template/node/src/main.rs +++ b/bin/node-template/node/src/main.rs @@ -8,16 +8,5 @@ mod cli; mod command; fn main() -> sc_cli::Result<()> { - let version = sc_cli::VersionInfo { - name: "Substrate Node", - commit: env!("VERGEN_SHA_SHORT"), - version: env!("CARGO_PKG_VERSION"), - executable_name: "node-template", - author: "Anonymous", - description: "Template Node", - support_url: "support.anonymous.an", - copyright_start_year: 2017, - }; - - command::run(version) + command::run() } diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index b8c1b32b56c385d52d649f375c570ce5a7870368..16e5271cce9436c3030e7897cfcc920ec546d67d 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -2,8 +2,8 @@ use std::sync::Arc; use std::time::Duration; -use sc_client::LongestChain; use sc_client_api::ExecutorProvider; +use sc_consensus::LongestChain; use node_template_runtime::{self, opaque::Block, RuntimeApi}; use sc_service::{error::{Error as ServiceError}, AbstractService, Configuration, ServiceBuilder}; use sp_inherents::InherentDataProviders; @@ -26,6 +26,8 @@ native_executor_instance!( macro_rules! new_full_start { ($config:expr) => {{ use std::sync::Arc; + use sp_consensus_aura::sr25519::AuthorityPair as AuraPair; + let mut import_setup = None; let inherent_data_providers = sp_inherents::InherentDataProviders::new(); @@ -33,11 +35,11 @@ macro_rules! new_full_start { node_template_runtime::opaque::Block, node_template_runtime::RuntimeApi, crate::service::Executor >($config)? .with_select_chain(|_config, backend| { - Ok(sc_client::LongestChain::new(backend.clone())) + Ok(sc_consensus::LongestChain::new(backend.clone())) })? - .with_transaction_pool(|config, client, _fetcher| { + .with_transaction_pool(|config, client, _fetcher, prometheus_registry| { let pool_api = sc_transaction_pool::FullChainApi::new(client.clone()); - Ok(sc_transaction_pool::BasicPool::new(config, std::sync::Arc::new(pool_api))) + Ok(sc_transaction_pool::BasicPool::new(config, std::sync::Arc::new(pool_api), prometheus_registry)) })? .with_import_queue(|_config, client, mut select_chain, _transaction_pool| { let select_chain = select_chain.take() @@ -69,12 +71,10 @@ macro_rules! new_full_start { } /// Builds a new service for a full client. -pub fn new_full(config: Configuration) - -> Result -{ +pub fn new_full(config: Configuration) -> Result { let role = config.role.clone(); let force_authoring = config.force_authoring; - let name = config.name.clone(); + let name = config.network.node_name.clone(); let disable_grandpa = config.disable_grandpa; let (builder, mut import_setup, inherent_data_providers) = new_full_start!(config); @@ -174,22 +174,20 @@ pub fn new_full(config: Configuration) } /// Builds a new service for a light client. -pub fn new_light(config: Configuration) - -> Result -{ +pub fn new_light(config: Configuration) -> Result { let inherent_data_providers = InherentDataProviders::new(); ServiceBuilder::new_light::(config)? .with_select_chain(|_config, backend| { Ok(LongestChain::new(backend.clone())) })? - .with_transaction_pool(|config, client, fetcher| { + .with_transaction_pool(|config, client, fetcher, prometheus_registry| { let fetcher = fetcher .ok_or_else(|| "Trying to start light transaction pool without active fetcher")?; let pool_api = sc_transaction_pool::LightChainApi::new(client.clone(), fetcher.clone()); let pool = sc_transaction_pool::BasicPool::with_revalidation_type( - config, Arc::new(pool_api), sc_transaction_pool::RevalidationType::Light, + config, Arc::new(pool_api), prometheus_registry, sc_transaction_pool::RevalidationType::Light, ); Ok(pool) })? diff --git a/bin/node-template/pallets/template/Cargo.toml b/bin/node-template/pallets/template/Cargo.toml index 69fcd843526806c9fe9a0b6e684a292fcbfe778b..01484c608cf66fc35534f6f8dc603bcd6a356c77 100644 --- a/bin/node-template/pallets/template/Cargo.toml +++ b/bin/node-template/pallets/template/Cargo.toml @@ -2,37 +2,40 @@ authors = ['Anonymous'] edition = '2018' name = 'pallet-template' -version = "2.0.0-alpha.5" +version = "2.0.0-dev" license = "Unlicense" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet template" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } [dependencies.frame-support] default-features = false -version = "2.0.0-alpha.5" +version = "2.0.0-dev" path = "../../../../frame/support" [dependencies.frame-system] default-features = false -version = "2.0.0-alpha.5" +version = "2.0.0-dev" path = "../../../../frame/system" [dev-dependencies.sp-core] default-features = false -version = "2.0.0-alpha.5" +version = "2.0.0-dev" path = "../../../../primitives/core" [dev-dependencies.sp-io] default-features = false -version = "2.0.0-alpha.5" +version = "2.0.0-dev" path = "../../../../primitives/io" [dev-dependencies.sp-runtime] default-features = false -version = "2.0.0-alpha.5" +version = "2.0.0-dev" path = "../../../../primitives/runtime" @@ -43,6 +46,3 @@ std = [ 'frame-support/std', 'frame-system/std' ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node-template/pallets/template/src/lib.rs b/bin/node-template/pallets/template/src/lib.rs index a0daecfb72c9a14ffa7e7819400f0345c542b5b8..1f82857c43e4769ad60ab6cc4001536ac8cd8dcb 100644 --- a/bin/node-template/pallets/template/src/lib.rs +++ b/bin/node-template/pallets/template/src/lib.rs @@ -75,7 +75,7 @@ decl_module! { /// Just a dummy entry point. /// function that can be called by the external world as an extrinsics call /// takes a parameter of the type `AccountId`, stores it, and emits an event - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn do_something(origin, something: u32) -> dispatch::DispatchResult { // Check it was signed and get the signer. See also: ensure_root and ensure_none let who = ensure_signed(origin)?; @@ -91,7 +91,7 @@ decl_module! { /// Another dummy entry point. /// takes no parameters, attempts to increment storage value, and possibly throws an error - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn cause_error(origin) -> dispatch::DispatchResult { // Check it was signed and get the signer. See also: ensure_root and ensure_none let _who = ensure_signed(origin)?; diff --git a/bin/node-template/pallets/template/src/mock.rs b/bin/node-template/pallets/template/src/mock.rs index a93ac0359e3a2f2b8f4e0e558a0b254e33eb1c7e..33c66e2a4e80d616a64255701279bdc9170a9240 100644 --- a/bin/node-template/pallets/template/src/mock.rs +++ b/bin/node-template/pallets/template/src/mock.rs @@ -36,6 +36,9 @@ impl system::Trait for Test { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/bin/node-template/runtime/Cargo.toml b/bin/node-template/runtime/Cargo.toml index e8653e6df70e82e503a54e79c638e9a796e41913..dfd517130b216215d8ded2098fd68ee2b7df93cc 100644 --- a/bin/node-template/runtime/Cargo.toml +++ b/bin/node-template/runtime/Cargo.toml @@ -1,40 +1,43 @@ [package] name = "node-template-runtime" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Anonymous"] edition = "2018" license = "Unlicense" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -aura = { version = "2.0.0-alpha.5", default-features = false, package = "pallet-aura", path = "../../../frame/aura" } -balances = { version = "2.0.0-alpha.5", default-features = false, package = "pallet-balances", path = "../../../frame/balances" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/support" } -grandpa = { version = "2.0.0-alpha.5", default-features = false, package = "pallet-grandpa", path = "../../../frame/grandpa" } -randomness-collective-flip = { version = "2.0.0-alpha.5", default-features = false, package = "pallet-randomness-collective-flip", path = "../../../frame/randomness-collective-flip" } -sudo = { version = "2.0.0-alpha.5", default-features = false, package = "pallet-sudo", path = "../../../frame/sudo" } -system = { version = "2.0.0-alpha.5", default-features = false, package = "frame-system", path = "../../../frame/system" } -timestamp = { version = "2.0.0-alpha.5", default-features = false, package = "pallet-timestamp", path = "../../../frame/timestamp" } -transaction-payment = { version = "2.0.0-alpha.5", default-features = false, package = "pallet-transaction-payment", path = "../../../frame/transaction-payment" } -frame-executive = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/executive" } +aura = { version = "2.0.0-dev", default-features = false, package = "pallet-aura", path = "../../../frame/aura" } +balances = { version = "2.0.0-dev", default-features = false, package = "pallet-balances", path = "../../../frame/balances" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../../../frame/support" } +grandpa = { version = "2.0.0-dev", default-features = false, package = "pallet-grandpa", path = "../../../frame/grandpa" } +randomness-collective-flip = { version = "2.0.0-dev", default-features = false, package = "pallet-randomness-collective-flip", path = "../../../frame/randomness-collective-flip" } +sudo = { version = "2.0.0-dev", default-features = false, package = "pallet-sudo", path = "../../../frame/sudo" } +system = { version = "2.0.0-dev", default-features = false, package = "frame-system", path = "../../../frame/system" } +timestamp = { version = "2.0.0-dev", default-features = false, package = "pallet-timestamp", path = "../../../frame/timestamp" } +transaction-payment = { version = "2.0.0-dev", default-features = false, package = "pallet-transaction-payment", path = "../../../frame/transaction-payment" } +frame-executive = { version = "2.0.0-dev", default-features = false, path = "../../../frame/executive" } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/api" } -sp-block-builder = { path = "../../../primitives/block-builder", default-features = false, version = "2.0.0-alpha.5"} -sp-consensus-aura = { version = "0.8.0-alpha.5", default-features = false, path = "../../../primitives/consensus/aura" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/core" } -sp-inherents = { path = "../../../primitives/inherents", default-features = false, version = "2.0.0-alpha.5"} -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/io" } -sp-offchain = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/offchain" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/runtime" } -sp-session = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/session" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/std" } -sp-transaction-pool = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/transaction-pool" } -sp-version = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/version" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/api" } +sp-block-builder = { path = "../../../primitives/block-builder", default-features = false, version = "2.0.0-dev"} +sp-consensus-aura = { version = "0.8.0-dev", default-features = false, path = "../../../primitives/consensus/aura" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/core" } +sp-inherents = { path = "../../../primitives/inherents", default-features = false, version = "2.0.0-dev"} +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/io" } +sp-offchain = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/offchain" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/runtime" } +sp-session = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/session" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/std" } +sp-transaction-pool = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/transaction-pool" } +sp-version = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/version" } -template = { version = "2.0.0-alpha.5", default-features = false, path = "../pallets/template", package = "pallet-template" } +template = { version = "2.0.0-dev", default-features = false, path = "../pallets/template", package = "pallet-template" } [build-dependencies] wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } @@ -68,6 +71,3 @@ std = [ "transaction-payment/std", "template/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index 94f033fd8f58ee231f412e7a2cf6f7f53f77d49a..fed1ee36db02e8c172800458c42285808db36746 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -34,7 +34,7 @@ pub use sp_runtime::{Permill, Perbill}; pub use frame_support::{ StorageValue, construct_runtime, parameter_types, traits::Randomness, - weights::Weight, + weights::{Weight, RuntimeDbWeight}, }; /// Importing a template pallet @@ -98,6 +98,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_version: 1, impl_version: 1, apis: RUNTIME_API_VERSIONS, + transaction_version: 1, }; pub const MILLISECS_PER_BLOCK: u64 = 6000; @@ -120,10 +121,18 @@ pub fn native_version() -> NativeVersion { parameter_types! { pub const BlockHashCount: BlockNumber = 250; - pub const MaximumBlockWeight: Weight = 1_000_000_000; + /// We allow for 2 seconds of compute with a 6 second average block time. + pub const MaximumBlockWeight: Weight = 2_000_000_000_000; + pub const ExtrinsicBaseWeight: Weight = 10_000_000; pub const AvailableBlockRatio: Perbill = Perbill::from_percent(75); pub const MaximumBlockLength: u32 = 5 * 1024 * 1024; pub const Version: RuntimeVersion = VERSION; + /// This probably should not be changed unless you have specific + /// disk i/o conditions for the node. + pub const DbWeight: RuntimeDbWeight = RuntimeDbWeight { + read: 60_000_000, // ~0.06 ms = ~60 µs + write: 200_000_000, // ~0.2 ms = 200 µs + }; } impl system::Trait for Runtime { @@ -151,6 +160,14 @@ impl system::Trait for Runtime { type BlockHashCount = BlockHashCount; /// Maximum weight of each block. type MaximumBlockWeight = MaximumBlockWeight; + /// The weight of database operations that the runtime can invoke. + type DbWeight = DbWeight; + /// The weight of the overhead invoked on the block import process, independent of the + /// extrinsics included in that block. + type BlockExecutionWeight = (); + /// The base weight of any extrinsic processed by the runtime, independent of the + /// logic of that extrinsic. (Signature verification, nonce increment, fee, etc...) + type ExtrinsicBaseWeight = ExtrinsicBaseWeight; /// Maximum size of all encoded transactions (in bytes) that are allowed in one block. type MaximumBlockLength = MaximumBlockLength; /// Portion of the block weight that is available to all normal transactions. @@ -203,14 +220,12 @@ impl balances::Trait for Runtime { } parameter_types! { - pub const TransactionBaseFee: Balance = 0; pub const TransactionByteFee: Balance = 1; } impl transaction_payment::Trait for Runtime { type Currency = balances::Module; type OnTransactionPayment = (); - type TransactionBaseFee = TransactionBaseFee; type TransactionByteFee = TransactionByteFee; type WeightToFee = ConvertInto; type FeeMultiplierUpdate = (); diff --git a/bin/node/bench/Cargo.toml b/bin/node/bench/Cargo.toml index 22e7fe51d848e1f44ca0c34eddb57d39ca0d0c32..e57383975562fa5f05001a8c72076fd1fdada67c 100644 --- a/bin/node/bench/Cargo.toml +++ b/bin/node/bench/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-bench" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "Substrate node integration benchmarks." edition = "2018" @@ -10,11 +10,23 @@ license = "GPL-3.0" [dependencies] log = "0.4.8" -node-primitives = { version = "2.0.0-alpha.5", path = "../primitives" } -node-testing = { version = "2.0.0-alpha.5", path = "../testing" } -sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api/" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } +node-primitives = { version = "2.0.0-dev", path = "../primitives" } +node-testing = { version = "2.0.0-dev", path = "../testing" } +sc-cli = { version = "0.8.0-dev", path = "../../../client/cli" } +sc-client-api = { version = "2.0.0-dev", path = "../../../client/api/" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-state-machine = { version = "0.8.0-dev", path = "../../../primitives/state-machine" } serde = "1.0.101" serde_json = "1.0.41" -structopt = "0.3" \ No newline at end of file +structopt = "0.3" +derive_more = "0.99.2" +kvdb = "0.5" +kvdb-rocksdb = "0.7" +sp-trie = { version = "2.0.0-alpha.5", path = "../../../primitives/trie" } +sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } +hash-db = "0.15.2" +tempfile = "3.1.0" +fs_extra = "1" +hex = "0.4.0" +rand = { version = "0.7.2", features = ["small_rng"] } +lazy_static = "1.4.0" diff --git a/bin/node/bench/src/core.rs b/bin/node/bench/src/core.rs index a8164db75a53833855ad996cd1aceaa24e232123..7a345f7a5bd99cccb37ff745bceb2fb252e25add 100644 --- a/bin/node/bench/src/core.rs +++ b/bin/node/bench/src/core.rs @@ -48,7 +48,7 @@ pub trait BenchmarkDescription { } pub trait Benchmark { - fn run(&mut self) -> std::time::Duration; + fn run(&mut self, mode: Mode) -> std::time::Duration; } #[derive(Debug, Clone, Serialize)] @@ -58,7 +58,7 @@ pub struct BenchmarkOutput { average: u64, } -struct NsFormatter(u64); +pub struct NsFormatter(pub u64); impl fmt::Display for NsFormatter { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { @@ -68,12 +68,12 @@ impl fmt::Display for NsFormatter { return write!(f, "{} ns", v) } - if self.0 < 10_000 { + if self.0 < 100_000 { return write!(f, "{:.1} µs", v as f64 / 1000.0) } if self.0 < 1_000_000 { - return write!(f, "{:.1} ms", v as f64 / 1_000_000.0) + return write!(f, "{:.2} ms", v as f64 / 1_000_000.0) } if self.0 < 100_000_000 { @@ -84,11 +84,28 @@ impl fmt::Display for NsFormatter { } } +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Mode { + Regular, + Profile, +} + +impl std::str::FromStr for Mode { + type Err = &'static str; + fn from_str(day: &str) -> Result { + match day { + "regular" => Ok(Mode::Regular), + "profile" => Ok(Mode::Profile), + _ => Err("Could not parse mode"), + } + } +} + impl fmt::Display for BenchmarkOutput { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!( f, - "({}: avg {}, w_avg {})", + "{}: avg {}, w_avg {}", self.name, NsFormatter(self.raw_average), NsFormatter(self.average), @@ -96,13 +113,16 @@ impl fmt::Display for BenchmarkOutput { } } -pub fn run_benchmark(benchmark: Box) -> BenchmarkOutput { +pub fn run_benchmark( + benchmark: Box, + mode: Mode, +) -> BenchmarkOutput { let name = benchmark.name().to_owned(); let mut benchmark = benchmark.setup(); let mut durations: Vec = vec![]; for _ in 0..50 { - let duration = benchmark.run(); + let duration = benchmark.run(mode); durations.push(duration.as_nanos()); } diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs new file mode 100644 index 0000000000000000000000000000000000000000..895f523497036d26dcd517da7700b9c10ef19f59 --- /dev/null +++ b/bin/node/bench/src/generator.rs @@ -0,0 +1,64 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::{collections::HashMap, sync::Arc}; + +use kvdb::KeyValueDB; +use node_primitives::Hash; +use sp_trie::{trie_types::TrieDBMut, TrieMut}; + +use crate::simple_trie::SimpleTrie; + +/// Generate trie from given `key_values`. +/// +/// Will fill your database `db` with trie data from `key_values` and +/// return root. +pub fn generate_trie( + db: Arc, + key_values: impl IntoIterator, Vec)>, +) -> Hash { + let mut root = Hash::default(); + + let (db, overlay) = { + let mut overlay = HashMap::new(); + overlay.insert( + hex::decode("03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314").expect("null key is valid"), + Some(vec![0]), + ); + let mut trie = SimpleTrie { db, overlay: &mut overlay }; + { + let mut trie_db = TrieDBMut::new(&mut trie, &mut root); + + for (key, value) in key_values { + trie_db.insert(&key, &value).expect("trie insertion failed"); + } + + trie_db.commit(); + } + ( trie.db, overlay ) + }; + + let mut transaction = db.transaction(); + for (key, value) in overlay.into_iter() { + match value { + Some(value) => transaction.put(0, &key[..], &value[..]), + None => transaction.delete(0, &key[..]), + } + } + db.write(transaction).expect("Failed to write transaction"); + + root +} diff --git a/bin/node/bench/src/import.rs b/bin/node/bench/src/import.rs index 20181bf4c7f7de816226ab76701d2c9339d56fa8..a6e4eb2514e890b6133400c37d2db67cef06ccd7 100644 --- a/bin/node/bench/src/import.rs +++ b/bin/node/bench/src/import.rs @@ -35,10 +35,19 @@ use node_primitives::Block; use sc_client_api::backend::Backend; use sp_runtime::generic::BlockId; -use crate::core::{self, Path}; - -#[derive(Clone, Copy, Debug)] -pub enum SizeType { Small, Medium, Large } +use crate::core::{self, Path, Mode}; + +#[derive(Clone, Copy, Debug, derive_more::Display)] +pub enum SizeType { + #[display(fmt = "small")] + Small, + #[display(fmt = "medium")] + Medium, + #[display(fmt = "large")] + Large, + #[display(fmt = "full")] + Full, +} impl SizeType { fn transactions(&self) -> usize { @@ -46,6 +55,7 @@ impl SizeType { SizeType::Small => 10, SizeType::Medium => 100, SizeType::Large => 500, + SizeType::Full => 4000, } } } @@ -77,18 +87,17 @@ impl core::BenchmarkDescription for ImportBenchmarkDescription { KeyTypes::Ed25519 => path.push("ed25519"), } - match self.size { - SizeType::Small => path.push("small"), - SizeType::Medium => path.push("medium"), - SizeType::Large => path.push("large"), - } + path.push(&format!("{}", self.size)); path } fn setup(self: Box) -> Box { let profile = self.profile; - let mut bench_db = BenchDb::with_key_types(self.size.transactions(), self.key_types); + let mut bench_db = BenchDb::with_key_types( + 50_000, + self.key_types + ); let block = bench_db.generate_block(BlockType::RandomTransfers(self.size.transactions())); Box::new(ImportBenchmark { database: bench_db, @@ -99,24 +108,38 @@ impl core::BenchmarkDescription for ImportBenchmarkDescription { fn name(&self) -> Cow<'static, str> { match self.profile { - Profile::Wasm => "Import benchmark (random transfers, wasm)".into(), - Profile::Native => "Import benchmark (random transfers, native)".into(), + Profile::Wasm => format!( + "Import benchmark (random transfers, wasm, {} block)", + self.size, + ).into(), + Profile::Native => format!( + "Import benchmark (random transfers, native, {} block)", + self.size, + ).into(), } } } impl core::Benchmark for ImportBenchmark { - fn run(&mut self) -> std::time::Duration { + fn run(&mut self, mode: Mode) -> std::time::Duration { let mut context = self.database.create_context(self.profile); let _ = context.client.runtime_version_at(&BlockId::Number(0)) .expect("Failed to get runtime version") .spec_version; + if mode == Mode::Profile { + std::thread::park_timeout(std::time::Duration::from_secs(3)); + } + let start = std::time::Instant::now(); context.import_block(self.block.clone()); let elapsed = start.elapsed(); + if mode == Mode::Profile { + std::thread::park_timeout(std::time::Duration::from_secs(1)); + } + log::info!( target: "bench-logistics", "imported block with {} tx, took: {:#?}", diff --git a/bin/node/bench/src/main.rs b/bin/node/bench/src/main.rs index 8f045465262856fcb907f9aa5dfee68f7f05fc88..7d92eabf4cbdfe9c3480d64159ffa440ac65da04 100644 --- a/bin/node/bench/src/main.rs +++ b/bin/node/bench/src/main.rs @@ -16,9 +16,15 @@ #[macro_use] mod core; mod import; +mod trie; +mod simple_trie; +mod generator; +mod tempdb; +mod state_sizes; -use crate::core::run_benchmark; +use crate::core::{run_benchmark, Mode as BenchmarkMode}; use import::{ImportBenchmarkDescription, SizeType}; +use trie::{TrieReadBenchmarkDescription, TrieWriteBenchmarkDescription, DatabaseSize}; use node_testing::bench::{Profile, KeyTypes}; use structopt::StructOpt; @@ -41,6 +47,15 @@ struct Opt { /// /// Run with `--list` for the hint of what to filter. filter: Option, + + /// Mode + /// + /// "regular" for regular becnhmark + /// + /// "profile" mode adds pauses between measurable runs, + /// so that actual interval can be selected in the profiler of choice. + #[structopt(short, long, default_value = "regular")] + mode: BenchmarkMode, } fn main() { @@ -62,14 +77,32 @@ fn main() { key_types: KeyTypes::Ed25519, size: SizeType::Medium, }, + ImportBenchmarkDescription { + profile: Profile::Wasm, + key_types: KeyTypes::Sr25519, + size: SizeType::Full, + }, + ImportBenchmarkDescription { + profile: Profile::Native, + key_types: KeyTypes::Sr25519, + size: SizeType::Full, + }, size in [SizeType::Small, SizeType::Large] => ImportBenchmarkDescription { profile: Profile::Native, key_types: KeyTypes::Sr25519, size: *size, }, + size in [ + DatabaseSize::Empty, DatabaseSize::Smallest, DatabaseSize::Small, + DatabaseSize::Medium, DatabaseSize::Large, DatabaseSize::Huge, + ] => TrieReadBenchmarkDescription { database_size: *size }, + size in [ + DatabaseSize::Empty, DatabaseSize::Smallest, DatabaseSize::Small, + DatabaseSize::Medium, DatabaseSize::Large, DatabaseSize::Huge, + ] => TrieWriteBenchmarkDescription { database_size: *size }, ); - + if opt.list { for benchmark in benchmarks.iter() { log::info!("{}: {}", benchmark.name(), benchmark.path().full()) @@ -81,7 +114,7 @@ fn main() { for benchmark in benchmarks { if opt.filter.as_ref().map(|f| benchmark.path().has(f)).unwrap_or(true) { log::info!("Starting {}", benchmark.name()); - let result = run_benchmark(benchmark); + let result = run_benchmark(benchmark, opt.mode); log::info!("{}", result); results.push(result); diff --git a/bin/node/bench/src/simple_trie.rs b/bin/node/bench/src/simple_trie.rs new file mode 100644 index 0000000000000000000000000000000000000000..50078a11df6b5290157bdbdfbc02389f1711261b --- /dev/null +++ b/bin/node/bench/src/simple_trie.rs @@ -0,0 +1,68 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::{collections::HashMap, sync::Arc}; + +use kvdb::KeyValueDB; +use node_primitives::Hash; +use sp_trie::DBValue; +use hash_db::{HashDB, AsHashDB, Prefix, Hasher as _}; + +pub type Hasher = sp_core::Blake2Hasher; + +/// Immutable generated trie database with root. +pub struct SimpleTrie<'a> { + pub db: Arc, + pub overlay: &'a mut HashMap, Option>>, +} + +impl<'a> AsHashDB for SimpleTrie<'a> { + fn as_hash_db(&self) -> &dyn hash_db::HashDB { &*self } + + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { + &mut *self + } +} + +impl<'a> HashDB for SimpleTrie<'a> { + fn get(&self, key: &Hash, prefix: Prefix) -> Option { + let key = sp_trie::prefixed_key::(key, prefix); + if let Some(value) = self.overlay.get(&key) { + return value.clone(); + } + self.db.get(0, &key).expect("Database backend error") + } + + fn contains(&self, hash: &Hash, prefix: Prefix) -> bool { + self.get(hash, prefix).is_some() + } + + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> Hash { + let key = Hasher::hash(value); + self.emplace(key, prefix, value.to_vec()); + key + } + + fn emplace(&mut self, key: Hash, prefix: Prefix, value: DBValue) { + let key = sp_trie::prefixed_key::(&key, prefix); + self.overlay.insert(key, Some(value)); + } + + fn remove(&mut self, key: &Hash, prefix: Prefix) { + let key = sp_trie::prefixed_key::(key, prefix); + self.overlay.insert(key, None); + } +} diff --git a/bin/node/bench/src/state_sizes.rs b/bin/node/bench/src/state_sizes.rs new file mode 100644 index 0000000000000000000000000000000000000000..d35989f61be3467960c0870fca1bd5969d571081 --- /dev/null +++ b/bin/node/bench/src/state_sizes.rs @@ -0,0 +1,4756 @@ +// Copyright 2015-2020 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +/// Kusama value size distribution +pub const KUSAMA_STATE_DISTRIBUTION: &'static[(u32, u32)] = &[ + (32, 35), + (33, 20035), + (34, 5369), + (35, 184), + (36, 54599), + (37, 1515056), + (38, 885), + (39, 69965), + (41, 210754), + (42, 467), + (43, 3241), + (44, 32660), + (45, 231141), + (46, 220016), + (47, 248931), + (48, 157232), + (49, 143236), + (50, 2428), + (51, 1476159), + (52, 31), + (53, 112), + (54, 711), + (55, 1934), + (56, 39), + (57, 407), + (58, 6929), + (59, 6568), + (60, 26), + (61, 268673), + (62, 118137), + (63, 84640), + (64, 193232), + (65, 2584210), + (66, 1002), + (67, 2993), + (68, 4266), + (69, 5633381), + (70, 277369), + (71, 5106), + (72, 722), + (73, 1882), + (74, 8178), + (75, 4045), + (76, 1596), + (77, 5335), + (78, 14591), + (79, 9645), + (80, 44171), + (81, 13474), + (82, 51090), + (83, 2595), + (84, 6276), + (85, 382195), + (86, 1062), + (87, 3846), + (88, 5663), + (89, 3811), + (90, 1580), + (91, 5729), + (92, 19144), + (93, 197), + (94, 235), + (95, 545), + (96, 54914), + (97, 3858), + (98, 1610), + (99, 635), + (100, 2481), + (101, 6457), + (102, 3753951), + (103, 11821), + (104, 11114), + (105, 2601), + (106, 2518), + (107, 521925), + (108, 297), + (109, 411), + (110, 668), + (111, 4500), + (112, 704), + (113, 316), + (114, 59), + (115, 291), + (116, 1727), + (117, 6010), + (118, 51874), + (119, 13969), + (120, 9496), + (121, 274), + (122, 810), + (123, 643), + (124, 69), + (125, 41), + (126, 329), + (127, 175435), + (128, 2641), + (129, 2658), + (130, 415277), + (131, 2705), + (132, 2314), + (133, 4290), + (134, 693), + (135, 1957478), + (136, 1111), + (137, 1474503), + (138, 3656), + (139, 940), + (140, 1755692), + (141, 61), + (142, 4140), + (143, 47), + (144, 6725), + (145, 610), + (146, 250), + (147, 48), + (148, 28), + (149, 132), + (150, 123489), + (151, 7476), + (152, 55), + (153, 68), + (154, 170), + (155, 566), + (156, 8110), + (157, 1243), + (158, 1445), + (159, 2569), + (160, 1096), + (161, 865), + (162, 634), + (163, 372411), + (164, 685), + (165, 3481), + (166, 1467), + (167, 2146), + (168, 556539), + (169, 566), + (170, 5080), + (171, 202), + (172, 123), + (173, 100750), + (174, 667), + (175, 433), + (176, 737), + (177, 315), + (178, 317), + (179, 656), + (180, 2522), + (181, 315), + (182, 406), + (183, 4680), + (184, 4941), + (185, 828), + (186, 782), + (187, 565), + (188, 584), + (189, 376), + (190, 321), + (191, 418), + (192, 167), + (193, 362), + (194, 2198), + (195, 180), + (196, 787), + (197, 2680), + (198, 501), + (199, 843), + (200, 287), + (201, 608362), + (202, 1157), + (203, 959), + (204, 1683623), + (205, 440), + (206, 756), + (207, 812), + (208, 1147), + (209, 723), + (210, 856), + (211, 496), + (212, 916), + (213, 615), + (214, 488), + (215, 522), + (216, 8265), + (217, 32574), + (218, 417), + (219, 247), + (220, 579), + (221, 68), + (222, 126), + (223, 306), + (224, 310), + (225, 24), + (226, 37), + (227, 160), + (228, 11), + (229, 3288), + (230, 349), + (231, 23), + (232, 14), + (233, 45), + (234, 452840), + (235, 118), + (236, 741), + (237, 390), + (238, 517), + (239, 694), + (240, 765), + (241, 542), + (242, 417), + (243, 617), + (244, 1307), + (245, 583), + (246, 1640), + (247, 735), + (248, 478), + (249, 4312), + (250, 5426), + (251, 1067), + (252, 435), + (253, 202), + (254, 122), + (255, 486), + (256, 180), + (257, 279), + (258, 406), + (259, 160), + (260, 2759), + (261, 2600), + (262, 686), + (263, 95), + (264, 164), + (265, 150), + (266, 1013), + (267, 552618), + (268, 217), + (269, 188), + (270, 284), + (271, 416), + (272, 453), + (273, 95), + (274, 42), + (275, 68), + (276, 90), + (277, 123), + (278, 340), + (279, 98), + (280, 2795), + (281, 261), + (282, 7370), + (283, 5768), + (284, 3285), + (285, 461), + (286, 363), + (287, 456), + (288, 1475), + (289, 211), + (290, 153), + (291, 282), + (292, 241), + (293, 2924), + (294, 261), + (295, 1070), + (296, 1301), + (297, 688), + (298, 592), + (299, 95), + (300, 686447), + (301, 42), + (302, 385), + (303, 24), + (304, 931), + (305, 49), + (306, 23), + (307, 67), + (308, 32), + (309, 38), + (310, 2), + (311, 7), + (312, 198), + (313, 11), + (314, 38), + (315, 3704), + (316, 7406), + (317, 116), + (318, 229), + (319, 100), + (320, 437), + (321, 244), + (322, 285), + (323, 433), + (324, 382), + (325, 3171), + (326, 761), + (327, 324), + (328, 2264), + (329, 340), + (330, 353), + (331, 110), + (332, 403), + (333, 731366), + (334, 223), + (335, 350), + (336, 600), + (337, 219), + (338, 112), + (339, 10), + (340, 761), + (341, 35), + (342, 99), + (343, 83), + (344, 136), + (345, 7), + (346, 836), + (347, 11), + (348, 10832), + (349, 8931), + (350, 33), + (351, 64), + (352, 66), + (353, 54), + (354, 78), + (355, 198), + (356, 722), + (357, 2647), + (358, 64), + (359, 71), + (360, 2242), + (361, 1462), + (362, 505), + (363, 444), + (364, 597), + (365, 372), + (366, 664852), + (367, 464), + (368, 605), + (369, 123), + (370, 64), + (371, 117), + (372, 328), + (373, 123), + (374, 227), + (375, 151), + (376, 881), + (377, 111), + (378, 30), + (379, 73), + (380, 2126), + (381, 3662), + (382, 9107), + (383, 18), + (384, 294), + (385, 12), + (386, 262), + (387, 127), + (388, 269), + (389, 2566), + (390, 14), + (391, 17), + (392, 80), + (393, 67), + (394, 1470), + (395, 25), + (396, 220), + (397, 131), + (398, 225), + (399, 484755), + (400, 597), + (401, 300), + (402, 253), + (403, 359), + (404, 523), + (405, 311), + (406, 238), + (407, 999), + (408, 424), + (409, 165), + (410, 96), + (411, 248), + (412, 1771), + (413, 139), + (414, 7374), + (415, 11186), + (416, 1355), + (417, 1283666), + (418, 9), + (419, 116), + (420, 3897), + (421, 2554), + (422, 1), + (423, 1), + (424, 16878), + (425, 3198212), + (426, 335), + (427, 1676), + (428, 80), + (429, 19), + (430, 47), + (431, 495), + (432, 421946), + (433, 73), + (434, 95), + (435, 105), + (436, 184), + (437, 56903), + (438, 132), + (439, 87), + (440, 207411), + (441, 230), + (442, 372), + (443, 361), + (444, 387), + (445, 299), + (446, 175), + (447, 7487), + (448, 16346), + (449, 37), + (450, 98313), + (451, 307), + (452, 304), + (453, 2675), + (454, 229), + (455, 130), + (456, 134), + (457, 50), + (458, 238), + (459, 2), + (460, 2267), + (461, 7), + (462, 1), + (463, 8), + (464, 395), + (465, 1279781), + (466, 9), + (467, 12), + (468, 633), + (469, 37), + (470, 13), + (471, 54), + (472, 247), + (473, 82), + (474, 119), + (475, 114), + (476, 332), + (477, 79), + (478, 116), + (479, 128), + (480, 4206), + (481, 20732), + (482, 311), + (483, 343), + (484, 527), + (485, 2750), + (486, 76), + (487, 152), + (488, 510), + (489, 63), + (490, 257), + (491, 79), + (492, 825), + (493, 4198), + (494, 389), + (495, 72), + (496, 1547), + (497, 34), + (498, 631996), + (499, 5), + (500, 2334), + (501, 34), + (502, 7), + (503, 7), + (504, 7682), + (505, 6), + (506, 26), + (507, 22), + (508, 461), + (509, 95), + (510, 36), + (511, 46), + (512, 2741), + (513, 38455), + (514, 29678), + (515, 179), + (516, 1637), + (517, 2597), + (518, 166), + (519, 230), + (520, 2736), + (521, 187), + (522, 361), + (523, 310), + (524, 3327), + (525, 76), + (526, 8070), + (527, 35), + (528, 3310), + (529, 118), + (530, 167), + (531, 214180), + (532, 4597), + (533, 153), + (534, 126), + (535, 23), + (536, 13920), + (537, 10), + (538, 11), + (539, 50), + (540, 50739), + (541, 8), + (542, 347), + (543, 77), + (544, 451575), + (545, 16), + (546, 218814), + (547, 1859026), + (548, 303), + (549, 2511), + (550, 27), + (551, 28), + (552, 188), + (553, 46), + (554, 216), + (555, 63), + (556, 202), + (557, 192), + (558, 257), + (559, 170377), + (560, 902), + (561, 424), + (562, 186), + (563, 145), + (564, 342), + (565, 76), + (566, 41), + (567, 26), + (568, 136), + (569, 1336), + (570, 988), + (571, 131), + (572, 766), + (573, 95), + (574, 57), + (575, 16), + (576, 47), + (577, 63), + (578, 5), + (579, 140), + (580, 1263808), + (581, 2498), + (583, 2), + (584, 706), + (585, 49), + (586, 502), + (587, 16), + (588, 115), + (589, 25), + (590, 31), + (591, 34), + (592, 818), + (593, 60), + (594, 84), + (595, 116), + (596, 446), + (597, 111), + (598, 151), + (599, 153), + (600, 1408), + (601, 165), + (602, 575), + (603, 163), + (604, 309), + (605, 52), + (606, 40), + (607, 116), + (608, 749), + (609, 231), + (610, 171), + (611, 218), + (612, 1145), + (613, 2572), + (614, 27), + (615, 26), + (616, 2060), + (617, 173), + (618, 1094), + (619, 66), + (620, 14235), + (622, 294), + (623, 2), + (624, 79374), + (625, 1), + (626, 3), + (627, 7), + (628, 335), + (629, 27), + (630, 47), + (631, 113), + (632, 589), + (633, 56), + (634, 75), + (635, 85), + (636, 740), + (637, 118), + (638, 180), + (639, 149), + (640, 1169), + (641, 135), + (642, 169), + (643, 170), + (644, 1802), + (645, 2481), + (646, 28), + (647, 78), + (648, 5585), + (649, 173), + (650, 135), + (651, 177), + (652, 6553), + (653, 129), + (654, 55), + (655, 6), + (656, 13250), + (657, 5), + (658, 15), + (659, 3), + (660, 39892), + (661, 28), + (663, 1), + (664, 575061), + (665, 1), + (666, 5), + (667, 73), + (668, 39), + (669, 62), + (670, 50), + (671, 27), + (672, 33), + (673, 48), + (674, 44), + (675, 151), + (676, 70), + (677, 2540), + (678, 150), + (679, 109), + (680, 117), + (681, 95), + (682, 80), + (683, 44), + (684, 34), + (685, 31), + (686, 125), + (687, 146), + (688, 423), + (689, 142), + (690, 154), + (691, 135), + (692, 194), + (693, 48), + (694, 6), + (695, 141), + (696, 47), + (697, 9), + (699, 1), + (701, 1), + (702, 2), + (703, 81), + (704, 3), + (705, 4), + (706, 23), + (707, 131), + (708, 31), + (709, 2458), + (710, 346), + (711, 43), + (712, 46), + (713, 48), + (714, 85), + (715, 119), + (716, 89), + (717, 97), + (718, 95), + (719, 137), + (720, 437), + (721, 64), + (722, 28), + (723, 29), + (724, 121), + (725, 162), + (726, 241), + (727, 219), + (728, 143), + (729, 92), + (730, 100), + (731, 42), + (732, 38), + (733, 60), + (734, 2), + (735, 71), + (736, 12), + (737, 9), + (738, 7), + (739, 193), + (740, 2), + (741, 2404), + (742, 3), + (743, 11), + (744, 5), + (745, 5), + (746, 9), + (747, 16), + (748, 27), + (749, 32), + (750, 57), + (751, 54), + (752, 383), + (753, 61), + (754, 48), + (755, 84), + (756, 108), + (757, 134), + (758, 121), + (759, 160), + (760, 80), + (761, 68), + (762, 192), + (763, 107), + (764, 270), + (765, 58), + (766, 125), + (767, 151), + (768, 75), + (769, 94), + (770, 91), + (771, 187), + (772, 57), + (773, 2371), + (774, 8), + (775, 93), + (776, 107), + (777, 20), + (779, 1), + (780, 22), + (781, 1), + (783, 6), + (784, 318), + (785, 25), + (786, 31), + (787, 23), + (788, 28), + (789, 62), + (790, 53), + (791, 41), + (792, 68), + (793, 60), + (794, 88), + (795, 108), + (796, 63), + (797, 100), + (798, 68), + (799, 72), + (800, 83), + (801, 46), + (802, 36), + (803, 157), + (804, 139), + (805, 2439), + (806, 73), + (807, 81), + (808, 99), + (809, 66), + (810, 45), + (811, 98), + (812, 1), + (814, 31), + (815, 1), + (816, 312), + (818, 155), + (819, 2), + (820, 12), + (821, 27), + (822, 97), + (823, 23), + (824, 7), + (825, 15), + (826, 37), + (827, 39), + (828, 28), + (829, 33), + (830, 53), + (831, 101), + (832, 189), + (833, 94), + (834, 66), + (835, 173), + (836, 74), + (837, 2402), + (838, 64), + (839, 28), + (840, 20), + (841, 13), + (842, 32), + (843, 72), + (844, 68), + (845, 50), + (846, 41), + (847, 114), + (848, 345), + (849, 33), + (850, 17), + (851, 6), + (852, 61), + (853, 101), + (854, 123), + (855, 28), + (856, 3), + (857, 3), + (858, 30), + (859, 12), + (860, 28), + (861, 16), + (862, 20), + (863, 7), + (864, 23), + (865, 28), + (866, 40), + (867, 159), + (868, 40), + (869, 2361), + (870, 92), + (871, 88), + (872, 193), + (873, 61), + (874, 58), + (875, 67), + (876, 65), + (877, 46), + (878, 55), + (879, 30), + (880, 334), + (881, 74), + (882, 121), + (883, 107), + (884, 36), + (885, 66), + (886, 22), + (887, 25), + (888, 24), + (889, 10), + (890, 44), + (891, 5), + (892, 84), + (893, 4), + (894, 1), + (895, 7), + (896, 3), + (897, 8), + (898, 3), + (899, 126), + (900, 13), + (901, 2280), + (902, 74), + (903, 36), + (904, 46), + (905, 52), + (906, 24), + (907, 23), + (908, 43), + (909, 31), + (910, 66), + (911, 65), + (912, 376), + (913, 77), + (914, 85), + (915, 60), + (916, 29), + (917, 64), + (918, 48), + (919, 135), + (920, 21), + (921, 34), + (922, 26), + (923, 22), + (924, 52), + (925, 28), + (926, 142), + (927, 18), + (928, 14), + (929, 30), + (930, 56), + (931, 113), + (933, 2264), + (934, 14), + (935, 4), + (936, 10), + (937, 18), + (938, 2), + (939, 30), + (940, 9), + (941, 29), + (942, 10), + (943, 17), + (944, 296), + (945, 31), + (946, 40), + (947, 26), + (948, 70), + (949, 66), + (950, 44), + (951, 57), + (952, 55), + (953, 56), + (954, 51), + (955, 133), + (956, 39), + (957, 49), + (958, 45), + (959, 26), + (960, 30), + (961, 35), + (962, 40), + (963, 148), + (964, 34), + (965, 2264), + (966, 50), + (967, 21), + (968, 2), + (970, 24), + (972, 45), + (973, 8), + (974, 11), + (975, 20), + (976, 287), + (977, 20), + (978, 6), + (979, 9), + (980, 99), + (981, 32), + (982, 10), + (983, 13), + (984, 26), + (985, 30), + (986, 31), + (987, 38), + (988, 25), + (989, 32), + (990, 44), + (991, 125), + (992, 58), + (993, 44), + (994, 25), + (995, 140), + (996, 25), + (997, 2222), + (998, 16), + (999, 25), + (1000, 38), + (1001, 66), + (1002, 31), + (1003, 38), + (1004, 38), + (1005, 10), + (1006, 7), + (1008, 283), + (1009, 3), + (1010, 1), + (1011, 17), + (1012, 4), + (1013, 51), + (1014, 1), + (1015, 1), + (1016, 3), + (1017, 12), + (1018, 11), + (1019, 21), + (1020, 31), + (1021, 14), + (1022, 14), + (1023, 23), + (1024, 25), + (1025, 42), + (1026, 39), + (1027, 220), + (1028, 33), + (1029, 2206), + (1030, 24), + (1031, 64), + (1032, 36), + (1033, 61), + (1034, 123), + (1035, 32), + (1036, 20), + (1037, 15), + (1038, 11), + (1039, 33), + (1040, 311), + (1041, 58), + (1042, 80), + (1043, 29), + (1044, 10), + (1045, 48), + (1046, 18), + (1047, 22), + (1048, 3), + (1049, 17), + (1050, 1), + (1051, 2), + (1052, 5), + (1053, 4), + (1054, 4), + (1055, 1), + (1056, 4), + (1057, 15), + (1058, 11), + (1059, 135), + (1060, 59), + (1061, 2132), + (1062, 32), + (1063, 116), + (1064, 37), + (1065, 44), + (1066, 42), + (1067, 28), + (1068, 10), + (1069, 36), + (1070, 59), + (1071, 48), + (1072, 332), + (1073, 59), + (1074, 43), + (1075, 19), + (1076, 19), + (1077, 31), + (1078, 31), + (1079, 20), + (1080, 38), + (1081, 58), + (1082, 37), + (1083, 47), + (1084, 19), + (1085, 24), + (1086, 12), + (1087, 26), + (1088, 89), + (1089, 3), + (1091, 108), + (1093, 2112), + (1094, 13), + (1095, 4), + (1096, 4), + (1097, 17), + (1098, 7), + (1099, 105), + (1100, 12), + (1101, 10), + (1102, 17), + (1103, 19), + (1104, 329), + (1105, 28), + (1106, 58), + (1107, 21), + (1108, 22), + (1109, 63), + (1110, 29), + (1111, 53), + (1112, 84), + (1113, 28), + (1114, 30), + (1115, 22), + (1116, 40), + (1117, 16), + (1118, 20), + (1119, 75), + (1120, 43), + (1121, 49), + (1122, 25), + (1123, 118), + (1124, 8), + (1125, 2083), + (1126, 21), + (1127, 3), + (1128, 43), + (1129, 1), + (1130, 1), + (1132, 3), + (1133, 1), + (1134, 3), + (1135, 83), + (1136, 266), + (1137, 7), + (1138, 22), + (1139, 14), + (1140, 30), + (1141, 54), + (1142, 125), + (1143, 44), + (1144, 34), + (1145, 19), + (1146, 21), + (1147, 19), + (1148, 46), + (1149, 45), + (1150, 54), + (1151, 22), + (1152, 30), + (1153, 20), + (1154, 7), + (1155, 143), + (1156, 23), + (1157, 2078), + (1158, 30), + (1159, 23), + (1160, 12), + (1161, 18), + (1162, 6), + (1164, 5), + (1165, 1), + (1168, 254), + (1169, 1), + (1170, 3), + (1171, 95), + (1172, 37), + (1173, 23), + (1174, 7), + (1175, 11), + (1176, 5), + (1177, 14), + (1178, 15), + (1179, 19), + (1180, 10), + (1181, 28), + (1182, 87), + (1183, 35), + (1184, 30), + (1185, 30), + (1186, 38), + (1187, 148), + (1188, 49), + (1189, 2056), + (1190, 42), + (1191, 41), + (1192, 14), + (1193, 36), + (1194, 37), + (1195, 22), + (1196, 108), + (1197, 62), + (1198, 55), + (1199, 43), + (1200, 261), + (1201, 16), + (1202, 1), + (1203, 9), + (1204, 3), + (1205, 32), + (1207, 81), + (1208, 3), + (1210, 3), + (1212, 4), + (1213, 9), + (1214, 5), + (1215, 6), + (1216, 4), + (1217, 8), + (1218, 13), + (1219, 120), + (1220, 11), + (1221, 1989), + (1222, 11), + (1223, 20), + (1224, 15), + (1225, 21), + (1226, 23), + (1227, 50), + (1228, 37), + (1229, 51), + (1230, 37), + (1231, 21), + (1232, 256), + (1233, 26), + (1234, 25), + (1235, 21), + (1236, 79), + (1237, 50), + (1238, 21), + (1239, 2), + (1240, 6), + (1241, 8), + (1243, 95), + (1244, 1), + (1247, 1), + (1248, 1), + (1249, 1), + (1250, 96), + (1251, 112), + (1252, 43), + (1253, 1960), + (1254, 7), + (1255, 13), + (1256, 16), + (1257, 20), + (1258, 19), + (1259, 17), + (1260, 12), + (1261, 5), + (1262, 12), + (1263, 29), + (1264, 272), + (1265, 63), + (1266, 37), + (1267, 36), + (1268, 25), + (1269, 55), + (1270, 38), + (1271, 7), + (1272, 37), + (1273, 10), + (1274, 16), + (1275, 28), + (1276, 18), + (1277, 11), + (1278, 8), + (1279, 91), + (1280, 1), + (1282, 1), + (1283, 110), + (1284, 20), + (1285, 1923), + (1287, 3), + (1288, 1), + (1290, 23), + (1291, 4), + (1292, 4), + (1293, 12), + (1294, 19), + (1295, 8), + (1296, 248), + (1297, 21), + (1298, 12), + (1299, 31), + (1300, 10), + (1301, 60), + (1302, 1), + (1303, 8), + (1304, 99), + (1305, 29), + (1306, 29), + (1307, 28), + (1308, 33), + (1309, 19), + (1310, 8), + (1311, 1), + (1313, 11), + (1314, 12), + (1315, 236), + (1316, 18), + (1317, 1891), + (1318, 2), + (1322, 21), + (1324, 1), + (1326, 8), + (1327, 3), + (1328, 235), + (1329, 4), + (1330, 1), + (1331, 2), + (1332, 5), + (1333, 38), + (1334, 2), + (1335, 30), + (1336, 18), + (1337, 31), + (1338, 8), + (1339, 5), + (1340, 11), + (1341, 9), + (1342, 12), + (1343, 11), + (1344, 79), + (1345, 37), + (1346, 19), + (1347, 136), + (1348, 9), + (1349, 1861), + (1350, 8), + (1351, 112), + (1352, 10), + (1353, 3), + (1354, 16), + (1355, 4), + (1356, 12), + (1357, 18), + (1358, 67), + (1359, 6), + (1360, 229), + (1361, 1), + (1362, 1), + (1364, 1), + (1365, 27), + (1366, 6), + (1368, 14), + (1370, 8), + (1371, 29), + (1372, 3), + (1373, 21), + (1374, 8), + (1375, 6), + (1376, 3), + (1377, 9), + (1378, 9), + (1379, 120), + (1380, 5), + (1381, 1833), + (1382, 45), + (1383, 35), + (1384, 23), + (1385, 25), + (1386, 26), + (1387, 159), + (1388, 24), + (1389, 16), + (1390, 16), + (1391, 14), + (1392, 273), + (1393, 17), + (1394, 9), + (1395, 5), + (1396, 14), + (1397, 24), + (1398, 27), + (1400, 2), + (1404, 5), + (1405, 8), + (1406, 3), + (1407, 25), + (1408, 2), + (1409, 22), + (1410, 10), + (1411, 111), + (1412, 89), + (1413, 1793), + (1414, 4), + (1415, 9), + (1416, 16), + (1417, 13), + (1418, 13), + (1419, 13), + (1420, 15), + (1421, 19), + (1422, 26), + (1423, 110), + (1424, 229), + (1425, 11), + (1426, 10), + (1427, 7), + (1428, 7), + (1429, 28), + (1430, 12), + (1431, 11), + (1432, 14), + (1433, 2), + (1434, 2), + (1436, 1), + (1437, 1), + (1438, 13), + (1439, 1), + (1440, 1), + (1441, 1), + (1442, 2), + (1443, 132), + (1444, 5), + (1445, 1795), + (1448, 11), + (1449, 10), + (1450, 11), + (1451, 8), + (1452, 47), + (1453, 6), + (1454, 8), + (1455, 12), + (1456, 229), + (1457, 15), + (1458, 12), + (1459, 121), + (1460, 15), + (1461, 48), + (1462, 49), + (1463, 22), + (1464, 11), + (1465, 9), + (1466, 81), + (1467, 1), + (1468, 1), + (1469, 6), + (1470, 6), + (1471, 6), + (1472, 9), + (1473, 12), + (1474, 2), + (1475, 109), + (1476, 5), + (1477, 1721), + (1478, 1), + (1479, 28), + (1480, 7), + (1481, 23), + (1482, 2), + (1483, 12), + (1484, 5), + (1485, 3), + (1486, 2), + (1487, 4), + (1488, 219), + (1489, 7), + (1490, 8), + (1491, 10), + (1492, 16), + (1493, 32), + (1494, 25), + (1495, 96), + (1496, 13), + (1497, 15), + (1498, 16), + (1499, 12), + (1500, 14), + (1501, 19), + (1502, 7), + (1503, 11), + (1504, 3), + (1505, 8), + (1506, 41), + (1507, 108), + (1508, 25), + (1509, 1719), + (1510, 8), + (1511, 10), + (1514, 2), + (1515, 25), + (1516, 2), + (1517, 32), + (1518, 6), + (1519, 7), + (1520, 273), + (1521, 2), + (1522, 6), + (1523, 5), + (1524, 6), + (1525, 36), + (1526, 3), + (1527, 12), + (1528, 7), + (1529, 9), + (1530, 12), + (1531, 107), + (1532, 44), + (1533, 17), + (1534, 12), + (1535, 18), + (1536, 12), + (1537, 26), + (1538, 35), + (1539, 131), + (1540, 15), + (1541, 1693), + (1542, 11), + (1543, 7), + (1544, 2), + (1545, 6), + (1546, 14), + (1547, 6), + (1548, 2), + (1549, 24), + (1550, 2), + (1551, 33), + (1552, 206), + (1553, 18), + (1555, 1), + (1556, 7), + (1557, 38), + (1558, 6), + (1559, 3), + (1560, 21), + (1562, 2), + (1563, 5), + (1564, 7), + (1565, 5), + (1566, 6), + (1567, 110), + (1568, 9), + (1569, 16), + (1570, 13), + (1571, 109), + (1572, 6), + (1573, 1664), + (1574, 53), + (1575, 14), + (1576, 21), + (1577, 31), + (1578, 42), + (1579, 13), + (1580, 10), + (1581, 12), + (1582, 11), + (1583, 85), + (1584, 202), + (1585, 7), + (1586, 6), + (1587, 25), + (1588, 5), + (1589, 41), + (1590, 4), + (1591, 5), + (1593, 1), + (1595, 5), + (1596, 11), + (1598, 1), + (1599, 1), + (1600, 1), + (1601, 4), + (1602, 19), + (1603, 200), + (1604, 10), + (1605, 1640), + (1606, 15), + (1607, 14), + (1608, 7), + (1609, 12), + (1610, 5), + (1611, 2), + (1612, 3), + (1613, 7), + (1614, 37), + (1615, 4), + (1616, 203), + (1617, 13), + (1618, 3), + (1619, 12), + (1620, 38), + (1621, 22), + (1622, 12), + (1623, 43), + (1624, 19), + (1625, 35), + (1626, 15), + (1627, 26), + (1628, 43), + (1629, 2), + (1630, 10), + (1631, 1), + (1633, 1), + (1634, 1), + (1635, 110), + (1637, 1612), + (1638, 1), + (1639, 107), + (1640, 1), + (1641, 2), + (1643, 7), + (1644, 9), + (1645, 8), + (1646, 3), + (1647, 19), + (1648, 206), + (1649, 2), + (1650, 9), + (1651, 8), + (1652, 19), + (1653, 22), + (1654, 4), + (1655, 13), + (1656, 3), + (1657, 5), + (1658, 5), + (1659, 35), + (1660, 10), + (1661, 26), + (1662, 8), + (1663, 10), + (1664, 7), + (1665, 4), + (1666, 2), + (1667, 110), + (1668, 12), + (1669, 1594), + (1670, 1), + (1671, 2), + (1672, 15), + (1673, 4), + (1674, 2), + (1675, 303), + (1676, 12), + (1678, 1), + (1680, 194), + (1681, 1), + (1682, 40), + (1683, 2), + (1684, 2), + (1685, 19), + (1686, 16), + (1687, 2), + (1688, 6), + (1689, 9), + (1690, 18), + (1691, 15), + (1692, 5), + (1693, 7), + (1694, 6), + (1695, 32), + (1696, 4), + (1697, 34), + (1698, 1), + (1699, 117), + (1700, 5), + (1701, 1590), + (1702, 20), + (1703, 4), + (1704, 6), + (1705, 20), + (1707, 2), + (1710, 3), + (1711, 89), + (1712, 195), + (1713, 4), + (1714, 2), + (1715, 1), + (1716, 3), + (1717, 16), + (1718, 9), + (1719, 2), + (1720, 3), + (1723, 18), + (1724, 1), + (1725, 2), + (1726, 3), + (1727, 3), + (1728, 9), + (1729, 5), + (1730, 7), + (1731, 132), + (1732, 28), + (1733, 1585), + (1734, 5), + (1735, 3), + (1736, 5), + (1737, 27), + (1738, 4), + (1739, 19), + (1740, 15), + (1741, 4), + (1742, 15), + (1743, 9), + (1744, 183), + (1745, 12), + (1747, 119), + (1748, 1), + (1749, 15), + (1750, 5), + (1754, 1), + (1757, 2), + (1758, 8), + (1759, 7), + (1760, 7), + (1761, 2), + (1762, 13), + (1763, 113), + (1764, 8), + (1765, 1547), + (1766, 7), + (1767, 21), + (1768, 3), + (1769, 34), + (1770, 5), + (1772, 6), + (1773, 7), + (1774, 12), + (1775, 9), + (1776, 189), + (1777, 25), + (1778, 10), + (1779, 4), + (1780, 1), + (1781, 21), + (1782, 3), + (1783, 186), + (1784, 2), + (1787, 1), + (1788, 10), + (1789, 8), + (1790, 1), + (1791, 34), + (1792, 1), + (1793, 1), + (1794, 1), + (1795, 108), + (1796, 4), + (1797, 1519), + (1798, 9), + (1799, 9), + (1800, 3), + (1801, 6), + (1802, 4), + (1803, 35), + (1804, 15), + (1805, 30), + (1806, 5), + (1807, 7), + (1808, 192), + (1809, 8), + (1811, 4), + (1812, 24), + (1813, 36), + (1814, 4), + (1815, 14), + (1816, 2), + (1817, 2), + (1818, 4), + (1819, 72), + (1820, 3), + (1822, 1), + (1823, 4), + (1825, 1), + (1826, 5), + (1827, 104), + (1828, 1), + (1829, 1494), + (1830, 11), + (1831, 5), + (1832, 2), + (1833, 2), + (1834, 2), + (1835, 4), + (1836, 9), + (1837, 1), + (1838, 14), + (1839, 33), + (1840, 188), + (1841, 27), + (1842, 13), + (1843, 10), + (1844, 28), + (1845, 52), + (1846, 17), + (1847, 40), + (1848, 35), + (1849, 6), + (1850, 6), + (1851, 2), + (1853, 4), + (1854, 6), + (1855, 77), + (1856, 1), + (1859, 106), + (1860, 2), + (1861, 1466), + (1863, 2), + (1866, 1), + (1869, 1), + (1870, 2), + (1872, 179), + (1873, 1), + (1874, 9), + (1875, 29), + (1876, 15), + (1877, 43), + (1878, 2), + (1880, 8), + (1881, 13), + (1882, 18), + (1883, 12), + (1884, 14), + (1885, 18), + (1886, 16), + (1887, 6), + (1888, 2), + (1889, 3), + (1890, 9), + (1891, 196), + (1892, 13), + (1893, 1456), + (1894, 14), + (1895, 8), + (1896, 2), + (1898, 1), + (1899, 17), + (1900, 5), + (1901, 1), + (1904, 175), + (1905, 1), + (1906, 2), + (1907, 3), + (1908, 6), + (1909, 10), + (1910, 3), + (1911, 22), + (1912, 6), + (1913, 22), + (1914, 6), + (1915, 10), + (1916, 5), + (1917, 2), + (1918, 6), + (1919, 4), + (1920, 7), + (1921, 14), + (1922, 4), + (1923, 107), + (1924, 10), + (1925, 1434), + (1926, 7), + (1927, 76), + (1928, 4), + (1929, 7), + (1930, 10), + (1931, 14), + (1932, 6), + (1933, 15), + (1934, 4), + (1935, 2), + (1936, 182), + (1937, 2), + (1939, 11), + (1940, 1), + (1941, 4), + (1942, 2), + (1943, 9), + (1944, 1), + (1947, 24), + (1949, 22), + (1952, 15), + (1953, 14), + (1954, 5), + (1955, 111), + (1956, 11), + (1957, 1435), + (1958, 5), + (1959, 5), + (1960, 10), + (1961, 6), + (1962, 11), + (1963, 95), + (1964, 11), + (1965, 7), + (1966, 7), + (1967, 2), + (1968, 182), + (1969, 6), + (1970, 15), + (1972, 7), + (1973, 11), + (1974, 6), + (1975, 2), + (1976, 6), + (1977, 3), + (1978, 2), + (1983, 24), + (1985, 26), + (1986, 3), + (1987, 109), + (1988, 3), + (1989, 1421), + (1990, 1), + (1991, 3), + (1992, 8), + (1993, 4), + (1994, 6), + (1995, 5), + (1996, 13), + (1997, 6), + (1998, 10), + (1999, 92), + (2000, 181), + (2001, 5), + (2002, 5), + (2003, 1), + (2004, 1), + (2005, 14), + (2006, 12), + (2007, 10), + (2008, 7), + (2009, 9), + (2010, 6), + (2011, 8), + (2012, 13), + (2013, 2), + (2014, 2), + (2018, 1), + (2019, 128), + (2021, 1429), + (2022, 4), + (2026, 2), + (2027, 2), + (2030, 7), + (2032, 175), + (2033, 1), + (2035, 90), + (2036, 3), + (2037, 11), + (2038, 2), + (2039, 4), + (2040, 3), + (2041, 2), + (2042, 1), + (2043, 2), + (2044, 5), + (2045, 1), + (2046, 3), + (2047, 21), + (2048, 5), + (2050, 16), + (2051, 120), + (2053, 1403), + (2054, 4), + (2055, 29), + (2057, 26), + (2058, 3), + (2059, 4), + (2060, 4), + (2061, 7), + (2063, 1), + (2065, 170), + (2066, 3), + (2067, 2), + (2068, 7), + (2069, 13), + (2071, 77), + (2072, 1), + (2075, 4), + (2077, 1), + (2078, 2), + (2079, 5), + (2080, 4), + (2081, 3), + (2082, 3), + (2083, 2), + (2084, 293), + (2085, 6), + (2086, 1395), + (2087, 2), + (2089, 4), + (2090, 10), + (2091, 26), + (2092, 14), + (2093, 25), + (2097, 170), + (2099, 2), + (2100, 1), + (2101, 8), + (2102, 5), + (2104, 2), + (2105, 2), + (2107, 90), + (2108, 1), + (2110, 15), + (2112, 1), + (2113, 1), + (2114, 3), + (2115, 8), + (2116, 3), + (2117, 5), + (2118, 1380), + (2119, 4), + (2120, 1), + (2121, 3), + (2122, 1), + (2123, 6), + (2124, 24), + (2125, 1), + (2127, 33), + (2128, 4), + (2129, 197), + (2132, 1), + (2133, 3), + (2134, 8), + (2141, 1), + (2143, 95), + (2144, 6), + (2146, 1), + (2147, 1), + (2148, 3), + (2150, 1369), + (2152, 1), + (2153, 1), + (2155, 5), + (2156, 7), + (2157, 12), + (2158, 2), + (2159, 6), + (2160, 7), + (2161, 174), + (2162, 22), + (2163, 27), + (2164, 5), + (2165, 24), + (2166, 6), + (2169, 8), + (2170, 2), + (2171, 1), + (2172, 1), + (2174, 8), + (2175, 10), + (2176, 2), + (2177, 3), + (2179, 72), + (2180, 4), + (2181, 1), + (2182, 1366), + (2183, 2), + (2184, 5), + (2185, 4), + (2188, 3), + (2191, 1), + (2192, 2), + (2193, 169), + (2198, 7), + (2199, 27), + (2201, 28), + (2205, 2), + (2206, 2), + (2209, 9), + (2213, 8), + (2214, 1364), + (2215, 95), + (2216, 1), + (2217, 2), + (2218, 1), + (2219, 1), + (2220, 3), + (2221, 2), + (2222, 3), + (2223, 41), + (2225, 168), + (2228, 1), + (2229, 6), + (2230, 8), + (2231, 1), + (2232, 2), + (2233, 6), + (2234, 1), + (2235, 41), + (2236, 2), + (2237, 17), + (2240, 7), + (2242, 6), + (2244, 1), + (2246, 1350), + (2249, 2), + (2250, 4), + (2251, 89), + (2252, 1), + (2257, 167), + (2260, 4), + (2261, 3), + (2262, 6), + (2265, 1), + (2269, 2), + (2270, 4), + (2271, 32), + (2273, 21), + (2274, 1), + (2275, 3), + (2276, 1), + (2277, 2), + (2278, 1344), + (2279, 2), + (2280, 1), + (2281, 1), + (2284, 1), + (2287, 98), + (2288, 2), + (2289, 168), + (2292, 3), + (2293, 3), + (2294, 4), + (2298, 3), + (2303, 9), + (2307, 26), + (2308, 1), + (2309, 30), + (2310, 1344), + (2314, 1), + (2318, 1), + (2321, 164), + (2323, 1), + (2324, 82), + (2325, 1), + (2326, 5), + (2327, 1), + (2334, 6), + (2338, 1), + (2339, 1), + (2340, 1), + (2342, 1337), + (2343, 55), + (2344, 27), + (2345, 6), + (2346, 25), + (2347, 1), + (2348, 18), + (2350, 1), + (2351, 3), + (2352, 2), + (2353, 166), + (2358, 6), + (2360, 87), + (2361, 3), + (2362, 1), + (2373, 9), + (2374, 1330), + (2376, 1), + (2377, 1), + (2378, 11), + (2379, 4), + (2380, 28), + (2382, 29), + (2383, 2), + (2384, 8), + (2385, 169), + (2386, 4), + (2387, 9), + (2388, 8), + (2389, 4), + (2390, 15), + (2392, 1), + (2396, 117), + (2397, 4), + (2399, 1), + (2406, 1330), + (2410, 1), + (2414, 1), + (2415, 4), + (2416, 26), + (2417, 164), + (2418, 31), + (2421, 3), + (2422, 4), + (2424, 6), + (2425, 3), + (2426, 3), + (2427, 5), + (2428, 1), + (2429, 2), + (2432, 100), + (2433, 1), + (2435, 1), + (2436, 1), + (2438, 1328), + (2441, 10), + (2443, 11), + (2448, 2), + (2449, 163), + (2451, 1), + (2452, 27), + (2453, 8), + (2454, 24), + (2455, 1), + (2456, 2), + (2457, 2), + (2460, 4), + (2465, 5), + (2466, 3), + (2468, 95), + (2469, 6), + (2470, 1324), + (2471, 1), + (2472, 1), + (2476, 2), + (2477, 2), + (2478, 2), + (2479, 4), + (2481, 163), + (2484, 2), + (2485, 6), + (2486, 2), + (2488, 23), + (2489, 1), + (2490, 26), + (2491, 1), + (2493, 1), + (2494, 1), + (2495, 3), + (2496, 1), + (2500, 3), + (2502, 1327), + (2503, 1), + (2504, 93), + (2505, 2), + (2506, 1), + (2511, 4), + (2513, 166), + (2516, 3), + (2517, 5), + (2518, 8), + (2519, 2), + (2521, 1), + (2524, 27), + (2526, 20), + (2532, 1), + (2534, 1320), + (2535, 1), + (2540, 114), + (2541, 1), + (2543, 1), + (2545, 163), + (2550, 3), + (2555, 3), + (2557, 4), + (2558, 3), + (2559, 2), + (2560, 26), + (2561, 6), + (2562, 26), + (2564, 5), + (2565, 1), + (2566, 1325), + (2567, 5), + (2568, 9), + (2569, 10), + (2570, 2), + (2571, 1), + (2576, 97), + (2577, 165), + (2582, 3), + (2583, 5), + (2593, 2), + (2596, 42), + (2597, 1), + (2598, 1336), + (2602, 1), + (2609, 163), + (2612, 97), + (2613, 1), + (2614, 2), + (2619, 1), + (2621, 2), + (2624, 2), + (2628, 2), + (2630, 1684946), + (2632, 27), + (2633, 2), + (2634, 25), + (2635, 1), + (2637, 4), + (2639, 1), + (2640, 1), + (2641, 163), + (2644, 1), + (2645, 3), + (2646, 2), + (2648, 112), + (2649, 1), + (2653, 5), + (2659, 3), + (2660, 1), + (2661, 1), + (2662, 1315), + (2664, 1), + (2668, 30), + (2669, 1), + (2670, 26), + (2673, 163), + (2674, 2), + (2675, 1), + (2678, 7), + (2679, 1), + (2680, 1), + (2684, 90), + (2685, 1), + (2686, 1), + (2694, 1315), + (2699, 1), + (2701, 1), + (2704, 30), + (2705, 163), + (2706, 27), + (2710, 2), + (2712, 1), + (2720, 112), + (2721, 2), + (2723, 5), + (2726, 1316), + (2736, 1), + (2737, 165), + (2738, 2), + (2740, 25), + (2742, 33), + (2745, 1), + (2756, 97), + (2757, 1), + (2758, 1315), + (2769, 163), + (2774, 3), + (2776, 32), + (2778, 34), + (2781, 1), + (2782, 1), + (2784, 1), + (2790, 1313), + (2792, 94), + (2793, 12), + (2796, 1), + (2800, 1), + (2801, 163), + (2804, 2), + (2805, 6), + (2806, 2), + (2807, 2), + (2809, 1), + (2810, 1), + (2812, 23), + (2814, 33), + (2815, 3), + (2816, 1), + (2820, 2), + (2821, 1), + (2822, 1314), + (2824, 1), + (2828, 104), + (2829, 1), + (2833, 163), + (2837, 6), + (2838, 4), + (2839, 1), + (2848, 32), + (2849, 4), + (2850, 32), + (2852, 4), + (2853, 1), + (2854, 1312), + (2861, 1), + (2863, 52), + (2864, 111), + (2865, 164), + (2868, 2), + (2869, 15), + (2870, 2), + (2871, 1), + (2884, 30), + (2886, 1333), + (2890, 2), + (2891, 2), + (2892, 3), + (2893, 4), + (2894, 2), + (2897, 163), + (2899, 3), + (2900, 230), + (2901, 1), + (2902, 2), + (2908, 2), + (2911, 1), + (2918, 1312), + (2920, 42), + (2922, 25), + (2923, 1), + (2925, 1), + (2929, 165), + (2930, 2), + (2931, 5), + (2932, 4), + (2933, 8), + (2934, 2), + (2936, 110), + (2937, 1), + (2938, 1), + (2939, 1), + (2948, 1), + (2950, 1313), + (2956, 38), + (2958, 32), + (2961, 163), + (2964, 1), + (2966, 4), + (2967, 2), + (2969, 1), + (2971, 1), + (2972, 151), + (2973, 1), + (2975, 3), + (2976, 4), + (2977, 3), + (2978, 1), + (2979, 1), + (2980, 1), + (2982, 1312), + (2992, 28), + (2993, 163), + (2994, 29), + (2998, 2), + (3006, 1), + (3007, 2), + (3008, 188), + (3009, 2), + (3014, 1311), + (3015, 5), + (3016, 9), + (3017, 1), + (3020, 1), + (3025, 164), + (3028, 27), + (3030, 31), + (3044, 223), + (3045, 1), + (3046, 1311), + (3048, 1), + (3057, 163), + (3061, 2), + (3062, 4), + (3064, 41), + (3066, 35), + (3076, 2), + (3078, 1310), + (3080, 151), + (3081, 2), + (3089, 163), + (3094, 2), + (3100, 35), + (3101, 2), + (3102, 38), + (3104, 2), + (3110, 1310), + (3116, 106), + (3117, 2), + (3121, 163), + (3125, 5), + (3126, 2), + (3132, 2), + (3136, 36), + (3138, 39), + (3140, 2), + (3141, 1), + (3142, 1309), + (3143, 1), + (3144, 1), + (3152, 120), + (3153, 164), + (3155, 1), + (3157, 1), + (3158, 2), + (3163, 1), + (3164, 1), + (3172, 34), + (3174, 1343), + (3185, 163), + (3188, 136), + (3189, 1), + (3190, 2), + (3203, 1), + (3204, 1), + (3206, 1308), + (3208, 53), + (3210, 52), + (3217, 163), + (3220, 38), + (3221, 114), + (3222, 2), + (3224, 141), + (3225, 5), + (3230, 1), + (3236, 38), + (3238, 1308), + (3244, 35), + (3246, 46), + (3249, 163), + (3254, 2), + (3260, 105), + (3261, 4), + (3263, 1), + (3270, 1308), + (3280, 38), + (3281, 163), + (3282, 28), + (3286, 3), + (3292, 1), + (3296, 138), + (3297, 1), + (3301, 1), + (3302, 1308), + (3304, 1), + (3313, 163), + (3316, 33), + (3318, 34), + (3329, 1), + (3331, 1), + (3332, 120), + (3333, 1), + (3334, 1309), + (3345, 163), + (3350, 3), + (3352, 34), + (3354, 31), + (3357, 1), + (3366, 1307), + (3368, 230), + (3369, 6), + (3377, 163), + (3382, 2), + (3388, 37), + (3390, 45), + (3398, 1307), + (3404, 3128), + (3405, 2), + (3409, 163), + (3414, 2), + (3424, 40), + (3426, 23), + (3430, 1307), + (3440, 117), + (3441, 164), + (3446, 2), + (3460, 30), + (3462, 1344), + (3469, 1), + (3473, 163), + (3476, 116), + (3477, 1), + (3478, 3), + (3494, 1305), + (3496, 36), + (3498, 38), + (3501, 2), + (3504, 2), + (3505, 163), + (3510, 2), + (3512, 124), + (3513, 4), + (3515, 1), + (3525, 1), + (3526, 1305), + (3532, 27), + (3534, 33), + (3537, 165), + (3541, 2), + (3542, 2), + (3544, 2), + (3548, 119), + (3549, 1), + (3558, 1305), + (3568, 29), + (3569, 163), + (3570, 53), + (3574, 2), + (3581, 6), + (3584, 115), + (3585, 2), + (3590, 1306), + (3601, 163), + (3604, 39), + (3606, 45), + (3620, 107), + (3621, 1), + (3622, 1304), + (3633, 163), + (3634, 1), + (3637, 1), + (3638, 2), + (3640, 43), + (3642, 35), + (3654, 1305), + (3656, 126), + (3657, 2), + (3661, 1), + (3664, 1), + (3665, 163), + (3670, 3), + (3676, 32), + (3678, 48), + (3679, 1), + (3686, 1303), + (3692, 128), + (3693, 2), + (3697, 163), + (3702, 3), + (3712, 33), + (3714, 28), + (3718, 1302), + (3728, 137), + (3729, 165), + (3734, 2), + (3748, 54), + (3749, 1), + (3750, 1333), + (3758, 1), + (3761, 163), + (3764, 125), + (3765, 2), + (3766, 3), + (3782, 1301), + (3784, 32), + (3786, 50), + (3793, 163), + (3798, 2), + (3800, 123), + (3801, 3), + (3805, 1), + (3814, 1301), + (3820, 53), + (3822, 30), + (3825, 163), + (3830, 2), + (3833, 1), + (3836, 109), + (3837, 3), + (3846, 1301), + (3856, 35), + (3857, 163), + (3858, 54), + (3860, 20), + (3861, 51), + (3862, 2), + (3872, 124), + (3873, 2), + (3876, 17), + (3878, 1302), + (3882, 1), + (3889, 163), + (3892, 45), + (3894, 47), + (3901, 2), + (3903, 1), + (3904, 2), + (3908, 138), + (3909, 2), + (3910, 1300), + (3917, 2), + (3921, 163), + (3926, 2), + (3928, 38), + (3930, 37), + (3942, 1300), + (3944, 137), + (3945, 2), + (3953, 163), + (3958, 2), + (3964, 66), + (3966, 37), + (3971, 1), + (3974, 1300), + (3980, 166), + (3981, 1), + (3985, 163), + (3990, 2), + (4000, 35), + (4002, 54), + (4006, 1300), + (4016, 150), + (4017, 164), + (4021, 38), + (4022, 2), + (4024, 38), + (4036, 47), + (4038, 1347), + (4049, 163), + (4052, 134), + (4053, 10), + (4054, 2), + (4068, 1), + (4070, 1300), + (4072, 52), + (4074, 40), + (4075, 1), + (4081, 163), + (4085, 7), + (4086, 2), + (4088, 123), + (4089, 4), + (4100, 2), + (4102, 1300), + (4108, 38), + (4110, 43), + (4113, 163), + (4118, 2), + (4119, 2), + (4124, 159), + (4125, 3), + (4128, 1), + (4134, 1299), + (4141, 1), + (4144, 51), + (4145, 163), + (4146, 41), + (4150, 2), + (4152, 30), + (4160, 153), + (4161, 1), + (4164, 2), + (4166, 1299), + (4177, 163), + (4180, 225), + (4181, 596), + (4182, 50), + (4187, 1), + (4196, 373), + (4197, 3), + (4198, 1299), + (4209, 163), + (4214, 2), + (4216, 66), + (4217, 3), + (4218, 69), + (4221, 1), + (4230, 1299), + (4232, 158), + (4233, 2), + (4241, 163), + (4246, 2), + (4252, 45), + (4253, 1), + (4254, 48), + (4262, 1300), + (4267, 2), + (4268, 145), + (4269, 3), + (4270, 1), + (4271, 1), + (4273, 163), + (4278, 3), + (4288, 75), + (4290, 36), + (4294, 1298), + (4301, 1), + (4304, 173), + (4305, 166), + (4309, 2), + (4310, 2), + (4324, 52), + (4326, 1359), + (4337, 163), + (4340, 195), + (4341, 2), + (4342, 3), + (4358, 1297), + (4360, 76), + (4362, 56), + (4365, 2), + (4369, 163), + (4374, 2), + (4376, 171), + (4377, 1), + (4390, 1298), + (4396, 52), + (4398, 49), + (4401, 163), + (4406, 3), + (4407, 2), + (4412, 170), + (4413, 2), + (4421, 1), + (4422, 1296), + (4432, 57), + (4433, 163), + (4434, 51), + (4436, 1), + (4438, 2), + (4448, 481), + (4449, 2), + (4451, 1), + (4454, 1295), + (4463, 1), + (4465, 163), + (4468, 74), + (4470, 92), + (4484, 448), + (4485, 3), + (4486, 1295), + (4487, 1), + (4497, 163), + (4502, 2), + (4504, 52), + (4506, 65), + (4518, 1295), + (4519, 2), + (4520, 631), + (4521, 3), + (4529, 164), + (4530, 1), + (4532, 1), + (4533, 3), + (4534, 2), + (4540, 55), + (4542, 48), + (4550, 1294), + (4556, 2358), + (4557, 3), + (4561, 163), + (4562, 1), + (4566, 2), + (4576, 58), + (4578, 74), + (4582, 1294), + (4592, 193), + (4593, 167), + (4598, 2), + (4612, 66), + (4614, 1363), + (4621, 2), + (4625, 163), + (4628, 218), + (4629, 3), + (4630, 2), + (4635, 3), + (4640, 1), + (4645, 1), + (4646, 1295), + (4648, 57), + (4650, 90), + (4657, 163), + (4662, 3), + (4664, 194), + (4665, 1), + (4678, 1295), + (4684, 49), + (4685, 1), + (4686, 85), + (4689, 163), + (4694, 4), + (4700, 183), + (4701, 3), + (4710, 1291), + (4720, 61), + (4721, 163), + (4722, 75), + (4726, 3), + (4736, 175), + (4737, 4), + (4742, 1291), + (4753, 163), + (4756, 84), + (4758, 53), + (4772, 210), + (4773, 4), + (4774, 1291), + (4785, 163), + (4790, 2), + (4792, 54), + (4794, 66), + (4799, 2), + (4806, 1292), + (4808, 180), + (4809, 6), + (4817, 164), + (4820, 32), + (4821, 132), + (4822, 3), + (4824, 17), + (4828, 70), + (4830, 62), + (4836, 42), + (4838, 1290), + (4844, 199), + (4845, 3), + (4849, 163), + (4854, 2), + (4864, 104), + (4866, 98), + (4870, 1290), + (4873, 1), + (4880, 184), + (4881, 164), + (4886, 2), + (4900, 88), + (4902, 1387), + (4909, 1), + (4913, 163), + (4916, 187), + (4917, 6), + (4918, 2), + (4934, 1290), + (4936, 65), + (4938, 59), + (4945, 163), + (4948, 1), + (4950, 2), + (4952, 198), + (4953, 3), + (4966, 1290), + (4972, 64), + (4974, 108), + (4977, 163), + (4982, 2), + (4988, 199), + (4989, 8), + (4998, 1290), + (5008, 82), + (5009, 163), + (5010, 113), + (5012, 3), + (5013, 9), + (5014, 2), + (5017, 1), + (5024, 228), + (5025, 2), + (5028, 4), + (5030, 1290), + (5041, 162), + (5044, 96), + (5046, 71), + (5060, 275), + (5061, 6), + (5062, 1291), + (5064, 1), + (5070, 1), + (5073, 162), + (5078, 3), + (5080, 66), + (5082, 153), + (5094, 1289), + (5096, 272), + (5097, 10), + (5101, 2), + (5104, 2), + (5105, 162), + (5110, 2), + (5116, 87), + (5118, 80), + (5126, 1289), + (5132, 266), + (5133, 5), + (5135, 1), + (5137, 162), + (5140, 190), + (5141, 681), + (5142, 2), + (5152, 104), + (5154, 184), + (5156, 238), + (5158, 1289), + (5168, 257), + (5169, 165), + (5174, 2), + (5188, 99), + (5190, 1435), + (5201, 162), + (5204, 228), + (5205, 6), + (5206, 2), + (5221, 206), + (5222, 1289), + (5224, 312), + (5226, 110), + (5231, 1), + (5233, 162), + (5238, 2), + (5240, 266), + (5241, 7), + (5254, 1289), + (5260, 87), + (5262, 243), + (5265, 162), + (5270, 2), + (5274, 8), + (5276, 318), + (5277, 7), + (5286, 1289), + (5288, 86), + (5296, 88), + (5297, 162), + (5298, 123), + (5302, 3), + (5312, 351), + (5313, 1), + (5318, 1289), + (5329, 162), + (5332, 115), + (5334, 173), + (5339, 6), + (5344, 1), + (5348, 313), + (5349, 3), + (5350, 1289), + (5352, 24), + (5353, 14), + (5361, 162), + (5366, 3), + (5368, 157), + (5370, 107), + (5374, 1), + (5382, 1289), + (5384, 293), + (5385, 4), + (5388, 4), + (5393, 162), + (5396, 1), + (5398, 2), + (5404, 142), + (5406, 201), + (5407, 1), + (5414, 1289), + (5417, 3), + (5420, 285), + (5421, 5), + (5423, 1), + (5425, 162), + (5430, 2), + (5436, 1), + (5440, 142), + (5442, 210), + (5444, 1), + (5446, 1294), + (5456, 318), + (5457, 166), + (5462, 3), + (5476, 123), + (5478, 1608), + (5482, 2), + (5489, 162), + (5492, 329), + (5493, 2), + (5494, 2), + (5504, 1), + (5506, 1), + (5510, 1289), + (5511, 1), + (5512, 165), + (5514, 167), + (5521, 163), + (5522, 1), + (5526, 2), + (5528, 367), + (5529, 8), + (5542, 1289), + (5548, 192), + (5550, 291), + (5553, 162), + (5558, 2), + (5564, 399), + (5565, 13), + (5574, 1289), + (5584, 188), + (5585, 163), + (5586, 356), + (5590, 2), + (5592, 1), + (5599, 1), + (5600, 375), + (5601, 3), + (5606, 1290), + (5608, 1), + (5617, 162), + (5618, 1), + (5620, 261), + (5622, 667), + (5623, 1), + (5626, 1), + (5633, 1), + (5636, 406), + (5637, 4), + (5638, 1289), + (5639, 1), + (5649, 162), + (5654, 2), + (5656, 468), + (5658, 1159), + (5662, 1), + (5670, 1289), + (5671, 1), + (5672, 349), + (5673, 8), + (5675, 1), + (5681, 162), + (5686, 2), + (5692, 321), + (5694, 3067), + (5702, 1289), + (5706, 1), + (5708, 443), + (5709, 7), + (5713, 162), + (5718, 2), + (5728, 496), + (5730, 4577), + (5734, 1289), + (5744, 383), + (5745, 165), + (5750, 3), + (5756, 1), + (5758, 1), + (5764, 5847), + (5766, 8966), + (5775, 1), + (5777, 162), + (5780, 616), + (5781, 240), + (5782, 2), + (5784, 1), + (5788, 1), + (5796, 81), + (5798, 1289), + (5799, 1), + (5800, 5543), + (5802, 13287), + (5809, 162), + (5814, 2), + (5816, 409), + (5817, 3), + (5830, 1289), + (5833, 1), + (5836, 123), + (5838, 59), + (5841, 162), + (5846, 2), + (5852, 480), + (5853, 10), + (5862, 1289), + (5872, 191), + (5873, 162), + (5874, 38), + (5878, 2), + (5888, 616), + (5889, 12), + (5894, 1289), + (5905, 162), + (5908, 139), + (5910, 54), + (5922, 1), + (5924, 675), + (5925, 9), + (5926, 1289), + (5937, 162), + (5942, 2), + (5944, 153), + (5946, 48), + (5958, 1289), + (5960, 614), + (5961, 33), + (5969, 162), + (5974, 2), + (5980, 140), + (5982, 95), + (5990, 1289), + (5996, 628), + (5997, 10), + (6001, 162), + (6006, 2), + (6016, 155), + (6018, 67), + (6021, 42), + (6022, 1289), + (6024, 42), + (6032, 772), + (6033, 177), + (6038, 2), + (6049, 1), + (6052, 109), + (6054, 1340), + (6065, 162), + (6068, 749), + (6069, 11), + (6070, 2), + (6086, 1289), + (6088, 364), + (6090, 49), + (6096, 1), + (6097, 162), + (6102, 2), + (6104, 975), + (6105, 4), + (6106, 1), + (6118, 1289), + (6124, 273), + (6126, 58), + (6129, 162), + (6134, 2), + (6138, 1), + (6140, 1053), + (6141, 13), + (6150, 1289), + (6152, 1), + (6153, 2), + (6160, 372), + (6161, 162), + (6162, 70), + (6164, 1), + (6166, 2), + (6172, 1), + (6176, 1088), + (6177, 96), + (6178, 1), + (6182, 1290), + (6188, 4), + (6193, 162), + (6194, 1), + (6196, 346), + (6198, 101), + (6206, 1), + (6212, 1352), + (6213, 4), + (6214, 1290), + (6219, 2), + (6223, 1), + (6225, 162), + (6230, 1), + (6232, 321), + (6234, 170), + (6246, 1290), + (6248, 1755), + (6249, 4), + (6257, 162), + (6261, 4), + (6262, 1), + (6264, 4), + (6268, 616), + (6270, 141), + (6275, 1), + (6278, 1289), + (6280, 1), + (6281, 1), + (6284, 2516), + (6285, 73), + (6289, 162), + (6294, 1), + (6304, 409), + (6306, 163), + (6310, 1289), + (6314, 2), + (6320, 2276), + (6321, 210), + (6326, 1), + (6340, 445), + (6342, 1437), + (6353, 162), + (6356, 4090), + (6357, 55), + (6358, 1), + (6364, 1), + (6374, 1290), + (6376, 929), + (6378, 270), + (6385, 162), + (6390, 1), + (6392, 6135), + (6393, 16), + (6400, 1), + (6406, 1289), + (6412, 607), + (6414, 386), + (6417, 162), + (6420, 1), + (6421, 238), + (6422, 1), + (6424, 238), + (6428, 15189), + (6429, 227), + (6438, 1289), + (6443, 1), + (6448, 1211), + (6449, 162), + (6450, 1135), + (6453, 2), + (6454, 1), + (6464, 66588), + (6465, 77), + (6470, 1289), + (6474, 31), + (6481, 162), + (6484, 21001), + (6486, 9926), + (6488, 95), + (6498, 1), + (6500, 51017), + (6501, 2547), + (6502, 1289), + (6513, 162), + (6518, 1), + (6520, 11978), + (6522, 2546), + (6534, 1289), + (6536, 1), + (6537, 4), + (6539, 7), + (6545, 162), + (6546, 1), + (6550, 1), + (6553, 27), + (6566, 1289), + (6572, 1), + (6573, 2), + (6574, 1), + (6577, 163), + (6582, 2), + (6587, 1), + (6588, 17), + (6598, 1289), + (6600, 1), + (6603, 1), + (6605, 1), + (6606, 2), + (6608, 1), + (6609, 163), + (6610, 1), + (6614, 1), + (6623, 4), + (6630, 1289), + (6631, 1), + (6633, 1), + (6635, 1), + (6640, 1), + (6641, 162), + (6644, 1), + (6645, 2), + (6646, 2), + (6662, 1289), + (6666, 1), + (6670, 1), + (6673, 162), + (6678, 1), + (6679, 1), + (6680, 1), + (6681, 5), + (6686, 1), + (6694, 1289), + (6705, 162), + (6710, 1), + (6711, 1), + (6714, 1), + (6716, 1), + (6717, 10), + (6726, 1289), + (6734, 1), + (6737, 163), + (6738, 1), + (6740, 2), + (6742, 1), + (6752, 1), + (6753, 1), + (6757, 1), + (6758, 1289), + (6769, 162), + (6770, 1), + (6774, 1), + (6775, 1), + (6788, 1), + (6789, 3), + (6790, 1289), + (6797, 1), + (6801, 162), + (6802, 1), + (6803, 1), + (6806, 1), + (6818, 1), + (6819, 1), + (6822, 1289), + (6824, 1), + (6825, 5), + (6833, 162), + (6834, 1), + (6837, 1), + (6838, 1), + (6844, 2), + (6854, 1289), + (6860, 1), + (6861, 5), + (6865, 163), + (6869, 1), + (6870, 1), + (6872, 1), + (6875, 1), + (6881, 3), + (6886, 1289), + (6896, 1), + (6897, 166), + (6902, 1), + (6915, 1), + (6918, 1289), + (6929, 162), + (6932, 2), + (6933, 1), + (6934, 1), + (6947, 1), + (6950, 1290), + (6961, 162), + (6966, 1), + (6969, 2), + (6982, 1289), + (6993, 162), + (6998, 1), + (7004, 1), + (7005, 1), + (7014, 1289), + (7025, 162), + (7030, 1), + (7032, 1), + (7034, 1), + (7040, 1), + (7041, 1), + (7046, 1289), + (7057, 162), + (7058, 1), + (7059, 1), + (7062, 1), + (7070, 1), + (7076, 1), + (7077, 3), + (7078, 1289), + (7084, 1), + (7089, 162), + (7094, 1), + (7110, 1289), + (7112, 1), + (7113, 5), + (7121, 162), + (7124, 1), + (7126, 1), + (7133, 1), + (7142, 1289), + (7148, 1), + (7149, 12), + (7153, 162), + (7158, 1), + (7174, 1289), + (7184, 1), + (7185, 170), + (7190, 1), + (7206, 1289), + (7217, 162), + (7220, 1), + (7221, 82), + (7222, 1), + (7224, 81), + (7229, 1), + (7237, 1), + (7238, 1289), + (7242, 1), + (7243, 1), + (7248, 1), + (7249, 162), + (7254, 1), + (7256, 1), + (7257, 1), + (7266, 4), + (7270, 1289), + (7274, 13), + (7280, 20), + (7281, 162), + (7286, 1), + (7288, 12), + (7292, 1), + (7293, 5), + (7296, 1), + (7302, 1289), + (7308, 1), + (7313, 162), + (7315, 1), + (7318, 1), + (7328, 1), + (7329, 1), + (7334, 1290), + (7345, 162), + (7349, 1), + (7350, 1), + (7353, 1), + (7364, 1), + (7365, 1), + (7366, 1290), + (7377, 162), + (7382, 1), + (7392, 1), + (7398, 1289), + (7400, 1), + (7401, 4), + (7406, 1), + (7409, 162), + (7411, 1), + (7414, 1), + (7430, 1289), + (7431, 3), + (7436, 1), + (7437, 2), + (7441, 162), + (7445, 5), + (7446, 1), + (7448, 1), + (7460, 1), + (7462, 1289), + (7472, 1), + (7473, 166), + (7474, 1), + (7478, 1), + (7494, 1289), + (7505, 162), + (7508, 3), + (7509, 2), + (7510, 2), + (7525, 1), + (7526, 1289), + (7532, 1), + (7537, 162), + (7542, 1), + (7544, 1), + (7545, 9), + (7546, 1), + (7558, 1289), + (7569, 162), + (7574, 1), + (7580, 1), + (7581, 6), + (7590, 1289), + (7601, 162), + (7606, 1), + (7616, 1), + (7617, 6), + (7622, 1289), + (7623, 1), + (7625, 1), + (7633, 162), + (7638, 1), + (7652, 1), + (7653, 11), + (7654, 1289), + (7657, 1), + (7665, 162), + (7670, 1), + (7686, 1289), + (7688, 1), + (7689, 1), + (7697, 162), + (7702, 1), + (7708, 1), + (7715, 1), + (7717, 2), + (7718, 1289), + (7724, 1), + (7725, 3), + (7729, 162), + (7734, 1), + (7746, 1), + (7750, 1289), + (7760, 1), + (7761, 167), + (7766, 1), + (7782, 1289), + (7793, 162), + (7794, 1), + (7796, 1), + (7797, 1), + (7798, 1), + (7814, 1289), + (7820, 1), + (7825, 162), + (7826, 1), + (7830, 1), + (7832, 1), + (7833, 14), + (7842, 1), + (7846, 1289), + (7857, 162), + (7862, 1), + (7863, 1), + (7868, 1), + (7869, 4), + (7878, 1289), + (7885, 1), + (7889, 162), + (7894, 1), + (7904, 1), + (7905, 2), + (7910, 1289), + (7921, 162), + (7926, 1), + (7929, 1), + (7940, 1), + (7941, 2), + (7942, 1289), + (7953, 162), + (7958, 1), + (7963, 1), + (7973, 1), + (7974, 1289), + (7976, 1), + (7977, 16), + (7985, 162), + (7989, 1), + (7990, 1), + (7991, 1), + (7997, 1), + (8000, 1), + (8006, 1289), + (8012, 1), + (8013, 14), + (8017, 162), + (8022, 1), + (8038, 1289), + (8048, 1), + (8049, 185), + (8054, 2), + (8070, 1289), + (8081, 162), + (8084, 1), + (8085, 24), + (8086, 1), + (8102, 1289), + (8113, 162), + (8118, 1), + (8119, 1), + (8120, 1), + (8121, 1), + (8126, 1), + (8134, 1289), + (8140, 1), + (8145, 162), + (8150, 1), + (8157, 20), + (8166, 1289), + (8177, 162), + (8182, 1), + (8192, 1), + (8193, 1), + (8198, 1289), + (8209, 162), + (8214, 1), + (8228, 1), + (8229, 32), + (8230, 1290), + (8246, 1), + (8264, 1), + (8265, 27), + (8269, 1), + (8276, 1), + (8282, 1), + (8300, 1), + (8301, 133), + (8336, 2), + (8337, 60), + (8348, 3), + (8356, 1), + (8358, 1), + (8372, 1), + (8373, 196), + (8408, 1), + (8444, 1), + (8468, 1), + (8480, 1), + (8499, 1), + (8516, 1), + (8552, 1), + (8555, 1), + (8588, 1), + (8624, 1), + (8660, 3), + (8675, 1), + (8696, 1), + (8704, 1), + (8724, 1), + (8732, 1), + (8768, 1), + (8779, 1), + (8804, 1), + (8840, 1), + (8852, 2), + (8876, 1), + (8912, 1), + (8948, 1), + (8984, 1), + (9020, 1), + (9128, 1), + (9164, 1), + (9192, 1), + (9200, 2), + (9236, 1), + (9272, 1), + (9308, 1), + (9344, 1), + (9380, 1), + (9416, 1), + (9452, 1), + (9524, 1), + (9560, 1), + (9589, 1), + (9632, 1), + (9642, 1), + (9704, 1), + (9776, 1), + (9848, 1), + (9992, 1), + (10064, 1), + (10100, 1), + (10136, 1), + (10172, 1), + (10208, 1), + (10244, 1), + (10280, 1), + (10316, 1), + (10388, 1), + (10532, 1), + (10572, 1), + (10620, 1), + (10640, 1), + (10669, 1), + (10748, 1), + (10856, 1), + (10964, 1), + (11067, 1), + (11072, 1), + (11180, 1), + (11216, 1), + (11252, 1), + (11288, 1), + (11324, 1), + (11348, 2), + (11360, 1), + (11396, 1), + (11432, 1), + (11468, 1), + (11504, 1), + (11540, 1), + (11576, 1), + (11612, 1), + (11648, 1), + (11756, 1), + (11792, 1), + (11828, 1), + (11864, 1), + (11936, 1), + (12008, 1), + (12080, 1), + (12152, 1), + (12188, 1), + (12224, 1), + (12260, 1), + (12296, 1), + (12332, 1), + (12360, 1), + (12368, 1), + (12404, 1), + (12440, 1), + (12476, 1), + (12501, 2), + (12512, 1), + (12548, 1), + (12584, 1), + (12620, 1), + (12656, 1), + (12693, 1), + (12728, 1), + (12885, 1), + (13123, 1), + (13269, 1), + (13461, 1), + (13653, 1), + (13664, 1), + (13740, 1), + (13872, 1), + (13946, 1), + (14109, 1), + (14613, 2), + (14805, 2), + (14945, 1), + (14997, 1), + (15176, 1), + (15276, 1), + (15384, 1), + (15492, 1), + (15600, 1), + (15708, 1), + (15716, 1), + (15765, 1), + (15816, 1), + (15924, 1), + (16068, 1), + (16104, 1), + (16140, 1), + (16176, 1), + (16212, 1), + (16248, 1), + (16284, 1), + (16320, 1), + (16356, 1), + (16392, 1), + (16430, 1), + (16468, 1), + (16504, 1), + (16540, 1), + (16727, 2), + (16728, 1), + (16919, 2), + (16921, 1), + (16938, 1), + (17111, 6), + (17413, 1), + (17430, 1), + (17495, 1), + (17880, 1), + (18647, 2), + (18672, 1), + (19223, 38), + (19680, 1), + (20436, 1), + (21156, 1), + (21732, 1), + (22380, 1), + (22992, 1), + (23063, 17), + (23244, 1), + (23532, 1), + (23892, 1), + (24108, 1), + (24215, 1), + (24324, 1), + (24407, 2), + (24504, 1), + (24720, 1), + (24900, 1), + (24983, 205), + (25440, 1), + (25620, 1), + (26088, 1), + (26268, 1), + (26448, 1), + (26664, 1), + (26988, 1), + (27276, 1), + (27492, 1), + (27744, 1), + (28032, 1), + (28284, 1), + (28536, 1), + (28823, 42), + (28896, 1), + (29184, 1), + (29292, 1), + (29400, 1), + (29796, 1), + (29975, 4), + (30156, 1), + (30228, 1), + (30743, 238), + (30768, 1), + (31056, 1), + (31092, 1), + (31416, 1), + (32100, 1), + (32712, 1), + (33144, 1), + (33324, 1), + (33792, 1), + (34008, 1), + (34440, 1), + (34583, 81), + (34656, 1), + (34872, 1), + (34944, 1), + (35160, 1), + (35304, 1), + (35376, 1), + (35412, 1), + (35556, 1), + (35628, 1), + (35664, 1), + (35808, 1), + (36204, 1), + (36744, 1), + (37788, 1), + (39372, 1), + (40956, 1), + (41640, 1), + (41892, 1), + (42144, 1), + (42576, 1), + (42936, 1), + (43476, 1), + (45096, 1), + (47256, 1), + (47760, 1), + (47796, 1), + (47868, 1), + (48228, 1), + (48948, 1), + (49128, 1), + (49452, 1), + (49560, 1), + (49668, 1), + (49776, 1), + (50352, 1), + (50964, 1), + (52008, 1), + (53880, 1), + (55284, 1), + (55860, 1), + (56040, 1), + (56400, 1), + (56904, 1), + (57444, 1), + (59424, 1), + (60156, 1), + (60626, 1), + (60641, 1), + (61260, 1), + (62520, 1), + (64392, 1), + (65976, 1), + (67308, 1), + (68064, 1), + (68748, 1), + (69216, 1), + (69504, 1), + (69648, 1), + (69684, 1), + (69720, 1), + (69756, 1), + (69792, 1), + (69828, 1), + (70224, 1), + (70620, 1), + (71016, 1), + (71412, 1), + (71772, 1), + (71952, 1), + (72024, 1), + (72096, 1), + (72168, 1), + (72240, 1), + (72312, 1), + (72348, 1), + (72420, 1), + (72492, 1), + (72600, 1), + (72672, 1), + (72780, 1), + (72996, 1), + (73320, 1), + (73356, 1), + (73500, 1), + (73536, 1), + (73572, 1), + (73608, 1), + (73680, 1), + (73716, 1), + (73788, 1), + (73896, 1), + (74040, 1), + (74112, 1), + (74170, 1), + (74184, 1), + (74185, 1), + (74220, 1), + (74256, 1), + (74292, 1), + (74328, 1), + (74364, 1), + (74400, 1), + (74436, 1), + (74472, 1), + (74616, 1), + (74976, 1), + (75156, 1), + (75228, 1), + (75336, 1), + (75408, 1), + (75588, 1), + (75696, 1), + (75804, 1), + (75984, 1), + (76056, 1), + (76164, 1), + (76308, 1), + (76452, 1), + (76560, 1), + (76776, 1), + (76920, 1), + (77064, 1), + (77208, 1), + (77316, 1), + (77532, 1), + (77676, 1), + (77748, 1), + (77820, 1), + (77928, 1), + (78000, 1), + (78036, 1), + (78072, 1), + (78108, 1), + (78180, 1), + (78324, 1), + (78396, 1), + (78576, 1), + (78684, 1), + (78828, 1), + (78864, 1), + (78900, 1), + (78972, 1), + (79080, 1), + (79116, 1), + (79152, 1), + (79512, 1), + (79872, 1), + (80268, 1), + (80592, 1), + (80700, 1), + (80916, 1), + (81168, 1), + (81276, 1), + (81528, 1), + (81708, 1), + (81816, 1), + (81888, 1), + (82068, 1), + (82176, 1), + (82284, 1), + (82356, 1), + (82716, 1), + (83004, 1), + (83312, 1), + (83436, 1), + (83688, 1), + (83904, 1), + (84012, 1), + (84408, 1), + (84660, 1), + (85056, 1), + (85488, 1), + (85776, 1), + (85992, 1), + (86172, 1), + (86424, 1), + (86615, 1), + (86640, 1), + (86928, 1), + (87072, 1), + (87288, 1), + (87576, 1), + (87684, 1), + (87756, 1), + (87972, 1), + (88044, 1), + (88152, 1), + (88368, 1), + (88728, 1), + (88836, 1), + (88944, 1), + (89088, 1), + (89448, 1), + (89592, 1), + (89700, 1), + (89808, 1), + (89952, 1), + (90060, 1), + (90204, 1), + (90348, 1), + (90528, 1), + (90636, 1), + (90744, 1), + (90816, 1), + (91032, 1), + (91068, 1), + (91140, 1), + (91212, 1), + (91284, 1), + (91860, 1), + (92112, 1), + (92292, 1), + (92400, 1), + (92544, 1), + (92652, 1), + (92796, 1), + (92904, 1), + (92976, 1), + (93192, 1), + (93300, 1), + (93444, 1), + (93516, 1), + (93624, 1), + (93696, 1), + (93840, 1), + (93984, 1), + (94056, 1), + (94128, 1), + (94164, 1), + (94200, 1), + (94236, 1), + (94272, 1), + (94344, 1), + (94452, 1), + (94524, 1), + (94596, 1), + (94704, 1), + (94776, 1), + (94884, 1), + (94956, 1), + (95172, 1), + (95244, 1), + (95280, 1), + (95316, 1), + (95352, 1), + (95388, 1), + (95424, 1), + (95460, 1), + (95496, 1), + (95604, 1), + (95676, 1), + (95784, 1), + (95856, 1), + (95928, 1), + (96000, 1), + (96036, 1), + (96072, 1), + (96108, 1), + (96144, 1), + (96180, 1), + (96216, 1), + (96288, 1), + (96576, 1), + (98029, 1), + (98304, 1), + (98527, 1), + (98628, 1), + (99276, 1), + (99528, 1), + (99780, 1), + (99996, 1), + (100212, 1), + (100428, 1), + (100680, 1), + (100752, 1), + (100788, 1), + (100860, 1), + (100932, 1), + (101004, 1), + (101076, 1), + (101148, 1), + (101220, 1), + (101256, 1), + (101328, 1), + (101364, 1), + (101400, 1), + (101436, 1), + (101472, 1), + (101508, 1), + (101544, 1), + (101616, 1), + (101652, 1), + (101724, 1), + (101832, 1), + (101904, 1), + (101940, 1), + (101976, 1), + (102012, 1), + (102048, 1), + (102084, 1), + (102120, 1), + (102264, 1), + (102516, 1), + (102588, 1), + (102624, 1), + (102660, 1), + (102696, 1), + (102732, 1), + (102768, 1), + (102804, 1), + (102840, 1), + (102876, 1), + (102912, 1), + (102948, 1), + (102984, 1), + (103056, 1), + (103092, 1), + (103128, 1), + (103164, 1), + (103200, 1), + (103236, 1), + (103272, 1), + (103308, 1), + (103344, 1), + (103380, 1), + (103452, 1), + (103560, 1), + (103596, 1), + (103632, 1), + (103668, 1), + (103704, 1), + (103740, 1), + (103776, 1), + (103848, 1), + (103920, 1), + (103956, 1), + (104028, 1), + (104100, 1), + (104136, 1), + (104208, 1), + (104244, 1), + (104316, 1), + (104352, 1), + (104388, 1), + (104424, 1), + (104460, 1), + (104496, 1), + (104532, 1), + (104568, 1), + (104604, 1), + (104676, 1), + (104712, 1), + (104748, 1), + (104784, 1), + (104820, 1), + (104856, 1), + (104892, 1), + (104928, 1), + (104964, 1), + (105000, 1), + (105036, 1), + (105072, 1), + (105108, 1), + (105216, 1), + (105324, 1), + (105360, 1), + (105396, 1), + (105432, 1), + (105468, 1), + (105504, 1), + (105540, 1), + (105576, 1), + (105612, 1), + (105648, 1), + (105684, 1), + (105720, 1), + (105756, 1), + (105792, 1), + (105828, 1), + (105864, 1), + (105900, 1), + (105936, 1), + (110580, 1), + (115224, 1), + (118788, 1), + (121056, 1), + (121452, 1), + (121848, 1), + (122244, 1), + (122604, 1), + (122928, 1), + (123252, 1), + (123288, 1), + (123360, 1), + (123432, 1), + (123468, 1), + (123504, 1), + (123540, 1), + (123612, 1), + (123684, 1), + (123756, 1), + (123828, 1), + (123900, 1), + (123972, 1), + (124080, 1), + (124188, 1), + (124296, 1), + (124404, 1), + (124548, 1), + (124764, 1), + (124872, 1), + (124980, 1), + (125088, 1), + (125196, 1), + (125304, 1), + (125412, 1), + (125448, 1), + (125520, 1), + (125628, 1), + (125700, 1), + (125772, 1), + (125844, 1), + (125880, 1), + (125916, 1), + (125952, 1), + (125988, 1), + (126024, 1), + (126060, 1), + (126096, 1), + (126168, 1), + (126276, 1), + (126312, 1), + (126348, 1), + (126420, 1), + (126492, 1), + (126564, 1), + (126636, 1), + (126708, 1), + (126780, 1), + (126852, 1), + (126960, 1), + (127068, 1), + (127176, 1), + (127212, 1), + (127248, 1), + (127284, 1), + (127320, 1), + (127356, 1), + (127392, 1), + (127464, 1), + (127536, 1), + (127608, 1), + (127644, 1), + (127680, 1), + (127716, 1), + (127788, 1), + (127860, 1), + (127932, 1), + (128004, 1), + (128076, 1), + (128148, 1), + (128220, 1), + (128256, 1), + (128292, 1), + (128328, 1), + (128364, 1), + (128400, 1), + (128436, 1), + (128472, 1), + (128508, 1), + (128544, 1), + (128580, 1), + (128616, 1), + (128652, 1), + (128688, 1), + (128724, 1), + (128760, 1), + (128832, 1), + (128904, 1), + (128976, 1), + (129048, 1), + (129120, 1), + (129192, 1), + (129228, 1), + (129264, 1), + (129300, 1), + (129372, 1), + (129408, 1), + (129444, 1), + (129480, 1), + (129516, 1), + (129552, 1), + (129588, 1), + (129660, 1), + (129732, 1), + (129768, 1), + (129804, 1), + (129840, 1), + (129876, 1), + (129912, 1), + (129948, 1), + (129984, 1), + (130020, 1), + (130056, 1), + (130128, 1), + (130200, 1), + (130236, 1), + (130272, 1), + (130308, 1), + (130380, 1), + (130452, 1), + (130524, 1), + (130560, 1), + (130596, 1), + (130632, 1), + (130668, 1), + (130704, 1), + (130776, 1), + (130812, 1), + (130848, 1), + (130920, 1), + (130992, 1), + (131064, 1), + (131136, 1), + (131172, 1), + (131208, 1), + (131244, 1), + (131316, 1), + (131388, 1), + (131424, 1), + (131532, 1), + (131640, 1), + (131784, 1), + (131892, 1), + (131964, 1), + (132036, 1), + (132108, 1), + (132180, 1), + (132252, 1), + (132324, 1), + (132360, 1), + (132432, 1), + (132504, 1), + (132576, 1), + (132684, 1), + (132792, 1), + (132900, 1), + (132972, 1), + (133044, 1), + (133116, 1), + (133188, 1), + (133260, 1), + (133332, 1), + (133368, 1), + (133404, 1), + (133440, 1), + (133476, 1), + (133512, 1), + (133548, 1), + (133620, 1), + (133692, 1), + (133764, 1), + (133836, 1), + (133908, 1), + (133980, 1), + (134016, 1), + (134052, 1), + (134088, 1), + (134124, 1), + (134160, 1), + (134196, 1), + (134232, 1), + (134268, 1), + (134304, 1), + (134340, 1), + (134376, 1), + (134412, 1), + (134484, 1), + (134592, 1), + (134700, 1), + (134808, 1), + (134916, 1), + (134988, 1), + (135024, 1), + (135060, 1), + (135096, 1), + (135132, 1), + (135168, 1), + (135204, 1), + (135240, 1), + (135276, 1), + (135312, 1), + (135348, 1), + (135384, 1), + (135456, 1), + (135492, 1), + (135528, 1), + (135564, 1), + (135600, 1), + (135636, 1), + (135672, 1), + (135708, 1), + (135744, 1), + (135780, 1), + (135816, 1), + (135852, 1), + (135888, 1), + (135924, 1), + (135960, 1), + (135996, 1), + (136032, 1), + (136068, 1), + (136140, 1), + (136212, 1), + (136284, 1), + (136356, 1), + (136428, 1), + (136500, 1), + (136572, 1), + (136608, 1), + (136644, 1), + (136680, 1), + (136716, 1), + (136752, 1), + (136788, 1), + (136824, 1), + (136860, 1), + (136896, 1), + (136932, 1), + (136968, 1), + (137004, 1), + (137040, 1), + (137076, 1), + (137112, 1), + (137148, 1), + (137184, 1), + (137256, 1), + (137328, 1), + (137400, 1), + (137472, 1), + (137544, 1), + (137580, 1), + (137616, 1), + (137652, 1), + (137688, 1), + (137724, 1), + (137796, 1), + (137832, 1), + (137868, 1), + (137904, 1), + (137940, 1), + (137976, 1), + (138012, 1), + (138048, 1), + (138084, 1), + (138120, 1), + (138228, 1), + (138300, 1), + (138336, 1), + (138372, 1), + (138408, 1), + (138444, 1), + (138480, 1), + (138516, 1), + (138552, 1), + (138588, 1), + (138624, 1), + (138696, 1), + (138768, 1), + (138840, 1), + (138912, 1), + (138948, 1), + (138984, 1), + (139020, 1), + (139056, 1), + (139092, 1), + (139128, 1), + (139164, 1), + (139200, 1), + (139272, 1), + (139308, 1), + (139380, 1), + (139452, 1), + (139488, 1), + (139524, 1), + (139596, 1), + (139632, 1), + (139668, 1), + (139704, 1), + (139740, 1), + (139776, 1), + (139848, 1), + (139884, 1), + (139920, 1), + (139956, 1), + (139992, 1), + (140028, 1), + (140064, 1), + (140136, 1), + (140172, 1), + (140208, 1), + (140244, 1), + (140280, 1), + (140316, 1), + (140352, 1), + (140424, 1), + (140460, 1), + (140496, 1), + (140532, 1), + (140604, 1), + (140640, 1), + (140676, 1), + (140712, 1), + (140748, 1), + (140784, 1), + (140820, 1), + (140856, 1), + (140928, 1), + (141036, 1), + (141072, 1), + (141108, 1), + (141144, 1), + (141180, 1), + (141216, 1), + (141252, 1), + (141324, 1), + (141396, 1), + (141432, 1), + (141468, 1), + (141504, 1), + (141612, 1), + (142152, 1), + (142188, 1), + (142260, 1), + (142296, 1), + (142800, 1), + (143304, 1), + (143376, 1), + (143448, 1), + (143520, 1), + (143592, 1), + (143664, 1), + (143700, 1), + (143736, 1), + (143772, 1), + (143808, 1), + (143844, 1), + (143880, 1), + (143952, 1), + (144096, 1), + (144240, 1), + (144348, 1), + (144456, 1), + (144564, 1), + (144672, 1), + (144708, 1), + (144744, 1), + (144780, 1), + (144816, 1), + (144852, 1), + (144888, 1), + (144924, 1), + (144960, 1), + (144996, 1), + (145032, 1), + (145068, 1), + (145104, 1), + (145140, 1), + (145176, 1), + (145212, 1), + (145248, 1), + (145284, 1), + (145320, 1), + (145356, 1), + (145392, 1), + (145464, 1), + (145500, 1), + (145536, 1), + (145572, 1), + (145644, 1), + (145716, 1), + (145752, 1), + (145788, 1), + (145824, 1), + (145860, 1), + (145896, 1), + (145932, 1), + (145968, 1), + (146004, 1), + (146040, 1), + (146076, 1), + (146112, 1), + (146148, 1), + (146184, 1), + (146220, 1), + (146256, 1), + (146292, 1), + (146328, 1), + (146364, 1), + (146400, 1), + (146436, 1), + (146472, 1), + (146508, 1), + (146544, 1), + (146580, 1), + (146616, 1), + (146652, 1), + (146688, 1), + (146724, 1), + (146760, 1), + (146796, 1), + (146832, 1), + (146868, 1), + (146940, 1), + (146976, 1), + (147012, 1), + (147048, 1), + (147084, 1), + (147120, 1), + (147156, 1), + (147192, 1), + (147228, 1), + (147264, 1), + (147300, 1), + (147336, 1), + (147372, 1), + (147408, 1), + (147444, 1), + (147480, 1), + (147516, 1), + (147552, 1), + (147588, 1), + (147624, 1), + (147660, 1), + (147732, 1), + (147768, 1), + (147804, 1), + (147840, 1), + (147876, 1), + (147912, 1), + (147948, 1), + (147984, 1), + (148020, 1), + (148056, 1), + (148092, 1), + (148128, 1), + (148164, 1), + (148200, 1), + (148236, 1), + (148272, 1), + (1070556, 1), + (1079378, 1), + (1085421, 1), + (1086835, 1), + (1121118, 1), + (1121208, 1), + (1124515, 1), + (1128287, 1), + (1128379, 1), + (1153308, 1), + (1153342, 4), + (1153344, 5), + (1153398, 1), + (1153571, 1), + (1153663, 1), + (1153670, 1), + (1153672, 3), + (1153688, 3), + (1154504, 1), + (1154538, 5), + (1154540, 6), + (1154596, 1), + (1164963, 1), + (1165053, 1), + (1166494, 1), + (1166586, 1), + (1175528, 1), + (1175636, 1), + (1177016, 1), + (1193653, 1), + (1193743, 1), + (1205060, 1), + (1205152, 1), + (1323322, 1), + (1323414, 1), + (1336354, 1), + (1336444, 1), + (1348925, 1), + (1349015, 1), + (1353326, 1), + (1353418, 1), + (1426757, 1), + (1426845, 1), + (1426847, 1), + (1426937, 1), + (1476463, 1), + (1476553, 1), + (1516580, 1), + (1516670, 1), + (1605731, 1), + (1605821, 1), +]; \ No newline at end of file diff --git a/bin/node/bench/src/tempdb.rs b/bin/node/bench/src/tempdb.rs new file mode 100644 index 0000000000000000000000000000000000000000..067db6a5652bf909940aad0a68ef2dfd720d31fc --- /dev/null +++ b/bin/node/bench/src/tempdb.rs @@ -0,0 +1,68 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::sync::Arc; +use kvdb::KeyValueDB; +use kvdb_rocksdb::{DatabaseConfig, Database}; + +pub struct TempDatabase(tempfile::TempDir); + +impl TempDatabase { + pub fn new() -> Self { + let dir = tempfile::tempdir().expect("temp dir creation failed"); + log::trace!( + target: "bench-logistics", + "Created temp db at {}", + dir.path().to_string_lossy(), + ); + + TempDatabase(dir) + } + + pub fn open(&mut self) -> Arc { + let db_cfg = DatabaseConfig::with_columns(1); + let db = Database::open(&db_cfg, &self.0.path().to_string_lossy()).expect("Database backend error"); + Arc::new(db) + } +} + +impl Clone for TempDatabase { + fn clone(&self) -> Self { + let new_dir = tempfile::tempdir().expect("temp dir creation failed"); + let self_dir = self.0.path(); + + log::trace!( + target: "bench-logistics", + "Cloning db ({}) to {}", + self_dir.to_string_lossy(), + new_dir.path().to_string_lossy(), + ); + let self_db_files = std::fs::read_dir(self_dir) + .expect("failed to list file in seed dir") + .map(|f_result| + f_result.expect("failed to read file in seed db") + .path() + .clone() + ).collect(); + fs_extra::copy_items( + &self_db_files, + new_dir.path(), + &fs_extra::dir::CopyOptions::new(), + ).expect("Copy of seed database is ok"); + + TempDatabase(new_dir) + } +} diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs new file mode 100644 index 0000000000000000000000000000000000000000..3280618fb6a557e5b73ac2d6c3c7285b197f10d0 --- /dev/null +++ b/bin/node/bench/src/trie.rs @@ -0,0 +1,366 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Trie benchmark (integrated). + +use std::{borrow::Cow, collections::HashMap, sync::Arc}; +use kvdb::KeyValueDB; +use lazy_static::lazy_static; +use rand::Rng; +use hash_db::Prefix; +use sp_state_machine::Backend as _; +use sp_trie::{trie_types::TrieDBMut, TrieMut as _}; + +use node_primitives::Hash; + +use crate::{ + core::{self, Mode, Path}, + generator::generate_trie, + simple_trie::SimpleTrie, + tempdb::TempDatabase, +}; + +pub const SAMPLE_SIZE: usize = 100; +pub const TEST_WRITE_SIZE: usize = 128; + +pub type KeyValue = (Vec, Vec); +pub type KeyValues = Vec; + +#[derive(Clone, Copy, Debug, derive_more::Display)] +pub enum DatabaseSize { + #[display(fmt = "empty")] + Empty, + #[display(fmt = "smallest")] + Smallest, + #[display(fmt = "small")] + Small, + #[display(fmt = "medium")] + Medium, + #[display(fmt = "large")] + Large, + #[display(fmt = "huge")] + Huge, +} + +lazy_static! { + static ref KUSAMA_STATE_DISTRIBUTION: SizePool = + SizePool::from_histogram(crate::state_sizes::KUSAMA_STATE_DISTRIBUTION); +} + +impl DatabaseSize { + /// Should be multiple of SAMPLE_SIZE! + fn keys(&self) -> usize { + let val = match *self { + Self::Empty => 200, // still need some keys to query + Self::Smallest => 1_000, + Self::Small => 10_000, + Self::Medium => 100_000, + Self::Large => 200_000, + Self::Huge => 1_000_000, + }; + + assert_eq!(val % SAMPLE_SIZE, 0); + + val + } +} + +fn pretty_print(v: usize) -> String { + let mut print = String::new(); + for (idx, val) in v.to_string().chars().rev().enumerate() { + if idx != 0 && idx % 3 == 0 { + print.insert(0, ','); + } + print.insert(0, val); + } + print +} + +pub struct TrieReadBenchmarkDescription { + pub database_size: DatabaseSize, +} + +pub struct TrieReadBenchmark { + database: TempDatabase, + root: Hash, + warmup_keys: KeyValues, + query_keys: KeyValues, +} + +impl core::BenchmarkDescription for TrieReadBenchmarkDescription { + fn path(&self) -> Path { + let mut path = Path::new(&["trie", "read"]); + path.push(&format!("{}", self.database_size)); + path + } + + fn setup(self: Box) -> Box { + let mut database = TempDatabase::new(); + + let mut rng = rand::thread_rng(); + let warmup_prefix = KUSAMA_STATE_DISTRIBUTION.key(&mut rng); + + let mut key_values = KeyValues::new(); + let mut warmup_keys = KeyValues::new(); + let mut query_keys = KeyValues::new(); + let every_x_key = self.database_size.keys() / SAMPLE_SIZE; + for idx in 0..self.database_size.keys() { + let kv = ( + KUSAMA_STATE_DISTRIBUTION.key(&mut rng).to_vec(), + KUSAMA_STATE_DISTRIBUTION.value(&mut rng), + ); + if idx % every_x_key == 0 { + // warmup keys go to separate tree with high prob + let mut actual_warmup_key = warmup_prefix.clone(); + actual_warmup_key[16..].copy_from_slice(&kv.0[16..]); + warmup_keys.push((actual_warmup_key.clone(), kv.1.clone())); + key_values.push((actual_warmup_key.clone(), kv.1.clone())); + } else if idx % every_x_key == 1 { + query_keys.push(kv.clone()); + } + + key_values.push(kv) + } + + assert_eq!(warmup_keys.len(), SAMPLE_SIZE); + assert_eq!(query_keys.len(), SAMPLE_SIZE); + + let root = generate_trie( + database.open(), + key_values, + ); + + Box::new(TrieReadBenchmark { + database, + root, + warmup_keys, + query_keys, + }) + } + + fn name(&self) -> Cow<'static, str> { + format!( + "Trie read benchmark({} database ({} keys))", + self.database_size, + pretty_print(self.database_size.keys()), + ).into() + } +} + +struct Storage(Arc); + +impl sp_state_machine::Storage for Storage { + fn get(&self, key: &Hash, prefix: Prefix) -> Result>, String> { + let key = sp_trie::prefixed_key::(key, prefix); + self.0.get(0, &key).map_err(|e| format!("Database backend error: {:?}", e)) + } +} + +impl core::Benchmark for TrieReadBenchmark { + fn run(&mut self, mode: Mode) -> std::time::Duration { + let mut db = self.database.clone(); + let storage: Arc> = + Arc::new(Storage(db.open())); + + let trie_backend = sp_state_machine::TrieBackend::new( + storage, + self.root, + ); + for (warmup_key, warmup_value) in self.warmup_keys.iter() { + let value = trie_backend.storage(&warmup_key[..]) + .expect("Failed to get key: db error") + .expect("Warmup key should exist"); + + // sanity for warmup keys + assert_eq!(&value, warmup_value); + } + + if mode == Mode::Profile { + std::thread::park_timeout(std::time::Duration::from_secs(3)); + } + + let started = std::time::Instant::now(); + for (key, _) in self.query_keys.iter() { + let _ = trie_backend.storage(&key[..]); + } + let elapsed = started.elapsed(); + + if mode == Mode::Profile { + std::thread::park_timeout(std::time::Duration::from_secs(1)); + } + + elapsed / (SAMPLE_SIZE as u32) + } +} + +pub struct TrieWriteBenchmarkDescription { + pub database_size: DatabaseSize, +} + +impl core::BenchmarkDescription for TrieWriteBenchmarkDescription { + fn path(&self) -> Path { + let mut path = Path::new(&["trie", "write"]); + path.push(&format!("{}", self.database_size)); + path + } + + fn setup(self: Box) -> Box { + let mut database = TempDatabase::new(); + + let mut rng = rand::thread_rng(); + let warmup_prefix = KUSAMA_STATE_DISTRIBUTION.key(&mut rng); + + let mut key_values = KeyValues::new(); + let mut warmup_keys = KeyValues::new(); + let every_x_key = self.database_size.keys() / SAMPLE_SIZE; + for idx in 0..self.database_size.keys() { + let kv = ( + KUSAMA_STATE_DISTRIBUTION.key(&mut rng).to_vec(), + KUSAMA_STATE_DISTRIBUTION.value(&mut rng), + ); + if idx % every_x_key == 0 { + // warmup keys go to separate tree with high prob + let mut actual_warmup_key = warmup_prefix.clone(); + actual_warmup_key[16..].copy_from_slice(&kv.0[16..]); + warmup_keys.push((actual_warmup_key.clone(), kv.1.clone())); + key_values.push((actual_warmup_key.clone(), kv.1.clone())); + } + + key_values.push(kv) + } + + assert_eq!(warmup_keys.len(), SAMPLE_SIZE); + + let root = generate_trie( + database.open(), + key_values, + ); + + Box::new(TrieWriteBenchmark { + database, + root, + warmup_keys, + }) + } + + fn name(&self) -> Cow<'static, str> { + format!( + "Trie write benchmark({} database ({} keys))", + self.database_size, + pretty_print(self.database_size.keys()), + ).into() + } +} + +struct TrieWriteBenchmark { + database: TempDatabase, + root: Hash, + warmup_keys: KeyValues, +} + +impl core::Benchmark for TrieWriteBenchmark { + fn run(&mut self, mode: Mode) -> std::time::Duration { + let mut rng = rand::thread_rng(); + let mut db = self.database.clone(); + let kvdb = db.open(); + + let mut new_root = self.root.clone(); + + let mut overlay = HashMap::new(); + let mut trie = SimpleTrie { + db: kvdb.clone(), + overlay: &mut overlay, + }; + let mut trie_db_mut = TrieDBMut::from_existing(&mut trie, &mut new_root) + .expect("Failed to create TrieDBMut"); + + for (warmup_key, warmup_value) in self.warmup_keys.iter() { + let value = trie_db_mut.get(&warmup_key[..]) + .expect("Failed to get key: db error") + .expect("Warmup key should exist"); + + // sanity for warmup keys + assert_eq!(&value, warmup_value); + } + + let test_key = random_vec(&mut rng, 32); + let test_val = random_vec(&mut rng, TEST_WRITE_SIZE); + + if mode == Mode::Profile { + std::thread::park_timeout(std::time::Duration::from_secs(3)); + } + + let started = std::time::Instant::now(); + + trie_db_mut.insert(&test_key, &test_val).expect("Should be inserted ok"); + trie_db_mut.commit(); + drop(trie_db_mut); + + let mut transaction = kvdb.transaction(); + for (key, value) in overlay.into_iter() { + match value { + Some(value) => transaction.put(0, &key[..], &value[..]), + None => transaction.delete(0, &key[..]), + } + } + kvdb.write(transaction).expect("Failed to write transaction"); + + let elapsed = started.elapsed(); + + // sanity check + assert!(new_root != self.root); + + if mode == Mode::Profile { + std::thread::park_timeout(std::time::Duration::from_secs(1)); + } + + elapsed + } +} + +fn random_vec(rng: &mut R, len: usize) -> Vec { + let mut val = vec![0u8; len]; + rng.fill_bytes(&mut val[..]); + val +} + +struct SizePool { + distribution: std::collections::BTreeMap, + total: u32, +} + +impl SizePool { + fn from_histogram(h: &[(u32, u32)]) -> SizePool { + let mut distribution = std::collections::BTreeMap::default(); + let mut total = 0; + for (size, count) in h { + total += count; + distribution.insert(total, *size); + } + SizePool { distribution, total } + } + + fn value(&self, rng: &mut R) -> Vec { + let sr = (rng.next_u64() % self.total as u64) as u32; + let mut range = self.distribution.range((std::ops::Bound::Included(sr), std::ops::Bound::Unbounded)); + let size = *range.next().unwrap().1 as usize; + random_vec(rng, size) + } + + fn key(&self, rng: &mut R) -> Vec { + random_vec(rng, 32) + } +} diff --git a/bin/node/browser-testing/Cargo.toml b/bin/node/browser-testing/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..4b8d08af90a59f4aa5e0864585e45148d0b0298a --- /dev/null +++ b/bin/node/browser-testing/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "node-browser-testing" +version = "2.0.0" +authors = ["Parity Technologies "] +description = "Tests for the in-browser light client." +edition = "2018" +license = "GPL-3.0" + +[dependencies] +futures-timer = "3.0.2" +libp2p = { version = "0.18.0", default-features = false } +jsonrpc-core = "14.0.5" +serde = "1.0.106" +serde_json = "1.0.48" +wasm-bindgen = { version = "0.2.60", features = ["serde-serialize"] } +wasm-bindgen-futures = "0.4.10" +wasm-bindgen-test = "0.3.10" +futures = "0.3.4" + +node-cli = { path = "../cli", default-features = false, features = ["browser"] } +sc-rpc-api = { path = "../../../client/rpc-api" } diff --git a/bin/node/browser-testing/src/lib.rs b/bin/node/browser-testing/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..65f1e4620ba9c8aa59164e8776f051d513f7489d --- /dev/null +++ b/bin/node/browser-testing/src/lib.rs @@ -0,0 +1,69 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! # Running +//! Running this test can be done with +//! ```text +//! wasm-pack test --firefox --release --headless bin/node/browser-testing +//! ``` +//! or (without `wasm-pack`) +//! ```text +//! CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER=wasm-bindgen-test-runner WASM_BINDGEN_TEST_TIMEOUT=60 cargo test --target wasm32-unknown-unknown +//! ``` +//! For debug infomation, such as the informant, run without the `--headless` +//! flag and open a browser to the url that `wasm-pack test` outputs. +//! For more infomation see https://rustwasm.github.io/docs/wasm-pack/. + +use wasm_bindgen_test::{wasm_bindgen_test, wasm_bindgen_test_configure}; +use wasm_bindgen_futures::JsFuture; +use wasm_bindgen::JsValue; +use jsonrpc_core::types::{MethodCall, Success, Version, Params, Id}; +use serde::de::DeserializeOwned; + +wasm_bindgen_test_configure!(run_in_browser); + +fn rpc_call(method: &str) -> String { + serde_json::to_string(&MethodCall { + jsonrpc: Some(Version::V2), + method: method.into(), + params: Params::None, + id: Id::Num(1) + }).unwrap() +} + +fn deserialize_rpc_result(js_value: JsValue) -> T { + let string = js_value.as_string().unwrap(); + let value = serde_json::from_str::(&string).unwrap().result; + // We need to convert a `Value::Object` into a proper type. + let value_string = serde_json::to_string(&value).unwrap(); + serde_json::from_str(&value_string).unwrap() +} + +#[wasm_bindgen_test] +async fn runs() { + let mut client = node_cli::start_client(None, "info".into()) + .await + .unwrap(); + + // Check that the node handles rpc calls. + // TODO: Re-add the code that checks if the node is syncing. + let chain_name: String = deserialize_rpc_result( + JsFuture::from(client.rpc_send(&rpc_call("system_chain"))) + .await + .unwrap() + ); + assert_eq!(chain_name, "Development"); +} diff --git a/bin/node/browser-testing/webdriver.json b/bin/node/browser-testing/webdriver.json new file mode 100644 index 0000000000000000000000000000000000000000..417ac35a7bccd35a3c18135787c9c1506b48a21b --- /dev/null +++ b/bin/node/browser-testing/webdriver.json @@ -0,0 +1,7 @@ +{ + "goog:chromeOptions": { + "args": [ + "--whitelisted-ips=127.0.0.1" + ] + } +} diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index c477471166156e650c07cc5094774e2c7b22f824..af71de7570a30295024fef0c6182bbfd6f3915ee 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "node-cli" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] -description = "Substrate node implementation in Rust." +description = "Generic Substrate node implementation in Rust." build = "build.rs" edition = "2018" license = "GPL-3.0" @@ -15,6 +15,9 @@ repository = "https://github.com/paritytech/substrate/" # https://github.com/rustwasm/wasm-pack/issues/781 etc. wasm-opt = false +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [badges] travis-ci = { repository = "paritytech/substrate" } maintenance = { status = "actively-developed" } @@ -42,98 +45,97 @@ structopt = { version = "0.3.8", optional = true } tracing = "0.1.10" # primitives -sp-authority-discovery = { version = "2.0.0-alpha.5", path = "../../../primitives/authority-discovery" } -sp-consensus-babe = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/babe" } -grandpa-primitives = { version = "2.0.0-alpha.5", package = "sp-finality-grandpa", path = "../../../primitives/finality-grandpa" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sp-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/timestamp" } -sp-finality-tracker = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/finality-tracker" } -sp-inherents = { version = "2.0.0-alpha.5", path = "../../../primitives/inherents" } -sp-keyring = { version = "2.0.0-alpha.5", path = "../../../primitives/keyring" } -sp-io = { version = "2.0.0-alpha.5", path = "../../../primitives/io" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } +sp-authority-discovery = { version = "2.0.0-dev", path = "../../../primitives/authority-discovery" } +sp-consensus-babe = { version = "0.8.0-dev", path = "../../../primitives/consensus/babe" } +grandpa-primitives = { version = "2.0.0-dev", package = "sp-finality-grandpa", path = "../../../primitives/finality-grandpa" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-timestamp = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/timestamp" } +sp-finality-tracker = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/finality-tracker" } +sp-inherents = { version = "2.0.0-dev", path = "../../../primitives/inherents" } +sp-keyring = { version = "2.0.0-dev", path = "../../../primitives/keyring" } +sp-io = { version = "2.0.0-dev", path = "../../../primitives/io" } +sp-consensus = { version = "0.8.0-dev", path = "../../../primitives/consensus/common" } +sp-transaction-pool = { version = "2.0.0-dev", path = "../../../primitives/transaction-pool" } # client dependencies -sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api" } -sc-client = { version = "0.8.0-alpha.5", path = "../../../client/" } -sc-chain-spec = { version = "2.0.0-alpha.5", path = "../../../client/chain-spec" } -sc-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../client/transaction-pool" } -sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../primitives/transaction-pool" } -sc-network = { version = "0.8.0-alpha.5", path = "../../../client/network" } -sc-consensus-babe = { version = "0.8.0-alpha.5", path = "../../../client/consensus/babe" } -grandpa = { version = "0.8.0-alpha.5", package = "sc-finality-grandpa", path = "../../../client/finality-grandpa" } -sc-client-db = { version = "0.8.0-alpha.5", default-features = false, path = "../../../client/db" } -sc-offchain = { version = "2.0.0-alpha.5", path = "../../../client/offchain" } -sc-rpc = { version = "2.0.0-alpha.5", path = "../../../client/rpc" } -sc-basic-authorship = { version = "0.8.0-alpha.5", path = "../../../client/basic-authorship" } -sc-service = { version = "0.8.0-alpha.5", default-features = false, path = "../../../client/service" } -sc-tracing = { version = "2.0.0-alpha.5", path = "../../../client/tracing" } -sc-telemetry = { version = "2.0.0-alpha.5", path = "../../../client/telemetry" } -sc-authority-discovery = { version = "0.8.0-alpha.5", path = "../../../client/authority-discovery" } +sc-client-api = { version = "2.0.0-dev", path = "../../../client/api" } +sc-chain-spec = { version = "2.0.0-dev", path = "../../../client/chain-spec" } +sc-consensus = { version = "0.8.0-dev", path = "../../../client/consensus/common" } +sc-transaction-pool = { version = "2.0.0-dev", path = "../../../client/transaction-pool" } +sc-network = { version = "0.8.0-dev", path = "../../../client/network" } +sc-consensus-babe = { version = "0.8.0-dev", path = "../../../client/consensus/babe" } +grandpa = { version = "0.8.0-dev", package = "sc-finality-grandpa", path = "../../../client/finality-grandpa" } +sc-client-db = { version = "0.8.0-dev", default-features = false, path = "../../../client/db" } +sc-offchain = { version = "2.0.0-dev", path = "../../../client/offchain" } +sc-rpc = { version = "2.0.0-dev", path = "../../../client/rpc" } +sc-basic-authorship = { version = "0.8.0-dev", path = "../../../client/basic-authorship" } +sc-service = { version = "0.8.0-dev", default-features = false, path = "../../../client/service" } +sc-tracing = { version = "2.0.0-dev", path = "../../../client/tracing" } +sc-telemetry = { version = "2.0.0-dev", path = "../../../client/telemetry" } +sc-authority-discovery = { version = "0.8.0-dev", path = "../../../client/authority-discovery" } # frame dependencies -pallet-indices = { version = "2.0.0-alpha.5", path = "../../../frame/indices" } -pallet-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/timestamp" } -pallet-contracts = { version = "2.0.0-alpha.5", path = "../../../frame/contracts" } -frame-system = { version = "2.0.0-alpha.5", path = "../../../frame/system" } -pallet-balances = { version = "2.0.0-alpha.5", path = "../../../frame/balances" } -pallet-transaction-payment = { version = "2.0.0-alpha.5", path = "../../../frame/transaction-payment" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/support" } -pallet-im-online = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/im-online" } -pallet-authority-discovery = { version = "2.0.0-alpha.5", path = "../../../frame/authority-discovery" } -pallet-staking = { version = "2.0.0-alpha.5", path = "../../../frame/staking" } +pallet-indices = { version = "2.0.0-dev", path = "../../../frame/indices" } +pallet-timestamp = { version = "2.0.0-dev", default-features = false, path = "../../../frame/timestamp" } +pallet-contracts = { version = "2.0.0-dev", path = "../../../frame/contracts" } +frame-system = { version = "2.0.0-dev", path = "../../../frame/system" } +pallet-balances = { version = "2.0.0-dev", path = "../../../frame/balances" } +pallet-transaction-payment = { version = "2.0.0-dev", path = "../../../frame/transaction-payment" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../../../frame/support" } +pallet-im-online = { version = "2.0.0-dev", default-features = false, path = "../../../frame/im-online" } +pallet-authority-discovery = { version = "2.0.0-dev", path = "../../../frame/authority-discovery" } +pallet-staking = { version = "2.0.0-dev", path = "../../../frame/staking" } # node-specific dependencies -node-runtime = { version = "2.0.0-alpha.5", path = "../runtime" } -node-rpc = { version = "2.0.0-alpha.5", path = "../rpc" } -node-primitives = { version = "2.0.0-alpha.5", path = "../primitives" } -node-executor = { version = "2.0.0-alpha.5", path = "../executor" } +node-runtime = { version = "2.0.0-dev", path = "../runtime" } +node-rpc = { version = "2.0.0-dev", path = "../rpc" } +node-primitives = { version = "2.0.0-dev", path = "../primitives" } +node-executor = { version = "2.0.0-dev", path = "../executor" } # CLI-specific dependencies -sc-cli = { version = "0.8.0-alpha.5", optional = true, path = "../../../client/cli" } -frame-benchmarking-cli = { version = "2.0.0-alpha.5", optional = true, path = "../../../utils/frame/benchmarking-cli" } -node-transaction-factory = { version = "0.8.0-alpha.5", optional = true, path = "../transaction-factory" } -node-inspect = { version = "0.8.0-alpha.5", optional = true, path = "../inspect" } +sc-cli = { version = "0.8.0-dev", optional = true, path = "../../../client/cli" } +frame-benchmarking-cli = { version = "2.0.0-dev", optional = true, path = "../../../utils/frame/benchmarking-cli" } +node-transaction-factory = { version = "0.8.0-dev", optional = true, path = "../transaction-factory" } +node-inspect = { version = "0.8.0-dev", optional = true, path = "../inspect" } # WASM-specific dependencies wasm-bindgen = { version = "0.2.57", optional = true } wasm-bindgen-futures = { version = "0.4.7", optional = true } -browser-utils = { package = "substrate-browser-utils", path = "../../../utils/browser", optional = true, version = "0.8.0-alpha.5"} +browser-utils = { package = "substrate-browser-utils", path = "../../../utils/browser", optional = true, version = "0.8.0-dev"} [target.'cfg(target_arch="x86_64")'.dependencies] -node-executor = { version = "2.0.0-alpha.4", path = "../executor", features = [ "wasmtime" ] } -sc-cli = { version = "0.8.0-alpha.4", optional = true, path = "../../../client/cli", features = [ "wasmtime" ] } -sc-service = { version = "0.8.0-alpha.4", default-features = false, path = "../../../client/service", features = [ "wasmtime" ] } +node-executor = { version = "2.0.0-dev", path = "../executor", features = [ "wasmtime" ] } +sc-cli = { version = "0.8.0-dev", optional = true, path = "../../../client/cli", features = [ "wasmtime" ] } +sc-service = { version = "0.8.0-dev", default-features = false, path = "../../../client/service", features = [ "wasmtime" ] } [dev-dependencies] -sc-keystore = { version = "2.0.0-alpha.5", path = "../../../client/keystore" } -sc-consensus-babe = { version = "0.8.0-alpha.5", features = ["test-helpers"], path = "../../../client/consensus/babe" } -sc-consensus-epochs = { version = "0.8.0-alpha.5", path = "../../../client/consensus/epochs" } +sc-keystore = { version = "2.0.0-dev", path = "../../../client/keystore" } +sc-consensus = { version = "0.8.0-dev", path = "../../../client/consensus/common" } +sc-consensus-babe = { version = "0.8.0-dev", features = ["test-helpers"], path = "../../../client/consensus/babe" } +sc-consensus-epochs = { version = "0.8.0-dev", path = "../../../client/consensus/epochs" } sc-service-test = { version = "2.0.0-dev", path = "../../../client/service/test" } futures = "0.3.4" tempfile = "3.1.0" assert_cmd = "1.0" nix = "0.17" serde_json = "1.0" +regex = "1" +platforms = "0.2.1" [build-dependencies] -build-script-utils = { version = "2.0.0-alpha.5", package = "substrate-build-script-utils", path = "../../../utils/build-script-utils" } structopt = { version = "0.3.8", optional = true } -node-transaction-factory = { version = "0.8.0-alpha.5", optional = true, path = "../transaction-factory" } -node-inspect = { version = "0.8.0-alpha.5", optional = true, path = "../inspect" } -frame-benchmarking-cli = { version = "2.0.0-alpha.5", optional = true, path = "../../../utils/frame/benchmarking-cli" } +node-transaction-factory = { version = "0.8.0-dev", optional = true, path = "../transaction-factory" } +node-inspect = { version = "0.8.0-dev", optional = true, path = "../inspect" } +frame-benchmarking-cli = { version = "2.0.0-dev", optional = true, path = "../../../utils/frame/benchmarking-cli" } +substrate-build-script-utils = { version = "2.0.0-dev", optional = true, path = "../../../utils/build-script-utils" } [build-dependencies.sc-cli] -version = "0.8.0-alpha.5" +version = "0.8.0-dev" package = "sc-cli" path = "../../../client/cli" optional = true -[build-dependencies.vergen] -version = "3.0.4" -optional = true - [features] default = [ "cli" ] browser = [ @@ -147,11 +149,11 @@ cli = [ "node-transaction-factory", "sc-cli", "frame-benchmarking-cli", - "sc-service/rocksdb", + "sc-service/db", "structopt", - "vergen", + "substrate-build-script-utils", +] +runtime-benchmarks = [ + "node-runtime/runtime-benchmarks", + "frame-benchmarking-cli", ] -runtime-benchmarks = [ "node-runtime/runtime-benchmarks" ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node/cli/bin/main.rs b/bin/node/cli/bin/main.rs index 8c4412667baceec56904864413178cbc88001497..cfad84a4cb5520f2542dc18da771d4cfa4a0a9ed 100644 --- a/bin/node/cli/bin/main.rs +++ b/bin/node/cli/bin/main.rs @@ -19,16 +19,5 @@ #![warn(missing_docs)] fn main() -> sc_cli::Result<()> { - let version = sc_cli::VersionInfo { - name: "Substrate Node", - commit: env!("VERGEN_SHA_SHORT"), - version: env!("CARGO_PKG_VERSION"), - executable_name: "substrate", - author: "Parity Technologies ", - description: "Generic substrate node", - support_url: "https://github.com/paritytech/substrate/issues/new", - copyright_start_year: 2017, - }; - - node_cli::run(std::env::args(), version) + node_cli::run() } diff --git a/bin/node/cli/browser-demo/README.md b/bin/node/cli/browser-demo/README.md index 2ff1cc54f5dbe308a423c0bf90bccf49ecf4a1d0..08f1646f114a3104fa1cac14e83c03306860e7c9 100644 --- a/bin/node/cli/browser-demo/README.md +++ b/bin/node/cli/browser-demo/README.md @@ -1,9 +1,6 @@ # How to run this demo ```sh -cargo install wasm-pack # If necessary - -wasm-pack build --target web --out-dir ./browser-demo/pkg --no-typescript --release ./.. -- --no-default-features --features "browser" - -xdg-open index.html +cargo install wasm-bindgen-cli # If necessary +./build.sh ``` diff --git a/bin/node/cli/browser-demo/build.sh b/bin/node/cli/browser-demo/build.sh index 059ed9fe423b0ff030a1096b108b5d80efb3772d..be52b7a523f0177728181bfa18b8aef614185a26 100755 --- a/bin/node/cli/browser-demo/build.sh +++ b/bin/node/cli/browser-demo/build.sh @@ -1,3 +1,4 @@ #!/usr/bin/env sh -wasm-pack build --target web --out-dir ./browser-demo/pkg --no-typescript --release ./.. -- --no-default-features --features "browser" +cargo +nightly build --release -p node-cli --target wasm32-unknown-unknown --no-default-features --features browser -Z features=itarget +wasm-bindgen ../../../../target/wasm32-unknown-unknown/release/node_cli.wasm --out-dir pkg --target web python -m http.server 8000 diff --git a/bin/node/cli/build.rs b/bin/node/cli/build.rs index e824b59be64f3dedba3b738a6a3908b9e48b5091..12e0cab58ada5797c069e9a24970be6daa06de4f 100644 --- a/bin/node/cli/build.rs +++ b/bin/node/cli/build.rs @@ -24,14 +24,14 @@ mod cli { include!("src/cli.rs"); use std::{fs, env, path::Path}; - use sc_cli::{structopt::clap::Shell}; - use vergen::{ConstantsFlags, generate_cargo_keys}; + use sc_cli::structopt::clap::Shell; + use substrate_build_script_utils::{generate_cargo_keys, rerun_if_git_head_changed}; pub fn main() { build_shell_completion(); - generate_cargo_keys(ConstantsFlags::all()).expect("Failed to generate metadata files"); + generate_cargo_keys(); - build_script_utils::rerun_if_git_head_changed(); + rerun_if_git_head_changed(); } /// Build shell completion scripts for all known shells diff --git a/bin/node/cli/res/flaming-fir.json b/bin/node/cli/res/flaming-fir.json index 7ed98239b54b62c2a3c6309d96b13dedcdb1e65d..3612d7284faba2dcaf6ec9e14dd79162164fb086 100644 --- a/bin/node/cli/res/flaming-fir.json +++ b/bin/node/cli/res/flaming-fir.json @@ -134,7 +134,7 @@ "0x5f3e4907f716ac89b6347d15ececedca0b6a45321efae92aea15e0740ec7afe7": "0x00000000", "0x5f3e4907f716ac89b6347d15ececedca9220e172bed316605f73f1ff7b4ade98e54094c2d5af8ae10b91e1288f4f59f2946d7738f2c509b7effd909e5e9ba0ad": "0x00" }, - "children": {} + "childrenDefault": {} } } } diff --git a/bin/node/cli/src/browser.rs b/bin/node/cli/src/browser.rs index e7dd1e78b4fb250eb6e1ead9932290ddab6fb0e9..861d37c6058e1aff0640da97a6ad2fdce622abd2 100644 --- a/bin/node/cli/src/browser.rs +++ b/bin/node/cli/src/browser.rs @@ -17,7 +17,6 @@ use crate::chain_spec::ChainSpec; use log::info; use wasm_bindgen::prelude::*; -use sc_service::Configuration; use browser_utils::{ Client, browser_configuration, set_console_error_panic_hook, init_console_log, @@ -26,25 +25,28 @@ use std::str::FromStr; /// Starts the client. #[wasm_bindgen] -pub async fn start_client(chain_spec: String, log_level: String) -> Result { +pub async fn start_client(chain_spec: Option, log_level: String) -> Result { start_inner(chain_spec, log_level) .await .map_err(|err| JsValue::from_str(&err.to_string())) } -async fn start_inner(chain_spec: String, log_level: String) -> Result> { +async fn start_inner(chain_spec: Option, log_level: String) -> Result> { set_console_error_panic_hook(); init_console_log(log::Level::from_str(&log_level)?)?; - let chain_spec = ChainSpec::from_json_bytes(chain_spec.as_bytes().to_vec()) - .map_err(|e| format!("{:?}", e))?; + let chain_spec = match chain_spec { + Some(chain_spec) => ChainSpec::from_json_bytes(chain_spec.as_bytes().to_vec()) + .map_err(|e| format!("{:?}", e))?, + None => crate::chain_spec::development_config(), + }; let config = browser_configuration(chain_spec).await?; info!("Substrate browser node"); - info!("✌️ version {}", config.full_version()); + info!("✌️ version {}", config.impl_version); info!("❤️ by Parity Technologies, 2017-2020"); - info!("📋 Chain specification: {}", config.expect_chain_spec().name()); - info!("🏷 Node name: {}", config.name); + info!("📋 Chain specification: {}", config.chain_spec.name()); + info!("🏷 Node name: {}", config.network.node_name); info!("👤 Role: {:?}", config.role); // Create the service. This is the most heavy initialization step. diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index 700556206d46072b778fc371a89fa60777e42e4f..ea3999fa377187958f7e28bf87376a7eaf83a93b 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -27,7 +27,7 @@ use node_runtime::{ }; use node_runtime::Block; use node_runtime::constants::currency::*; -use sc_service; +use sc_service::ChainType; use hex_literal::hex; use sc_telemetry::TelemetryEndpoints; use grandpa_primitives::{AuthorityId as GrandpaId}; @@ -51,9 +51,9 @@ const STAGING_TELEMETRY_URL: &str = "wss://telemetry.polkadot.io/submit/"; #[serde(rename_all = "camelCase")] pub struct Extensions { /// Block numbers with known hashes. - pub fork_blocks: sc_client::ForkBlocks, + pub fork_blocks: sc_client_api::ForkBlocks, /// Known bad block hashes. - pub bad_blocks: sc_client::BadBlocks, + pub bad_blocks: sc_client_api::BadBlocks, } /// Specialized `ChainSpec`. @@ -158,6 +158,7 @@ pub fn staging_testnet_config() -> ChainSpec { ChainSpec::from_genesis( "Staging Testnet", "staging_testnet", + ChainType::Live, staging_testnet_config_genesis, boot_nodes, Some(TelemetryEndpoints::new(vec![(STAGING_TELEMETRY_URL.to_string(), 0)]) @@ -183,7 +184,7 @@ pub fn get_account_id_from_seed(seed: &str) -> AccountId where } /// Helper function to generate stash, controller and session key from seed -pub fn get_authority_keys_from_seed(seed: &str) -> ( +pub fn authority_keys_from_seed(seed: &str) -> ( AccountId, AccountId, GrandpaId, @@ -291,7 +292,6 @@ pub fn testnet_genesis( enable_println, // this should only be enabled on development chains ..Default::default() }, - gas_price: 1 * MILLICENTS, }), pallet_sudo: Some(SudoConfig { key: root_key, @@ -325,7 +325,7 @@ pub fn testnet_genesis( fn development_config_genesis() -> GenesisConfig { testnet_genesis( vec![ - get_authority_keys_from_seed("Alice"), + authority_keys_from_seed("Alice"), ], get_account_id_from_seed::("Alice"), None, @@ -338,6 +338,7 @@ pub fn development_config() -> ChainSpec { ChainSpec::from_genesis( "Development", "dev", + ChainType::Development, development_config_genesis, vec![], None, @@ -350,8 +351,8 @@ pub fn development_config() -> ChainSpec { fn local_testnet_genesis() -> GenesisConfig { testnet_genesis( vec![ - get_authority_keys_from_seed("Alice"), - get_authority_keys_from_seed("Bob"), + authority_keys_from_seed("Alice"), + authority_keys_from_seed("Bob"), ], get_account_id_from_seed::("Alice"), None, @@ -364,6 +365,7 @@ pub fn local_testnet_config() -> ChainSpec { ChainSpec::from_genesis( "Local Testnet", "local_testnet", + ChainType::Local, local_testnet_genesis, vec![], None, @@ -383,7 +385,7 @@ pub(crate) mod tests { fn local_testnet_genesis_instant_single() -> GenesisConfig { testnet_genesis( vec![ - get_authority_keys_from_seed("Alice"), + authority_keys_from_seed("Alice"), ], get_account_id_from_seed::("Alice"), None, @@ -396,6 +398,7 @@ pub(crate) mod tests { ChainSpec::from_genesis( "Integration Test", "test", + ChainType::Development, local_testnet_genesis_instant_single, vec![], None, @@ -410,6 +413,7 @@ pub(crate) mod tests { ChainSpec::from_genesis( "Integration Test", "test", + ChainType::Development, local_testnet_genesis, vec![], None, diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index b6db9c3deb7e31f8f8da62d30b6971bf572b30d1..44b18fd716337c0857ebdf50f442001db12034c0 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use sc_cli::{SharedParams, ImportParams, RunCmd}; +use sc_cli::{ImportParams, RunCmd, SharedParams}; use structopt::StructOpt; /// An overarching CLI command definition. @@ -50,10 +50,7 @@ pub enum Subcommand { Inspect(node_inspect::cli::InspectCmd), /// The custom benchmark subcommmand benchmarking runtime pallets. - #[structopt( - name = "benchmark", - about = "Benchmark runtime pallets." - )] + #[structopt(name = "benchmark", about = "Benchmark runtime pallets.")] Benchmark(frame_benchmarking_cli::BenchmarkCmd), } @@ -62,11 +59,11 @@ pub enum Subcommand { #[derive(Debug, StructOpt, Clone)] pub struct FactoryCmd { /// Number of blocks to generate. - #[structopt(long="blocks", default_value = "1")] + #[structopt(long = "blocks", default_value = "1")] pub blocks: u32, /// Number of transactions to push per block. - #[structopt(long="transactions", default_value = "8")] + #[structopt(long = "transactions", default_value = "8")] pub transactions: u32, #[allow(missing_docs)] diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 84d680b6f8024b94e9941620c1f6f676f1d2a7da..ab7d6ea65e8c276b3f1aaba552960265a3abd0f1 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -14,104 +14,136 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use sc_cli::VersionInfo; -use sc_service::{Role as ServiceRole}; +use crate::{chain_spec, factory_impl::FactoryState, service, Cli, FactoryCmd, Subcommand}; +use node_executor::Executor; +use node_runtime::{Block, RuntimeApi}; use node_transaction_factory::RuntimeAdapter; -use crate::{Cli, service, ChainSpec, load_spec, Subcommand, factory_impl::FactoryState}; +use sc_cli::{CliConfiguration, ImportParams, Result, SharedParams, SubstrateCli}; +use sc_service::Configuration; + +impl SubstrateCli for Cli { + fn impl_name() -> &'static str { + "Substrate Node" + } + + fn impl_version() -> &'static str { + env!("SUBSTRATE_CLI_IMPL_VERSION") + } + + fn description() -> &'static str { + env!("CARGO_PKG_DESCRIPTION") + } + + fn author() -> &'static str { + env!("CARGO_PKG_AUTHORS") + } + + fn support_url() -> &'static str { + "https://github.com/paritytech/substrate/issues/new" + } + + fn copyright_start_year() -> i32 { + 2017 + } + + fn executable_name() -> &'static str { + "substrate" + } + + fn load_spec(&self, id: &str) -> std::result::Result, String> { + Ok(match id { + "dev" => Box::new(chain_spec::development_config()), + "local" => Box::new(chain_spec::local_testnet_config()), + "" | "fir" | "flaming-fir" => Box::new(chain_spec::flaming_fir_config()?), + "staging" => Box::new(chain_spec::staging_testnet_config()), + path => Box::new(chain_spec::ChainSpec::from_json_file( + std::path::PathBuf::from(path), + )?), + }) + } +} /// Parse command line arguments into service configuration. -pub fn run(args: I, version: VersionInfo) -> sc_cli::Result<()> -where - I: Iterator, - T: Into + Clone, -{ +pub fn run() -> Result<()> { sc_cli::reset_signal_pipe_handler()?; - let args: Vec<_> = args.collect(); - let opt = sc_cli::from_iter::(args.clone(), &version); - - let mut config = sc_service::Configuration::from_version(&version); + let cli = Cli::from_args(); - match opt.subcommand { + match &cli.subcommand { None => { - opt.run.init(&version)?; - opt.run.update_config(&mut config, load_spec, &version)?; - opt.run.run( - config, + let runner = cli.create_runner(&cli.run)?; + runner.run_node( service::new_light, service::new_full, - &version, + node_runtime::VERSION ) - }, + } Some(Subcommand::Inspect(cmd)) => { - cmd.init(&version)?; - cmd.update_config(&mut config, load_spec, &version)?; + let runner = cli.create_runner(cmd)?; - let client = sc_service::new_full_client::< - node_runtime::Block, node_runtime::RuntimeApi, node_executor::Executor, - >(&config)?; - let inspect = node_inspect::Inspector::::new(client); - - cmd.run(inspect) - }, + runner.sync_run(|config| cmd.run::(config)) + } Some(Subcommand::Benchmark(cmd)) => { - cmd.init(&version)?; - cmd.update_config(&mut config, load_spec, &version)?; - - cmd.run::(config) - }, - Some(Subcommand::Factory(cli_args)) => { - cli_args.shared_params.init(&version)?; - cli_args.shared_params.update_config(&mut config, load_spec, &version)?; - cli_args.import_params.update_config( - &mut config, - &ServiceRole::Full, - cli_args.shared_params.dev, - )?; - - config.use_in_memory_keystore()?; - - match ChainSpec::from(config.expect_chain_spec().id()) { - Some(ref c) if c == &ChainSpec::Development || c == &ChainSpec::LocalTestnet => {}, - _ => return Err( - "Factory is only supported for development and local testnet.".into() - ), + if cfg!(feature = "runtime-benchmarks") { + let runner = cli.create_runner(cmd)?; + + runner.sync_run(|config| cmd.run::(config)) + } else { + println!("Benchmarking wasn't enabled when building the node. \ + You can enable it with `--features runtime-benchmarks`."); + Ok(()) } + } + Some(Subcommand::Factory(cmd)) => { + let runner = cli.create_runner(cmd)?; - // Setup tracing. - if let Some(tracing_targets) = cli_args.import_params.tracing_targets.as_ref() { - let subscriber = sc_tracing::ProfilingSubscriber::new( - cli_args.import_params.tracing_receiver.into(), tracing_targets - ); - if let Err(e) = tracing::subscriber::set_global_default(subscriber) { - return Err( - format!("Unable to set global default subscriber {}", e).into() - ); - } - } + runner.sync_run(|config| cmd.run(config)) + } + Some(Subcommand::Base(subcommand)) => { + let runner = cli.create_runner(subcommand)?; - let factory_state = FactoryState::new( - cli_args.blocks, - cli_args.transactions, - ); + runner.run_subcommand(subcommand, |config| Ok(new_full_start!(config).0)) + } + } +} - let service_builder = new_full_start!(config).0; - node_transaction_factory::factory( - factory_state, - service_builder.client(), - service_builder.select_chain() - .expect("The select_chain is always initialized by new_full_start!; QED") - ).map_err(|e| format!("Error in transaction factory: {}", e))?; +impl CliConfiguration for FactoryCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } - Ok(()) - }, - Some(Subcommand::Base(subcommand)) => { - subcommand.init(&version)?; - subcommand.update_config(&mut config, load_spec, &version)?; - subcommand.run( - config, - |config: sc_service::Configuration| Ok(new_full_start!(config).0), - ) - }, + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) + } +} + +impl FactoryCmd { + fn run(&self, config: Configuration) -> Result<()> { + match config.chain_spec.id() { + "dev" | "local" => {} + _ => return Err("Factory is only supported for development and local testnet.".into()), + } + + // Setup tracing. + if let Some(tracing_targets) = self.import_params.tracing_targets.as_ref() { + let subscriber = sc_tracing::ProfilingSubscriber::new( + self.import_params.tracing_receiver.into(), + tracing_targets, + ); + if let Err(e) = tracing::subscriber::set_global_default(subscriber) { + return Err(format!("Unable to set global default subscriber {}", e).into()); + } + } + + let factory_state = FactoryState::new(self.blocks, self.transactions); + + let service_builder = new_full_start!(config).0; + node_transaction_factory::factory( + factory_state, + service_builder.client(), + service_builder + .select_chain() + .expect("The select_chain is always initialized by new_full_start!; qed"), + ) } } diff --git a/bin/node/cli/src/factory_impl.rs b/bin/node/cli/src/factory_impl.rs index cd7e3022e0b9a66e510707245118dbc123edee0d..bc7653538247f05c33381516be2a756463dc3c2a 100644 --- a/bin/node/cli/src/factory_impl.rs +++ b/bin/node/cli/src/factory_impl.rs @@ -57,8 +57,6 @@ impl FactoryState { frame_system::CheckNonce::from(index), frame_system::CheckWeight::new(), pallet_transaction_payment::ChargeTransactionPayment::from(0), - Default::default(), - Default::default(), ) } } @@ -123,7 +121,7 @@ impl RuntimeAdapter for FactoryState { (*amount).into() ) ) - }, key, (version, genesis_hash.clone(), prior_block_hash.clone(), (), (), (), (), ())) + }, key, (version, genesis_hash.clone(), prior_block_hash.clone(), (), (), ())) } fn inherent_extrinsics(&self) -> InherentData { diff --git a/bin/node/cli/src/lib.rs b/bin/node/cli/src/lib.rs index 6b3644856c6b027172ad62b132c65fcb92d05146..1e2c790bfa7d21a8857ba1bf2f90a3a419701585 100644 --- a/bin/node/cli/src/lib.rs +++ b/bin/node/cli/src/lib.rs @@ -47,45 +47,3 @@ pub use browser::*; pub use cli::*; #[cfg(feature = "cli")] pub use command::*; - -/// The chain specification option. -#[derive(Clone, Debug, PartialEq)] -pub enum ChainSpec { - /// Whatever the current runtime is, with just Alice as an auth. - Development, - /// Whatever the current runtime is, with simple Alice/Bob auths. - LocalTestnet, - /// The Flaming Fir testnet. - FlamingFir, - /// Whatever the current runtime is with the "global testnet" defaults. - StagingTestnet, -} - -/// Get a chain config from a spec setting. -impl ChainSpec { - pub(crate) fn load(self) -> Result { - Ok(match self { - ChainSpec::FlamingFir => chain_spec::flaming_fir_config()?, - ChainSpec::Development => chain_spec::development_config(), - ChainSpec::LocalTestnet => chain_spec::local_testnet_config(), - ChainSpec::StagingTestnet => chain_spec::staging_testnet_config(), - }) - } - - pub(crate) fn from(s: &str) -> Option { - match s { - "dev" => Some(ChainSpec::Development), - "local" => Some(ChainSpec::LocalTestnet), - "" | "fir" | "flaming-fir" => Some(ChainSpec::FlamingFir), - "staging" => Some(ChainSpec::StagingTestnet), - _ => None, - } - } -} - -fn load_spec(id: &str) -> Result, String> { - Ok(match ChainSpec::from(id) { - Some(spec) => Box::new(spec.load()?), - None => Box::new(chain_spec::ChainSpec::from_json_file(std::path::PathBuf::from(id))?), - }) -} diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 3e09802ccd8f653585d2c34cbe1e62d9ec7fafba..c8b0e50c4ff2731666c6b1ca143e074eb9c14758 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -21,7 +21,6 @@ use std::sync::Arc; use sc_consensus_babe; -use sc_client::{self, LongestChain}; use grandpa::{self, FinalityProofProvider as GrandpaFinalityProofProvider, StorageAndProofProvider}; use node_executor; use node_primitives::Block; @@ -30,14 +29,7 @@ use sc_service::{ AbstractService, ServiceBuilder, config::Configuration, error::{Error as ServiceError}, }; use sp_inherents::InherentDataProviders; - -use sc_service::{Service, NetworkStatus}; -use sc_client::{Client, LocalCallExecutor}; -use sc_client_db::Backend; -use sp_runtime::traits::Block as BlockT; -use node_executor::NativeExecutor; -use sc_network::NetworkService; -use sc_offchain::OffchainWorkers; +use sc_consensus::LongestChain; /// Starts a `ServiceBuilder` for a full service. /// @@ -54,11 +46,11 @@ macro_rules! new_full_start { node_primitives::Block, node_runtime::RuntimeApi, node_executor::Executor >($config)? .with_select_chain(|_config, backend| { - Ok(sc_client::LongestChain::new(backend.clone())) + Ok(sc_consensus::LongestChain::new(backend.clone())) })? - .with_transaction_pool(|config, client, _fetcher| { + .with_transaction_pool(|config, client, _fetcher, prometheus_registry| { let pool_api = sc_transaction_pool::FullChainApi::new(client.clone()); - Ok(sc_transaction_pool::BasicPool::new(config, std::sync::Arc::new(pool_api))) + Ok(sc_transaction_pool::BasicPool::new(config, std::sync::Arc::new(pool_api), prometheus_registry)) })? .with_import_queue(|_config, client, mut select_chain, _transaction_pool| { let select_chain = select_chain.take() @@ -88,7 +80,7 @@ macro_rules! new_full_start { import_setup = Some((block_import, grandpa_link, babe_link)); Ok(import_queue) })? - .with_rpc_extensions(|builder| -> Result { + .with_rpc_extensions(|builder| -> std::result::Result { let babe_link = import_setup.as_ref().map(|s| &s.2) .expect("BabeLink is present for full services or set up failed; qed."); let deps = node_rpc::FullDeps { @@ -127,7 +119,7 @@ macro_rules! new_full { ) = ( $config.role.clone(), $config.force_authoring, - $config.name.clone(), + $config.network.node_name.clone(), $config.disable_grandpa, ); @@ -146,7 +138,7 @@ macro_rules! new_full { ($with_startup_data)(&block_import, &babe_link); - if let sc_service::config::Role::Authority { sentry_nodes } = &role { + if let sc_service::config::Role::Authority { .. } = &role { let proposer = sc_basic_authorship::ProposerFactory::new( service.client(), service.transaction_pool() @@ -174,18 +166,35 @@ macro_rules! new_full { let babe = sc_consensus_babe::start_babe(babe_config)?; service.spawn_essential_task("babe-proposer", babe); + } + + // Spawn authority discovery module. + if matches!(role, sc_service::config::Role::Authority{..} | sc_service::config::Role::Sentry {..}) { + let (sentries, authority_discovery_role) = match role { + sc_service::config::Role::Authority { ref sentry_nodes } => ( + sentry_nodes.clone(), + sc_authority_discovery::Role::Authority ( + service.keystore(), + ), + ), + sc_service::config::Role::Sentry {..} => ( + vec![], + sc_authority_discovery::Role::Sentry, + ), + _ => unreachable!("Due to outer matches! constraint; qed.") + }; let network = service.network(); - let dht_event_stream = network.event_stream().filter_map(|e| async move { match e { + let dht_event_stream = network.event_stream("authority-discovery").filter_map(|e| async move { match e { Event::Dht(e) => Some(e), _ => None, }}).boxed(); let authority_discovery = sc_authority_discovery::AuthorityDiscovery::new( service.client(), network, - sentry_nodes.clone(), - service.keystore(), + sentries, dht_event_stream, + authority_discovery_role, service.prometheus_registry(), ); @@ -249,38 +258,9 @@ macro_rules! new_full { }} } -type ConcreteBlock = node_primitives::Block; -type ConcreteClient = - Client< - Backend, - LocalCallExecutor, NativeExecutor>, - ConcreteBlock, - node_runtime::RuntimeApi - >; -type ConcreteBackend = Backend; -type ConcreteTransactionPool = sc_transaction_pool::BasicPool< - sc_transaction_pool::FullChainApi, - ConcreteBlock ->; - /// Builds a new service for a full client. pub fn new_full(config: Configuration) --> Result< - Service< - ConcreteBlock, - ConcreteClient, - LongestChain, - NetworkStatus, - NetworkService::Hash>, - ConcreteTransactionPool, - OffchainWorkers< - ConcreteClient, - >::OffchainStorage, - ConcreteBlock, - > - >, - ServiceError, -> +-> Result { new_full!(config).map(|(service, _)| service) } @@ -295,12 +275,12 @@ pub fn new_light(config: Configuration) .with_select_chain(|_config, backend| { Ok(LongestChain::new(backend.clone())) })? - .with_transaction_pool(|config, client, fetcher| { + .with_transaction_pool(|config, client, fetcher, prometheus_registry| { let fetcher = fetcher .ok_or_else(|| "Trying to start light transaction pool without active fetcher")?; let pool_api = sc_transaction_pool::LightChainApi::new(client.clone(), fetcher.clone()); let pool = sc_transaction_pool::BasicPool::with_revalidation_type( - config, Arc::new(pool_api), sc_transaction_pool::RevalidationType::Light, + config, Arc::new(pool_api), prometheus_registry, sc_transaction_pool::RevalidationType::Light, ); Ok(pool) })? @@ -399,7 +379,7 @@ mod tests { use sp_core::ed25519::Pair; use {service_test, Factory}; - use sc_client::{BlockImportParams, BlockOrigin}; + use sp_consensus::{BlockImportParams, BlockOrigin}; let alice: Arc = Arc::new(Keyring::Alice.into()); let bob: Arc = Arc::new(Keyring::Bob.into()); @@ -616,13 +596,11 @@ mod tests { check_nonce, check_weight, payment, - Default::default(), - Default::default(), ); let raw_payload = SignedPayload::from_raw( function, extra, - (version, genesis_hash, genesis_hash, (), (), (), (), ()) + (version, genesis_hash, genesis_hash, (), (), ()) ); let signature = raw_payload.using_encoded(|payload| { signer.sign(payload) diff --git a/bin/node/cli/tests/version.rs b/bin/node/cli/tests/version.rs new file mode 100644 index 0000000000000000000000000000000000000000..6f99d7c24ae1434ff84e81ba84ee406ec6248ac4 --- /dev/null +++ b/bin/node/cli/tests/version.rs @@ -0,0 +1,83 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use assert_cmd::cargo::cargo_bin; +use platforms::*; +use regex::Regex; +use std::process::Command; + +fn expected_regex() -> Regex { + Regex::new(r"^substrate (\d+\.\d+\.\d+(?:-.+?)?)-([a-f\d]+|unknown-commit)-(.+?)-(.+?)(?:-(.+))?$").unwrap() +} + +#[test] +fn version_is_full() { + let expected = expected_regex(); + let output = Command::new(cargo_bin("substrate")) + .args(&["--version"]) + .output() + .unwrap(); + + assert!( + output.status.success(), + "command returned with non-success exit code" + ); + + let output = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + let captures = expected + .captures(output.as_str()) + .expect("could not parse version in output"); + + assert_eq!(&captures[1], env!("CARGO_PKG_VERSION")); + assert_eq!(&captures[3], TARGET_ARCH.as_str()); + assert_eq!(&captures[4], TARGET_OS.as_str()); + assert_eq!( + captures.get(5).map(|x| x.as_str()), + TARGET_ENV.map(|x| x.as_str()) + ); +} + +#[test] +fn test_regex_matches_properly() { + let expected = expected_regex(); + + let captures = expected + .captures("substrate 2.0.0-da487d19d-x86_64-linux-gnu") + .unwrap(); + assert_eq!(&captures[1], "2.0.0"); + assert_eq!(&captures[2], "da487d19d"); + assert_eq!(&captures[3], "x86_64"); + assert_eq!(&captures[4], "linux"); + assert_eq!(captures.get(5).map(|x| x.as_str()), Some("gnu")); + + let captures = expected + .captures("substrate 2.0.0-alpha.5-da487d19d-x86_64-linux-gnu") + .unwrap(); + assert_eq!(&captures[1], "2.0.0-alpha.5"); + assert_eq!(&captures[2], "da487d19d"); + assert_eq!(&captures[3], "x86_64"); + assert_eq!(&captures[4], "linux"); + assert_eq!(captures.get(5).map(|x| x.as_str()), Some("gnu")); + + let captures = expected + .captures("substrate 2.0.0-alpha.5-da487d19d-x86_64-linux") + .unwrap(); + assert_eq!(&captures[1], "2.0.0-alpha.5"); + assert_eq!(&captures[2], "da487d19d"); + assert_eq!(&captures[3], "x86_64"); + assert_eq!(&captures[4], "linux"); + assert_eq!(captures.get(5).map(|x| x.as_str()), None); +} diff --git a/bin/node/executor/Cargo.toml b/bin/node/executor/Cargo.toml index 2f1060a99884505ba94f1692f493bf8cdb01f43a..99bd83bb4a2bf0c4e8f3181dcd739eccaa685690 100644 --- a/bin/node/executor/Cargo.toml +++ b/bin/node/executor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-executor" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] description = "Substrate node implementation in Rust." edition = "2018" @@ -8,35 +8,38 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0" } -node-primitives = { version = "2.0.0-alpha.5", path = "../primitives" } -node-runtime = { version = "2.0.0-alpha.5", path = "../runtime" } -sc-executor = { version = "0.8.0-alpha.5", path = "../../../client/executor" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-io = { version = "2.0.0-alpha.5", path = "../../../primitives/io" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../../primitives/state-machine" } -sp-trie = { version = "2.0.0-alpha.5", path = "../../../primitives/trie" } +node-primitives = { version = "2.0.0-dev", path = "../primitives" } +node-runtime = { version = "2.0.0-dev", path = "../runtime" } +sc-executor = { version = "0.8.0-dev", path = "../../../client/executor" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-io = { version = "2.0.0-dev", path = "../../../primitives/io" } +sp-state-machine = { version = "0.8.0-dev", path = "../../../primitives/state-machine" } +sp-trie = { version = "2.0.0-dev", path = "../../../primitives/trie" } trie-root = "0.16.0" -frame-benchmarking = { version = "2.0.0-alpha.5", path = "../../../frame/benchmarking" } +frame-benchmarking = { version = "2.0.0-dev", path = "../../../frame/benchmarking" } [dev-dependencies] criterion = "0.3.0" -frame-support = { version = "2.0.0-alpha.5", path = "../../../frame/support" } -frame-system = { version = "2.0.0-alpha.5", path = "../../../frame/system" } -node-testing = { version = "2.0.0-alpha.5", path = "../testing" } -pallet-balances = { version = "2.0.0-alpha.5", path = "../../../frame/balances" } -pallet-contracts = { version = "2.0.0-alpha.5", path = "../../../frame/contracts" } -pallet-grandpa = { version = "2.0.0-alpha.5", path = "../../../frame/grandpa" } -pallet-im-online = { version = "2.0.0-alpha.5", path = "../../../frame/im-online" } -pallet-indices = { version = "2.0.0-alpha.5", path = "../../../frame/indices" } -pallet-session = { version = "2.0.0-alpha.5", path = "../../../frame/session" } -pallet-timestamp = { version = "2.0.0-alpha.5", path = "../../../frame/timestamp" } -pallet-transaction-payment = { version = "2.0.0-alpha.5", path = "../../../frame/transaction-payment" } -pallet-treasury = { version = "2.0.0-alpha.5", path = "../../../frame/treasury" } -sp-application-crypto = { version = "2.0.0-alpha.5", path = "../../../primitives/application-crypto" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sp-externalities = { version = "0.8.0-alpha.5", path = "../../../primitives/externalities" } +frame-support = { version = "2.0.0-dev", path = "../../../frame/support" } +frame-system = { version = "2.0.0-dev", path = "../../../frame/system" } +node-testing = { version = "2.0.0-dev", path = "../testing" } +pallet-balances = { version = "2.0.0-dev", path = "../../../frame/balances" } +pallet-contracts = { version = "2.0.0-dev", path = "../../../frame/contracts" } +pallet-grandpa = { version = "2.0.0-dev", path = "../../../frame/grandpa" } +pallet-im-online = { version = "2.0.0-dev", path = "../../../frame/im-online" } +pallet-indices = { version = "2.0.0-dev", path = "../../../frame/indices" } +pallet-session = { version = "2.0.0-dev", path = "../../../frame/session" } +pallet-timestamp = { version = "2.0.0-dev", path = "../../../frame/timestamp" } +pallet-transaction-payment = { version = "2.0.0-dev", path = "../../../frame/transaction-payment" } +pallet-treasury = { version = "2.0.0-dev", path = "../../../frame/treasury" } +sp-application-crypto = { version = "2.0.0-dev", path = "../../../primitives/application-crypto" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-externalities = { version = "0.8.0-dev", path = "../../../primitives/externalities" } substrate-test-client = { version = "2.0.0-dev", path = "../../../test-utils/client" } wabt = "0.9.2" @@ -52,6 +55,3 @@ stress-test = [] [[bench]] name = "bench" harness = false - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node/executor/tests/basic.rs b/bin/node/executor/tests/basic.rs index fccf4a62cc21dbe81f3fa72f6ab2fde8ae6fceda..b357e2886ca7108aeb9f22cc44bea5caa6add827 100644 --- a/bin/node/executor/tests/basic.rs +++ b/bin/node/executor/tests/basic.rs @@ -18,13 +18,13 @@ use codec::{Encode, Decode, Joiner}; use frame_support::{ StorageValue, StorageMap, traits::Currency, - weights::{GetDispatchInfo, DispatchInfo, DispatchClass}, + weights::{GetDispatchInfo, DispatchInfo, DispatchClass, Pays}, }; use sp_core::{ NeverNativeValue, map, traits::Externalities, storage::{well_known_keys, Storage}, }; use sp_runtime::{ - ApplyExtrinsicResult, Fixed64, + ApplyExtrinsicResult, Fixed128, traits::{Hash as HashT, Convert, BlakeTwo256}, transaction_validity::InvalidTransaction, }; @@ -33,7 +33,7 @@ use frame_system::{self, EventRecord, Phase}; use node_runtime::{ Header, Block, UncheckedExtrinsic, CheckedExtrinsic, Call, Runtime, Balances, - System, TransactionPayment, Event, TransactionBaseFee, TransactionByteFee, + System, TransactionPayment, Event, TransactionByteFee, ExtrinsicBaseWeight, constants::currency::*, }; use node_primitives::{Balance, Hash}; @@ -51,14 +51,14 @@ use self::common::{*, sign}; pub const BLOATY_CODE: &[u8] = node_runtime::WASM_BINARY_BLOATY; /// Default transfer fee -fn transfer_fee(extrinsic: &E, fee_multiplier: Fixed64) -> Balance { +fn transfer_fee(extrinsic: &E, fee_multiplier: Fixed128) -> Balance { let length_fee = TransactionByteFee::get() * (extrinsic.encode().len() as Balance); + let base_weight = ExtrinsicBaseWeight::get(); + let base_fee = ::WeightToFee::convert(base_weight); let weight = default_transfer_call().get_dispatch_info().weight; - let weight_fee = - ::WeightToFee::convert(weight); + let weight_fee = ::WeightToFee::convert(weight); - let base_fee = TransactionBaseFee::get(); base_fee + fee_multiplier.saturated_multiply_accumulate(length_fee + weight_fee) } @@ -170,7 +170,7 @@ fn panic_execution_with_foreign_code_gives_error() { vec![0u8; 32] } ], - children: map![], + children_default: map![], }); let r = executor_call:: _>( @@ -206,7 +206,7 @@ fn bad_extrinsic_with_native_equivalent_code_gives_error() { vec![0u8; 32] } ], - children: map![], + children_default: map![], }); let r = executor_call:: _>( @@ -240,7 +240,7 @@ fn successful_execution_with_native_equivalent_code_gives_ok() { }, >::hashed_key_for(0) => vec![0u8; 32] ], - children: map![], + children_default: map![], }); let r = executor_call:: _>( @@ -282,7 +282,7 @@ fn successful_execution_with_foreign_code_gives_ok() { }, >::hashed_key_for(0) => vec![0u8; 32] ], - children: map![], + children_default: map![], }); let r = executor_call:: _>( @@ -338,15 +338,10 @@ fn full_native_block_import_works() { EventRecord { phase: Phase::ApplyExtrinsic(0), event: Event::frame_system(frame_system::RawEvent::ExtrinsicSuccess( - DispatchInfo { weight: 10000, class: DispatchClass::Mandatory, pays_fee: true } + DispatchInfo { weight: 0, class: DispatchClass::Mandatory, ..Default::default() } )), topics: vec![], }, - EventRecord { - phase: Phase::ApplyExtrinsic(1), - event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(fees * 8 / 10)), - topics: vec![], - }, EventRecord { phase: Phase::ApplyExtrinsic(1), event: Event::pallet_balances(pallet_balances::RawEvent::Transfer( @@ -356,10 +351,15 @@ fn full_native_block_import_works() { )), topics: vec![], }, + EventRecord { + phase: Phase::ApplyExtrinsic(1), + event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(fees * 8 / 10)), + topics: vec![], + }, EventRecord { phase: Phase::ApplyExtrinsic(1), event: Event::frame_system(frame_system::RawEvent::ExtrinsicSuccess( - DispatchInfo { weight: 1000000, class: DispatchClass::Normal, pays_fee: true } + DispatchInfo { weight: 460_000_000, ..Default::default() } )), topics: vec![], }, @@ -391,15 +391,10 @@ fn full_native_block_import_works() { EventRecord { phase: Phase::ApplyExtrinsic(0), event: Event::frame_system(frame_system::RawEvent::ExtrinsicSuccess( - DispatchInfo { weight: 10000, class: DispatchClass::Mandatory, pays_fee: true } + DispatchInfo { weight: 0, class: DispatchClass::Mandatory, pays_fee: Pays::Yes } )), topics: vec![], }, - EventRecord { - phase: Phase::ApplyExtrinsic(1), - event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(fees * 8 / 10)), - topics: vec![], - }, EventRecord { phase: Phase::ApplyExtrinsic(1), event: Event::pallet_balances( @@ -413,14 +408,14 @@ fn full_native_block_import_works() { }, EventRecord { phase: Phase::ApplyExtrinsic(1), - event: Event::frame_system(frame_system::RawEvent::ExtrinsicSuccess( - DispatchInfo { weight: 1000000, class: DispatchClass::Normal, pays_fee: true } - )), + event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(fees * 8 / 10)), topics: vec![], }, EventRecord { - phase: Phase::ApplyExtrinsic(2), - event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(fees * 8 / 10)), + phase: Phase::ApplyExtrinsic(1), + event: Event::frame_system(frame_system::RawEvent::ExtrinsicSuccess( + DispatchInfo { weight: 460_000_000, ..Default::default() } + )), topics: vec![], }, EventRecord { @@ -434,10 +429,15 @@ fn full_native_block_import_works() { ), topics: vec![], }, + EventRecord { + phase: Phase::ApplyExtrinsic(2), + event: Event::pallet_treasury(pallet_treasury::RawEvent::Deposit(fees * 8 / 10)), + topics: vec![], + }, EventRecord { phase: Phase::ApplyExtrinsic(2), event: Event::frame_system(frame_system::RawEvent::ExtrinsicSuccess( - DispatchInfo { weight: 1000000, class: DispatchClass::Normal, pays_fee: true } + DispatchInfo { weight: 460_000_000, ..Default::default() } )), topics: vec![], }, @@ -606,13 +606,18 @@ fn deploying_wasm_contract_should_work() { CheckedExtrinsic { signed: Some((charlie(), signed_extra(0, 0))), function: Call::Contracts( - pallet_contracts::Call::put_code::(10_000, transfer_code) + pallet_contracts::Call::put_code::(transfer_code) ), }, CheckedExtrinsic { signed: Some((charlie(), signed_extra(1, 0))), function: Call::Contracts( - pallet_contracts::Call::instantiate::(1 * DOLLARS, 10_000, transfer_ch, Vec::new()) + pallet_contracts::Call::instantiate::( + 1 * DOLLARS, + 500_000_000, + transfer_ch, + Vec::new() + ) ), }, CheckedExtrinsic { @@ -621,7 +626,7 @@ fn deploying_wasm_contract_should_work() { pallet_contracts::Call::call::( pallet_indices::address::Address::Id(addr.clone()), 10, - 10_000, + 500_000_000, vec![0x00, 0x01, 0x02, 0x03] ) ), @@ -704,7 +709,7 @@ fn panic_execution_gives_error() { }, >::hashed_key_for(0) => vec![0u8; 32] ], - children: map![], + children_default: map![], }); let r = executor_call:: _>( @@ -738,7 +743,7 @@ fn successful_execution_gives_ok() { }, >::hashed_key_for(0) => vec![0u8; 32] ], - children: map![], + children_default: map![], }); let r = executor_call:: _>( @@ -817,5 +822,3 @@ fn should_import_block_with_test_client() { client.import(BlockOrigin::Own, block).unwrap(); } - - diff --git a/bin/node/executor/tests/common.rs b/bin/node/executor/tests/common.rs index 6b6ef272f8a35bac1140e77ed1411adc52f0e8cf..5a51e4312c5e80315c3b05fc5a82bc72a6842fb2 100644 --- a/bin/node/executor/tests/common.rs +++ b/bin/node/executor/tests/common.rs @@ -15,10 +15,21 @@ // along with Substrate. If not, see . use codec::{Encode, Decode}; +use frame_system::offchain::AppCrypto; use frame_support::Hashable; use sp_state_machine::TestExternalities as CoreTestExternalities; -use sp_core::{NeverNativeValue, NativeOrEncoded, traits::{CodeExecutor, RuntimeCode}}; -use sp_runtime::{ApplyExtrinsicResult, traits::{Header as HeaderT, BlakeTwo256}}; +use sp_core::{ + NeverNativeValue, NativeOrEncoded, + crypto::KeyTypeId, + sr25519::Signature, + traits::{CodeExecutor, RuntimeCode}, +}; +use sp_runtime::{ + ApplyExtrinsicResult, + MultiSigner, + MultiSignature, + traits::{Header as HeaderT, BlakeTwo256}, +}; use sc_executor::{NativeExecutor, WasmExecutionMethod}; use sc_executor::error::Result; @@ -31,6 +42,25 @@ use node_primitives::{Hash, BlockNumber}; use node_testing::keyring::*; use sp_externalities::Externalities; +pub const TEST_KEY_TYPE_ID: KeyTypeId = KeyTypeId(*b"test"); + +pub mod sr25519 { + mod app_sr25519 { + use sp_application_crypto::{app_crypto, sr25519}; + use super::super::TEST_KEY_TYPE_ID; + app_crypto!(sr25519, TEST_KEY_TYPE_ID); + } + + pub type AuthorityId = app_sr25519::Public; +} + +pub struct TestAuthorityId; +impl AppCrypto for TestAuthorityId { + type RuntimeAppPublic = sr25519::AuthorityId; + type GenericSignature = Signature; + type GenericPublic = sp_core::sr25519::Public; +} + /// The wasm runtime code. /// /// `compact` since it is after post-processing with wasm-gc which performs tree-shaking thus diff --git a/bin/node/executor/tests/fees.rs b/bin/node/executor/tests/fees.rs index ea9d740a05ef86e77fa923c8863d55a984cdbca7..9c1b89d045f6d76ec706fd6c67a1de2f0b794780 100644 --- a/bin/node/executor/tests/fees.rs +++ b/bin/node/executor/tests/fees.rs @@ -21,11 +21,11 @@ use frame_support::{ weights::GetDispatchInfo, }; use sp_core::{NeverNativeValue, map, storage::Storage}; -use sp_runtime::{Fixed64, Perbill, traits::{Convert, BlakeTwo256}}; +use sp_runtime::{Fixed128, Perbill, traits::{Convert, BlakeTwo256}}; use node_runtime::{ - CheckedExtrinsic, Call, Runtime, Balances, TransactionPayment, TransactionBaseFee, + CheckedExtrinsic, Call, Runtime, Balances, TransactionPayment, TransactionByteFee, WeightFeeCoefficient, - constants::currency::*, + constants::currency::*, ExtrinsicBaseWeight, }; use node_runtime::impls::LinearWeightToFee; use node_primitives::Balance; @@ -39,7 +39,7 @@ fn fee_multiplier_increases_and_decreases_on_big_weight() { let mut t = new_test_ext(COMPACT_CODE, false); // initial fee multiplier must be zero - let mut prev_multiplier = Fixed64::from_parts(0); + let mut prev_multiplier = Fixed128::from_parts(0); t.execute_with(|| { assert_eq!(TransactionPayment::next_fee_multiplier(), prev_multiplier); @@ -143,7 +143,7 @@ fn transaction_fee_is_correct_ultimate() { }, >::hashed_key_for(0) => vec![0u8; 32] ], - children: map![], + children_default: map![], }); let tip = 1_000_000; @@ -173,22 +173,27 @@ fn transaction_fee_is_correct_ultimate() { t.execute_with(|| { assert_eq!(Balances::total_balance(&bob()), (10 + 69) * DOLLARS); // Components deducted from alice's balances: + // - Base fee // - Weight fee // - Length fee // - Tip // - Creation-fee of bob's account. let mut balance_alice = (100 - 69) * DOLLARS; - let length_fee = TransactionBaseFee::get() + - TransactionByteFee::get() * - (xt.clone().encode().len() as Balance); + let base_weight = ExtrinsicBaseWeight::get(); + let base_fee = LinearWeightToFee::::convert(base_weight); + + let length_fee = TransactionByteFee::get() * (xt.clone().encode().len() as Balance); balance_alice -= length_fee; let weight = default_transfer_call().get_dispatch_info().weight; let weight_fee = LinearWeightToFee::::convert(weight); // we know that weight to fee multiplier is effect-less in block 1. - assert_eq!(weight_fee as Balance, MILLICENTS); + // current weight of transfer = 200_000_000 + // Linear weight to fee is 1:1 right now (1 weight = 1 unit of balance) + assert_eq!(weight_fee, weight as Balance); + balance_alice -= base_fee; balance_alice -= weight_fee; balance_alice -= tip; diff --git a/bin/node/executor/tests/submit_transaction.rs b/bin/node/executor/tests/submit_transaction.rs index 536cf486e38ae28e0df13fadd65ac9b5c20e99a0..a4e89ca738904e5d5d51ed64b6df1239fa1edc07 100644 --- a/bin/node/executor/tests/submit_transaction.rs +++ b/bin/node/executor/tests/submit_transaction.rs @@ -15,24 +15,29 @@ // along with Substrate. If not, see . use node_runtime::{ - Call, Executive, Indices, Runtime, TransactionSubmitterOf, UncheckedExtrinsic, + Executive, Indices, Runtime, UncheckedExtrinsic, }; use sp_application_crypto::AppKey; use sp_core::testing::KeyStore; -use sp_core::traits::KeystoreExt; -use sp_core::offchain::{ - TransactionPoolExt, - testing::TestTransactionPoolExt, +use sp_core::{ + offchain::{ + TransactionPoolExt, + testing::TestTransactionPoolExt, + }, + traits::KeystoreExt, +}; +use frame_system::{ + offchain::{ + Signer, + SubmitTransaction, + SendSignedTransaction, + } }; -use frame_system::offchain::{SubmitSignedTransaction, SubmitUnsignedTransaction}; -use pallet_im_online::sr25519::AuthorityPair as Key; use codec::Decode; pub mod common; use self::common::*; -type SubmitTransaction = TransactionSubmitterOf; - #[test] fn should_submit_unsigned_transaction() { let mut t = new_test_ext(COMPACT_CODE, false); @@ -46,11 +51,11 @@ fn should_submit_unsigned_transaction() { network_state: Default::default(), session_index: 1, authority_index: 0, + validators_len: 0, }; let call = pallet_im_online::Call::heartbeat(heartbeat_data, signature); - > - ::submit_unsigned(call) + SubmitTransaction::>::submit_unsigned_transaction(call.into()) .unwrap(); assert_eq!(state.read().transactions.len(), 1) @@ -66,23 +71,16 @@ fn should_submit_signed_transaction() { t.register_extension(TransactionPoolExt::new(pool)); let keystore = KeyStore::new(); - keystore.write().sr25519_generate_new(Key::ID, Some(&format!("{}/hunter1", PHRASE))).unwrap(); - keystore.write().sr25519_generate_new(Key::ID, Some(&format!("{}/hunter2", PHRASE))).unwrap(); - keystore.write().sr25519_generate_new(Key::ID, Some(&format!("{}/hunter3", PHRASE))).unwrap(); + keystore.write().sr25519_generate_new(sr25519::AuthorityId::ID, Some(&format!("{}/hunter1", PHRASE))).unwrap(); + keystore.write().sr25519_generate_new(sr25519::AuthorityId::ID, Some(&format!("{}/hunter2", PHRASE))).unwrap(); + keystore.write().sr25519_generate_new(sr25519::AuthorityId::ID, Some(&format!("{}/hunter3", PHRASE))).unwrap(); t.register_extension(KeystoreExt(keystore)); t.execute_with(|| { - let keys = > - ::find_all_local_keys(); - assert_eq!(keys.len(), 3, "Missing keys: {:?}", keys); - - let can_sign = > - ::can_sign(); - assert!(can_sign, "Since there are keys, `can_sign` should return true"); - - let call = pallet_balances::Call::transfer(Default::default(), Default::default()); - let results = - >::submit_signed(call); + let results = Signer::::all_accounts() + .send_signed_transaction(|_| { + pallet_balances::Call::transfer(Default::default(), Default::default()) + }); let len = results.len(); assert_eq!(len, 3); @@ -98,27 +96,26 @@ fn should_submit_signed_twice_from_the_same_account() { t.register_extension(TransactionPoolExt::new(pool)); let keystore = KeyStore::new(); - keystore.write().sr25519_generate_new(Key::ID, Some(&format!("{}/hunter1", PHRASE))).unwrap(); + keystore.write().sr25519_generate_new(sr25519::AuthorityId::ID, Some(&format!("{}/hunter1", PHRASE))).unwrap(); + keystore.write().sr25519_generate_new(sr25519::AuthorityId::ID, Some(&format!("{}/hunter2", PHRASE))).unwrap(); t.register_extension(KeystoreExt(keystore)); t.execute_with(|| { - let call = pallet_balances::Call::transfer(Default::default(), Default::default()); - let results = - >::submit_signed(call); + let result = Signer::::any_account() + .send_signed_transaction(|_| { + pallet_balances::Call::transfer(Default::default(), Default::default()) + }); - let len = results.len(); - assert_eq!(len, 1); - assert_eq!(results.into_iter().filter_map(|x| x.1.ok()).count(), len); + assert!(result.is_some()); assert_eq!(state.read().transactions.len(), 1); // submit another one from the same account. The nonce should be incremented. - let call = pallet_balances::Call::transfer(Default::default(), Default::default()); - let results = - >::submit_signed(call); + let result = Signer::::any_account() + .send_signed_transaction(|_| { + pallet_balances::Call::transfer(Default::default(), Default::default()) + }); - let len = results.len(); - assert_eq!(len, 1); - assert_eq!(results.into_iter().filter_map(|x| x.1.ok()).count(), len); + assert!(result.is_some()); assert_eq!(state.read().transactions.len(), 2); // now check that the transaction nonces are not equal @@ -136,6 +133,60 @@ fn should_submit_signed_twice_from_the_same_account() { }); } +#[test] +fn should_submit_signed_twice_from_all_accounts() { + let mut t = new_test_ext(COMPACT_CODE, false); + let (pool, state) = TestTransactionPoolExt::new(); + t.register_extension(TransactionPoolExt::new(pool)); + + let keystore = KeyStore::new(); + keystore.write().sr25519_generate_new(sr25519::AuthorityId::ID, Some(&format!("{}/hunter1", PHRASE))).unwrap(); + keystore.write().sr25519_generate_new(sr25519::AuthorityId::ID, Some(&format!("{}/hunter2", PHRASE))).unwrap(); + t.register_extension(KeystoreExt(keystore)); + + t.execute_with(|| { + let results = Signer::::all_accounts() + .send_signed_transaction(|_| { + pallet_balances::Call::transfer(Default::default(), Default::default()) + }); + + let len = results.len(); + assert_eq!(len, 2); + assert_eq!(results.into_iter().filter_map(|x| x.1.ok()).count(), len); + assert_eq!(state.read().transactions.len(), 2); + + // submit another one from the same account. The nonce should be incremented. + let results = Signer::::all_accounts() + .send_signed_transaction(|_| { + pallet_balances::Call::transfer(Default::default(), Default::default()) + }); + + let len = results.len(); + assert_eq!(len, 2); + assert_eq!(results.into_iter().filter_map(|x| x.1.ok()).count(), len); + assert_eq!(state.read().transactions.len(), 4); + + // now check that the transaction nonces are not equal + let s = state.read(); + fn nonce(tx: UncheckedExtrinsic) -> frame_system::CheckNonce { + let extra = tx.signature.unwrap().2; + extra.3 + } + let nonce1 = nonce(UncheckedExtrinsic::decode(&mut &*s.transactions[0]).unwrap()); + let nonce2 = nonce(UncheckedExtrinsic::decode(&mut &*s.transactions[1]).unwrap()); + let nonce3 = nonce(UncheckedExtrinsic::decode(&mut &*s.transactions[2]).unwrap()); + let nonce4 = nonce(UncheckedExtrinsic::decode(&mut &*s.transactions[3]).unwrap()); + assert!( + nonce1 != nonce3, + "Transactions should have different nonces. Got: 1st tx nonce: {:?}, 2nd nonce: {:?}", nonce1, nonce3 + ); + assert!( + nonce2 != nonce4, + "Transactions should have different nonces. Got: 1st tx nonce: {:?}, 2nd tx nonce: {:?}", nonce2, nonce4 + ); + }); +} + #[test] fn submitted_transaction_should_be_valid() { use codec::Encode; @@ -148,13 +199,14 @@ fn submitted_transaction_should_be_valid() { t.register_extension(TransactionPoolExt::new(pool)); let keystore = KeyStore::new(); - keystore.write().sr25519_generate_new(Key::ID, Some(&format!("{}/hunter1", PHRASE))).unwrap(); + keystore.write().sr25519_generate_new(sr25519::AuthorityId::ID, Some(&format!("{}/hunter1", PHRASE))).unwrap(); t.register_extension(KeystoreExt(keystore)); t.execute_with(|| { - let call = pallet_balances::Call::transfer(Default::default(), Default::default()); - let results = - >::submit_signed(call); + let results = Signer::::all_accounts() + .send_signed_transaction(|_| { + pallet_balances::Call::transfer(Default::default(), Default::default()) + }); let len = results.len(); assert_eq!(len, 1); assert_eq!(results.into_iter().filter_map(|x| x.1.ok()).count(), len); @@ -178,7 +230,7 @@ fn submitted_transaction_should_be_valid() { let res = Executive::validate_transaction(source, extrinsic); assert_eq!(res.unwrap(), ValidTransaction { - priority: 2_411_002_000_000, + priority: 1_411_390_000_000, requires: vec![], provides: vec![(address, 0).encode()], longevity: 128, diff --git a/bin/node/inspect/Cargo.toml b/bin/node/inspect/Cargo.toml index 9e94fe74d68469d9570fe76292d004e9106ef049..03d3a94b6277822397af4945b53ed41ddd27f2ce 100644 --- a/bin/node/inspect/Cargo.toml +++ b/bin/node/inspect/Cargo.toml @@ -1,23 +1,23 @@ [package] name = "node-inspect" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0" } derive_more = "0.99" log = "0.4.8" -sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api" } -sc-service = { version = "0.8.0-alpha.5", default-features = false, path = "../../../client/service" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } +sc-cli = { version = "0.8.0-dev", path = "../../../client/cli" } +sc-client-api = { version = "2.0.0-dev", path = "../../../client/api" } +sc-service = { version = "0.8.0-dev", default-features = false, path = "../../../client/service" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } structopt = "0.3.8" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node/inspect/src/command.rs b/bin/node/inspect/src/command.rs index 335b6c85319a23c2c8e5ffd17d52b593f9602cd7..2212907f763138feef1607b2236bdbbf7e0867a3 100644 --- a/bin/node/inspect/src/command.rs +++ b/bin/node/inspect/src/command.rs @@ -16,186 +16,48 @@ //! Command ran by the CLI -use std::{ - fmt::Debug, - str::FromStr, -}; - use crate::cli::{InspectCmd, InspectSubCmd}; -use crate::{Inspector, PrettyPrinter}; +use crate::Inspector; +use sc_cli::{CliConfiguration, ImportParams, Result, SharedParams}; +use sc_service::{new_full_client, Configuration, NativeExecutionDispatch}; +use sp_runtime::traits::Block; +use std::str::FromStr; impl InspectCmd { - /// Initialize - pub fn init(&self, version: &sc_cli::VersionInfo) -> sc_cli::Result<()> { - self.shared_params.init(version) - } - - /// Parse CLI arguments and initialize given config. - pub fn update_config( - &self, - mut config: &mut sc_service::config::Configuration, - spec_factory: impl FnOnce(&str) -> Result, String>, - version: &sc_cli::VersionInfo, - ) -> sc_cli::Result<()> { - self.shared_params.update_config(config, spec_factory, version)?; - - // make sure to configure keystore - config.use_in_memory_keystore()?; - - // and all import params (especially pruning that has to match db meta) - self.import_params.update_config( - &mut config, - &sc_service::Role::Full, - self.shared_params.dev, - )?; - - Ok(()) - } - /// Run the inspect command, passing the inspector. - pub fn run( - self, - inspect: Inspector, - ) -> sc_cli::Result<()> where - B: sp_runtime::traits::Block, + pub fn run(&self, config: Configuration) -> Result<()> + where + B: Block, B::Hash: FromStr, - P: PrettyPrinter, + RA: Send + Sync + 'static, + EX: NativeExecutionDispatch + 'static, { - match self.command { + let client = new_full_client::(&config)?; + let inspect = Inspector::::new(client); + + match &self.command { InspectSubCmd::Block { input } => { let input = input.parse()?; - let res = inspect.block(input) - .map_err(|e| format!("{}", e))?; + let res = inspect.block(input).map_err(|e| format!("{}", e))?; println!("{}", res); Ok(()) - }, + } InspectSubCmd::Extrinsic { input } => { let input = input.parse()?; - let res = inspect.extrinsic(input) - .map_err(|e| format!("{}", e))?; + let res = inspect.extrinsic(input).map_err(|e| format!("{}", e))?; println!("{}", res); Ok(()) - }, - } - } -} - -/// A block to retrieve. -#[derive(Debug, Clone, PartialEq)] -pub enum BlockAddress { - /// Get block by hash. - Hash(Hash), - /// Get block by number. - Number(Number), - /// Raw SCALE-encoded bytes. - Bytes(Vec), -} - -impl FromStr for BlockAddress { - type Err = String; - - fn from_str(s: &str) -> Result { - // try to parse hash first - if let Ok(hash) = s.parse() { - return Ok(Self::Hash(hash)) - } - - // then number - if let Ok(number) = s.parse() { - return Ok(Self::Number(number)) + } } - - // then assume it's bytes (hex-encoded) - sp_core::bytes::from_hex(s) - .map(Self::Bytes) - .map_err(|e| format!( - "Given string does not look like hash or number. It could not be parsed as bytes either: {}", - e - )) - } -} - -/// An extrinsic address to decode and print out. -#[derive(Debug, Clone, PartialEq)] -pub enum ExtrinsicAddress { - /// Extrinsic as part of existing block. - Block(BlockAddress, usize), - /// Raw SCALE-encoded extrinsic bytes. - Bytes(Vec), -} - -impl FromStr for ExtrinsicAddress { - type Err = String; - - fn from_str(s: &str) -> Result { - // first try raw bytes - if let Ok(bytes) = sp_core::bytes::from_hex(s).map(Self::Bytes) { - return Ok(bytes) - } - - // split by a bunch of different characters - let mut it = s.split(|c| c == '.' || c == ':' || c == ' '); - let block = it.next() - .expect("First element of split iterator is never empty; qed") - .parse()?; - - let index = it.next() - .ok_or_else(|| format!("Extrinsic index missing: example \"5:0\""))? - .parse() - .map_err(|e| format!("Invalid index format: {}", e))?; - - Ok(Self::Block(block, index)) } } -#[cfg(test)] -mod tests { - use super::*; - use sp_core::hash::H160 as Hash; - - #[test] - fn should_parse_block_strings() { - type BlockAddress = super::BlockAddress; - - let b0 = BlockAddress::from_str("3BfC20f0B9aFcAcE800D73D2191166FF16540258"); - let b1 = BlockAddress::from_str("1234"); - let b2 = BlockAddress::from_str("0"); - let b3 = BlockAddress::from_str("0x0012345f"); - - - assert_eq!(b0, Ok(BlockAddress::Hash( - "3BfC20f0B9aFcAcE800D73D2191166FF16540258".parse().unwrap() - ))); - assert_eq!(b1, Ok(BlockAddress::Number(1234))); - assert_eq!(b2, Ok(BlockAddress::Number(0))); - assert_eq!(b3, Ok(BlockAddress::Bytes(vec![0, 0x12, 0x34, 0x5f]))); +impl CliConfiguration for InspectCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params } - #[test] - fn should_parse_extrinsic_address() { - type BlockAddress = super::BlockAddress; - type ExtrinsicAddress = super::ExtrinsicAddress; - - let e0 = ExtrinsicAddress::from_str("1234"); - let b0 = ExtrinsicAddress::from_str("3BfC20f0B9aFcAcE800D73D2191166FF16540258:5"); - let b1 = ExtrinsicAddress::from_str("1234:0"); - let b2 = ExtrinsicAddress::from_str("0 0"); - let b3 = ExtrinsicAddress::from_str("0x0012345f"); - - - assert_eq!(e0, Err("Extrinsic index missing: example \"5:0\"".into())); - assert_eq!(b0, Ok(ExtrinsicAddress::Block( - BlockAddress::Hash("3BfC20f0B9aFcAcE800D73D2191166FF16540258".parse().unwrap()), - 5 - ))); - assert_eq!(b1, Ok(ExtrinsicAddress::Block( - BlockAddress::Number(1234), - 0 - ))); - assert_eq!(b2, Ok(ExtrinsicAddress::Block( - BlockAddress::Number(0), - 0 - ))); - assert_eq!(b3, Ok(ExtrinsicAddress::Bytes(vec![0, 0x12, 0x34, 0x5f]))); + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) } } diff --git a/bin/node/inspect/src/lib.rs b/bin/node/inspect/src/lib.rs index c82682d6021dbbac42f51c0367343d1dec8dcb65..b8101d98a31ce717b4a57f9ee939abe2444253a9 100644 --- a/bin/node/inspect/src/lib.rs +++ b/bin/node/inspect/src/lib.rs @@ -27,7 +27,9 @@ pub mod command; use std::{ fmt, - marker::PhantomData + fmt::Debug, + marker::PhantomData, + str::FromStr, }; use codec::{Encode, Decode}; use sc_client_api::BlockBackend; @@ -38,8 +40,6 @@ use sp_runtime::{ traits::{Block, HashFor, NumberFor, Hash} }; -use command::{BlockAddress, ExtrinsicAddress}; - /// A helper type for a generic block input. pub type BlockAddressFor = BlockAddress< as Hash>::Output, @@ -205,3 +205,123 @@ impl> Inspector Ok(format!("{}", ExtrinsicPrinter(ext, &self.printer))) } } + +/// A block to retrieve. +#[derive(Debug, Clone, PartialEq)] +pub enum BlockAddress { + /// Get block by hash. + Hash(Hash), + /// Get block by number. + Number(Number), + /// Raw SCALE-encoded bytes. + Bytes(Vec), +} + +impl FromStr for BlockAddress { + type Err = String; + + fn from_str(s: &str) -> Result { + // try to parse hash first + if let Ok(hash) = s.parse() { + return Ok(Self::Hash(hash)) + } + + // then number + if let Ok(number) = s.parse() { + return Ok(Self::Number(number)) + } + + // then assume it's bytes (hex-encoded) + sp_core::bytes::from_hex(s) + .map(Self::Bytes) + .map_err(|e| format!( + "Given string does not look like hash or number. It could not be parsed as bytes either: {}", + e + )) + } +} + +/// An extrinsic address to decode and print out. +#[derive(Debug, Clone, PartialEq)] +pub enum ExtrinsicAddress { + /// Extrinsic as part of existing block. + Block(BlockAddress, usize), + /// Raw SCALE-encoded extrinsic bytes. + Bytes(Vec), +} + +impl FromStr for ExtrinsicAddress { + type Err = String; + + fn from_str(s: &str) -> Result { + // first try raw bytes + if let Ok(bytes) = sp_core::bytes::from_hex(s).map(Self::Bytes) { + return Ok(bytes) + } + + // split by a bunch of different characters + let mut it = s.split(|c| c == '.' || c == ':' || c == ' '); + let block = it.next() + .expect("First element of split iterator is never empty; qed") + .parse()?; + + let index = it.next() + .ok_or_else(|| format!("Extrinsic index missing: example \"5:0\""))? + .parse() + .map_err(|e| format!("Invalid index format: {}", e))?; + + Ok(Self::Block(block, index)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use sp_core::hash::H160 as Hash; + + #[test] + fn should_parse_block_strings() { + type BlockAddress = super::BlockAddress; + + let b0 = BlockAddress::from_str("3BfC20f0B9aFcAcE800D73D2191166FF16540258"); + let b1 = BlockAddress::from_str("1234"); + let b2 = BlockAddress::from_str("0"); + let b3 = BlockAddress::from_str("0x0012345f"); + + + assert_eq!(b0, Ok(BlockAddress::Hash( + "3BfC20f0B9aFcAcE800D73D2191166FF16540258".parse().unwrap() + ))); + assert_eq!(b1, Ok(BlockAddress::Number(1234))); + assert_eq!(b2, Ok(BlockAddress::Number(0))); + assert_eq!(b3, Ok(BlockAddress::Bytes(vec![0, 0x12, 0x34, 0x5f]))); + } + + #[test] + fn should_parse_extrinsic_address() { + type BlockAddress = super::BlockAddress; + type ExtrinsicAddress = super::ExtrinsicAddress; + + let e0 = ExtrinsicAddress::from_str("1234"); + let b0 = ExtrinsicAddress::from_str("3BfC20f0B9aFcAcE800D73D2191166FF16540258:5"); + let b1 = ExtrinsicAddress::from_str("1234:0"); + let b2 = ExtrinsicAddress::from_str("0 0"); + let b3 = ExtrinsicAddress::from_str("0x0012345f"); + + + assert_eq!(e0, Err("Extrinsic index missing: example \"5:0\"".into())); + assert_eq!(b0, Ok(ExtrinsicAddress::Block( + BlockAddress::Hash("3BfC20f0B9aFcAcE800D73D2191166FF16540258".parse().unwrap()), + 5 + ))); + assert_eq!(b1, Ok(ExtrinsicAddress::Block( + BlockAddress::Number(1234), + 0 + ))); + assert_eq!(b2, Ok(ExtrinsicAddress::Block( + BlockAddress::Number(0), + 0 + ))); + assert_eq!(b3, Ok(ExtrinsicAddress::Bytes(vec![0, 0x12, 0x34, 0x5f]))); + } +} diff --git a/bin/node/primitives/Cargo.toml b/bin/node/primitives/Cargo.toml index 81c5009f394d57332d5c9ef65ed8c6bb2cb6f120..11959bde75c642422adc25ca11420ea71e548cf2 100644 --- a/bin/node/primitives/Cargo.toml +++ b/bin/node/primitives/Cargo.toml @@ -1,18 +1,21 @@ [package] name = "node-primitives" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/runtime" } [dev-dependencies] -sp-serializer = { version = "2.0.0-alpha.5", path = "../../../primitives/serializer" } +sp-serializer = { version = "2.0.0-dev", path = "../../../primitives/serializer" } pretty_assertions = "0.6.1" [features] @@ -21,6 +24,3 @@ std = [ "sp-core/std", "sp-runtime/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node/rpc-client/Cargo.toml b/bin/node/rpc-client/Cargo.toml index df095bc5bb10d2574a01790f3ed32ea9c8585ebe..1df3590d1fe833f2e2c93c98cf4dfc2aa5501c2f 100644 --- a/bin/node/rpc-client/Cargo.toml +++ b/bin/node/rpc-client/Cargo.toml @@ -1,20 +1,20 @@ [package] name = "node-rpc-client" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] env_logger = "0.7.0" futures = "0.1.29" hyper = "0.12.35" jsonrpc-core-client = { version = "14.0.5", default-features = false, features = ["http"] } log = "0.4.8" -node-primitives = { version = "2.0.0-alpha.5", path = "../primitives" } -sc-rpc = { version = "2.0.0-alpha.5", path = "../../../client/rpc" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +node-primitives = { version = "2.0.0-dev", path = "../primitives" } +sc-rpc = { version = "2.0.0-dev", path = "../../../client/rpc" } diff --git a/bin/node/rpc/Cargo.toml b/bin/node/rpc/Cargo.toml index f1d230af90dc5e9e65da0ba4284355fa5292435a..76d9998831cc57d18d554ac4e298eb88500de652 100644 --- a/bin/node/rpc/Cargo.toml +++ b/bin/node/rpc/Cargo.toml @@ -1,30 +1,30 @@ [package] name = "node-rpc" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" -[dependencies] -sc-client = { version = "0.8.0-alpha.5", path = "../../../client/" } -jsonrpc-core = "14.0.3" -node-primitives = { version = "2.0.0-alpha.5", path = "../primitives" } -node-runtime = { version = "2.0.0-alpha.5", path = "../runtime" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.5", path = "../../../primitives/api" } -pallet-contracts-rpc = { version = "0.8.0-alpha.5", path = "../../../frame/contracts/rpc/" } -pallet-transaction-payment-rpc = { version = "2.0.0-alpha.5", path = "../../../frame/transaction-payment/rpc/" } -substrate-frame-rpc-system = { version = "2.0.0-alpha.5", path = "../../../utils/frame/rpc/system" } -sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../primitives/transaction-pool" } -sc-consensus-babe = { version = "0.8.0-alpha.5", path = "../../../client/consensus/babe" } -sc-consensus-babe-rpc = { version = "0.8.0-alpha.5", path = "../../../client/consensus/babe/rpc" } -sp-consensus-babe = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/babe" } -sc-keystore = { version = "2.0.0-alpha.5", path = "../../../client/keystore" } -sc-consensus-epochs = { version = "0.8.0-alpha.5", path = "../../../client/consensus/epochs" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } - [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +sc-client-api = { version = "2.0.0-dev", path = "../../../client/api" } +jsonrpc-core = "14.0.3" +node-primitives = { version = "2.0.0-dev", path = "../primitives" } +node-runtime = { version = "2.0.0-dev", path = "../runtime" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-dev", path = "../../../primitives/api" } +pallet-contracts-rpc = { version = "0.8.0-dev", path = "../../../frame/contracts/rpc/" } +pallet-transaction-payment-rpc = { version = "2.0.0-dev", path = "../../../frame/transaction-payment/rpc/" } +substrate-frame-rpc-system = { version = "2.0.0-dev", path = "../../../utils/frame/rpc/system" } +sp-transaction-pool = { version = "2.0.0-dev", path = "../../../primitives/transaction-pool" } +sc-consensus-babe = { version = "0.8.0-dev", path = "../../../client/consensus/babe" } +sc-consensus-babe-rpc = { version = "0.8.0-dev", path = "../../../client/consensus/babe/rpc" } +sp-consensus-babe = { version = "0.8.0-dev", path = "../../../primitives/consensus/babe" } +sc-keystore = { version = "2.0.0-dev", path = "../../../client/keystore" } +sc-consensus-epochs = { version = "0.8.0-dev", path = "../../../client/consensus/epochs" } +sp-consensus = { version = "0.8.0-dev", path = "../../../primitives/consensus/common" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } diff --git a/bin/node/rpc/src/lib.rs b/bin/node/rpc/src/lib.rs index 4e1cfa56733a7bbbbb50fd944241410465d488c1..297dc129aeada73e1b03548d5e670dde0b99a6ab 100644 --- a/bin/node/rpc/src/lib.rs +++ b/bin/node/rpc/src/lib.rs @@ -50,7 +50,7 @@ pub struct LightDeps { /// Transaction pool instance. pub pool: Arc

, /// Remote access to the blockchain (async). - pub remote_blockchain: Arc>, + pub remote_blockchain: Arc>, /// Fetcher instance. pub fetcher: Arc, } @@ -135,9 +135,9 @@ pub fn create_full( pub fn create_light( deps: LightDeps, ) -> jsonrpc_core::IoHandler where - C: sc_client::blockchain::HeaderBackend, + C: sp_blockchain::HeaderBackend, C: Send + Sync + 'static, - F: sc_client::light::fetcher::Fetcher + 'static, + F: sc_client_api::light::Fetcher + 'static, P: TransactionPool + 'static, M: jsonrpc_core::Metadata + Default, { diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 67e50e53f69e6755481de5c5fe615d968a67d2d5..bf360821c84572eaeb6bb2f9952efc4a8b7bd9c3 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-runtime" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" @@ -8,6 +8,9 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] # third-party dependencies @@ -16,66 +19,67 @@ integer-sqrt = { version = "0.1.2" } serde = { version = "1.0.102", optional = true } # primitives -sp-authority-discovery = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/authority-discovery" } -sp-consensus-babe = { version = "0.8.0-alpha.5", default-features = false, path = "../../../primitives/consensus/babe" } -sp-block-builder = { path = "../../../primitives/block-builder", default-features = false, version = "2.0.0-alpha.5"} -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/inherents" } -node-primitives = { version = "2.0.0-alpha.5", default-features = false, path = "../primitives" } -sp-offchain = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/offchain" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/core" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/std" } -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/api" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/runtime" } -sp-staking = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/staking" } -sp-keyring = { version = "2.0.0-alpha.5", optional = true, path = "../../../primitives/keyring" } -sp-session = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/session" } -sp-transaction-pool = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/transaction-pool" } -sp-version = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/version" } +sp-authority-discovery = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/authority-discovery" } +sp-consensus-babe = { version = "0.8.0-dev", default-features = false, path = "../../../primitives/consensus/babe" } +sp-block-builder = { path = "../../../primitives/block-builder", default-features = false, version = "2.0.0-dev"} +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/inherents" } +node-primitives = { version = "2.0.0-dev", default-features = false, path = "../primitives" } +sp-offchain = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/offchain" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/core" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/std" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/api" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/runtime" } +sp-staking = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/staking" } +sp-keyring = { version = "2.0.0-dev", optional = true, path = "../../../primitives/keyring" } +sp-session = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/session" } +sp-transaction-pool = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/transaction-pool" } +sp-version = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/version" } # frame dependencies -frame-executive = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/executive" } -frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/benchmarking", optional = true } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/system" } -frame-system-rpc-runtime-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } -pallet-authority-discovery = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/authority-discovery" } -pallet-authorship = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/authorship" } -pallet-babe = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/babe" } -pallet-balances = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/balances" } -pallet-collective = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/collective" } -pallet-contracts = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/contracts" } -pallet-contracts-primitives = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/contracts/common/" } -pallet-contracts-rpc-runtime-api = { version = "0.8.0-alpha.5", default-features = false, path = "../../../frame/contracts/rpc/runtime-api/" } -pallet-democracy = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/democracy" } -pallet-elections-phragmen = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/elections-phragmen" } -pallet-finality-tracker = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/finality-tracker" } -pallet-grandpa = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/grandpa" } -pallet-im-online = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/im-online" } -pallet-indices = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/indices" } -pallet-identity = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/identity" } -pallet-membership = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/membership" } -pallet-offences = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/offences" } -pallet-randomness-collective-flip = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/randomness-collective-flip" } -pallet-recovery = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/recovery" } -pallet-session = { version = "2.0.0-alpha.5", features = ["historical"], path = "../../../frame/session", default-features = false } -pallet-session-benchmarking = { version = "2.0.0-alpha.5", path = "../../../frame/session/benchmarking", default-features = false, optional = true } -pallet-staking = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/staking" } -pallet-staking-reward-curve = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/staking/reward-curve" } -pallet-scheduler = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/scheduler" } -pallet-society = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/society" } -pallet-sudo = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/sudo" } -pallet-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/timestamp" } -pallet-treasury = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/treasury" } -pallet-utility = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/utility" } -pallet-transaction-payment = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/transaction-payment" } -pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" } -pallet-vesting = { version = "2.0.0-alpha.5", default-features = false, path = "../../../frame/vesting" } +frame-executive = { version = "2.0.0-dev", default-features = false, path = "../../../frame/executive" } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../../../frame/benchmarking", optional = true } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../../../frame/support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../../../frame/system" } +frame-system-rpc-runtime-api = { version = "2.0.0-dev", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } +pallet-authority-discovery = { version = "2.0.0-dev", default-features = false, path = "../../../frame/authority-discovery" } +pallet-authorship = { version = "2.0.0-dev", default-features = false, path = "../../../frame/authorship" } +pallet-babe = { version = "2.0.0-dev", default-features = false, path = "../../../frame/babe" } +pallet-balances = { version = "2.0.0-dev", default-features = false, path = "../../../frame/balances" } +pallet-collective = { version = "2.0.0-dev", default-features = false, path = "../../../frame/collective" } +pallet-contracts = { version = "2.0.0-dev", default-features = false, path = "../../../frame/contracts" } +pallet-contracts-primitives = { version = "2.0.0-dev", default-features = false, path = "../../../frame/contracts/common/" } +pallet-contracts-rpc-runtime-api = { version = "0.8.0-dev", default-features = false, path = "../../../frame/contracts/rpc/runtime-api/" } +pallet-democracy = { version = "2.0.0-dev", default-features = false, path = "../../../frame/democracy" } +pallet-elections-phragmen = { version = "2.0.0-dev", default-features = false, path = "../../../frame/elections-phragmen" } +pallet-finality-tracker = { version = "2.0.0-dev", default-features = false, path = "../../../frame/finality-tracker" } +pallet-grandpa = { version = "2.0.0-dev", default-features = false, path = "../../../frame/grandpa" } +pallet-im-online = { version = "2.0.0-dev", default-features = false, path = "../../../frame/im-online" } +pallet-indices = { version = "2.0.0-dev", default-features = false, path = "../../../frame/indices" } +pallet-identity = { version = "2.0.0-dev", default-features = false, path = "../../../frame/identity" } +pallet-membership = { version = "2.0.0-dev", default-features = false, path = "../../../frame/membership" } +pallet-offences = { version = "2.0.0-dev", default-features = false, path = "../../../frame/offences" } +pallet-offences-benchmarking = { version = "2.0.0-dev", path = "../../../frame/offences/benchmarking", default-features = false, optional = true } +pallet-randomness-collective-flip = { version = "2.0.0-dev", default-features = false, path = "../../../frame/randomness-collective-flip" } +pallet-recovery = { version = "2.0.0-dev", default-features = false, path = "../../../frame/recovery" } +pallet-session = { version = "2.0.0-dev", features = ["historical"], path = "../../../frame/session", default-features = false } +pallet-session-benchmarking = { version = "2.0.0-dev", path = "../../../frame/session/benchmarking", default-features = false, optional = true } +pallet-staking = { version = "2.0.0-dev", default-features = false, path = "../../../frame/staking" } +pallet-staking-reward-curve = { version = "2.0.0-dev", default-features = false, path = "../../../frame/staking/reward-curve" } +pallet-scheduler = { version = "2.0.0-dev", default-features = false, path = "../../../frame/scheduler" } +pallet-society = { version = "2.0.0-dev", default-features = false, path = "../../../frame/society" } +pallet-sudo = { version = "2.0.0-dev", default-features = false, path = "../../../frame/sudo" } +pallet-timestamp = { version = "2.0.0-dev", default-features = false, path = "../../../frame/timestamp" } +pallet-treasury = { version = "2.0.0-dev", default-features = false, path = "../../../frame/treasury" } +pallet-utility = { version = "2.0.0-dev", default-features = false, path = "../../../frame/utility" } +pallet-transaction-payment = { version = "2.0.0-dev", default-features = false, path = "../../../frame/transaction-payment" } +pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-dev", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" } +pallet-vesting = { version = "2.0.0-dev", default-features = false, path = "../../../frame/vesting" } [build-dependencies] wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } [dev-dependencies] -sp-io = { version = "2.0.0-alpha.5", path = "../../../primitives/io" } +sp-io = { version = "2.0.0-dev", path = "../../../primitives/io" } [features] default = ["std"] @@ -149,9 +153,6 @@ runtime-benchmarks = [ "pallet-treasury/runtime-benchmarks", "pallet-utility/runtime-benchmarks", "pallet-vesting/runtime-benchmarks", - "pallet-collective/runtime-benchmarks", + "pallet-offences-benchmarking", "pallet-session-benchmarking", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/node/runtime/src/impls.rs b/bin/node/runtime/src/impls.rs index 646dc24f578de221339d08d27393b5457e7fda88..f613dc5af5048cfdd5ea214600099d60df83097f 100644 --- a/bin/node/runtime/src/impls.rs +++ b/bin/node/runtime/src/impls.rs @@ -16,9 +16,10 @@ //! Some configurable implementations as associated type for the substrate runtime. +use core::num::NonZeroI128; use node_primitives::Balance; use sp_runtime::traits::{Convert, Saturating}; -use sp_runtime::{Fixed64, Perbill}; +use sp_runtime::{Fixed128, Perquintill}; use frame_support::{traits::{OnUnbalanced, Currency, Get}, weights::Weight}; use crate::{Balances, System, Authorship, MaximumBlockWeight, NegativeImbalance}; @@ -51,8 +52,7 @@ pub struct LinearWeightToFee(sp_std::marker::PhantomData); impl> Convert for LinearWeightToFee { fn convert(w: Weight) -> Balance { - // substrate-node a weight of 10_000 (smallest non-zero weight) to be mapped to 10^7 units of - // fees, hence: + // setting this to zero will disable the weight fee. let coefficient = C::get(); Balance::from(w).saturating_mul(coefficient) } @@ -60,16 +60,16 @@ impl> Convert for LinearWeightToFee { /// Update the given multiplier based on the following formula /// -/// diff = (previous_block_weight - target_weight) +/// diff = (previous_block_weight - target_weight)/max_weight /// v = 0.00004 -/// next_weight = weight * (1 + (v . diff) + (v . diff)^2 / 2) +/// next_weight = weight * (1 + (v * diff) + (v * diff)^2 / 2) /// /// Where `target_weight` must be given as the `Get` implementation of the `T` generic type. /// https://research.web3.foundation/en/latest/polkadot/Token%20Economics/#relay-chain-transaction-fees pub struct TargetedFeeAdjustment(sp_std::marker::PhantomData); -impl> Convert for TargetedFeeAdjustment { - fn convert(multiplier: Fixed64) -> Fixed64 { +impl> Convert for TargetedFeeAdjustment { + fn convert(multiplier: Fixed128) -> Fixed128 { let block_weight = System::all_extrinsics_weight(); let max_weight = MaximumBlockWeight::get(); let target_weight = (T::get() * max_weight) as u128; @@ -78,19 +78,20 @@ impl> Convert for TargetedFeeAdjustment { // determines if the first_term is positive let positive = block_weight >= target_weight; let diff_abs = block_weight.max(target_weight) - block_weight.min(target_weight); - // diff is within u32, safe. - let diff = Fixed64::from_rational(diff_abs as i64, max_weight as u64); + // safe, diff_abs cannot exceed u64 and it can always be computed safely even with the lossy + // `Fixed128::from_rational`. + let diff = Fixed128::from_rational( + diff_abs as i128, + NonZeroI128::new(max_weight.max(1) as i128).unwrap(), + ); let diff_squared = diff.saturating_mul(diff); // 0.00004 = 4/100_000 = 40_000/10^9 - let v = Fixed64::from_rational(4, 100_000); - // 0.00004^2 = 16/10^10 ~= 2/10^9. Taking the future /2 into account, then it is just 1 - // parts from a billionth. - let v_squared_2 = Fixed64::from_rational(1, 1_000_000_000); + let v = Fixed128::from_rational(4, NonZeroI128::new(100_000).unwrap()); + // 0.00004^2 = 16/10^10 Taking the future /2 into account... 8/10^10 + let v_squared_2 = Fixed128::from_rational(8, NonZeroI128::new(10_000_000_000).unwrap()); let first_term = v.saturating_mul(diff); - // It is very unlikely that this will exist (in our poor perbill estimate) but we are giving - // it a shot. let second_term = v_squared_2.saturating_mul(diff_squared); if positive { @@ -99,15 +100,15 @@ impl> Convert for TargetedFeeAdjustment { let excess = first_term.saturating_add(second_term); multiplier.saturating_add(excess) } else { - // Proof: first_term > second_term. Safe subtraction. - let negative = first_term - second_term; + // Defensive-only: first_term > second_term. Safe subtraction. + let negative = first_term.saturating_sub(second_term); multiplier.saturating_sub(negative) // despite the fact that apply_to saturates weight (final fee cannot go below 0) // it is crucially important to stop here and don't further reduce the weight fee // multiplier. While at -1, it means that the network is so un-congested that all // transactions have no weight fee. We stop here and only increase if the network // became more busy. - .max(Fixed64::from_rational(-1, 1)) + .max(Fixed128::from_natural(-1)) } } } @@ -119,6 +120,7 @@ mod tests { use crate::{MaximumBlockWeight, AvailableBlockRatio, Runtime}; use crate::{constants::currency::*, TransactionPayment, TargetBlockFullness}; use frame_support::weights::Weight; + use core::num::NonZeroI128; fn max() -> Weight { MaximumBlockWeight::get() @@ -129,26 +131,22 @@ mod tests { } // poc reference implementation. - fn fee_multiplier_update(block_weight: Weight, previous: Fixed64) -> Fixed64 { - let block_weight = block_weight as f32; - let v: f32 = 0.00004; + fn fee_multiplier_update(block_weight: Weight, previous: Fixed128) -> Fixed128 { + let block_weight = block_weight as f64; + let v: f64 = 0.00004; // maximum tx weight - let m = max() as f32; + let m = max() as f64; // Ideal saturation in terms of weight - let ss = target() as f32; + let ss = target() as f64; // Current saturation in terms of weight let s = block_weight; let fm = v * (s/m - ss/m) + v.powi(2) * (s/m - ss/m).powi(2) / 2.0; - let addition_fm = Fixed64::from_parts((fm * 1_000_000_000_f32).round() as i64); + let addition_fm = Fixed128::from_parts((fm * Fixed128::accuracy() as f64).round() as i128); previous.saturating_add(addition_fm) } - fn feemul(parts: i64) -> Fixed64 { - Fixed64::from_parts(parts) - } - fn run_with_system_weight(w: Weight, assertions: F) where F: Fn() -> () { let mut t: sp_io::TestExternalities = frame_system::GenesisConfig::default().build_storage::().unwrap().into(); @@ -160,7 +158,7 @@ mod tests { #[test] fn fee_multiplier_update_poc_works() { - let fm = Fixed64::from_rational(0, 1); + let fm = Fixed128::from_rational(0, NonZeroI128::new(1).unwrap()); let test_set = vec![ (0, fm.clone()), (100, fm.clone()), @@ -171,9 +169,10 @@ mod tests { test_set.into_iter().for_each(|(w, fm)| { run_with_system_weight(w, || { assert_eq_error_rate!( - fee_multiplier_update(w, fm).into_inner(), - TargetedFeeAdjustment::::convert(fm).into_inner(), - 5, + fee_multiplier_update(w, fm), + TargetedFeeAdjustment::::convert(fm), + // Error is only 1 in 10^18 + Fixed128::from_parts(1), ); }) }) @@ -184,12 +183,12 @@ mod tests { // just a few txs per_block. let block_weight = 0; run_with_system_weight(block_weight, || { - let mut fm = Fixed64::default(); + let mut fm = Fixed128::default(); let mut iterations: u64 = 0; loop { let next = TargetedFeeAdjustment::::convert(fm); fm = next; - if fm == Fixed64::from_rational(-1, 1) { break; } + if fm == Fixed128::from_natural(-1) { break; } iterations += 1; } println!("iteration {}, new fm = {:?}. Weight fee is now zero", iterations, fm); @@ -217,7 +216,7 @@ mod tests { run_with_system_weight(block_weight, || { // initial value configured on module - let mut fm = Fixed64::default(); + let mut fm = Fixed128::default(); assert_eq!(fm, TransactionPayment::next_fee_multiplier()); let mut iterations: u64 = 0; @@ -245,79 +244,94 @@ mod tests { #[test] fn stateless_weight_mul() { + // This test will show that heavy blocks have a weight multiplier greater than 0 + // and light blocks will have a weight multiplier less than 0. run_with_system_weight(target() / 4, || { - // Light block. Fee is reduced a little. + // `fee_multiplier_update` is enough as it is the absolute truth value. + let next = TargetedFeeAdjustment::::convert(Fixed128::default()); assert_eq!( - TargetedFeeAdjustment::::convert(Fixed64::default()), - feemul(-7500), + next, + fee_multiplier_update(target() / 4 ,Fixed128::default()) ); + + // Light block. Fee is reduced a little. + assert!(next < Fixed128::zero()) }); run_with_system_weight(target() / 2, || { - // a bit more. Fee is decreased less, meaning that the fee increases as the block grows. + let next = TargetedFeeAdjustment::::convert(Fixed128::default()); assert_eq!( - TargetedFeeAdjustment::::convert(Fixed64::default()), - feemul(-5000), + next, + fee_multiplier_update(target() / 2 ,Fixed128::default()) ); + // Light block. Fee is reduced a little. + assert!(next < Fixed128::zero()) + }); run_with_system_weight(target(), || { // ideal. Original fee. No changes. - assert_eq!( - TargetedFeeAdjustment::::convert(Fixed64::default()), - feemul(0), - ); + let next = TargetedFeeAdjustment::::convert(Fixed128::default()); + assert_eq!(next, Fixed128::zero()) }); run_with_system_weight(target() * 2, || { - // // More than ideal. Fee is increased. + // More than ideal. Fee is increased. + let next = TargetedFeeAdjustment::::convert(Fixed128::default()); assert_eq!( - TargetedFeeAdjustment::::convert(Fixed64::default()), - feemul(10000), + next, + fee_multiplier_update(target() * 2 ,Fixed128::default()) ); + + // Heavy block. Fee is increased a little. + assert!(next > Fixed128::zero()) }); } #[test] fn stateful_weight_mul_grow_to_infinity() { run_with_system_weight(target() * 2, || { - assert_eq!( - TargetedFeeAdjustment::::convert(Fixed64::default()), - feemul(10000) - ); - assert_eq!( - TargetedFeeAdjustment::::convert(feemul(10000)), - feemul(20000) - ); - assert_eq!( - TargetedFeeAdjustment::::convert(feemul(20000)), - feemul(30000) - ); - // ... - assert_eq!( - TargetedFeeAdjustment::::convert(feemul(1_000_000_000)), - feemul(1_000_000_000 + 10000) - ); + let mut original = Fixed128::default(); + let mut next = Fixed128::default(); + + (0..1_000).for_each(|_| { + next = TargetedFeeAdjustment::::convert(original); + assert_eq!( + next, + fee_multiplier_update(target() * 2, original), + ); + // must always increase + assert!(next > original); + original = next; + }); }); } #[test] fn stateful_weight_mil_collapse_to_minus_one() { run_with_system_weight(0, || { + let mut original = Fixed128::default(); // 0 + let mut next; + + // decreases + next = TargetedFeeAdjustment::::convert(original); assert_eq!( - TargetedFeeAdjustment::::convert(Fixed64::default()), - feemul(-10000) - ); - assert_eq!( - TargetedFeeAdjustment::::convert(feemul(-10000)), - feemul(-20000) + next, + fee_multiplier_update(0, original), ); + assert!(next < original); + original = next; + + // keeps decreasing + next = TargetedFeeAdjustment::::convert(original); assert_eq!( - TargetedFeeAdjustment::::convert(feemul(-20000)), - feemul(-30000) + next, + fee_multiplier_update(0, original), ); - // ... + assert!(next < original); + + // ... stops going down at -1 assert_eq!( - TargetedFeeAdjustment::::convert(feemul(1_000_000_000 * -1)), - feemul(-1_000_000_000) + TargetedFeeAdjustment::::convert(Fixed128::from_natural(-1)), + Fixed128::from_natural(-1) ); }) } @@ -326,7 +340,7 @@ mod tests { fn weight_to_fee_should_not_overflow_on_large_weights() { let kb = 1024 as Weight; let mb = kb * kb; - let max_fm = Fixed64::from_natural(i64::max_value()); + let max_fm = Fixed128::from_natural(i128::max_value()); // check that for all values it can compute, correctly. vec![ @@ -339,13 +353,17 @@ mod tests { 100 * kb, mb, 10 * mb, + 2147483647, + 4294967295, + MaximumBlockWeight::get() / 2, + MaximumBlockWeight::get(), Weight::max_value() / 2, Weight::max_value(), ].into_iter().for_each(|i| { run_with_system_weight(i, || { - let next = TargetedFeeAdjustment::::convert(Fixed64::default()); - let truth = fee_multiplier_update(i, Fixed64::default()); - assert_eq_error_rate!(truth.into_inner(), next.into_inner(), 5); + let next = TargetedFeeAdjustment::::convert(Fixed128::default()); + let truth = fee_multiplier_update(i, Fixed128::default()); + assert_eq_error_rate!(truth, next, Fixed128::from_parts(50_000_000)); }); }); diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 309f4ed0245e4b6e3006dd6b87613aff69ec1159..641cbf44ebe0e524e620456cce2789b82d7ac361 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -12,7 +12,7 @@ // GNU General Public License for more details. // You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . +// along with Substrate. If not, see . //! The Substrate runtime. This can be compiled with ``#[no_std]`, ready for Wasm. @@ -23,19 +23,19 @@ use sp_std::prelude::*; use frame_support::{ construct_runtime, parameter_types, debug, - weights::Weight, - traits::{Currency, Randomness, OnUnbalanced, Imbalance}, + weights::{Weight, RuntimeDbWeight}, + traits::{Currency, Randomness, OnUnbalanced, Imbalance, LockIdentifier}, }; use sp_core::u32_trait::{_1, _2, _3, _4}; pub use node_primitives::{AccountId, Signature}; use node_primitives::{AccountIndex, Balance, BlockNumber, Hash, Index, Moment}; use sp_api::impl_runtime_apis; use sp_runtime::{ - Permill, Perbill, Percent, ApplyExtrinsicResult, - impl_opaque_keys, generic, create_runtime_str, + Permill, Perbill, Perquintill, Percent, ApplyExtrinsicResult, + impl_opaque_keys, generic, create_runtime_str, ModuleId, }; use sp_runtime::curve::PiecewiseLinear; -use sp_runtime::transaction_validity::{TransactionValidity, TransactionSource}; +use sp_runtime::transaction_validity::{TransactionValidity, TransactionSource, TransactionPriority}; use sp_runtime::traits::{ self, BlakeTwo256, Block as BlockT, StaticLookup, SaturatedConversion, ConvertInto, OpaqueKeys, @@ -46,11 +46,10 @@ use sp_version::NativeVersion; use sp_core::OpaqueMetadata; use pallet_grandpa::AuthorityList as GrandpaAuthorityList; use pallet_grandpa::fg_primitives; -use pallet_im_online::sr25519::{AuthorityId as ImOnlineId}; +use pallet_im_online::sr25519::AuthorityId as ImOnlineId; use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId; use pallet_transaction_payment_rpc_runtime_api::RuntimeDispatchInfo; use pallet_contracts_rpc_runtime_api::ContractExecResult; -use frame_system::offchain::TransactionSubmitter; use sp_inherents::{InherentData, CheckInherentsResult}; #[cfg(any(feature = "std", test))] @@ -59,7 +58,8 @@ pub use pallet_timestamp::Call as TimestampCall; pub use pallet_balances::Call as BalancesCall; pub use pallet_contracts::Gas; pub use frame_support::StorageValue; -pub use pallet_staking::{StakerStatus, LockStakingStatus}; +pub use pallet_staking::StakerStatus; +use codec::Encode; /// Implementations of some helper traits passed into runtime modules as associated types. pub mod impls; @@ -73,52 +73,6 @@ use constants::{time::*, currency::*}; #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); -/// A transaction submitter with the given key type. -pub type TransactionSubmitterOf = TransactionSubmitter; - -/// Submits transaction with the node's public and signature type. Adheres to the signed extension -/// format of the chain. -impl frame_system::offchain::CreateTransaction for Runtime { - type Public = ::Signer; - type Signature = Signature; - - fn create_transaction>( - call: Call, - public: Self::Public, - account: AccountId, - index: Index, - ) -> Option<(Call, ::SignaturePayload)> { - // take the biggest period possible. - let period = BlockHashCount::get() - .checked_next_power_of_two() - .map(|c| c / 2) - .unwrap_or(2) as u64; - let current_block = System::block_number() - .saturated_into::() - // The `System::block_number` is initialized with `n+1`, - // so the actual block number is `n`. - .saturating_sub(1); - let tip = 0; - let extra: SignedExtra = ( - frame_system::CheckVersion::::new(), - frame_system::CheckGenesis::::new(), - frame_system::CheckEra::::from(generic::Era::mortal(period, current_block)), - frame_system::CheckNonce::::from(index), - frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(tip), - Default::default(), - Default::default(), - ); - let raw_payload = SignedPayload::new(call, extra).map_err(|e| { - debug::warn!("Unable to create signed payload: {:?}", e); - }).ok()?; - let signature = TSigner::sign(public, &raw_payload)?; - let address = Indices::unlookup(account); - let (call, extra, _) = raw_payload.deconstruct(); - Some((call, (address, signature, extra))) - } -} - /// Runtime version. pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("node"), @@ -128,9 +82,10 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // and set impl_version to 0. If only runtime // implementation changes and behavior does not, then leave spec_version as // is and increment impl_version. - spec_version: 240, - impl_version: 1, + spec_version: 244, + impl_version: 3, apis: RUNTIME_API_VERSIONS, + transaction_version: 1, }; /// Native version. @@ -162,10 +117,16 @@ impl OnUnbalanced for DealWithFees { parameter_types! { pub const BlockHashCount: BlockNumber = 250; - pub const MaximumBlockWeight: Weight = 1_000_000_000; + /// We allow for 2 seconds of compute with a 6 second average block time. + pub const MaximumBlockWeight: Weight = 2_000_000_000_000; + pub const ExtrinsicBaseWeight: Weight = 10_000_000; pub const MaximumBlockLength: u32 = 5 * 1024 * 1024; pub const Version: RuntimeVersion = VERSION; pub const AvailableBlockRatio: Perbill = Perbill::from_percent(75); + pub const DbWeight: RuntimeDbWeight = RuntimeDbWeight { + read: 60_000_000, // ~0.06 ms = ~60 µs + write: 200_000_000, // ~0.2 ms = 200 µs + }; } impl frame_system::Trait for Runtime { @@ -181,6 +142,9 @@ impl frame_system::Trait for Runtime { type Event = Event; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = DbWeight; + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = ExtrinsicBaseWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = Version; @@ -253,18 +217,17 @@ impl pallet_balances::Trait for Runtime { } parameter_types! { - pub const TransactionBaseFee: Balance = 1 * CENTS; pub const TransactionByteFee: Balance = 10 * MILLICENTS; - // setting this to zero will disable the weight fee. - pub const WeightFeeCoefficient: Balance = 1_000; + // In the Substrate node, a weight of 10_000_000 (smallest non-zero weight) + // is mapped to 10_000_000 units of fees, hence: + pub const WeightFeeCoefficient: Balance = 1; // for a sane configuration, this should always be less than `AvailableBlockRatio`. - pub const TargetBlockFullness: Perbill = Perbill::from_percent(25); + pub const TargetBlockFullness: Perquintill = Perquintill::from_percent(25); } impl pallet_transaction_payment::Trait for Runtime { type Currency = Balances; type OnTransactionPayment = DealWithFees; - type TransactionBaseFee = TransactionBaseFee; type TransactionByteFee = TransactionByteFee; type WeightToFee = LinearWeightToFee; type FeeMultiplierUpdate = TargetedFeeAdjustment; @@ -337,8 +300,9 @@ parameter_types! { pub const BondingDuration: pallet_staking::EraIndex = 24 * 28; pub const SlashDeferDuration: pallet_staking::EraIndex = 24 * 7; // 1/4 the bonding duration. pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; - pub const ElectionLookahead: BlockNumber = 25; // 10 minutes per session => 100 block. + pub const ElectionLookahead: BlockNumber = EPOCH_DURATION_IN_BLOCKS / 4; pub const MaxNominatorRewardedPerValidator: u32 = 64; + pub const MaxIterations: u32 = 5; } impl pallet_staking::Trait for Runtime { @@ -359,8 +323,9 @@ impl pallet_staking::Trait for Runtime { type NextNewSession = Session; type ElectionLookahead = ElectionLookahead; type Call = Call; - type SubmitTransaction = TransactionSubmitterOf<()>; + type MaxIterations = MaxIterations; type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; + type UnsignedPriority = StakingUnsignedPriority; } parameter_types! { @@ -425,9 +390,11 @@ parameter_types! { pub const TermDuration: BlockNumber = 7 * DAYS; pub const DesiredMembers: u32 = 13; pub const DesiredRunnersUp: u32 = 7; + pub const ElectionsPhragmenModuleId: LockIdentifier = *b"phrelect"; } impl pallet_elections_phragmen::Trait for Runtime { + type ModuleId = ElectionsPhragmenModuleId; type Event = Event; type Currency = Balances; type ChangeMembers = Council; @@ -477,6 +444,7 @@ parameter_types! { pub const TipFindersFee: Percent = Percent::from_percent(20); pub const TipReportDepositBase: Balance = 1 * DOLLARS; pub const TipReportDepositPerByte: Balance = 1 * CENTS; + pub const TreasuryModuleId: ModuleId = ModuleId(*b"py/trsry"); } impl pallet_treasury::Trait for Runtime { @@ -494,12 +462,10 @@ impl pallet_treasury::Trait for Runtime { type ProposalBondMinimum = ProposalBondMinimum; type SpendPeriod = SpendPeriod; type Burn = Burn; + type ModuleId = TreasuryModuleId; } parameter_types! { - pub const ContractTransactionBaseFee: Balance = 1 * CENTS; - pub const ContractTransactionByteFee: Balance = 10 * MILLICENTS; - pub const ContractFee: Balance = 1 * CENTS; pub const TombstoneDeposit: Balance = 1 * DOLLARS; pub const RentByteFee: Balance = 1 * DOLLARS; pub const RentDepositOffset: Balance = 1000 * DOLLARS; @@ -507,15 +473,12 @@ parameter_types! { } impl pallet_contracts::Trait for Runtime { - type Currency = Balances; type Time = Timestamp; type Randomness = RandomnessCollectiveFlip; type Call = Call; type Event = Event; type DetermineContractAddress = pallet_contracts::SimpleAddressDeterminer; - type ComputeDispatchFee = pallet_contracts::DefaultDispatchFeeComputor; type TrieIdGenerator = pallet_contracts::TrieIdFromParentCounter; - type GasPayment = (); type RentPayment = (); type SignedClaimHandicap = pallet_contracts::DefaultSignedClaimHandicap; type TombstoneDeposit = TombstoneDeposit; @@ -523,14 +486,8 @@ impl pallet_contracts::Trait for Runtime { type RentByteFee = RentByteFee; type RentDepositOffset = RentDepositOffset; type SurchargeReward = SurchargeReward; - type TransactionBaseFee = ContractTransactionBaseFee; - type TransactionByteFee = ContractTransactionByteFee; - type ContractFee = ContractFee; - type CallBaseFee = pallet_contracts::DefaultCallBaseFee; - type InstantiateBaseFee = pallet_contracts::DefaultInstantiateBaseFee; type MaxDepth = pallet_contracts::DefaultMaxDepth; type MaxValueSize = pallet_contracts::DefaultMaxValueSize; - type BlockGasLimit = pallet_contracts::DefaultBlockGasLimit; } impl pallet_sudo::Trait for Runtime { @@ -540,15 +497,70 @@ impl pallet_sudo::Trait for Runtime { parameter_types! { pub const SessionDuration: BlockNumber = EPOCH_DURATION_IN_SLOTS as _; + pub const ImOnlineUnsignedPriority: TransactionPriority = TransactionPriority::max_value(); + /// We prioritize im-online heartbeats over phragmen solution submission. + pub const StakingUnsignedPriority: TransactionPriority = TransactionPriority::max_value() / 2; +} + + +impl frame_system::offchain::CreateSignedTransaction for Runtime where + Call: From, +{ + fn create_transaction>( + call: Call, + public: ::Signer, + account: AccountId, + nonce: Index, + ) -> Option<(Call, ::SignaturePayload)> { + // take the biggest period possible. + let period = BlockHashCount::get() + .checked_next_power_of_two() + .map(|c| c / 2) + .unwrap_or(2) as u64; + let current_block = System::block_number() + .saturated_into::() + // The `System::block_number` is initialized with `n+1`, + // so the actual block number is `n`. + .saturating_sub(1); + let tip = 0; + let extra: SignedExtra = ( + frame_system::CheckVersion::::new(), + frame_system::CheckGenesis::::new(), + frame_system::CheckEra::::from(generic::Era::mortal(period, current_block)), + frame_system::CheckNonce::::from(nonce), + frame_system::CheckWeight::::new(), + pallet_transaction_payment::ChargeTransactionPayment::::from(tip), + ); + let raw_payload = SignedPayload::new(call, extra).map_err(|e| { + debug::warn!("Unable to create signed payload: {:?}", e); + }).ok()?; + let signature = raw_payload.using_encoded(|payload| { + C::sign(payload, public) + })?; + let address = Indices::unlookup(account); + let (call, extra, _) = raw_payload.deconstruct(); + Some((call, (address, signature.into(), extra))) + } +} + +impl frame_system::offchain::SigningTypes for Runtime { + type Public = ::Signer; + type Signature = Signature; +} + +impl frame_system::offchain::SendTransactionTypes for Runtime where + Call: From, +{ + type OverarchingCall = Call; + type Extrinsic = UncheckedExtrinsic; } impl pallet_im_online::Trait for Runtime { type AuthorityId = ImOnlineId; type Event = Event; - type Call = Call; - type SubmitTransaction = TransactionSubmitterOf; type SessionDuration = SessionDuration; type ReportUnresponsiveness = Offences; + type UnsignedPriority = ImOnlineUnsignedPriority; } impl pallet_offences::Trait for Runtime { @@ -580,6 +592,7 @@ parameter_types! { pub const SubAccountDeposit: Balance = 2 * DOLLARS; // 53 bytes on-chain pub const MaxSubAccounts: u32 = 100; pub const MaxAdditionalFields: u32 = 100; + pub const MaxRegistrars: u32 = 20; } impl pallet_identity::Trait for Runtime { @@ -590,6 +603,7 @@ impl pallet_identity::Trait for Runtime { type SubAccountDeposit = SubAccountDeposit; type MaxSubAccounts = MaxSubAccounts; type MaxAdditionalFields = MaxAdditionalFields; + type MaxRegistrars = MaxRegistrars; type Slashed = Treasury; type ForceOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; type RegistrarOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; @@ -620,6 +634,7 @@ parameter_types! { pub const PeriodSpend: Balance = 500 * DOLLARS; pub const MaxLockDuration: BlockNumber = 36 * 30 * DAYS; pub const ChallengePeriod: BlockNumber = 7 * DAYS; + pub const SocietyModuleId: ModuleId = ModuleId(*b"py/socie"); } impl pallet_society::Trait for Runtime { @@ -636,6 +651,7 @@ impl pallet_society::Trait for Runtime { type FounderSetOrigin = pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; type SuspensionJudgementOrigin = pallet_society::EnsureFounder; type ChallengePeriod = ChallengePeriod; + type ModuleId = SocietyModuleId; } parameter_types! { @@ -673,7 +689,7 @@ construct_runtime!( FinalityTracker: pallet_finality_tracker::{Module, Call, Inherent}, Grandpa: pallet_grandpa::{Module, Call, Storage, Config, Event}, Treasury: pallet_treasury::{Module, Call, Storage, Config, Event}, - Contracts: pallet_contracts::{Module, Call, Config, Storage, Event}, + Contracts: pallet_contracts::{Module, Call, Config, Storage, Event}, Sudo: pallet_sudo::{Module, Call, Config, Storage, Event}, ImOnline: pallet_im_online::{Module, Call, Storage, Event, ValidateUnsigned, Config}, AuthorityDiscovery: pallet_authority_discovery::{Module, Call, Config}, @@ -705,8 +721,6 @@ pub type SignedExtra = ( frame_system::CheckNonce, frame_system::CheckWeight, pallet_transaction_payment::ChargeTransactionPayment, - pallet_contracts::CheckBlockGasLimit, - pallet_staking::LockStakingStatus, ); /// Unchecked extrinsic type as expected by this runtime. pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; @@ -782,19 +796,19 @@ impl_runtime_apis! { } impl sp_consensus_babe::BabeApi for Runtime { - fn configuration() -> sp_consensus_babe::BabeConfiguration { + fn configuration() -> sp_consensus_babe::BabeGenesisConfiguration { // The choice of `c` parameter (where `1 - c` represents the // probability of a slot being empty), is done in accordance to the // slot duration and expected target block time, for safely // resisting network delays of maximum two seconds. // - sp_consensus_babe::BabeConfiguration { + sp_consensus_babe::BabeGenesisConfiguration { slot_duration: Babe::slot_duration(), epoch_length: EpochDuration::get(), c: PRIMARY_PROBABILITY, genesis_authorities: Babe::authorities(), randomness: Babe::randomness(), - secondary_slots: true, + allowed_slots: sp_consensus_babe::AllowedSlots::PrimaryAndSecondaryPlainSlots, } } @@ -887,21 +901,27 @@ impl_runtime_apis! { // To get around that, we separated the Session benchmarks into its own crate, which is why // we need these two lines below. use pallet_session_benchmarking::Module as SessionBench; + use pallet_offences_benchmarking::Module as OffencesBench; + impl pallet_session_benchmarking::Trait for Runtime {} + impl pallet_offences_benchmarking::Trait for Runtime {} let mut batches = Vec::::new(); let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat); + add_benchmark!(params, batches, b"balances", Balances); add_benchmark!(params, batches, b"collective", Council); add_benchmark!(params, batches, b"democracy", Democracy); add_benchmark!(params, batches, b"identity", Identity); add_benchmark!(params, batches, b"im-online", ImOnline); + add_benchmark!(params, batches, b"offences", OffencesBench::); add_benchmark!(params, batches, b"session", SessionBench::); add_benchmark!(params, batches, b"staking", Staking); add_benchmark!(params, batches, b"timestamp", Timestamp); add_benchmark!(params, batches, b"treasury", Treasury); add_benchmark!(params, batches, b"utility", Utility); add_benchmark!(params, batches, b"vesting", Vesting); + if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) } Ok(batches) } @@ -911,28 +931,14 @@ impl_runtime_apis! { #[cfg(test)] mod tests { use super::*; - use frame_system::offchain::{SignAndSubmitTransaction, SubmitSignedTransaction}; + use frame_system::offchain::CreateSignedTransaction; #[test] fn validate_transaction_submitter_bounds() { fn is_submit_signed_transaction() where - T: SubmitSignedTransaction< - Runtime, - Call, - >, - {} - - fn is_sign_and_submit_transaction() where - T: SignAndSubmitTransaction< - Runtime, - Call, - Extrinsic=UncheckedExtrinsic, - CreateTransaction=Runtime, - Signer=ImOnlineId, - >, + T: CreateSignedTransaction, {} - is_submit_signed_transaction::>(); - is_sign_and_submit_transaction::>(); + is_submit_signed_transaction::(); } } diff --git a/bin/node/testing/Cargo.toml b/bin/node/testing/Cargo.toml index df73e20070d2cedca064206d6b6dfc9b796272f2..af375c774de425c3a940efd43e75ac49924a9606 100644 --- a/bin/node/testing/Cargo.toml +++ b/bin/node/testing/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "node-testing" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] description = "Test utilities for Substrate node." edition = "2018" @@ -9,49 +9,49 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" publish = true +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -pallet-balances = { version = "2.0.0-alpha.5", path = "../../../frame/balances" } -sc-client = { version = "0.8.0-alpha.5", path = "../../../client/" } -sc-client-db = { version = "0.8.0-alpha.5", path = "../../../client/db/", features = ["kvdb-rocksdb"] } -sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api/" } +pallet-balances = { version = "2.0.0-dev", path = "../../../frame/balances" } +sc-service = { version = "0.8.0-dev", features = ["test-helpers", "db"], path = "../../../client/service" } +sc-client-db = { version = "0.8.0-dev", path = "../../../client/db/", features = ["kvdb-rocksdb", "parity-db"] } +sc-client-api = { version = "2.0.0-dev", path = "../../../client/api/" } codec = { package = "parity-scale-codec", version = "1.3.0" } -pallet-contracts = { version = "2.0.0-alpha.5", path = "../../../frame/contracts" } -pallet-grandpa = { version = "2.0.0-alpha.5", path = "../../../frame/grandpa" } -pallet-indices = { version = "2.0.0-alpha.5", path = "../../../frame/indices" } -sp-keyring = { version = "2.0.0-alpha.5", path = "../../../primitives/keyring" } -node-executor = { version = "2.0.0-alpha.5", path = "../executor" } -node-primitives = { version = "2.0.0-alpha.5", path = "../primitives" } -node-runtime = { version = "2.0.0-alpha.5", path = "../runtime" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-io = { version = "2.0.0-alpha.5", path = "../../../primitives/io" } -frame-support = { version = "2.0.0-alpha.5", path = "../../../frame/support" } -pallet-session = { version = "2.0.0-alpha.5", path = "../../../frame/session" } -pallet-society = { version = "2.0.0-alpha.5", path = "../../../frame/society" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -pallet-staking = { version = "2.0.0-alpha.5", path = "../../../frame/staking" } -sc-executor = { version = "0.8.0-alpha.5", path = "../../../client/executor", features = ["wasmtime"] } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } -frame-system = { version = "2.0.0-alpha.5", path = "../../../frame/system" } +pallet-contracts = { version = "2.0.0-dev", path = "../../../frame/contracts" } +pallet-grandpa = { version = "2.0.0-dev", path = "../../../frame/grandpa" } +pallet-indices = { version = "2.0.0-dev", path = "../../../frame/indices" } +sp-keyring = { version = "2.0.0-dev", path = "../../../primitives/keyring" } +node-executor = { version = "2.0.0-dev", path = "../executor" } +node-primitives = { version = "2.0.0-dev", path = "../primitives" } +node-runtime = { version = "2.0.0-dev", path = "../runtime" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-io = { version = "2.0.0-dev", path = "../../../primitives/io" } +frame-support = { version = "2.0.0-dev", path = "../../../frame/support" } +pallet-session = { version = "2.0.0-dev", path = "../../../frame/session" } +pallet-society = { version = "2.0.0-dev", path = "../../../frame/society" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +pallet-staking = { version = "2.0.0-dev", path = "../../../frame/staking" } +sc-executor = { version = "0.8.0-dev", path = "../../../client/executor", features = ["wasmtime"] } +sp-consensus = { version = "0.8.0-dev", path = "../../../primitives/consensus/common" } +frame-system = { version = "2.0.0-dev", path = "../../../frame/system" } substrate-test-client = { version = "2.0.0-dev", path = "../../../test-utils/client" } -pallet-timestamp = { version = "2.0.0-alpha.5", path = "../../../frame/timestamp" } -pallet-transaction-payment = { version = "2.0.0-alpha.5", path = "../../../frame/transaction-payment" } -pallet-treasury = { version = "2.0.0-alpha.5", path = "../../../frame/treasury" } +pallet-timestamp = { version = "2.0.0-dev", path = "../../../frame/timestamp" } +pallet-transaction-payment = { version = "2.0.0-dev", path = "../../../frame/transaction-payment" } +pallet-treasury = { version = "2.0.0-dev", path = "../../../frame/treasury" } wabt = "0.9.2" -sp-api = { version = "2.0.0-alpha.5", path = "../../../primitives/api" } -sp-finality-tracker = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/finality-tracker" } -sp-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/timestamp" } -sp-block-builder = { version = "2.0.0-alpha.5", path = "../../../primitives/block-builder" } -sc-block-builder = { version = "0.8.0-alpha.5", path = "../../../client/block-builder" } -sp-inherents = { version = "2.0.0-alpha.5", path = "../../../primitives/inherents" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } +sp-api = { version = "2.0.0-dev", path = "../../../primitives/api" } +sp-finality-tracker = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/finality-tracker" } +sp-timestamp = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/timestamp" } +sp-block-builder = { version = "2.0.0-dev", path = "../../../primitives/block-builder" } +sc-block-builder = { version = "0.8.0-dev", path = "../../../client/block-builder" } +sp-inherents = { version = "2.0.0-dev", path = "../../../primitives/inherents" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } log = "0.4.8" tempfile = "3.1.0" fs_extra = "1" +futures = "0.3.1" [dev-dependencies] criterion = "0.3.0" -sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" } -sc-service = { version = "0.8.0-alpha.5", path = "../../../client/service", features = ["rocksdb"] } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sc-cli = { version = "0.8.0-dev", path = "../../../client/cli" } diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index 3bc8d483ea2afe4c7cadc0ad9a74e746b14121ea..d05bb9a0950ea355d04641fc14dc9d8b45aba845 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -47,7 +47,7 @@ use node_runtime::{ AccountId, Signature, }; -use sp_core::{ExecutionContext, blake2_256}; +use sp_core::{ExecutionContext, blake2_256, traits::CloneableSpawn}; use sp_api::ProvideRuntimeApi; use sp_block_builder::BlockBuilder; use sp_inherents::InherentData; @@ -57,6 +57,7 @@ use sc_client_api::{ }; use sp_core::{Pair, Public, sr25519, ed25519}; use sc_block_builder::BlockBuilderProvider; +use futures::{executor, task}; /// Keyring full of accounts for benching. /// @@ -142,6 +143,36 @@ impl BlockType { } } +/// Benchmarking task executor. +/// +/// Uses multiple threads as the regular executable. +#[derive(Debug, Clone)] +pub struct TaskExecutor { + pool: executor::ThreadPool, +} + +impl TaskExecutor { + fn new() -> Self { + Self { + pool: executor::ThreadPool::new() + .expect("Failed to create task executor") + } + } +} + +impl task::Spawn for TaskExecutor { + fn spawn_obj(&self, future: task::FutureObj<'static, ()>) + -> Result<(), task::SpawnError> { + self.pool.spawn_obj(future) + } +} + +impl CloneableSpawn for TaskExecutor { + fn clone(&self) -> Box { + Box::new(Clone::clone(self)) + } +} + impl BenchDb { /// New immutable benchmarking database. /// @@ -168,8 +199,8 @@ impl BenchDb { /// and keep it there until struct is dropped. /// /// You can `clone` this database or you can `create_context` from it - /// (which also do `clone`) to run actual operation against new database - /// which will be identical to this. + /// (which also does `clone`) to run actual operation against new database + /// which will be identical to the original. pub fn new(keyring_length: usize) -> Self { Self::with_key_types(keyring_length, KeyTypes::Sr25519) } @@ -184,21 +215,22 @@ impl BenchDb { state_cache_size: 16*1024*1024, state_cache_child_ratio: Some((0, 100)), pruning: PruningMode::ArchiveAll, - source: sc_client_db::DatabaseSettingsSrc::Path { + source: sc_client_db::DatabaseSettingsSrc::RocksDb { path: dir.into(), - cache_size: None, + cache_size: 512, }, }; - let (client, backend) = sc_client_db::new_client( + let (client, backend) = sc_service::new_client( db_config, NativeExecutor::new(WasmExecutionMethod::Compiled, None, 8), &keyring.generate_genesis(), None, None, ExecutionExtensions::new(profile.into_execution_strategies(), None), - sp_core::tasks::executor(), + Box::new(TaskExecutor::new()), None, + Default::default(), ).expect("Should not fail"); (client, backend) diff --git a/bin/node/testing/src/client.rs b/bin/node/testing/src/client.rs index 963bac7041b7d8ac25686e8e872b72d9210fb23b..69583e37dc90f25a73dfa0660b5d692f9520d57d 100644 --- a/bin/node/testing/src/client.rs +++ b/bin/node/testing/src/client.rs @@ -17,7 +17,7 @@ //! Utilities to build a `TestClient` for `node-runtime`. use sp_runtime::BuildStorage; - +use sc_service::client; /// Re-export test-client utilities. pub use substrate_test_client::*; @@ -28,9 +28,9 @@ pub type Executor = sc_executor::NativeExecutor; pub type Backend = sc_client_db::Backend; /// Test client type. -pub type Client = sc_client::Client< +pub type Client = client::Client< Backend, - sc_client::LocalCallExecutor, + client::LocalCallExecutor, node_primitives::Block, node_runtime::RuntimeApi, >; @@ -61,7 +61,7 @@ pub trait TestClientBuilderExt: Sized { impl TestClientBuilderExt for substrate_test_client::TestClientBuilder< node_primitives::Block, - sc_client::LocalCallExecutor, + client::LocalCallExecutor, Backend, GenesisParameters, > { diff --git a/bin/node/testing/src/genesis.rs b/bin/node/testing/src/genesis.rs index 8a57010770f3dedf9795e0da462455943fb6f74d..0f72d2c5471bd835be225a9d046d02867ca6fbc4 100644 --- a/bin/node/testing/src/genesis.rs +++ b/bin/node/testing/src/genesis.rs @@ -97,7 +97,6 @@ pub fn config_endowed( }), pallet_contracts: Some(ContractsConfig { current_schedule: Default::default(), - gas_price: 1 * MILLICENTS, }), pallet_babe: Some(Default::default()), pallet_grandpa: Some(GrandpaConfig { diff --git a/bin/node/testing/src/keyring.rs b/bin/node/testing/src/keyring.rs index 5fa1e48b0321813d3c9bec039ec3212c5be4216a..5eebc09f4b7af4a3c7be434f53b2baa3586d705a 100644 --- a/bin/node/testing/src/keyring.rs +++ b/bin/node/testing/src/keyring.rs @@ -74,8 +74,6 @@ pub fn signed_extra(nonce: Index, extra_fee: Balance) -> SignedExtra { frame_system::CheckNonce::from(nonce), frame_system::CheckWeight::new(), pallet_transaction_payment::ChargeTransactionPayment::from(extra_fee), - Default::default(), - Default::default(), ) } diff --git a/bin/node/transaction-factory/Cargo.toml b/bin/node/transaction-factory/Cargo.toml index 33ebeb767ae37e75c04309c958e3de79096bee67..5aa803a7e18ee4d7702b6d9939ac8df2df6acb74 100644 --- a/bin/node/transaction-factory/Cargo.toml +++ b/bin/node/transaction-factory/Cargo.toml @@ -1,26 +1,25 @@ [package] name = "node-transaction-factory" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-block-builder = { version = "2.0.0-alpha.5", path = "../../../primitives/block-builder" } -sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api" } -sc-block-builder = { version = "0.8.0-alpha.5", path = "../../../client/block-builder" } -sc-client = { version = "0.8.0-alpha.5", path = "../../../client" } +sp-block-builder = { version = "2.0.0-dev", path = "../../../primitives/block-builder" } +sc-cli = { version = "0.8.0-dev", path = "../../../client/cli" } +sc-client-api = { version = "2.0.0-dev", path = "../../../client/api" } +sc-block-builder = { version = "0.8.0-dev", path = "../../../client/block-builder" } codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } +sp-consensus = { version = "0.8.0-dev", path = "../../../primitives/consensus/common" } log = "0.4.8" -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-api = { version = "2.0.0-alpha.5", path = "../../../primitives/api" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sc-service = { version = "0.8.0-alpha.5", default-features = false, path = "../../../client/service" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-api = { version = "2.0.0-dev", path = "../../../primitives/api" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sc-service = { version = "0.8.0-dev", default-features = false, path = "../../../client/service" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } diff --git a/bin/utils/chain-spec-builder/Cargo.toml b/bin/utils/chain-spec-builder/Cargo.toml index 9b03dd17b76de499094287fc126a0607c866a3a3..a6c2b671fe687f3d103c4673842a9b8a3a73266e 100644 --- a/bin/utils/chain-spec-builder/Cargo.toml +++ b/bin/utils/chain-spec-builder/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chain-spec-builder" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" @@ -8,13 +8,14 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] ansi_term = "0.12.1" -sc-keystore = { version = "2.0.0-alpha.5", path = "../../../client/keystore" } -node-cli = { version = "2.0.0-alpha.5", path = "../../node/cli" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } +sc-keystore = { version = "2.0.0-dev", path = "../../../client/keystore" } +sc-chain-spec = { version = "2.0.0-dev", path = "../../../client/chain-spec" } +node-cli = { version = "2.0.0-dev", path = "../../node/cli" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } rand = "0.7.2" structopt = "0.3.8" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/utils/chain-spec-builder/src/main.rs b/bin/utils/chain-spec-builder/src/main.rs index 3673909706cdb765508e4e72230d66c0ffc46635..144018701460e79254db45b81cd1ef2431e2b866 100644 --- a/bin/utils/chain-spec-builder/src/main.rs +++ b/bin/utils/chain-spec-builder/src/main.rs @@ -87,7 +87,7 @@ fn genesis_constructor( let authorities = authority_seeds .iter() .map(AsRef::as_ref) - .map(chain_spec::get_authority_keys_from_seed) + .map(chain_spec::authority_keys_from_seed) .collect::>(); let enable_println = true; @@ -120,6 +120,7 @@ fn generate_chain_spec( let chain_spec = chain_spec::ChainSpec::from_genesis( "Custom", "custom", + sc_chain_spec::ChainType::Live, move || genesis_constructor(&authority_seeds, &endowed_accounts, &sudo_account), vec![], None, @@ -142,7 +143,7 @@ fn generate_authority_keys_and_store( ).map_err(|err| err.to_string())?; let (_, _, grandpa, babe, im_online, authority_discovery) = - chain_spec::get_authority_keys_from_seed(seed); + chain_spec::authority_keys_from_seed(seed); let insert_key = |key_type, public| { keystore.write().insert_unknown( diff --git a/bin/utils/subkey/Cargo.toml b/bin/utils/subkey/Cargo.toml index 9bf20146a990d8dd876495caa867812d331949a5..b0c642ec43e7c3c4748b35050a02574a2c96ff0e 100644 --- a/bin/utils/subkey/Cargo.toml +++ b/bin/utils/subkey/Cargo.toml @@ -1,18 +1,21 @@ [package] name = "subkey" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] futures = "0.1.29" -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -node-runtime = { version = "2.0.0-alpha.5", path = "../../node/runtime" } -node-primitives = { version = "2.0.0-alpha.5", path = "../../node/primitives" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +node-runtime = { version = "2.0.0-dev", path = "../../node/runtime" } +node-primitives = { version = "2.0.0-dev", path = "../../node/primitives" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } rand = "0.7.2" clap = "2.33.0" tiny-bip39 = "0.7" @@ -20,20 +23,17 @@ substrate-bip39 = "0.4.1" hex = "0.4.0" hex-literal = "0.2.1" codec = { package = "parity-scale-codec", version = "1.3.0" } -frame-system = { version = "2.0.0-alpha.5", path = "../../../frame/system" } -pallet-balances = { version = "2.0.0-alpha.5", path = "../../../frame/balances" } -pallet-transaction-payment = { version = "2.0.0-alpha.5", path = "../../../frame/transaction-payment" } +frame-system = { version = "2.0.0-dev", path = "../../../frame/system" } +pallet-balances = { version = "2.0.0-dev", path = "../../../frame/balances" } +pallet-transaction-payment = { version = "2.0.0-dev", path = "../../../frame/transaction-payment" } rpassword = "4.0.1" itertools = "0.8.2" derive_more = { version = "0.99.2" } -sc-rpc = { version = "2.0.0-alpha.5", path = "../../../client/rpc" } +sc-rpc = { version = "2.0.0-dev", path = "../../../client/rpc" } jsonrpc-core-client = { version = "14.0.3", features = ["http"] } hyper = "0.12.35" -libp2p = "0.16.2" +libp2p = "0.18.1" serde_json = "1.0" [features] bench = [] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/bin/utils/subkey/README.adoc b/bin/utils/subkey/README.adoc index 07533a002f80ef1f1912b352829c147be81bad88..1fa0753312f0cd6dc1f9a94c42866326bdf4f59d 100644 --- a/bin/utils/subkey/README.adoc +++ b/bin/utils/subkey/README.adoc @@ -68,3 +68,16 @@ subkey sign-transaction \ ``` Will output a signed and encoded `UncheckedMortalCompactExtrinsic` as hex. + +=== Inspecting a module ID + +```bash +subkey --network kusama moduleid "py/trsry" + +OUTPUT: +Public Key URI `F3opxRbN5ZbjJNU511Kj2TLuzFcDq9BGduA9TgiECafpg29` is account: + Network ID/version: kusama + Public key (hex): 0x6d6f646c70792f74727372790000000000000000000000000000000000000000 + Account ID: 0x6d6f646c70792f74727372790000000000000000000000000000000000000000 + SS58 Address: F3opxRbN5ZbjJNU511Kj2TLuzFcDq9BGduA9TgiECafpg29 +``` \ No newline at end of file diff --git a/bin/utils/subkey/src/main.rs b/bin/utils/subkey/src/main.rs index 237cc68df2f4657808152d493d2e26ac9866231f..22706ebd822daea88f34693e5ed475bad8ecda27 100644 --- a/bin/utils/subkey/src/main.rs +++ b/bin/utils/subkey/src/main.rs @@ -31,7 +31,7 @@ use sp_core::{ crypto::{set_default_ss58_version, Ss58AddressFormat, Ss58Codec}, ed25519, sr25519, ecdsa, Pair, Public, H256, hexdisplay::HexDisplay, }; -use sp_runtime::{traits::{IdentifyAccount, Verify}, generic::Era}; +use sp_runtime::{traits::{AccountIdConversion, IdentifyAccount, Verify}, generic::Era, ModuleId}; use std::{ convert::{TryInto, TryFrom}, io::{stdin, Read}, str::FromStr, path::PathBuf, fs, fmt, }; @@ -318,6 +318,11 @@ fn get_app<'a, 'b>(usage: &'a str) -> App<'a, 'b> { 'Key type, examples: \"gran\", or \"imon\" ' [node-url] 'Node JSON-RPC endpoint, default \"http:://localhost:9933\"' "), + SubCommand::with_name("moduleid") + .about("Inspect a module ID address") + .args_from_usage(" + 'The module ID used to derive the account' + ") ]) } @@ -507,6 +512,20 @@ where sp_core::Bytes(pair.public().as_ref().to_vec()), ); } + ("moduleid", Some(matches)) => { + let id = get_uri("id", &matches)?; + if id.len() != 8 { + Err("a module id must be a string of 8 characters")? + } + + let id_fixed_array: [u8; 8] = id.as_bytes().try_into() + .map_err(|_| Error::Static("Cannot convert argument to moduleid: argument should be 8-character string"))?; + + let account_id: AccountId = ModuleId(id_fixed_array).into_account(); + let v = maybe_network.unwrap_or(Ss58AddressFormat::SubstrateAccount); + + C::print_from_uri(&account_id.to_ss58check_with_version(v), password, maybe_network, output); + } _ => print_usage(&matches), } @@ -689,8 +708,6 @@ fn create_extrinsic( frame_system::CheckNonce::::from(i), frame_system::CheckWeight::::new(), pallet_transaction_payment::ChargeTransactionPayment::::from(f), - Default::default(), - Default::default(), ) }; let raw_payload = SignedPayload::from_raw( @@ -703,8 +720,6 @@ fn create_extrinsic( (), (), (), - (), - (), ), ); let signature = raw_payload.using_encoded(|payload| signer.sign(payload)).into_runtime(); diff --git a/client/Cargo.toml b/client/Cargo.toml deleted file mode 100644 index 5d4b5927111ea220c33256c50c8250fe198ec840..0000000000000000000000000000000000000000 --- a/client/Cargo.toml +++ /dev/null @@ -1,49 +0,0 @@ -[package] -name = "sc-client" -version = "0.8.0-alpha.5" -authors = ["Parity Technologies "] -edition = "2018" -license = "GPL-3.0" -homepage = "https://substrate.dev" -repository = "https://github.com/paritytech/substrate/" -description = "Substrate Client and associated logic." - -[dependencies] -sc-block-builder = { version = "0.8.0-alpha.5", path = "block-builder" } -sc-client-api = { version = "2.0.0-alpha.5", path = "api" } -codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } -sp-consensus = { version = "0.8.0-alpha.5", path = "../primitives/consensus/common" } -derive_more = { version = "0.99.2" } -sc-executor = { version = "0.8.0-alpha.5", path = "executor" } -sp-externalities = { version = "0.8.0-alpha.5", path = "../primitives/externalities" } -fnv = { version = "1.0.6" } -futures = { version = "0.3.1", features = ["compat"] } -hash-db = { version = "0.15.2" } -hex-literal = { version = "0.2.1" } -sp-inherents = { version = "2.0.0-alpha.5", path = "../primitives/inherents" } -sp-keyring = { version = "2.0.0-alpha.5", path = "../primitives/keyring" } -kvdb = "0.5.0" -log = { version = "0.4.8" } -parking_lot = "0.10.0" -sp-core = { version = "2.0.0-alpha.5", path = "../primitives/core" } -sp-std = { version = "2.0.0-alpha.5", path = "../primitives/std" } -sp-version = { version = "2.0.0-alpha.5", path = "../primitives/version" } -sp-api = { version = "2.0.0-alpha.5", path = "../primitives/api" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../primitives/runtime" } -sp-utils = { version = "2.0.0-alpha.5", path = "../primitives/utils" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../primitives/blockchain" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../primitives/state-machine" } -sc-telemetry = { version = "2.0.0-alpha.5", path = "telemetry" } -sp-trie = { version = "2.0.0-alpha.5", path = "../primitives/trie" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.8.0-alpha.5", path = "../utils/prometheus" } -tracing = "0.1.10" - -[dev-dependencies] -env_logger = "0.7.0" -tempfile = "3.1.0" -substrate-test-runtime-client = { version = "2.0.0-dev", path = "../test-utils/runtime/client" } -kvdb-memorydb = "0.5.0" -sp-panic-handler = { version = "2.0.0-alpha.5", path = "../primitives/panic-handler" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/api/Cargo.toml b/client/api/Cargo.toml index dbe6afe9c7cae00c99abca8464dd4acd4c4f8a3c..3e9cbc1a9db2bf3588956afaf86f50e259663ec2 100644 --- a/client/api/Cargo.toml +++ b/client/api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-client-api" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,38 +9,41 @@ repository = "https://github.com/paritytech/substrate/" description = "Substrate client interfaces." documentation = "https://docs.rs/sc-client-api" - [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/common" } +sp-consensus = { version = "0.8.0-dev", path = "../../primitives/consensus/common" } derive_more = { version = "0.99.2" } -sc-executor = { version = "0.8.0-alpha.5", path = "../executor" } -sp-externalities = { version = "0.8.0-alpha.5", path = "../../primitives/externalities" } +sc-executor = { version = "0.8.0-dev", path = "../executor" } +sp-externalities = { version = "0.8.0-dev", path = "../../primitives/externalities" } fnv = { version = "1.0.6" } futures = { version = "0.3.1" } hash-db = { version = "0.15.2", default-features = false } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } +sp-blockchain = { version = "2.0.0-dev", path = "../../primitives/blockchain" } hex-literal = { version = "0.2.1" } -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" } -sp-keyring = { version = "2.0.0-alpha.5", path = "../../primitives/keyring" } +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../../primitives/inherents" } +sp-keyring = { version = "2.0.0-dev", path = "../../primitives/keyring" } kvdb = "0.5.0" log = { version = "0.4.8" } parking_lot = "0.10.0" lazy_static = "1.4.0" -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-version = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/version" } -sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } -sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../primitives/state-machine" } -sc-telemetry = { version = "2.0.0-alpha.5", path = "../telemetry" } -sp-trie = { version = "2.0.0-alpha.5", path = "../../primitives/trie" } -sp-storage = { version = "2.0.0-alpha.5", path = "../../primitives/storage" } -sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../primitives/transaction-pool" } +sp-database = { version = "2.0.0-dev", path = "../../primitives/database" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-version = { version = "2.0.0-dev", default-features = false, path = "../../primitives/version" } +sp-api = { version = "2.0.0-dev", path = "../../primitives/api" } +sp-utils = { version = "2.0.0-dev", path = "../../primitives/utils" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-state-machine = { version = "0.8.0-dev", path = "../../primitives/state-machine" } +sc-telemetry = { version = "2.0.0-dev", path = "../telemetry" } +sp-trie = { version = "2.0.0-dev", path = "../../primitives/trie" } +sp-storage = { version = "2.0.0-dev", path = "../../primitives/storage" } +sp-transaction-pool = { version = "2.0.0-dev", path = "../../primitives/transaction-pool" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.8.0-dev", path = "../../utils/prometheus" } [dev-dependencies] +kvdb-memorydb = "0.5.0" sp-test-primitives = { version = "2.0.0-dev", path = "../../primitives/test-primitives" } +substrate-test-runtime = { version = "2.0.0-dev", path = "../../test-utils/runtime" } [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index d10e62cc54920e56509ddbb06a51f38296b417c1..8aaeb944833908bf26149b818fd99ded46da42e4 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -19,14 +19,14 @@ use std::sync::Arc; use std::collections::HashMap; use sp_core::ChangesTrieConfigurationRange; -use sp_core::offchain::OffchainStorage; +use sp_core::offchain::{OffchainStorage,storage::OffchainOverlayedChanges}; use sp_runtime::{generic::BlockId, Justification, Storage}; use sp_runtime::traits::{Block as BlockT, NumberFor, HashFor}; use sp_state_machine::{ ChangesTrieState, ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction, StorageCollection, ChildStorageCollection, }; -use sp_storage::{StorageData, StorageKey, ChildInfo}; +use sp_storage::{StorageData, StorageKey, PrefixedStorageKey, ChildInfo}; use crate::{ blockchain::{ Backend as BlockchainBackend, well_known_cache_keys @@ -79,6 +79,25 @@ pub struct ClientImportOperation> { pub notify_finalized: Vec, } +/// Helper function to apply auxiliary data insertion into an operation. +pub fn apply_aux<'a, 'b: 'a, 'c: 'a, B, Block, D, I>( + operation: &mut ClientImportOperation, + insert: I, + delete: D, +) -> sp_blockchain::Result<()> + where + Block: BlockT, + B: Backend, + I: IntoIterator, + D: IntoIterator, +{ + operation.op.insert_aux( + insert.into_iter() + .map(|(k, v)| (k.to_vec(), Some(v.to_vec()))) + .chain(delete.into_iter().map(|k| (k.to_vec(), None))) + ) +} + /// State of a new block. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum NewBlockState { @@ -148,6 +167,14 @@ pub trait BlockImportOperation { child_update: ChildStorageCollection, ) -> sp_blockchain::Result<()>; + /// Write offchain storage changes to the database. + fn update_offchain_storage( + &mut self, + _offchain_update: OffchainOverlayedChanges, + ) -> sp_blockchain::Result<()> { + Ok(()) + } + /// Inject changes trie data into the database. fn update_changes_trie( &mut self, @@ -280,6 +307,7 @@ impl<'a, State, Block> Iterator for KeyIterator<'a, State, Block> where Some(StorageKey(next_key)) } } + /// Provides acess to storage primitives pub trait StorageProvider> { /// Given a `BlockId` and a key, return the value under the key in that block. @@ -310,8 +338,7 @@ pub trait StorageProvider> { fn child_storage( &self, id: &BlockId, - storage_key: &StorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: &StorageKey ) -> sp_blockchain::Result>; @@ -319,8 +346,7 @@ pub trait StorageProvider> { fn child_storage_keys( &self, id: &BlockId, - child_storage_key: &StorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key_prefix: &StorageKey ) -> sp_blockchain::Result>; @@ -328,8 +354,7 @@ pub trait StorageProvider> { fn child_storage_hash( &self, id: &BlockId, - storage_key: &StorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: &StorageKey ) -> sp_blockchain::Result>; @@ -351,7 +376,7 @@ pub trait StorageProvider> { &self, first: NumberFor, last: BlockId, - storage_key: Option<&StorageKey>, + storage_key: Option<&PrefixedStorageKey>, key: &StorageKey ) -> sp_blockchain::Result, u32)>>; } diff --git a/client/api/src/call_executor.rs b/client/api/src/call_executor.rs index 3afd29be8d49c7404bde565b300376ef6a953ee8..00711e83b75ac639f08246527e0aa08d0ce210e2 100644 --- a/client/api/src/call_executor.rs +++ b/client/api/src/call_executor.rs @@ -26,7 +26,7 @@ use sp_state_machine::{ }; use sc_executor::{RuntimeVersion, NativeVersion}; use sp_externalities::Extensions; -use sp_core::NativeOrEncoded; +use sp_core::{NativeOrEncoded,offchain::storage::OffchainOverlayedChanges}; use sp_api::{ProofRecorder, InitializeBlock, StorageTransactionCache}; use crate::execution_extensions::ExecutionExtensions; @@ -84,6 +84,7 @@ pub trait CallExecutor { method: &str, call_data: &[u8], changes: &RefCell, + offchain_changes: &RefCell, storage_transaction_cache: Option<&RefCell< StorageTransactionCache>::State>, >>, diff --git a/client/src/cht.rs b/client/api/src/cht.rs similarity index 99% rename from client/src/cht.rs rename to client/api/src/cht.rs index de67280632302395e2363f6d309b2fecbf75d9ad..3eba63e7026a3a27f8e62c95ae888173436da937 100644 --- a/client/src/cht.rs +++ b/client/api/src/cht.rs @@ -331,9 +331,10 @@ pub fn decode_cht_value(value: &[u8]) -> Option { #[cfg(test)] mod tests { - use substrate_test_runtime_client::runtime::Header; - use sp_runtime::traits::BlakeTwo256; use super::*; + use sp_runtime::{generic, traits::BlakeTwo256}; + + type Header = generic::Header; #[test] fn is_build_required_works() { diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index 10d33c20e679c6eeddb4368da3c18d429d2ade6b..55ffc3794c4eab029461d46102dc3526a23ad266 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -177,7 +177,7 @@ impl ExecutionExtensions { if let ExecutionContext::OffchainCall(Some(ext)) = context { extensions.register( OffchainExt::new(offchain::LimitedExternalities::new(capabilities, ext.0)) - ) + ); } (manager, extensions) diff --git a/client/src/in_mem.rs b/client/api/src/in_mem.rs similarity index 89% rename from client/src/in_mem.rs rename to client/api/src/in_mem.rs index 3672da1822df1b30f377f64a3bf1d8c8d85dd8d7..d5b4800f4e421dfdf55321059c036944ca05111d 100644 --- a/client/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -32,14 +32,15 @@ use sp_state_machine::{ }; use sp_blockchain::{CachedHeaderMetadata, HeaderMetadata}; -use sc_client_api::{ +use crate::{ backend::{self, NewBlockState}, blockchain::{ self, BlockStatus, HeaderBackend, well_known_cache_keys::Id as CacheKeyId }, UsageInfo, + light, + leaves::LeafSet, }; -use crate::leaves::LeafSet; struct PendingBlock { block: StoredBlock, @@ -400,7 +401,7 @@ impl backend::AuxStore for Blockchain { } } -impl sc_client_api::light::Storage for Blockchain +impl light::Storage for Blockchain where Block::Hash: From<[u8; 32]>, { @@ -454,7 +455,7 @@ impl sc_client_api::light::Storage for Blockchain None } - fn usage_info(&self) -> Option { + fn usage_info(&self) -> Option { None } } @@ -464,7 +465,7 @@ pub struct BlockImportOperation { pending_block: Option>, pending_cache: HashMap>, old_state: InMemoryBackend>, - new_state: Option>>, + new_state: Option<> as StateBackend>>::Transaction>, aux: Vec<(Vec, Option>)>, finalized_blocks: Vec<(BlockId, Option)>, set_head: Option>, @@ -502,7 +503,7 @@ impl backend::BlockImportOperation for BlockImportOperatio &mut self, update: > as StateBackend>>::Transaction, ) -> sp_blockchain::Result<()> { - self.new_state = Some(self.old_state.update(update)); + self.new_state = Some(update); Ok(()) } @@ -516,16 +517,16 @@ impl backend::BlockImportOperation for BlockImportOperatio fn reset_storage(&mut self, storage: Storage) -> sp_blockchain::Result { check_genesis_storage(&storage)?; - let child_delta = storage.children.into_iter() - .map(|(storage_key, child_content)| - (storage_key, child_content.data.into_iter().map(|(k, v)| (k, Some(v))), child_content.child_info)); + let child_delta = storage.children_default.into_iter() + .map(|(_storage_key, child_content)| + (child_content.child_info, child_content.data.into_iter().map(|(k, v)| (k, Some(v))))); let (root, transaction) = self.old_state.full_storage_root( storage.top.into_iter().map(|(k, v)| (k, Some(v))), child_delta ); - self.new_state = Some(InMemoryBackend::from(transaction)); + self.new_state = Some(transaction); Ok(root) } @@ -638,7 +639,12 @@ impl backend::Backend for Backend where Block::Hash let hash = header.hash(); - self.states.write().insert(hash, operation.new_state.unwrap_or_else(|| old_state.clone())); + let new_state = match operation.new_state { + Some(state) => old_state.update_backend(*header.state_root(), state), + None => old_state.clone(), + }; + + self.states.write().insert(hash, new_state); self.blockchain.insert(hash, header, justification, body, pending_block.state)?; } @@ -714,7 +720,7 @@ impl backend::RemoteBackend for Backend where Block .unwrap_or(false) } - fn remote_blockchain(&self) -> Arc> { + fn remote_blockchain(&self) -> Arc> { unimplemented!() } } @@ -725,52 +731,10 @@ pub fn check_genesis_storage(storage: &Storage) -> sp_blockchain::Result<()> { return Err(sp_blockchain::Error::GenesisInvalid.into()); } - if storage.children.keys().any(|child_key| !well_known_keys::is_child_storage_key(&child_key)) { - return Err(sp_blockchain::Error::GenesisInvalid.into()); + if storage.children_default.keys() + .any(|child_key| !well_known_keys::is_child_storage_key(&child_key)) { + return Err(sp_blockchain::Error::GenesisInvalid.into()); } Ok(()) } - -#[cfg(test)] -mod tests { - use sp_core::offchain::{OffchainStorage, storage::InMemOffchainStorage}; - use std::sync::Arc; - - type TestBackend = substrate_test_runtime_client::sc_client::in_mem::Backend; - - #[test] - fn test_leaves_with_complex_block_tree() { - let backend = Arc::new(TestBackend::new()); - - substrate_test_runtime_client::trait_tests::test_leaves_for_backend(backend); - } - - #[test] - fn test_blockchain_query_by_number_gets_canonical() { - let backend = Arc::new(TestBackend::new()); - - substrate_test_runtime_client::trait_tests::test_blockchain_query_by_number_gets_canonical(backend); - } - - #[test] - fn in_memory_offchain_storage() { - - let mut storage = InMemOffchainStorage::default(); - assert_eq!(storage.get(b"A", b"B"), None); - assert_eq!(storage.get(b"B", b"A"), None); - - storage.set(b"A", b"B", b"C"); - assert_eq!(storage.get(b"A", b"B"), Some(b"C".to_vec())); - assert_eq!(storage.get(b"B", b"A"), None); - - storage.compare_and_set(b"A", b"B", Some(b"X"), b"D"); - assert_eq!(storage.get(b"A", b"B"), Some(b"C".to_vec())); - storage.compare_and_set(b"A", b"B", Some(b"C"), b"D"); - assert_eq!(storage.get(b"A", b"B"), Some(b"D".to_vec())); - - assert!(!storage.compare_and_set(b"B", b"A", Some(b""), b"Y")); - assert!(storage.compare_and_set(b"B", b"A", None, b"X")); - assert_eq!(storage.get(b"B", b"A"), Some(b"X".to_vec())); - } -} diff --git a/client/src/leaves.rs b/client/api/src/leaves.rs similarity index 84% rename from client/src/leaves.rs rename to client/api/src/leaves.rs index 7c169488ddcf4ef2031fd4d45a1e121afb3f622e..a555943444ad1a120ad11c3ad40433af54560e25 100644 --- a/client/src/leaves.rs +++ b/client/api/src/leaves.rs @@ -18,11 +18,13 @@ use std::collections::BTreeMap; use std::cmp::Reverse; -use kvdb::{KeyValueDB, DBTransaction}; +use sp_database::{Database, Transaction}; use sp_runtime::traits::AtLeast32Bit; use codec::{Encode, Decode}; use sp_blockchain::{Error, Result}; +type DbHash = [u8; 32]; + #[derive(Debug, Clone, PartialEq, Eq)] struct LeafSetItem { hash: H, @@ -59,7 +61,7 @@ impl FinalizationDisplaced { #[derive(Debug, Clone, PartialEq, Eq)] pub struct LeafSet { storage: BTreeMap, Vec>, - pending_added: Vec>, + pending_added: Vec<(H, N)>, pending_removed: Vec, } @@ -77,21 +79,20 @@ impl LeafSet where } /// Read the leaf list from the DB, using given prefix for keys. - pub fn read_from_db(db: &dyn KeyValueDB, column: u32, prefix: &[u8]) -> Result { + pub fn read_from_db(db: &dyn Database, column: u32, prefix: &[u8]) -> Result { let mut storage = BTreeMap::new(); - for (key, value) in db.iter_from_prefix(column, prefix) { - if !key.starts_with(prefix) { break } - let raw_hash = &mut &key[prefix.len()..]; - let hash = match Decode::decode(raw_hash) { - Ok(hash) => hash, - Err(_) => return Err(Error::Backend("Error decoding hash".into())), - }; - let number = match Decode::decode(&mut &value[..]) { - Ok(number) => number, - Err(_) => return Err(Error::Backend("Error decoding number".into())), - }; - storage.entry(Reverse(number)).or_insert_with(Vec::new).push(hash); + match db.get(column, prefix) { + Some(leaves) => { + let vals: Vec<_> = match Decode::decode(&mut leaves.as_ref()) { + Ok(vals) => vals, + Err(_) => return Err(Error::Backend("Error decoding leaves".into())), + }; + for (number, hashes) in vals.into_iter() { + storage.insert(Reverse(number), hashes); + } + } + None => {}, } Ok(Self { storage, @@ -124,7 +125,7 @@ impl LeafSet where }; self.insert_leaf(Reverse(number.clone()), hash.clone()); - self.pending_added.push(LeafSetItem { hash, number: Reverse(number) }); + self.pending_added.push((hash, number)); displaced } @@ -185,7 +186,7 @@ impl LeafSet where // this is an invariant of regular block import. if !leaves_contains_best { self.insert_leaf(best_number.clone(), best_hash.clone()); - self.pending_added.push(LeafSetItem { hash: best_hash, number: best_number }); + self.pending_added.push((best_hash, best_number.0)); } } @@ -201,18 +202,11 @@ impl LeafSet where } /// Write the leaf list to the database transaction. - pub fn prepare_transaction(&mut self, tx: &mut DBTransaction, column: u32, prefix: &[u8]) { - let mut buf = prefix.to_vec(); - for LeafSetItem { hash, number } in self.pending_added.drain(..) { - hash.using_encoded(|s| buf.extend(s)); - tx.put_vec(column, &buf[..], number.0.encode()); - buf.truncate(prefix.len()); // reuse allocation. - } - for hash in self.pending_removed.drain(..) { - hash.using_encoded(|s| buf.extend(s)); - tx.delete(column, &buf[..]); - buf.truncate(prefix.len()); // reuse allocation. - } + pub fn prepare_transaction(&mut self, tx: &mut Transaction, column: u32, prefix: &[u8]) { + let leaves: Vec<_> = self.storage.iter().map(|(n, h)| (n.0.clone(), h.clone())).collect(); + tx.set_from_vec(column, prefix, leaves.encode()); + self.pending_added.clear(); + self.pending_removed.clear(); } #[cfg(test)] @@ -281,6 +275,7 @@ impl<'a, H: 'a, N: 'a> Drop for Undo<'a, H, N> { #[cfg(test)] mod tests { use super::*; + use std::sync::Arc; #[test] fn it_works() { @@ -305,7 +300,7 @@ mod tests { #[test] fn flush_to_disk() { const PREFIX: &[u8] = b"abcdefg"; - let db = ::kvdb_memorydb::create(1); + let db = Arc::new(sp_database::MemDb::default()); let mut set = LeafSet::new(); set.import(0u32, 0u32, 0u32); @@ -314,12 +309,12 @@ mod tests { set.import(2_1, 2, 1_1); set.import(3_1, 3, 2_1); - let mut tx = DBTransaction::new(); + let mut tx = Transaction::new(); set.prepare_transaction(&mut tx, 0, PREFIX); - db.write(tx).unwrap(); + db.commit(tx); - let set2 = LeafSet::read_from_db(&db, 0, PREFIX).unwrap(); + let set2 = LeafSet::read_from_db(&*db, 0, PREFIX).unwrap(); assert_eq!(set, set2); } @@ -339,7 +334,7 @@ mod tests { #[test] fn finalization_consistent_with_disk() { const PREFIX: &[u8] = b"prefix"; - let db = ::kvdb_memorydb::create(1); + let db = Arc::new(sp_database::MemDb::default()); let mut set = LeafSet::new(); set.import(10_1u32, 10u32, 0u32); @@ -349,21 +344,21 @@ mod tests { assert!(set.contains(10, 10_1)); - let mut tx = DBTransaction::new(); + let mut tx = Transaction::new(); set.prepare_transaction(&mut tx, 0, PREFIX); - db.write(tx).unwrap(); + db.commit(tx); let _ = set.finalize_height(11); - let mut tx = DBTransaction::new(); + let mut tx = Transaction::new(); set.prepare_transaction(&mut tx, 0, PREFIX); - db.write(tx).unwrap(); + db.commit(tx); assert!(set.contains(11, 11_1)); assert!(set.contains(11, 11_2)); assert!(set.contains(12, 12_1)); assert!(!set.contains(10, 10_1)); - let set2 = LeafSet::read_from_db(&db, 0, PREFIX).unwrap(); + let set2 = LeafSet::read_from_db(&*db, 0, PREFIX).unwrap(); assert_eq!(set, set2); } diff --git a/client/api/src/lib.rs b/client/api/src/lib.rs index e4080323c188eb5ded7f85ab14e229e9beae117a..bad61f7687a63293175da3e29a4c2b21049deff4 100644 --- a/client/api/src/lib.rs +++ b/client/api/src/lib.rs @@ -20,8 +20,11 @@ pub mod backend; pub mod call_executor; pub mod client; +pub mod cht; pub mod execution_extensions; +pub mod in_mem; pub mod light; +pub mod leaves; pub mod notifications; pub mod proof_provider; @@ -36,6 +39,13 @@ pub use proof_provider::*; pub use sp_state_machine::{StorageProof, ExecutionStrategy, CloneableSpawn}; +/// Usage Information Provider interface +/// +pub trait UsageProvider { + /// Get usage info about current client. + fn usage_info(&self) -> ClientInfo; +} + /// Utility methods for the client. pub mod utils { use sp_blockchain::{HeaderBackend, HeaderMetadata, Error}; diff --git a/client/api/src/light.rs b/client/api/src/light.rs index c0bebc1740a8a0047e1f1d93c259cb3d237994d3..b359c1149eea645e27f584122dfaf328c168ac47 100644 --- a/client/api/src/light.rs +++ b/client/api/src/light.rs @@ -26,7 +26,7 @@ use sp_runtime::{ }, generic::BlockId }; -use sp_core::ChangesTrieConfigurationRange; +use sp_core::{ChangesTrieConfigurationRange, storage::PrefixedStorageKey}; use sp_state_machine::StorageProof; use sp_blockchain::{ HeaderMetadata, well_known_cache_keys, HeaderBackend, Cache as BlockchainCache, @@ -81,12 +81,7 @@ pub struct RemoteReadChildRequest { /// Header of block at which read is performed. pub header: Header, /// Storage key for child. - pub storage_key: Vec, - /// Child trie source information. - pub child_info: Vec, - /// Child type, its required to resolve `child_info` - /// content and choose child implementation. - pub child_type: u32, + pub storage_key: PrefixedStorageKey, /// Child storage key to read. pub keys: Vec>, /// Number of times to retry request. None means that default RETRY_COUNT is used. @@ -110,7 +105,7 @@ pub struct RemoteChangesRequest { /// Proofs for roots of ascendants of tries_roots.0 are provided by the remote node. pub tries_roots: (Header::Number, Header::Hash, Vec), /// Optional Child Storage key to read. - pub storage_key: Option>, + pub storage_key: Option, /// Storage key to read. pub key: Vec, /// Number of times to retry request. None means that default RETRY_COUNT is used. @@ -301,7 +296,25 @@ pub trait RemoteBlockchain: Send + Sync { >>; } - +/// Returns future that resolves header either locally, or remotely. +pub fn future_header>( + blockchain: &dyn RemoteBlockchain, + fetcher: &F, + id: BlockId, +) -> impl Future, ClientError>> { + use futures::future::{ready, Either, FutureExt}; + + match blockchain.header(id) { + Ok(LocalOrRemote::Remote(request)) => Either::Left( + fetcher + .remote_header(request) + .then(|header| ready(header.map(Some))) + ), + Ok(LocalOrRemote::Unknown) => Either::Right(ready(Ok(None))), + Ok(LocalOrRemote::Local(local_header)) => Either::Right(ready(Ok(Some(local_header)))), + Err(err) => Either::Right(ready(Err(err))), + } +} #[cfg(test)] pub mod tests { diff --git a/client/api/src/notifications.rs b/client/api/src/notifications.rs index f154eade44d5e37cd76a3bad97e21d46e3ed779d..412fe8adc1e5ba1e8bc36b6e3cb7e7da8cc04619 100644 --- a/client/api/src/notifications.rs +++ b/client/api/src/notifications.rs @@ -25,6 +25,7 @@ use fnv::{FnvHashSet, FnvHashMap}; use sp_core::storage::{StorageKey, StorageData}; use sp_runtime::traits::Block as BlockT; use sp_utils::mpsc::{TracingUnboundedSender, TracingUnboundedReceiver, tracing_unbounded}; +use prometheus_endpoint::{Registry, CounterVec, Opts, U64, register}; /// Storage change set #[derive(Debug)] @@ -71,9 +72,12 @@ pub type StorageEventStream = TracingUnboundedReceiver<(H, StorageChangeSet)> type SubscriberId = u64; +type SubscribersGauge = CounterVec; + /// Manages storage listeners. #[derive(Debug)] pub struct StorageNotifications { + metrics: Option, next_id: SubscriberId, wildcard_listeners: FnvHashSet, listeners: HashMap>, @@ -90,7 +94,8 @@ pub struct StorageNotifications { impl Default for StorageNotifications { fn default() -> Self { - StorageNotifications { + Self { + metrics: Default::default(), next_id: Default::default(), wildcard_listeners: Default::default(), listeners: Default::default(), @@ -101,6 +106,29 @@ impl Default for StorageNotifications { } impl StorageNotifications { + /// Initialize a new StorageNotifications + /// optionally pass a prometheus registry to send subscriber metrics to + pub fn new(prometheus_registry: Option) -> Self { + let metrics = prometheus_registry.and_then(|r| + CounterVec::new( + Opts::new( + "storage_notification_subscribers", + "Number of subscribers in storage notification sytem" + ), + &["action"], //added | removed + ).and_then(|g| register(g, &r)) + .ok() + ); + + StorageNotifications { + metrics, + next_id: Default::default(), + wildcard_listeners: Default::default(), + listeners: Default::default(), + child_listeners: Default::default(), + sinks: Default::default(), + } + } /// Trigger notification to all listeners. /// /// Note the changes are going to be filtered by listener's filter key. @@ -113,6 +141,7 @@ impl StorageNotifications { Item=(Vec, impl Iterator, Option>)>) >, ) { + let has_wildcard = !self.wildcard_listeners.is_empty(); // early exit if no listeners @@ -169,21 +198,32 @@ impl StorageNotifications { let changes = Arc::new(changes); let child_changes = Arc::new(child_changes); // Trigger the events - for subscriber in subscribers { - let should_remove = { - let &(ref sink, ref filter, ref child_filters) = self.sinks.get(&subscriber) - .expect("subscribers returned from self.listeners are always in self.sinks; qed"); - sink.unbounded_send((hash.clone(), StorageChangeSet { - changes: changes.clone(), - child_changes: child_changes.clone(), - filter: filter.clone(), - child_filters: child_filters.clone(), - })).is_err() - }; - - if should_remove { - self.remove_subscriber(subscriber); - } + + let to_remove = self.sinks + .iter() + .filter_map(|(subscriber, &(ref sink, ref filter, ref child_filters))| { + let should_remove = { + if subscribers.contains(subscriber) { + sink.unbounded_send((hash.clone(), StorageChangeSet { + changes: changes.clone(), + child_changes: child_changes.clone(), + filter: filter.clone(), + child_filters: child_filters.clone(), + })).is_err() + } else { + sink.is_closed() + } + }; + + if should_remove { + Some(subscriber.clone()) + } else { + None + } + }).collect::>(); + + for sub_id in to_remove { + self.remove_subscriber(sub_id); } } @@ -241,6 +281,9 @@ impl StorageNotifications { } } } + if let Some(m) = self.metrics.as_ref() { + m.with_label_values(&[&"removed"]).inc(); + } } } @@ -301,6 +344,11 @@ impl StorageNotifications { // insert sink let (tx, rx) = tracing_unbounded("mpsc_storage_notification_items"); self.sinks.insert(current_id, (tx, keys, child_keys)); + + if let Some(m) = self.metrics.as_ref() { + m.with_label_values(&[&"added"]).inc(); + } + rx } } diff --git a/client/api/src/proof_provider.rs b/client/api/src/proof_provider.rs index 2d9876f7ad278dd7067199f9090122d5c9c073ab..93160855eaebe733a3058f7373b3e78a11f5813b 100644 --- a/client/api/src/proof_provider.rs +++ b/client/api/src/proof_provider.rs @@ -19,7 +19,7 @@ use sp_runtime::{ traits::{Block as BlockT}, }; use crate::{StorageProof, ChangesProof}; -use sp_storage::{ChildInfo, StorageKey}; +use sp_storage::{ChildInfo, StorageKey, PrefixedStorageKey}; /// Interface for providing block proving utilities. pub trait ProofProvider { @@ -35,8 +35,7 @@ pub trait ProofProvider { fn read_child_proof( &self, id: &BlockId, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, keys: &mut dyn Iterator, ) -> sp_blockchain::Result; @@ -65,7 +64,7 @@ pub trait ProofProvider { last: Block::Hash, min: Block::Hash, max: Block::Hash, - storage_key: Option<&StorageKey>, + storage_key: Option<&PrefixedStorageKey>, key: &StorageKey, ) -> sp_blockchain::Result>; } diff --git a/client/authority-discovery/Cargo.toml b/client/authority-discovery/Cargo.toml index 3fe4de13e33b9daa1e37083dcfd9b53bb69ade84..1a0c3d89afce7c47078e3949bb04a30990ae7170 100644 --- a/client/authority-discovery/Cargo.toml +++ b/client/authority-discovery/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-authority-discovery" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" build = "build.rs" @@ -9,6 +9,9 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Substrate authority discovery." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [build-dependencies] prost-build = "0.6.1" @@ -18,26 +21,23 @@ codec = { package = "parity-scale-codec", default-features = false, version = "1 derive_more = "0.99.2" futures = "0.3.4" futures-timer = "3.0.1" -libp2p = { version = "0.16.2", default-features = false, features = ["secp256k1", "libp2p-websocket"] } +libp2p = { version = "0.18.1", default-features = false, features = ["secp256k1", "libp2p-websocket"] } log = "0.4.8" -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-alpha.5"} +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-dev"} prost = "0.6.1" rand = "0.7.2" -sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } -sc-keystore = { version = "2.0.0-alpha.5", path = "../keystore" } -sc-network = { version = "0.8.0-alpha.5", path = "../network" } +sc-client-api = { version = "2.0.0-dev", path = "../api" } +sc-keystore = { version = "2.0.0-dev", path = "../keystore" } +sc-network = { version = "0.8.0-dev", path = "../network" } serde_json = "1.0.41" -sp-authority-discovery = { version = "2.0.0-alpha.5", path = "../../primitives/authority-discovery" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } +sp-authority-discovery = { version = "2.0.0-dev", path = "../../primitives/authority-discovery" } +sp-blockchain = { version = "2.0.0-dev", path = "../../primitives/blockchain" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } +sp-api = { version = "2.0.0-dev", path = "../../primitives/api" } [dev-dependencies] env_logger = "0.7.0" quickcheck = "0.9.0" -sc-peerset = { version = "2.0.0-alpha.5", path = "../peerset" } +sc-peerset = { version = "2.0.0-dev", path = "../peerset" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client"} - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/authority-discovery/src/error.rs b/client/authority-discovery/src/error.rs index 216f8c26bf07e397a80612c17ca8ea82cd2275da..b1358485c37eeb25edd77d1c5273a8f1316dcaf2 100644 --- a/client/authority-discovery/src/error.rs +++ b/client/authority-discovery/src/error.rs @@ -44,13 +44,13 @@ pub enum Error { EncodingProto(prost::EncodeError), /// Failed to decode a protobuf payload. DecodingProto(prost::DecodeError), - /// Failed to encode or decode scale payload + /// Failed to encode or decode scale payload. EncodingDecodingScale(codec::Error), /// Failed to parse a libp2p multi address. ParsingMultiaddress(libp2p::core::multiaddr::Error), - /// Failed to sign using a specific public key + /// Failed to sign using a specific public key. MissingSignature(CryptoTypePublicPair), - /// Failed to sign using all public keys + /// Failed to sign using all public keys. Signing, /// Failed to register Prometheus metric. Prometheus(prometheus_endpoint::PrometheusError), diff --git a/client/authority-discovery/src/lib.rs b/client/authority-discovery/src/lib.rs index 176e8b2e81dcd2e53bb13ed906e54a78a68f85d3..e5b7c986a44ceab46e72c33a4d52309f7adf70f2 100644 --- a/client/authority-discovery/src/lib.rs +++ b/client/authority-discovery/src/lib.rs @@ -25,13 +25,11 @@ //! //! 1. **Makes itself discoverable** //! -//! 1. Retrieves its external addresses. +//! 1. Retrieves its external addresses (including peer id) or the ones of its sentry nodes. //! -//! 2. Adds its network peer id to the addresses. +//! 2. Signs the above. //! -//! 3. Signs the above. -//! -//! 4. Puts the signature and the addresses on the libp2p Kademlia DHT. +//! 3. Puts the signature and the addresses on the libp2p Kademlia DHT. //! //! //! 2. **Discovers other authorities** @@ -43,6 +41,12 @@ //! 3. Validates the signatures of the retrieved key value pairs. //! //! 4. Adds the retrieved external addresses as priority nodes to the peerset. +//! +//! When run as a sentry node, the authority discovery module does not +//! publish any addresses to the DHT but still discovers validators and +//! sentry nodes of validators, i.e. only step 2 (Discovers other authorities) +//! is executed. + use std::collections::{HashMap, HashSet}; use std::convert::TryInto; use std::marker::PhantomData; @@ -51,18 +55,18 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use futures::task::{Context, Poll}; -use futures::{Future, FutureExt, Stream, StreamExt}; +use futures::{Future, FutureExt, ready, Stream, StreamExt}; use futures_timer::Delay; use codec::{Decode, Encode}; use error::{Error, Result}; -use log::{debug, error, log_enabled, warn}; +use log::{debug, error, log_enabled}; use prometheus_endpoint::{Counter, CounterVec, Gauge, Opts, U64, register}; use prost::Message; use sc_client_api::blockchain::HeaderBackend; use sc_network::{Multiaddr, config::MultiaddrWithPeerId, DhtEvent, ExHashT, NetworkStateInfo}; use sp_authority_discovery::{AuthorityDiscoveryApi, AuthorityId, AuthoritySignature, AuthorityPair}; -use sp_core::crypto::{key_types, CryptoTypePublicPair, Pair}; +use sp_core::crypto::{key_types, Pair}; use sp_core::traits::BareCryptoStorePtr; use sp_runtime::{traits::Block as BlockT, generic::BlockId}; use sp_api::ProvideRuntimeApi; @@ -80,6 +84,8 @@ mod schema { type Interval = Box + Unpin + Send + Sync>; +const LOG_TARGET: &'static str = "sub-authority-discovery"; + /// Upper bound estimation on how long one should wait before accessing the Kademlia DHT. const LIBP2P_KADEMLIA_BOOTSTRAP_TIME: Duration = Duration::from_secs(30); @@ -87,51 +93,15 @@ const LIBP2P_KADEMLIA_BOOTSTRAP_TIME: Duration = Duration::from_secs(30); /// discovery module. const AUTHORITIES_PRIORITY_GROUP_NAME: &'static str = "authorities"; -/// Prometheus metrics for an `AuthorityDiscovery`. -#[derive(Clone)] -pub(crate) struct Metrics { - publish: Counter, - amount_last_published: Gauge, - request: Counter, - dht_event_received: CounterVec, -} - -impl Metrics { - pub(crate) fn register(registry: &prometheus_endpoint::Registry) -> Result { - Ok(Self { - publish: register( - Counter::new( - "authority_discovery_times_published_total", - "Number of times authority discovery has published external addresses." - )?, - registry, - )?, - amount_last_published: register( - Gauge::new( - "authority_discovery_amount_external_addresses_last_published", - "Number of external addresses published when authority discovery last published addresses ." - )?, - registry, - )?, - request: register( - Counter::new( - "authority_discovery_times_requested_total", - "Number of times authority discovery has requested external addresses." - )?, - registry, - )?, - dht_event_received: register( - CounterVec::new( - Opts::new( - "authority_discovery_dht_event_received", - "Number of dht events received by authority discovery." - ), - &["name"], - )?, - registry, - )?, - }) - } +/// Role an authority discovery module can run as. +pub enum Role { + /// Actual authority as well as a reference to its key store. + Authority(BareCryptoStorePtr), + /// Sentry node that guards an authority. + /// + /// No reference to its key store needed, as sentry nodes don't have an identity to sign + /// addresses with in the first place. + Sentry, } /// An `AuthorityDiscovery` makes a given authority discoverable and discovers other authorities. @@ -156,8 +126,6 @@ where /// Channel we receive Dht events on. dht_event_rx: Pin + Send>>, - key_store: BareCryptoStorePtr, - /// Interval to be proactive, publishing own addresses. publish_interval: Interval, /// Interval on which to query for addresses of other authorities. @@ -167,6 +135,8 @@ where metrics: Option, + role: Role, + phantom: PhantomData, } @@ -187,8 +157,8 @@ where client: Arc, network: Arc, sentry_nodes: Vec, - key_store: BareCryptoStorePtr, dht_event_rx: Pin + Send>>, + role: Role, prometheus_registry: Option, ) -> Self { // Kademlia's default time-to-live for Dht records is 36h, republishing records every 24h. @@ -221,7 +191,7 @@ where match Metrics::register(®istry) { Ok(metrics) => Some(metrics), Err(e) => { - error!(target: "sub-authority-discovery", "Failed to register metrics: {:?}", e); + error!(target: LOG_TARGET, "Failed to register metrics: {:?}", e); None }, } @@ -234,10 +204,10 @@ where network, sentry_nodes, dht_event_rx, - key_store, publish_interval, query_interval, addr_cache, + role, metrics, phantom: PhantomData, } @@ -245,6 +215,14 @@ where /// Publish either our own or if specified the public addresses of our sentry nodes. fn publish_ext_addresses(&mut self) -> Result<()> { + let key_store = match &self.role { + Role::Authority(key_store) => key_store, + // Only authority nodes can put addresses (their own or the ones of their sentry nodes) + // on the Dht. Sentry nodes don't have a known identity to authenticate such addresses, + // thus `publish_ext_addresses` becomes a no-op. + Role::Sentry => return Ok(()), + }; + if let Some(metrics) = &self.metrics { metrics.publish.inc() } @@ -271,13 +249,12 @@ where .encode(&mut serialized_addresses) .map_err(Error::EncodingProto)?; - let keys: Vec = self.get_own_public_keys_within_authority_set()? - .into_iter() - .map(Into::into) - .collect(); + let keys = AuthorityDiscovery::get_own_public_keys_within_authority_set( + &key_store, + &self.client, + )?.into_iter().map(Into::into).collect::>(); - let signatures = self.key_store - .read() + let signatures = key_store.read() .sign_with_all( key_types::AUTHORITY_DISCOVERY, keys.clone(), @@ -300,7 +277,7 @@ where .map_err(Error::EncodingProto)?; self.network.put_value( - hash_authority_id(key.1.as_ref())?, + hash_authority_id(key.1.as_ref()), signed_addresses, ); } @@ -309,10 +286,6 @@ where } fn request_addresses_of_others(&mut self) -> Result<()> { - if let Some(metrics) = &self.metrics { - metrics.request.inc(); - } - let id = BlockId::hash(self.client.info().best_hash); let authorities = self @@ -322,17 +295,26 @@ where .map_err(Error::CallingRuntime)?; for authority_id in authorities.iter() { + if let Some(metrics) = &self.metrics { + metrics.request.inc(); + } + self.network - .get_value(&hash_authority_id(authority_id.as_ref())?); + .get_value(&hash_authority_id(authority_id.as_ref())); } Ok(()) } - fn handle_dht_events(&mut self, cx: &mut Context) -> Result<()> { - while let Poll::Ready(Some(event)) = self.dht_event_rx.poll_next_unpin(cx) { - match event { - DhtEvent::ValueFound(v) => { + /// Handle incoming Dht events. + /// + /// Returns either: + /// - Poll::Pending when there are no more events to handle or + /// - Poll::Ready(()) when the dht event stream terminated. + fn handle_dht_events(&mut self, cx: &mut Context) -> Poll<()>{ + loop { + match ready!(self.dht_event_rx.poll_next_unpin(cx)) { + Some(DhtEvent::ValueFound(v)) => { if let Some(metrics) = &self.metrics { metrics.dht_event_received.with_label_values(&["value_found"]).inc(); } @@ -340,47 +322,58 @@ where if log_enabled!(log::Level::Debug) { let hashes = v.iter().map(|(hash, _value)| hash.clone()); debug!( - target: "sub-authority-discovery", + target: LOG_TARGET, "Value for hash '{:?}' found on Dht.", hashes, ); } - self.handle_dht_value_found_event(v)?; + if let Err(e) = self.handle_dht_value_found_event(v) { + if let Some(metrics) = &self.metrics { + metrics.handle_value_found_event_failure.inc(); + } + + debug!( + target: LOG_TARGET, + "Failed to handle Dht value found event: {:?}", e, + ); + } } - DhtEvent::ValueNotFound(hash) => { + Some(DhtEvent::ValueNotFound(hash)) => { if let Some(metrics) = &self.metrics { metrics.dht_event_received.with_label_values(&["value_not_found"]).inc(); } debug!( - target: "sub-authority-discovery", + target: LOG_TARGET, "Value for hash '{:?}' not found on Dht.", hash ) }, - DhtEvent::ValuePut(hash) => { + Some(DhtEvent::ValuePut(hash)) => { if let Some(metrics) = &self.metrics { metrics.dht_event_received.with_label_values(&["value_put"]).inc(); } debug!( - target: "sub-authority-discovery", + target: LOG_TARGET, "Successfully put hash '{:?}' on Dht.", hash, ) }, - DhtEvent::ValuePutFailed(hash) => { + Some(DhtEvent::ValuePutFailed(hash)) => { if let Some(metrics) = &self.metrics { metrics.dht_event_received.with_label_values(&["value_put_failed"]).inc(); } - warn!( - target: "sub-authority-discovery", + debug!( + target: LOG_TARGET, "Failed to put hash '{:?}' on Dht.", hash ) }, + None => { + debug!(target: LOG_TARGET, "Dht event stream terminated."); + return Poll::Ready(()); + }, } } - - Ok(()) } fn handle_dht_value_found_event( @@ -408,8 +401,8 @@ where self.addr_cache.retain_ids(&authorities); authorities .into_iter() - .map(|id| hash_authority_id(id.as_ref()).map(|h| (h, id))) - .collect::>>()? + .map(|id| (hash_authority_id(id.as_ref()), id)) + .collect::>() }; // Check if the event origins from an authority in the current authority set. @@ -457,17 +450,17 @@ where // one for the upcoming session. In addition it could be participating in the current authority // set with two keys. The function does not return all of the local authority discovery public // keys, but only the ones intersecting with the current authority set. - fn get_own_public_keys_within_authority_set(&mut self) -> Result> { - let local_pub_keys = self.key_store - .read() + fn get_own_public_keys_within_authority_set( + key_store: &BareCryptoStorePtr, + client: &Client, + ) -> Result> { + let local_pub_keys = key_store.read() .sr25519_public_keys(key_types::AUTHORITY_DISCOVERY) .into_iter() .collect::>(); - let id = BlockId::hash(self.client.info().best_hash); - let current_authorities = self - .client - .runtime_api() + let id = BlockId::hash(client.info().best_hash); + let current_authorities = client.runtime_api() .authorities(&id) .map_err(Error::CallingRuntime)? .into_iter() @@ -483,16 +476,22 @@ where } /// Update the peer set 'authority' priority group. - // fn update_peer_set_priority_group(&self) -> Result<()> { let addresses = self.addr_cache.get_subset(); + if let Some(metrics) = &self.metrics { + metrics.priority_group_size.set(addresses.len().try_into().unwrap_or(std::u64::MAX)); + } + debug!( - target: "sub-authority-discovery", + target: LOG_TARGET, "Applying priority group {:?} to peerset.", addresses, ); self.network - .set_priority_group(AUTHORITIES_PRIORITY_GROUP_NAME.to_string(), addresses.into_iter().collect()) + .set_priority_group( + AUTHORITIES_PRIORITY_GROUP_NAME.to_string(), + addresses.into_iter().collect(), + ) .map_err(Error::SettingPeersetPriorityGroup)?; Ok(()) @@ -510,40 +509,41 @@ where type Output = (); fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { - let mut inner = || -> Result<()> { - // Process incoming events before triggering new ones. - self.handle_dht_events(cx)?; - - if let Poll::Ready(_) = self.publish_interval.poll_next_unpin(cx) { - // Make sure to call interval.poll until it returns Async::NotReady once. Otherwise, - // in case one of the function calls within this block do a `return`, we don't call - // `interval.poll` again and thereby the underlying Tokio task is never registered - // with Tokio's Reactor to be woken up on the next interval tick. - while let Poll::Ready(_) = self.publish_interval.poll_next_unpin(cx) {} - - self.publish_ext_addresses()?; - } + // Process incoming events. + if let Poll::Ready(()) = self.handle_dht_events(cx) { + // `handle_dht_events` returns `Poll::Ready(())` when the Dht event stream terminated. + // Termination of the Dht event stream implies that the underlying network terminated, + // thus authority discovery should terminate as well. + return Poll::Ready(()); + } - if let Poll::Ready(_) = self.query_interval.poll_next_unpin(cx) { - // Make sure to call interval.poll until it returns Async::NotReady once. Otherwise, - // in case one of the function calls within this block do a `return`, we don't call - // `interval.poll` again and thereby the underlying Tokio task is never registered - // with Tokio's Reactor to be woken up on the next interval tick. - while let Poll::Ready(_) = self.query_interval.poll_next_unpin(cx) {} - self.request_addresses_of_others()?; + // Publish own addresses. + if let Poll::Ready(_) = self.publish_interval.poll_next_unpin(cx) { + // Register waker of underlying task for next interval. + while let Poll::Ready(_) = self.publish_interval.poll_next_unpin(cx) {} + + if let Err(e) = self.publish_ext_addresses() { + error!( + target: LOG_TARGET, + "Failed to publish external addresses: {:?}", e, + ); } + } - Ok(()) - }; + // Request addresses of authorities. + if let Poll::Ready(_) = self.query_interval.poll_next_unpin(cx) { + // Register waker of underlying task for next interval. + while let Poll::Ready(_) = self.query_interval.poll_next_unpin(cx) {} - match inner() { - Ok(()) => {} - Err(e) => error!(target: "sub-authority-discovery", "Poll failure: {:?}", e), - }; + if let Err(e) = self.request_addresses_of_others() { + error!( + target: LOG_TARGET, + "Failed to request addresses of authorities: {:?}", e, + ); + } + } - // Make sure to always return NotReady as this is a long running task with the same lifetime - // as the node itself. Poll::Pending } } @@ -586,10 +586,8 @@ where } } -fn hash_authority_id(id: &[u8]) -> Result { - libp2p::multihash::encode(libp2p::multihash::Hash::SHA2256, id) - .map(|k| libp2p::kad::record::Key::new(&k)) - .map_err(Error::HashingAuthorityId) +fn hash_authority_id(id: &[u8]) -> libp2p::kad::record::Key { + libp2p::kad::record::Key::new(&libp2p::multihash::Sha2_256::digest(id)) } fn interval_at(start: Instant, duration: Duration) -> Interval { @@ -601,3 +599,68 @@ fn interval_at(start: Instant, duration: Duration) -> Interval { Box::new(stream) } + +/// Prometheus metrics for an `AuthorityDiscovery`. +#[derive(Clone)] +pub(crate) struct Metrics { + publish: Counter, + amount_last_published: Gauge, + request: Counter, + dht_event_received: CounterVec, + handle_value_found_event_failure: Counter, + priority_group_size: Gauge, +} + +impl Metrics { + pub(crate) fn register(registry: &prometheus_endpoint::Registry) -> Result { + Ok(Self { + publish: register( + Counter::new( + "authority_discovery_times_published_total", + "Number of times authority discovery has published external addresses." + )?, + registry, + )?, + amount_last_published: register( + Gauge::new( + "authority_discovery_amount_external_addresses_last_published", + "Number of external addresses published when authority discovery last \ + published addresses." + )?, + registry, + )?, + request: register( + Counter::new( + "authority_discovery_authority_addresses_requested_total", + "Number of times authority discovery has requested external addresses of a \ + single authority." + )?, + registry, + )?, + dht_event_received: register( + CounterVec::new( + Opts::new( + "authority_discovery_dht_event_received", + "Number of dht events received by authority discovery." + ), + &["name"], + )?, + registry, + )?, + handle_value_found_event_failure: register( + Counter::new( + "authority_discovery_handle_value_found_event_failure", + "Number of times handling a dht value found event failed." + )?, + registry, + )?, + priority_group_size: register( + Gauge::new( + "authority_discovery_priority_group_size", + "Number of addresses passed to the peer set as a priority group." + )?, + registry, + )?, + }) + } +} diff --git a/client/authority-discovery/src/tests.rs b/client/authority-discovery/src/tests.rs index d23836d6fac9cc3b0c677dafbae4da048159ddf6..c9b5e392d82b5e71046ee1c688a5fb6763a766fa 100644 --- a/client/authority-discovery/src/tests.rs +++ b/client/authority-discovery/src/tests.rs @@ -17,8 +17,11 @@ use std::{iter::FromIterator, sync::{Arc, Mutex}}; use futures::channel::mpsc::channel; -use futures::executor::block_on; -use futures::future::poll_fn; +use futures::executor::{block_on, LocalPool}; +use futures::future::{poll_fn, FutureExt}; +use futures::sink::SinkExt; +use futures::task::LocalSpawn; +use futures::poll; use libp2p::{kad, PeerId}; use sp_api::{ProvideRuntimeApi, ApiRef}; @@ -213,8 +216,8 @@ fn new_registers_metrics() { test_api, network.clone(), vec![], - key_store, dht_event_rx.boxed(), + Role::Authority(key_store), Some(registry.clone()), ); @@ -238,8 +241,8 @@ fn publish_ext_addresses_puts_record_on_dht() { test_api, network.clone(), vec![], - key_store, dht_event_rx.boxed(), + Role::Authority(key_store), None, ); @@ -269,8 +272,8 @@ fn request_addresses_of_others_triggers_dht_get_query() { test_api, network.clone(), vec![], - key_store, dht_event_rx.boxed(), + Role::Authority(key_store), None, ); @@ -283,6 +286,7 @@ fn request_addresses_of_others_triggers_dht_get_query() { #[test] fn handle_dht_events_with_value_found_should_call_set_priority_group() { let _ = ::env_logger::try_init(); + // Create authority discovery. let (mut dht_event_tx, dht_event_rx) = channel(1000); @@ -297,14 +301,14 @@ fn handle_dht_events_with_value_found_should_call_set_priority_group() { test_api, network.clone(), vec![], - key_store, dht_event_rx.boxed(), + Role::Authority(key_store), None, ); // Create sample dht event. - let authority_id_1 = hash_authority_id(key_pair.public().as_ref()).unwrap(); + let authority_id_1 = hash_authority_id(key_pair.public().as_ref()); let address_1: Multiaddr = "/ip6/2001:db8::".parse().unwrap(); let mut serialized_addresses = vec![]; @@ -318,7 +322,7 @@ fn handle_dht_events_with_value_found_should_call_set_priority_group() { let mut signed_addresses = vec![]; schema::SignedAuthorityAddresses { addresses: serialized_addresses, - signature: signature, + signature, } .encode(&mut signed_addresses) .unwrap(); @@ -328,7 +332,9 @@ fn handle_dht_events_with_value_found_should_call_set_priority_group() { // Make authority discovery handle the event. let f = |cx: &mut Context<'_>| -> Poll<()> { - authority_discovery.handle_dht_events(cx).unwrap(); + if let Poll::Ready(e) = authority_discovery.handle_dht_events(cx) { + panic!("Unexpected error: {:?}", e); + } // Expect authority discovery to set the priority set. assert_eq!(network.set_priority_group_call.lock().unwrap().len(), 1); @@ -346,3 +352,107 @@ fn handle_dht_events_with_value_found_should_call_set_priority_group() { let _ = block_on(poll_fn(f)); } + +#[test] +fn terminate_when_event_stream_terminates() { + let (dht_event_tx, dht_event_rx) = channel(1000); + let network: Arc = Arc::new(Default::default()); + let key_store = KeyStore::new(); + let test_api = Arc::new(TestApi { + authorities: vec![], + }); + + let mut authority_discovery = AuthorityDiscovery::new( + test_api, + network.clone(), + vec![], + dht_event_rx.boxed(), + Role::Authority(key_store), + None, + ); + + block_on(async { + assert_eq!(Poll::Pending, poll!(&mut authority_discovery)); + + // Simulate termination of the network through dropping the sender side of the dht event + // channel. + drop(dht_event_tx); + + assert_eq!( + Poll::Ready(()), poll!(&mut authority_discovery), + "Expect the authority discovery module to terminate once the sending side of the dht \ + event channel is terminated.", + ); + }); +} + +#[test] +fn dont_stop_polling_when_error_is_returned() { + #[derive(PartialEq, Debug)] + enum Event { + Processed, + End, + }; + + let (mut dht_event_tx, dht_event_rx) = channel(1000); + let (mut discovery_update_tx, mut discovery_update_rx) = channel(1000); + let network: Arc = Arc::new(Default::default()); + let key_store = KeyStore::new(); + let test_api = Arc::new(TestApi { + authorities: vec![], + }); + let mut pool = LocalPool::new(); + + let mut authority_discovery = AuthorityDiscovery::new( + test_api, + network.clone(), + vec![], + dht_event_rx.boxed(), + Role::Authority(key_store), + None, + ); + + // Spawn the authority discovery to make sure it is polled independently. + // + // As this is a local pool, only one future at a time will have the CPU and + // can make progress until the future returns `Pending`. + pool.spawner().spawn_local_obj( + futures::future::poll_fn(move |ctx| { + match std::pin::Pin::new(&mut authority_discovery).poll(ctx) { + Poll::Ready(()) => {}, + Poll::Pending => { + discovery_update_tx.send(Event::Processed).now_or_never(); + return Poll::Pending; + }, + } + let _ = discovery_update_tx.send(Event::End).now_or_never().unwrap(); + Poll::Ready(()) + }).boxed_local().into(), + ).expect("Spawns authority discovery"); + + pool.run_until( + // The future that drives the event stream + async { + // Send an event that should generate an error + let _ = dht_event_tx.send(DhtEvent::ValueFound(Default::default())).now_or_never(); + // Send the same event again to make sure that the event stream needs to be polled twice + // to be woken up again. + let _ = dht_event_tx.send(DhtEvent::ValueFound(Default::default())).now_or_never(); + + // Now we call `await` and give the control to the authority discovery future. + assert_eq!(Some(Event::Processed), discovery_update_rx.next().await); + + // Drop the event rx to stop the authority discovery. If it was polled correctly, it + // should end properly. + drop(dht_event_tx); + + assert!( + discovery_update_rx.collect::>() + .await + .into_iter() + .any(|evt| evt == Event::End), + "The authority discovery should have ended", + ); + } + ); +} diff --git a/client/basic-authorship/Cargo.toml b/client/basic-authorship/Cargo.toml index 040370ac491cf09932ef1e7467ebeb96847b5af8..a9ac58ef3211be3bc835bdc73754344964247934 100644 --- a/client/basic-authorship/Cargo.toml +++ b/client/basic-authorship/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-basic-authorship" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,27 +8,27 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Basic implementation of block-authoring logic." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] log = "0.4.8" futures = "0.3.4" codec = { package = "parity-scale-codec", version = "1.3.0" } -sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/common" } -sp-inherents = { version = "2.0.0-alpha.5", path = "../../primitives/inherents" } -sc-telemetry = { version = "2.0.0-alpha.5", path = "../telemetry" } -sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../primitives/transaction-pool" } -sc-block-builder = { version = "0.8.0-alpha.5", path = "../block-builder" } +sp-api = { version = "2.0.0-dev", path = "../../primitives/api" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-blockchain = { version = "2.0.0-dev", path = "../../primitives/blockchain" } +sc-client-api = { version = "2.0.0-dev", path = "../api" } +sp-consensus = { version = "0.8.0-dev", path = "../../primitives/consensus/common" } +sp-inherents = { version = "2.0.0-dev", path = "../../primitives/inherents" } +sc-telemetry = { version = "2.0.0-dev", path = "../telemetry" } +sp-transaction-pool = { version = "2.0.0-dev", path = "../../primitives/transaction-pool" } +sc-block-builder = { version = "0.8.0-dev", path = "../block-builder" } tokio-executor = { version = "0.2.0-alpha.6", features = ["blocking"] } futures-timer = "3.0.1" [dev-dependencies] -sc-transaction-pool = { version = "2.0.0-alpha.5", path = "../../client/transaction-pool" } +sc-transaction-pool = { version = "2.0.0-dev", path = "../../client/transaction-pool" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } parking_lot = "0.10.0" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/basic-authorship/src/basic_authorship.rs b/client/basic-authorship/src/basic_authorship.rs index 37bb34a4b67d993fb02bc6b1e5229db0c7fa7f81..e1e99938e375eb54ba16607938087626680df826 100644 --- a/client/basic-authorship/src/basic_authorship.rs +++ b/client/basic-authorship/src/basic_authorship.rs @@ -360,7 +360,11 @@ mod tests { // given let client = Arc::new(substrate_test_runtime_client::new()); let txpool = Arc::new( - BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone()))).0 + BasicPool::new( + Default::default(), + Arc::new(FullChainApi::new(client.clone())), + None, + ).0 ); futures::executor::block_on( @@ -408,7 +412,11 @@ mod tests { fn should_not_panic_when_deadline_is_reached() { let client = Arc::new(substrate_test_runtime_client::new()); let txpool = Arc::new( - BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone()))).0 + BasicPool::new( + Default::default(), + Arc::new(FullChainApi::new(client.clone())), + None, + ).0 ); let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone()); @@ -440,8 +448,13 @@ mod tests { .build_with_backend(); let client = Arc::new(client); let txpool = Arc::new( - BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone()))).0 + BasicPool::new( + Default::default(), + Arc::new(FullChainApi::new(client.clone())), + None, + ).0 ); + let genesis_hash = client.info().best_hash; let block_id = BlockId::Hash(genesis_hash); @@ -493,7 +506,11 @@ mod tests { // given let mut client = Arc::new(substrate_test_runtime_client::new()); let txpool = Arc::new( - BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone()))).0 + BasicPool::new( + Default::default(), + Arc::new(FullChainApi::new(client.clone())), + None, + ).0 ); futures::executor::block_on( diff --git a/client/basic-authorship/src/lib.rs b/client/basic-authorship/src/lib.rs index 5ec0bc6f9a520c5d8292ce21cc219b5a8a658bab..5eb60f1cd586c53099b0aedb6542db2298e0dbc6 100644 --- a/client/basic-authorship/src/lib.rs +++ b/client/basic-authorship/src/lib.rs @@ -26,7 +26,7 @@ //! # use substrate_test_runtime_client::{self, runtime::{Extrinsic, Transfer}, AccountKeyring}; //! # use sc_transaction_pool::{BasicPool, FullChainApi}; //! # let client = Arc::new(substrate_test_runtime_client::new()); -//! # let txpool = Arc::new(BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone()))).0); +//! # let txpool = Arc::new(BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone())), None).0); //! // The first step is to create a `ProposerFactory`. //! let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone()); //! diff --git a/client/block-builder/Cargo.toml b/client/block-builder/Cargo.toml index 5f9ef7c75e8591baaada2b2ff1a46fbebc72f231..15cc3c13934fa7e5fda805ce5d6fc5082aac25f4 100644 --- a/client/block-builder/Cargo.toml +++ b/client/block-builder/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-block-builder" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,21 +8,21 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Substrate block builder" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../primitives/state-machine" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/common" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-block-builder = { version = "2.0.0-alpha.5", path = "../../primitives/block-builder" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } +sp-state-machine = { version = "0.8.0-dev", path = "../../primitives/state-machine" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } +sp-api = { version = "2.0.0-dev", path = "../../primitives/api" } +sp-consensus = { version = "0.8.0-dev", path = "../../primitives/consensus/common" } +sp-blockchain = { version = "2.0.0-dev", path = "../../primitives/blockchain" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-block-builder = { version = "2.0.0-dev", path = "../../primitives/block-builder" } +sc-client-api = { version = "2.0.0-dev", path = "../api" } codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } [dev-dependencies] substrate-test-runtime-client = { path = "../../test-utils/runtime/client" } -sp-trie = { version = "2.0.0-alpha.5", path = "../../primitives/trie" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sp-trie = { version = "2.0.0-dev", path = "../../primitives/trie" } diff --git a/client/chain-spec/Cargo.toml b/client/chain-spec/Cargo.toml index 5d65cbd842b5430064ae2249fe63f9f0b4cf2b8c..6906b1ecdad434884437af619548e2b34daedf4c 100644 --- a/client/chain-spec/Cargo.toml +++ b/client/chain-spec/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-chain-spec" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,15 +8,16 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Substrate chain configurations." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sc-chain-spec-derive = { version = "2.0.0-alpha.5", path = "./derive" } +sc-chain-spec-derive = { version = "2.0.0-dev", path = "./derive" } impl-trait-for-tuples = "0.1.3" -sc-network = { version = "0.8.0-alpha.5", path = "../network" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } +sc-network = { version = "0.8.0-dev", path = "../network" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } -sc-telemetry = { version = "2.0.0-alpha.5", path = "../telemetry" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } +sp-chain-spec = { version = "2.0.0-dev", path = "../../primitives/chain-spec" } +sc-telemetry = { version = "2.0.0-dev", path = "../telemetry" } diff --git a/client/chain-spec/derive/Cargo.toml b/client/chain-spec/derive/Cargo.toml index 9343c9a6de1c95919abe695b4918bd576117efbf..66058b3f729963d100b846527a538d5c0c8915b1 100644 --- a/client/chain-spec/derive/Cargo.toml +++ b/client/chain-spec/derive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-chain-spec-derive" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,6 +8,9 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Macros to derive chain spec extension traits implementation." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [lib] proc-macro = true @@ -18,6 +21,3 @@ quote = "1.0.3" syn = "1.0.7" [dev-dependencies] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index a7cf1d8aac4143ff758ec67b39e32c39eb6cc379..fbe7b7e7a8effee0b8fd188d9ec3abbd2f0429aa 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -25,8 +25,7 @@ use serde::{Serialize, Deserialize}; use sp_core::storage::{StorageKey, StorageData, ChildInfo, Storage, StorageChild}; use sp_runtime::BuildStorage; use serde_json as json; -use crate::RuntimeGenesis; -use crate::extension::GetExtension; +use crate::{RuntimeGenesis, ChainType, extension::GetExtension, Properties}; use sc_network::config::MultiaddrWithPeerId; use sc_telemetry::TelemetryEndpoints; @@ -75,17 +74,14 @@ impl BuildStorage for ChainSpec { fn build_storage(&self) -> Result { match self.genesis.resolve()? { Genesis::Runtime(gc) => gc.build_storage(), - Genesis::Raw(RawGenesis { top: map, children: children_map }) => Ok(Storage { + Genesis::Raw(RawGenesis { top: map, children_default: children_map }) => Ok(Storage { top: map.into_iter().map(|(k, v)| (k.0, v.0)).collect(), - children: children_map.into_iter().map(|(sk, child_content)| { - let child_info = ChildInfo::resolve_child_info( - child_content.child_type, - child_content.child_info.as_slice(), - ).expect("chain spec contains correct content").to_owned(); + children_default: children_map.into_iter().map(|(storage_key, child_content)| { + let child_info = ChildInfo::new_default(storage_key.0.as_slice()); ( - sk.0, + storage_key.0, StorageChild { - data: child_content.data.into_iter().map(|(k, v)| (k.0, v.0)).collect(), + data: child_content.into_iter().map(|(k, v)| (k.0, v.0)).collect(), child_info, }, ) @@ -104,22 +100,13 @@ impl BuildStorage for ChainSpec { type GenesisStorage = HashMap; -#[derive(Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -#[serde(deny_unknown_fields)] -struct ChildRawStorage { - data: GenesisStorage, - child_info: Vec, - child_type: u32, -} - #[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] /// Storage content for genesis block. struct RawGenesis { top: GenesisStorage, - children: HashMap, + children_default: HashMap, } #[derive(Serialize, Deserialize)] @@ -137,6 +124,8 @@ enum Genesis { struct ClientSpec { name: String, id: String, + #[serde(default)] + chain_type: ChainType, boot_nodes: Vec, telemetry_endpoints: Option, protocol_id: Option, @@ -149,9 +138,6 @@ struct ClientSpec { genesis: serde::de::IgnoredAny, } -/// Arbitrary properties defined in chain spec as a JSON object -pub type Properties = json::map::Map; - /// A type denoting empty extensions. /// /// We use `Option` here since `()` is not flattenable by serde. @@ -219,6 +205,7 @@ impl ChainSpec { pub fn from_genesis G + 'static + Send + Sync>( name: &str, id: &str, + chain_type: ChainType, constructor: F, boot_nodes: Vec, telemetry_endpoints: Option, @@ -229,6 +216,7 @@ impl ChainSpec { let client_spec = ClientSpec { name: name.to_owned(), id: id.to_owned(), + chain_type, boot_nodes, telemetry_endpoints, protocol_id: protocol_id.map(str::to_owned), @@ -243,6 +231,11 @@ impl ChainSpec { genesis: GenesisSource::Factory(Arc::new(constructor)), } } + + /// Type of the chain. + fn chain_type(&self) -> ChainType { + self.client_spec.chain_type.clone() + } } impl ChainSpec { @@ -286,23 +279,16 @@ impl ChainSpec { let top = storage.top.into_iter() .map(|(k, v)| (StorageKey(k), StorageData(v))) .collect(); - let children = storage.children.into_iter() - .map(|(sk, child)| { - let info = child.child_info.as_ref(); - let (info, ci_type) = info.info(); - ( - StorageKey(sk), - ChildRawStorage { - data: child.data.into_iter() - .map(|(k, v)| (StorageKey(k), StorageData(v))) - .collect(), - child_info: info.to_vec(), - child_type: ci_type, - }, - )}) + let children_default = storage.children_default.into_iter() + .map(|(sk, child)| ( + StorageKey(sk), + child.data.into_iter() + .map(|(k, v)| (StorageKey(k), StorageData(v))) + .collect(), + )) .collect(); - Genesis::Raw(RawGenesis { top, children }) + Genesis::Raw(RawGenesis { top, children_default }) }, (_, genesis) => genesis, }; @@ -332,6 +318,10 @@ where ChainSpec::id(self) } + fn chain_type(&self) -> ChainType { + ChainSpec::chain_type(self) + } + fn telemetry_endpoints(&self) -> &Option { ChainSpec::telemetry_endpoints(self) } @@ -392,6 +382,7 @@ mod tests { ).unwrap(); assert_eq!(spec1.as_json(false), spec2.as_json(false)); + assert_eq!(spec2.chain_type(), ChainType::Live) } #[derive(Debug, Serialize, Deserialize)] diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index b7875fdbb4670ee97fb9ccd14971d3b0e3c3d95a..de83e170e039b807f7e283b14f613ec90de3e4fc 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -107,13 +107,13 @@ //! pub type MyChainSpec = GenericChainSpec; //! ``` - mod chain_spec; mod extension; -pub use chain_spec::{ChainSpec as GenericChainSpec, Properties, NoExtension}; +pub use chain_spec::{ChainSpec as GenericChainSpec, NoExtension}; pub use extension::{Group, Fork, Forks, Extension, GetExtension, get_extension}; pub use sc_chain_spec_derive::{ChainSpecExtension, ChainSpecGroup}; +pub use sp_chain_spec::{Properties, ChainType}; use serde::{Serialize, de::DeserializeOwned}; use sp_runtime::BuildStorage; @@ -124,12 +124,14 @@ use sc_telemetry::TelemetryEndpoints; pub trait RuntimeGenesis: Serialize + DeserializeOwned + BuildStorage {} impl RuntimeGenesis for T {} -/// Common interface to `GenericChainSpec` +/// Common interface of a chain specification. pub trait ChainSpec: BuildStorage + Send { /// Spec name. fn name(&self) -> &str; /// Spec id. fn id(&self) -> &str; + /// Type of the chain. + fn chain_type(&self) -> ChainType; /// A list of bootnode addresses. fn boot_nodes(&self) -> &[MultiaddrWithPeerId]; /// Telemetry endpoints (if any) diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml index 89247cf016d100792a6ca49e3882529c165472c4..198a9df5b53fdbef22f16d7cec3b8741153aeaaa 100644 --- a/client/cli/Cargo.toml +++ b/client/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-cli" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "Substrate CLI interface." edition = "2018" @@ -8,6 +8,9 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] clap = "2.33.0" derive_more = "0.99.2" @@ -23,24 +26,25 @@ tokio = { version = "0.2.9", features = [ "signal", "rt-core", "rt-threaded" ] } futures = "0.3.4" fdlimit = "0.1.4" serde_json = "1.0.41" -sc-informant = { version = "0.8.0-alpha.5", path = "../informant" } -sp-panic-handler = { version = "2.0.0-alpha.5", path = "../../primitives/panic-handler" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } -sc-network = { version = "0.8.0-alpha.5", path = "../network" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } -sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sc-service = { version = "0.8.0-alpha.5", default-features = false, path = "../service" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../primitives/state-machine" } -sc-telemetry = { version = "2.0.0-alpha.5", path = "../telemetry" } -substrate-prometheus-endpoint = { path = "../../utils/prometheus" , version = "0.8.0-alpha.5"} -sp-keyring = { version = "2.0.0-alpha.5", path = "../../primitives/keyring" } +sc-informant = { version = "0.8.0-dev", path = "../informant" } +sp-panic-handler = { version = "2.0.0-dev", path = "../../primitives/panic-handler" } +sc-client-api = { version = "2.0.0-dev", path = "../api" } +sp-blockchain = { version = "2.0.0-dev", path = "../../primitives/blockchain" } +sc-network = { version = "0.8.0-dev", path = "../network" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-dev", path = "../../primitives/utils" } +sp-version = { version = "2.0.0-dev", path = "../../primitives/version" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sc-service = { version = "0.8.0-dev", default-features = false, path = "../service" } +sp-state-machine = { version = "0.8.0-dev", path = "../../primitives/state-machine" } +sc-telemetry = { version = "2.0.0-dev", path = "../telemetry" } +substrate-prometheus-endpoint = { path = "../../utils/prometheus" , version = "0.8.0-dev"} +sp-keyring = { version = "2.0.0-dev", path = "../../primitives/keyring" } names = "0.11.0" structopt = "0.3.8" -sc-tracing = { version = "2.0.0-alpha.5", path = "../tracing" } +sc-tracing = { version = "2.0.0-dev", path = "../tracing" } chrono = "0.4.10" -parity-util-mem = { version = "0.6.0", default-features = false, features = ["primitive-types"] } +parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } [target.'cfg(not(target_os = "unknown"))'.dependencies] rpassword = "4.0.1" @@ -55,6 +59,3 @@ tempfile = "3.1.0" wasmtime = [ "sc-service/wasmtime", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/cli/src/arg_enums.rs b/client/cli/src/arg_enums.rs index 384087bec0dbe4538c7227e36a6dbce7d789586b..f0eeca4c8bef1f1a564d96d97bf16cd15e419aa7 100644 --- a/client/cli/src/arg_enums.rs +++ b/client/cli/src/arg_enums.rs @@ -45,7 +45,9 @@ impl WasmExecutionMethod { impl Into for WasmExecutionMethod { fn into(self) -> sc_service::config::WasmExecutionMethod { match self { - WasmExecutionMethod::Interpreted => sc_service::config::WasmExecutionMethod::Interpreted, + WasmExecutionMethod::Interpreted => { + sc_service::config::WasmExecutionMethod::Interpreted + } #[cfg(feature = "wasmtime")] WasmExecutionMethod::Compiled => sc_service::config::WasmExecutionMethod::Compiled, #[cfg(not(feature = "wasmtime"))] @@ -120,6 +122,32 @@ impl ExecutionStrategy { } } +arg_enum! { + /// Database backend + #[allow(missing_docs)] + #[derive(Debug, Clone, Copy)] + pub enum Database { + // Facebooks RocksDB + RocksDb, + // Subdb. https://github.com/paritytech/subdb/ + SubDb, + // ParityDb. https://github.com/paritytech/parity-db/ + ParityDb, + } +} + + +arg_enum! { + /// Whether off-chain workers are enabled. + #[allow(missing_docs)] + #[derive(Debug, Clone)] + pub enum OffchainWorkerEnabled { + Always, + Never, + WhenValidating, + } +} + /// Default value for the `--execution-syncing` parameter. pub const DEFAULT_EXECUTION_SYNCING: ExecutionStrategy = ExecutionStrategy::NativeElseWasm; /// Default value for the `--execution-import-block` parameter. diff --git a/client/cli/src/commands/build_spec_cmd.rs b/client/cli/src/commands/build_spec_cmd.rs index 67aaf998fca6f5d6c198fcc44c5120e5c587660c..a01101fa7965563795eb3058bf3be0f752801517 100644 --- a/client/cli/src/commands/build_spec_cmd.rs +++ b/client/cli/src/commands/build_spec_cmd.rs @@ -14,15 +14,14 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use structopt::StructOpt; -use log::info; -use sc_network::config::{build_multiaddr, MultiaddrWithPeerId}; -use sc_service::{Configuration, ChainSpec}; - use crate::error; -use crate::VersionInfo; -use crate::params::SharedParams; use crate::params::NodeKeyParams; +use crate::params::SharedParams; +use crate::CliConfiguration; +use log::info; +use sc_network::config::build_multiaddr; +use sc_service::{config::MultiaddrWithPeerId, Configuration}; +use structopt::StructOpt; /// The `build-spec` command used to build a specification. #[derive(Debug, StructOpt, Clone)] @@ -49,12 +48,9 @@ pub struct BuildSpecCmd { impl BuildSpecCmd { /// Run the build-spec command - pub fn run( - self, - config: Configuration, - ) -> error::Result<()> { + pub fn run(&self, config: Configuration) -> error::Result<()> { info!("Building chain spec"); - let mut spec = config.chain_spec.expect("`chain_spec` is set to `Some` in `update_config`"); + let mut spec = config.chain_spec; let raw_output = self.raw; if spec.boot_nodes().is_empty() && !self.disable_default_bootnode { @@ -73,25 +69,14 @@ impl BuildSpecCmd { Ok(()) } +} - /// Update and prepare a `Configuration` with command line parameters - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> where - F: FnOnce(&str) -> Result, String>, - { - self.shared_params.update_config(&mut config, spec_factory, version)?; - - let net_config_path = config - .in_chain_config_dir(crate::commands::DEFAULT_NETWORK_CONFIG_PATH) - .expect("We provided a base_path"); - - self.node_key_params.update_config(&mut config, Some(&net_config_path))?; +impl CliConfiguration for BuildSpecCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } - Ok(()) + fn node_key_params(&self) -> Option<&NodeKeyParams> { + Some(&self.node_key_params) } } - diff --git a/client/cli/src/commands/check_block_cmd.rs b/client/cli/src/commands/check_block_cmd.rs index ba267bbf4bbc54489229a15dde20103a7e1e382a..ac4fe63da950618f25948c9d2aeabe8c96f3e291 100644 --- a/client/cli/src/commands/check_block_cmd.rs +++ b/client/cli/src/commands/check_block_cmd.rs @@ -14,20 +14,16 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +use crate::error; +use crate::params::ImportParams; +use crate::params::SharedParams; +use crate::CliConfiguration; +use sc_service::{Configuration, ServiceBuilderCommand}; +use sp_runtime::generic::BlockId; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use std::fmt::Debug; use std::str::FromStr; use structopt::StructOpt; -use sc_service::{ - Configuration, ServiceBuilderCommand, Role, ChainSpec, -}; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; -use sp_runtime::generic::BlockId; - -use crate::error; -use crate::VersionInfo; -use crate::runtime::run_until_exit; -use crate::params::SharedParams; -use crate::params::ImportParams; /// The `check-block` command used to validate blocks. #[derive(Debug, StructOpt, Clone)] @@ -53,8 +49,8 @@ pub struct CheckBlockCmd { impl CheckBlockCmd { /// Run the check-block command - pub fn run( - self, + pub async fn run( + &self, config: Configuration, builder: B, ) -> error::Result<()> @@ -65,37 +61,37 @@ impl CheckBlockCmd { <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, ::Hash: std::str::FromStr, { - let input = if self.input.starts_with("0x") { &self.input[2..] } else { &self.input[..] }; + let input = if self.input.starts_with("0x") { + &self.input[2..] + } else { + &self.input[..] + }; let block_id = match FromStr::from_str(input) { Ok(hash) => BlockId::hash(hash), Err(_) => match self.input.parse::() { Ok(n) => BlockId::number((n as u32).into()), - Err(_) => return Err(error::Error::Input("Invalid hash or number specified".into())), - } + Err(_) => { + return Err(error::Error::Input( + "Invalid hash or number specified".into(), + )) + } + }, }; let start = std::time::Instant::now(); - run_until_exit(config, |config| { - Ok(builder(config)?.check_block(block_id)) - })?; + builder(config)?.check_block(block_id).await?; println!("Completed in {} ms.", start.elapsed().as_millis()); Ok(()) } +} - /// Update and prepare a `Configuration` with command line parameters - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> where - F: FnOnce(&str) -> Result, String>, - { - self.shared_params.update_config(&mut config, spec_factory, version)?; - self.import_params.update_config(&mut config, &Role::Full, self.shared_params.dev)?; - config.use_in_memory_keystore()?; +impl CliConfiguration for CheckBlockCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } - Ok(()) + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) } } diff --git a/client/cli/src/commands/export_blocks_cmd.rs b/client/cli/src/commands/export_blocks_cmd.rs index 26cfcf61bfabe909320121d5c63a2cfffe40b8a7..297d83506be3781ecb97d881539cb5c1e21c085a 100644 --- a/client/cli/src/commands/export_blocks_cmd.rs +++ b/client/cli/src/commands/export_blocks_cmd.rs @@ -14,22 +14,19 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use std::io; -use std::fs; -use std::path::PathBuf; -use std::fmt::Debug; +use crate::error; +use crate::params::{BlockNumber, PruningParams, SharedParams}; +use crate::CliConfiguration; use log::info; -use structopt::StructOpt; use sc_service::{ - Configuration, ServiceBuilderCommand, ChainSpec, - config::DatabaseConfig, Role, + config::DatabaseConfig, Configuration, ServiceBuilderCommand, }; use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; - -use crate::error; -use crate::VersionInfo; -use crate::runtime::run_until_exit; -use crate::params::{SharedParams, BlockNumber, PruningParams}; +use std::fmt::Debug; +use std::fs; +use std::io; +use std::path::PathBuf; +use structopt::StructOpt; /// The `export-blocks` command used to export blocks. #[derive(Debug, StructOpt, Clone)] @@ -65,8 +62,8 @@ pub struct ExportBlocksCmd { impl ExportBlocksCmd { /// Run the export-blocks command - pub fn run( - self, + pub async fn run( + &self, config: Configuration, builder: B, ) -> error::Result<()> @@ -77,9 +74,10 @@ impl ExportBlocksCmd { <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, ::Hash: std::str::FromStr, { - if let DatabaseConfig::Path { ref path, .. } = config.expect_database() { + if let DatabaseConfig::RocksDb { ref path, .. } = &config.database { info!("DB path: {}", path.display()); } + let from = self.from.as_ref().and_then(|f| f.parse().ok()).unwrap_or(1); let to = self.to.as_ref().and_then(|t| t.parse().ok()); @@ -90,24 +88,19 @@ impl ExportBlocksCmd { None => Box::new(io::stdout()), }; - run_until_exit(config, |config| { - Ok(builder(config)?.export_blocks(file, from.into(), to, binary)) - }) + builder(config)? + .export_blocks(file, from.into(), to, binary) + .await + .map_err(Into::into) } +} - /// Update and prepare a `Configuration` with command line parameters - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> where - F: FnOnce(&str) -> Result, String>, - { - self.shared_params.update_config(&mut config, spec_factory, version)?; - self.pruning_params.update_config(&mut config, &Role::Full, true)?; - config.use_in_memory_keystore()?; +impl CliConfiguration for ExportBlocksCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } - Ok(()) + fn pruning_params(&self) -> Option<&PruningParams> { + Some(&self.pruning_params) } } diff --git a/client/cli/src/commands/import_blocks_cmd.rs b/client/cli/src/commands/import_blocks_cmd.rs index 5dc8debe064f3435479849463eef9daaf87f9fd9..ce95640f469ceb3f380f07727f4ed5bc09d431ae 100644 --- a/client/cli/src/commands/import_blocks_cmd.rs +++ b/client/cli/src/commands/import_blocks_cmd.rs @@ -14,21 +14,17 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +use crate::error; +use crate::params::ImportParams; +use crate::params::SharedParams; +use crate::CliConfiguration; +use sc_service::{Configuration, ServiceBuilderCommand}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use std::fmt::Debug; -use std::io::{Read, Seek, self}; use std::fs; +use std::io::{self, Read, Seek}; use std::path::PathBuf; use structopt::StructOpt; -use sc_service::{ - Configuration, ServiceBuilderCommand, ChainSpec, Role, -}; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; - -use crate::error; -use crate::VersionInfo; -use crate::runtime::run_until_exit; -use crate::params::SharedParams; -use crate::params::ImportParams; /// The `import-blocks` command used to import blocks. #[derive(Debug, StructOpt, Clone)] @@ -59,8 +55,8 @@ impl ReadPlusSeek for T {} impl ImportBlocksCmd { /// Run the import-blocks command - pub fn run( - self, + pub async fn run( + &self, config: Configuration, builder: B, ) -> error::Result<()> @@ -77,27 +73,22 @@ impl ImportBlocksCmd { let mut buffer = Vec::new(); io::stdin().read_to_end(&mut buffer)?; Box::new(io::Cursor::new(buffer)) - }, + } }; - run_until_exit(config, |config| { - Ok(builder(config)?.import_blocks(file, false)) - }) + builder(config)? + .import_blocks(file, false) + .await + .map_err(Into::into) } +} - /// Update and prepare a `Configuration` with command line parameters - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> where - F: FnOnce(&str) -> Result, String>, - { - self.shared_params.update_config(&mut config, spec_factory, version)?; - self.import_params.update_config(&mut config, &Role::Full, self.shared_params.dev)?; - config.use_in_memory_keystore()?; +impl CliConfiguration for ImportBlocksCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } - Ok(()) + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) } } diff --git a/client/cli/src/commands/mod.rs b/client/cli/src/commands/mod.rs index d87b08f7f49828ef48bb469c016d2ac65e507906..3e8c2166b20e8aa2e8d1dd706f0ca0b2cfd4f105 100644 --- a/client/cli/src/commands/mod.rs +++ b/client/cli/src/commands/mod.rs @@ -14,34 +14,23 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -mod runcmd; -mod export_blocks_cmd; mod build_spec_cmd; -mod import_blocks_cmd; mod check_block_cmd; -mod revert_cmd; +mod export_blocks_cmd; +mod import_blocks_cmd; mod purge_chain_cmd; +mod revert_cmd; +mod runcmd; -use std::fmt::Debug; -use structopt::StructOpt; - -use sc_service::{ Configuration, ServiceBuilderCommand, ChainSpec }; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; - -use crate::error; -use crate::VersionInfo; -use crate::params::SharedParams; - -pub use crate::commands::runcmd::RunCmd; -pub use crate::commands::export_blocks_cmd::ExportBlocksCmd; pub use crate::commands::build_spec_cmd::BuildSpecCmd; -pub use crate::commands::import_blocks_cmd::ImportBlocksCmd; pub use crate::commands::check_block_cmd::CheckBlockCmd; -pub use crate::commands::revert_cmd::RevertCmd; +pub use crate::commands::export_blocks_cmd::ExportBlocksCmd; +pub use crate::commands::import_blocks_cmd::ImportBlocksCmd; pub use crate::commands::purge_chain_cmd::PurgeChainCmd; - -/// default sub directory to store network config -const DEFAULT_NETWORK_CONFIG_PATH : &'static str = "network"; +pub use crate::commands::revert_cmd::RevertCmd; +pub use crate::commands::runcmd::RunCmd; +use std::fmt::Debug; +use structopt::StructOpt; /// All core commands that are provided by default. /// @@ -51,89 +40,357 @@ const DEFAULT_NETWORK_CONFIG_PATH : &'static str = "network"; #[derive(Debug, Clone, StructOpt)] pub enum Subcommand { /// Build a spec.json file, outputs to stdout. - BuildSpec(build_spec_cmd::BuildSpecCmd), + BuildSpec(BuildSpecCmd), /// Export blocks to a file. - ExportBlocks(export_blocks_cmd::ExportBlocksCmd), + ExportBlocks(ExportBlocksCmd), /// Import blocks from file. - ImportBlocks(import_blocks_cmd::ImportBlocksCmd), + ImportBlocks(ImportBlocksCmd), /// Validate a single block. - CheckBlock(check_block_cmd::CheckBlockCmd), + CheckBlock(CheckBlockCmd), /// Revert chain to the previous state. - Revert(revert_cmd::RevertCmd), + Revert(RevertCmd), /// Remove the whole chain data. - PurgeChain(purge_chain_cmd::PurgeChainCmd), + PurgeChain(PurgeChainCmd), } -impl Subcommand { - /// Get the shared parameters of a `CoreParams` command. - pub fn get_shared_params(&self) -> &SharedParams { - use Subcommand::*; - - match self { - BuildSpec(params) => ¶ms.shared_params, - ExportBlocks(params) => ¶ms.shared_params, - ImportBlocks(params) => ¶ms.shared_params, - CheckBlock(params) => ¶ms.shared_params, - Revert(params) => ¶ms.shared_params, - PurgeChain(params) => ¶ms.shared_params, - } - } +// TODO: move to config.rs? +/// Macro that helps implement CliConfiguration on an enum of subcommand automatically +/// +/// # Example +/// +/// ``` +/// # #[macro_use] extern crate sc_cli; +/// +/// # struct EmptyVariant {} +/// +/// # impl sc_cli::CliConfiguration for EmptyVariant { +/// # fn shared_params(&self) -> &sc_cli::SharedParams { unimplemented!() } +/// # fn chain_id(&self, _: bool) -> sc_cli::Result { Ok("test-chain-id".to_string()) } +/// # } +/// +/// # fn main() { +/// enum Subcommand { +/// Variant1(EmptyVariant), +/// Variant2(EmptyVariant), +/// } +/// +/// substrate_cli_subcommands!( +/// Subcommand => Variant1, Variant2 +/// ); +/// +/// # use sc_cli::CliConfiguration; +/// # assert_eq!(Subcommand::Variant1(EmptyVariant {}).chain_id(false).unwrap(), "test-chain-id"); +/// +/// # } +/// ``` +/// +/// Which will expand to: +/// +/// ```ignore +/// impl CliConfiguration for Subcommand { +/// fn base_path(&self) -> Result> { +/// match self { +/// Subcommand::Variant1(cmd) => cmd.base_path(), +/// Subcommand::Variant2(cmd) => cmd.base_path(), +/// } +/// } +/// +/// fn is_dev(&self) -> Result { +/// match self { +/// Subcommand::Variant1(cmd) => cmd.is_dev(), +/// Subcommand::Variant2(cmd) => cmd.is_dev(), +/// } +/// } +/// +/// // ... +/// } +/// ``` +#[macro_export] +macro_rules! substrate_cli_subcommands { + ($enum:ident => $($variant:ident),*) => { + impl $crate::CliConfiguration for $enum { + fn shared_params(&self) -> &$crate::SharedParams { + match self { + $($enum::$variant(cmd) => cmd.shared_params()),* + } + } - /// Run any `CoreParams` command. - pub fn run( - self, - config: Configuration, - builder: B, - ) -> error::Result<()> - where - B: FnOnce(Configuration) -> Result, - BC: ServiceBuilderCommand + Unpin, - BB: sp_runtime::traits::Block + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, - { - match self { - Subcommand::BuildSpec(cmd) => cmd.run(config), - Subcommand::ExportBlocks(cmd) => cmd.run(config, builder), - Subcommand::ImportBlocks(cmd) => cmd.run(config, builder), - Subcommand::CheckBlock(cmd) => cmd.run(config, builder), - Subcommand::PurgeChain(cmd) => cmd.run(config), - Subcommand::Revert(cmd) => cmd.run(config, builder), - } - } + fn import_params(&self) -> Option<&$crate::ImportParams> { + match self { + $($enum::$variant(cmd) => cmd.import_params()),* + } + } - /// Update and prepare a `Configuration` with command line parameters. - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> where - F: FnOnce(&str) -> Result, String>, - { - match self { - Subcommand::BuildSpec(cmd) => cmd.update_config(&mut config, spec_factory, version), - Subcommand::ExportBlocks(cmd) => cmd.update_config(&mut config, spec_factory, version), - Subcommand::ImportBlocks(cmd) => cmd.update_config(&mut config, spec_factory, version), - Subcommand::CheckBlock(cmd) => cmd.update_config(&mut config, spec_factory, version), - Subcommand::PurgeChain(cmd) => cmd.update_config(&mut config, spec_factory, version), - Subcommand::Revert(cmd) => cmd.update_config(&mut config, spec_factory, version), - } - } + fn pruning_params(&self) -> Option<&$crate::PruningParams> { + match self { + $($enum::$variant(cmd) => cmd.pruning_params()),* + } + } + + fn keystore_params(&self) -> Option<&$crate::KeystoreParams> { + match self { + $($enum::$variant(cmd) => cmd.keystore_params()),* + } + } + + fn network_params(&self) -> Option<&$crate::NetworkParams> { + match self { + $($enum::$variant(cmd) => cmd.network_params()),* + } + } + + fn offchain_worker_params(&self) -> Option<&$crate::OffchainWorkerParams> { + match self { + $($enum::$variant(cmd) => cmd.offchain_worker_params()),* + } + } + + fn base_path(&self) -> $crate::Result<::std::option::Option<::std::path::PathBuf>> { + match self { + $($enum::$variant(cmd) => cmd.base_path()),* + } + } + + fn is_dev(&self) -> $crate::Result { + match self { + $($enum::$variant(cmd) => cmd.is_dev()),* + } + } + + fn role(&self, is_dev: bool) -> $crate::Result<::sc_service::Role> { + match self { + $($enum::$variant(cmd) => cmd.role(is_dev)),* + } + } + + fn transaction_pool(&self) + -> $crate::Result<::sc_service::config::TransactionPoolOptions> { + match self { + $($enum::$variant(cmd) => cmd.transaction_pool()),* + } + } + + fn network_config( + &self, + chain_spec: &::std::boxed::Box, + is_dev: bool, + net_config_dir: ::std::path::PathBuf, + client_id: &str, + node_name: &str, + node_key: ::sc_service::config::NodeKeyConfig, + ) -> $crate::Result<::sc_service::config::NetworkConfiguration> { + match self { + $( + $enum::$variant(cmd) => cmd.network_config( + chain_spec, is_dev, net_config_dir, client_id, node_name, node_key + ) + ),* + } + } + + fn keystore_config(&self, base_path: &::std::path::PathBuf) + -> $crate::Result<::sc_service::config::KeystoreConfig> { + match self { + $($enum::$variant(cmd) => cmd.keystore_config(base_path)),* + } + } + + fn database_cache_size(&self) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.database_cache_size()),* + } + } + + fn database_config( + &self, + base_path: &::std::path::PathBuf, + cache_size: usize, + database: $crate::Database, + ) -> $crate::Result<::sc_service::config::DatabaseConfig> { + match self { + $($enum::$variant(cmd) => cmd.database_config(base_path, cache_size, database)),* + } + } + + fn database(&self) -> $crate::Result<::std::option::Option<$crate::Database>> { + match self { + $($enum::$variant(cmd) => cmd.database()),* + } + } + + fn state_cache_size(&self) -> $crate::Result { + match self { + $($enum::$variant(cmd) => cmd.state_cache_size()),* + } + } + + fn state_cache_child_ratio(&self) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.state_cache_child_ratio()),* + } + } + + fn pruning(&self, unsafe_pruning: bool, role: &::sc_service::Role) + -> $crate::Result<::sc_service::config::PruningMode> { + match self { + $($enum::$variant(cmd) => cmd.pruning(unsafe_pruning, role)),* + } + } + + fn chain_id(&self, is_dev: bool) -> $crate::Result { + match self { + $($enum::$variant(cmd) => cmd.chain_id(is_dev)),* + } + } + + fn init(&self) -> $crate::Result<()> { + match self { + $($enum::$variant(cmd) => cmd.init::()),* + } + } + + fn node_name(&self) -> $crate::Result { + match self { + $($enum::$variant(cmd) => cmd.node_name()),* + } + } + + fn wasm_method(&self) -> $crate::Result<::sc_service::config::WasmExecutionMethod> { + match self { + $($enum::$variant(cmd) => cmd.wasm_method()),* + } + } - /// Initialize substrate. This must be done only once. - /// - /// This method: - /// - /// 1. Set the panic handler - /// 2. Raise the FD limit - /// 3. Initialize the logger - pub fn init(&self, version: &VersionInfo) -> error::Result<()> { - self.get_shared_params().init(version) + fn execution_strategies(&self, is_dev: bool) + -> $crate::Result<::sc_client_api::execution_extensions::ExecutionStrategies> { + match self { + $($enum::$variant(cmd) => cmd.execution_strategies(is_dev)),* + } + } + + fn rpc_http(&self) -> $crate::Result<::std::option::Option<::std::net::SocketAddr>> { + match self { + $($enum::$variant(cmd) => cmd.rpc_http()),* + } + } + + fn rpc_ws(&self) -> $crate::Result<::std::option::Option<::std::net::SocketAddr>> { + match self { + $($enum::$variant(cmd) => cmd.rpc_ws()),* + } + } + + fn unsafe_rpc_expose(&self) -> $crate::Result { + match self { + $($enum::$variant(cmd) => cmd.unsafe_rpc_expose()),* + } + } + + fn rpc_ws_max_connections(&self) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.rpc_ws_max_connections()),* + } + } + + fn rpc_cors(&self, is_dev: bool) + -> $crate::Result<::std::option::Option<::std::vec::Vec>> { + match self { + $($enum::$variant(cmd) => cmd.rpc_cors(is_dev)),* + } + } + + fn prometheus_config(&self) + -> $crate::Result<::std::option::Option<::sc_service::config::PrometheusConfig>> { + match self { + $($enum::$variant(cmd) => cmd.prometheus_config()),* + } + } + + fn telemetry_endpoints( + &self, + chain_spec: &Box, + ) -> $crate::Result<::std::option::Option<::sc_service::config::TelemetryEndpoints>> { + match self { + $($enum::$variant(cmd) => cmd.telemetry_endpoints(chain_spec)),* + } + } + + fn telemetry_external_transport(&self) + -> $crate::Result<::std::option::Option<::sc_service::config::ExtTransport>> { + match self { + $($enum::$variant(cmd) => cmd.telemetry_external_transport()),* + } + } + + fn default_heap_pages(&self) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.default_heap_pages()),* + } + } + + fn offchain_worker(&self, role: &::sc_service::Role) -> $crate::Result<::sc_service::config::OffchainWorkerConfig> { + match self { + $($enum::$variant(cmd) => cmd.offchain_worker(role)),* + } + } + + fn force_authoring(&self) -> $crate::Result { + match self { + $($enum::$variant(cmd) => cmd.force_authoring()),* + } + } + + fn disable_grandpa(&self) -> $crate::Result { + match self { + $($enum::$variant(cmd) => cmd.disable_grandpa()),* + } + } + + fn dev_key_seed(&self, is_dev: bool) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.dev_key_seed(is_dev)),* + } + } + + fn tracing_targets(&self) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.tracing_targets()),* + } + } + + fn tracing_receiver(&self) -> $crate::Result<::sc_service::TracingReceiver> { + match self { + $($enum::$variant(cmd) => cmd.tracing_receiver()),* + } + } + + fn node_key(&self, net_config_dir: &::std::path::PathBuf) + -> $crate::Result<::sc_service::config::NodeKeyConfig> { + match self { + $($enum::$variant(cmd) => cmd.node_key(net_config_dir)),* + } + } + + fn max_runtime_instances(&self) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.max_runtime_instances()),* + } + } + + fn log_filters(&self) -> $crate::Result<::std::option::Option> { + match self { + $($enum::$variant(cmd) => cmd.log_filters()),* + } + } + } } } + +substrate_cli_subcommands!( + Subcommand => BuildSpec, ExportBlocks, ImportBlocks, CheckBlock, Revert, PurgeChain +); diff --git a/client/cli/src/commands/purge_chain_cmd.rs b/client/cli/src/commands/purge_chain_cmd.rs index e12a50bf24f174ccfcc868fa4099bdf11fb45292..3be2883bd509335caac9bcc53a01b1c2d11443ed 100644 --- a/client/cli/src/commands/purge_chain_cmd.rs +++ b/client/cli/src/commands/purge_chain_cmd.rs @@ -14,15 +14,14 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +use crate::error; +use crate::params::SharedParams; +use crate::CliConfiguration; +use sc_service::{config::DatabaseConfig, Configuration}; use std::fmt::Debug; -use std::io::{Write, self}; use std::fs; +use std::io::{self, Write}; use structopt::StructOpt; -use sc_service::{ Configuration, ChainSpec, config::{DatabaseConfig} }; - -use crate::error; -use crate::VersionInfo; -use crate::params::SharedParams; /// The `purge-chain` command used to remove the whole chain. #[derive(Debug, StructOpt, Clone)] @@ -38,12 +37,9 @@ pub struct PurgeChainCmd { impl PurgeChainCmd { /// Run the purge command - pub fn run( - self, - config: Configuration, - ) -> error::Result<()> { - let db_path = match config.expect_database() { - DatabaseConfig::Path { path, .. } => path, + pub fn run(&self, config: Configuration) -> error::Result<()> { + let db_path = match &config.database { + DatabaseConfig::RocksDb { path, .. } => path, _ => { eprintln!("Cannot purge custom database implementation"); return Ok(()); @@ -76,22 +72,13 @@ impl PurgeChainCmd { eprintln!("{:?} did not exist.", &db_path); Ok(()) }, - Err(err) => Result::Err(err.into()) + Err(err) => Result::Err(err.into()), } } +} - /// Update and prepare a `Configuration` with command line parameters - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> where - F: FnOnce(&str) -> Result, String>, - { - self.shared_params.update_config(&mut config, spec_factory, version)?; - config.use_in_memory_keystore()?; - - Ok(()) +impl CliConfiguration for PurgeChainCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params } } diff --git a/client/cli/src/commands/revert_cmd.rs b/client/cli/src/commands/revert_cmd.rs index 9617f8eda40f3446a5f082a403a50da2d918ee5b..f7629ff2f6357e71ed417b0bad6d6ad1fc96851b 100644 --- a/client/cli/src/commands/revert_cmd.rs +++ b/client/cli/src/commands/revert_cmd.rs @@ -14,16 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +use crate::error; +use crate::params::{BlockNumber, PruningParams, SharedParams}; +use crate::CliConfiguration; +use sc_service::{Configuration, ServiceBuilderCommand}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use std::fmt::Debug; use structopt::StructOpt; -use sc_service::{ - Configuration, ServiceBuilderCommand, ChainSpec, Role, -}; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; - -use crate::error; -use crate::VersionInfo; -use crate::params::{BlockNumber, SharedParams, PruningParams}; /// The `revert` command used revert the chain to a previous state. #[derive(Debug, StructOpt, Clone)] @@ -43,11 +40,7 @@ pub struct RevertCmd { impl RevertCmd { /// Run the revert command - pub fn run( - self, - config: Configuration, - builder: B, - ) -> error::Result<()> + pub fn run(&self, config: Configuration, builder: B) -> error::Result<()> where B: FnOnce(Configuration) -> Result, BC: ServiceBuilderCommand + Unpin, @@ -60,20 +53,14 @@ impl RevertCmd { Ok(()) } +} - /// Update and prepare a `Configuration` with command line parameters - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> where - F: FnOnce(&str) -> Result, String>, - { - self.shared_params.update_config(&mut config, spec_factory, version)?; - self.pruning_params.update_config(&mut config, &Role::Full, true)?; - config.use_in_memory_keystore()?; +impl CliConfiguration for RevertCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } - Ok(()) + fn pruning_params(&self) -> Option<&PruningParams> { + Some(&self.pruning_params) } } diff --git a/client/cli/src/commands/runcmd.rs b/client/cli/src/commands/runcmd.rs index 30cefa5d0ce7c187ceed1b89ae4eda185b46d467..a24cadcd4f78bc4739f38b500854442d883e256d 100644 --- a/client/cli/src/commands/runcmd.rs +++ b/client/cli/src/commands/runcmd.rs @@ -14,45 +14,22 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use std::path::PathBuf; -use std::net::SocketAddr; -use std::fs; -use std::fmt; -use log::info; -use structopt::{StructOpt, clap::arg_enum}; -use names::{Generator, Name}; +use crate::error::{Error, Result}; +use crate::params::ImportParams; +use crate::params::KeystoreParams; +use crate::params::NetworkParams; +use crate::params::SharedParams; +use crate::params::TransactionPoolParams; +use crate::params::OffchainWorkerParams; +use crate::CliConfiguration; use regex::Regex; -use chrono::prelude::*; use sc_service::{ - AbstractService, Configuration, ChainSpec, Role, - config::{MultiaddrWithPeerId, KeystoreConfig, PrometheusConfig}, + config::{MultiaddrWithPeerId, PrometheusConfig, TransactionPoolOptions}, + ChainSpec, Role, }; use sc_telemetry::TelemetryEndpoints; - -use crate::VersionInfo; -use crate::error; -use crate::params::ImportParams; -use crate::params::SharedParams; -use crate::params::NetworkConfigurationParams; -use crate::params::TransactionPoolParams; -use crate::runtime::run_service_until_exit; - -/// The maximum number of characters for a node name. -const NODE_NAME_MAX_LENGTH: usize = 32; - -/// default sub directory for the key store -const DEFAULT_KEYSTORE_CONFIG_PATH : &'static str = "keystore"; - -arg_enum! { - /// Whether off-chain workers are enabled. - #[allow(missing_docs)] - #[derive(Debug, Clone)] - pub enum OffchainWorkerEnabled { - Always, - Never, - WhenValidating, - } -} +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use structopt::StructOpt; /// The `run` command used to run a node. #[derive(Debug, StructOpt, Clone)] @@ -106,6 +83,14 @@ pub struct RunCmd { #[structopt(long = "unsafe-rpc-external")] pub unsafe_rpc_external: bool, + /// Don't deny potentially unsafe RPCs when listening on external interfaces. + /// + /// Default is false. This allows exposing RPC methods publicly (same as `--unsafe-{rpc,ws}-external` ) + /// but will allow doing so even on validator nodes, which is prohibited by default. + /// Please do this if you know what you're doing. + #[structopt(long = "unsafe-rpc-expose")] + pub unsafe_rpc_expose: bool, + /// Listen to all Websocket interfaces. /// /// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use an RPC proxy @@ -178,17 +163,9 @@ pub struct RunCmd { #[structopt(long = "telemetry-url", value_name = "URL VERBOSITY", parse(try_from_str = parse_telemetry_endpoints))] pub telemetry_endpoints: Vec<(String, u8)>, - /// Should execute offchain workers on every block. - /// - /// By default it's only enabled for nodes that are authoring new blocks. - #[structopt( - long = "offchain-worker", - value_name = "ENABLED", - possible_values = &OffchainWorkerEnabled::variants(), - case_insensitive = true, - default_value = "WhenValidating" - )] - pub offchain_worker: OffchainWorkerEnabled, + #[allow(missing_docs)] + #[structopt(flatten)] + pub offchain_worker_params: OffchainWorkerParams, #[allow(missing_docs)] #[structopt(flatten)] @@ -200,7 +177,7 @@ pub struct RunCmd { #[allow(missing_docs)] #[structopt(flatten)] - pub network_config: NetworkConfigurationParams, + pub network_params: NetworkParams, #[allow(missing_docs)] #[structopt(flatten)] @@ -242,38 +219,23 @@ pub struct RunCmd { #[structopt(long = "force-authoring")] pub force_authoring: bool, - /// Specify custom keystore path. - #[structopt(long = "keystore-path", value_name = "PATH", parse(from_os_str))] - pub keystore_path: Option, - - /// Use interactive shell for entering the password used by the keystore. - #[structopt( - long = "password-interactive", - conflicts_with_all = &[ "password", "password-filename" ] - )] - pub password_interactive: bool, - - /// Password used by the keystore. - #[structopt( - long = "password", - conflicts_with_all = &[ "password-interactive", "password-filename" ] - )] - pub password: Option, - - /// File that contains the password used by the keystore. - #[structopt( - long = "password-filename", - value_name = "PATH", - parse(from_os_str), - conflicts_with_all = &[ "password-interactive", "password" ] - )] - pub password_filename: Option, + #[allow(missing_docs)] + #[structopt(flatten)] + pub keystore_params: KeystoreParams, /// The size of the instances cache for each runtime. /// /// The default value is 8 and the values higher than 256 are ignored. - #[structopt(long = "max-runtime-instances", default_value = "8")] - pub max_runtime_instances: usize, + #[structopt(long)] + pub max_runtime_instances: Option, + + /// Specify a list of sentry node public addresses. + #[structopt( + long = "sentry-nodes", + value_name = "ADDR", + conflicts_with_all = &[ "sentry" ] + )] + pub sentry_nodes: Vec, } impl RunCmd { @@ -281,219 +243,201 @@ impl RunCmd { pub fn get_keyring(&self) -> Option { use sp_keyring::Sr25519Keyring::*; - if self.alice { Some(Alice) } - else if self.bob { Some(Bob) } - else if self.charlie { Some(Charlie) } - else if self.dave { Some(Dave) } - else if self.eve { Some(Eve) } - else if self.ferdie { Some(Ferdie) } - else if self.one { Some(One) } - else if self.two { Some(Two) } - else { None } - } - - /// Update and prepare a `Configuration` with command line parameters of `RunCmd` and `VersionInfo`. - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<()> - where - F: FnOnce(&str) -> Result, String>, - { - self.shared_params.update_config(&mut config, spec_factory, version)?; - - let password = if self.password_interactive { - #[cfg(not(target_os = "unknown"))] - { - Some(input_keystore_password()?.into()) - } - #[cfg(target_os = "unknown")] - None - } else if let Some(ref file) = self.password_filename { - Some(fs::read_to_string(file).map_err(|e| format!("{}", e))?.into()) - } else if let Some(ref password) = self.password { - Some(password.clone().into()) + if self.alice { + Some(Alice) + } else if self.bob { + Some(Bob) + } else if self.charlie { + Some(Charlie) + } else if self.dave { + Some(Dave) + } else if self.eve { + Some(Eve) + } else if self.ferdie { + Some(Ferdie) + } else if self.one { + Some(One) + } else if self.two { + Some(Two) } else { None - }; + } + } +} - let path = self.keystore_path.clone().or( - config.in_chain_config_dir(DEFAULT_KEYSTORE_CONFIG_PATH) - ); +impl CliConfiguration for RunCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } - config.keystore = KeystoreConfig::Path { - path: path.ok_or_else(|| "No `base_path` provided to create keystore path!".to_string())?, - password, - }; + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) + } - let keyring = self.get_keyring(); - let is_dev = self.shared_params.dev; - let is_light = self.light; - let is_authority = (self.validator || is_dev || keyring.is_some()) - && !is_light; - let role = - if is_light { - sc_service::Role::Light - } else if is_authority { - sc_service::Role::Authority { sentry_nodes: self.network_config.sentry_nodes.clone() } - } else if !self.sentry.is_empty() { - sc_service::Role::Sentry { validators: self.sentry.clone() } - } else { - sc_service::Role::Full - }; + fn network_params(&self) -> Option<&NetworkParams> { + Some(&self.network_params) + } + + fn keystore_params(&self) -> Option<&KeystoreParams> { + Some(&self.keystore_params) + } - self.import_params.update_config(&mut config, &role, is_dev)?; + fn offchain_worker_params(&self) -> Option<&OffchainWorkerParams> { + Some(&self.offchain_worker_params) + } - config.name = match (self.name.as_ref(), keyring) { + fn node_name(&self) -> Result { + let name: String = match (self.name.as_ref(), self.get_keyring()) { (Some(name), _) => name.to_string(), (_, Some(keyring)) => keyring.to_string(), - (None, None) => generate_node_name(), + (None, None) => crate::generate_node_name(), }; - if let Err(msg) = is_node_name_valid(&config.name) { - return Err(error::Error::Input( - format!("Invalid node name '{}'. Reason: {}. If unsure, use none.", - config.name, - msg, - ) + + is_node_name_valid(&name).map_err(|msg| { + Error::Input(format!( + "Invalid node name '{}'. Reason: {}. If unsure, use none.", + name, msg )); - } + })?; - config.offchain_worker = match (&self.offchain_worker, &role) { - (OffchainWorkerEnabled::WhenValidating, sc_service::Role::Authority { .. }) => true, - (OffchainWorkerEnabled::Always, _) => true, - (OffchainWorkerEnabled::Never, _) => false, - (OffchainWorkerEnabled::WhenValidating, _) => false, - }; + Ok(name) + } - config.role = role; - config.disable_grandpa = self.no_grandpa; - - let client_id = config.client_id(); - let network_path = config - .in_chain_config_dir(crate::commands::DEFAULT_NETWORK_CONFIG_PATH) - .expect("We provided a basepath"); - self.network_config.update_config( - &mut config, - network_path, - client_id, - is_dev, - )?; + fn dev_key_seed(&self, is_dev: bool) -> Result> { + Ok(self.get_keyring().map(|a| format!("//{}", a)).or_else(|| { + if is_dev && !self.light { + Some("//Alice".into()) + } else { + None + } + })) + } - self.pool_config.update_config(&mut config)?; + fn telemetry_endpoints( + &self, + chain_spec: &Box, + ) -> Result> { + Ok(if self.no_telemetry { + None + } else if !self.telemetry_endpoints.is_empty() { + Some( + TelemetryEndpoints::new(self.telemetry_endpoints.clone()) + .map_err(|e| e.to_string())?, + ) + } else { + chain_spec.telemetry_endpoints().clone() + }) + } - config.dev_key_seed = keyring - .map(|a| format!("//{}", a)).or_else(|| { - if is_dev && !is_light { - Some("//Alice".into()) - } else { - None - } - }); + fn role(&self, is_dev: bool) -> Result { + let keyring = self.get_keyring(); + let is_light = self.light; + let is_authority = (self.validator || is_dev || keyring.is_some()) && !is_light; - if config.rpc_http.is_none() || self.rpc_port.is_some() { - let rpc_interface: &str = interface_str(self.rpc_external, self.unsafe_rpc_external, self.validator)?; - config.rpc_http = Some(parse_address(&format!("{}:{}", rpc_interface, 9933), self.rpc_port)?); - } - if config.rpc_ws.is_none() || self.ws_port.is_some() { - let ws_interface: &str = interface_str(self.ws_external, self.unsafe_ws_external, self.validator)?; - config.rpc_ws = Some(parse_address(&format!("{}:{}", ws_interface, 9944), self.ws_port)?); - } + Ok(if is_light { + sc_service::Role::Light + } else if is_authority { + sc_service::Role::Authority { + sentry_nodes: self.sentry_nodes.clone(), + } + } else if !self.sentry.is_empty() { + sc_service::Role::Sentry { + validators: self.sentry.clone(), + } + } else { + sc_service::Role::Full + }) + } - config.rpc_ws_max_connections = self.ws_max_connections; - config.rpc_cors = self.rpc_cors.clone().unwrap_or_else(|| if is_dev { - log::warn!("Running in --dev mode, RPC CORS has been disabled."); - Cors::All + fn force_authoring(&self) -> Result { + // Imply forced authoring on --dev + Ok(self.shared_params.dev || self.force_authoring) + } + + fn prometheus_config(&self) -> Result> { + Ok(if self.no_prometheus { + None } else { - Cors::List(vec![ - "http://localhost:*".into(), - "http://127.0.0.1:*".into(), - "https://localhost:*".into(), - "https://127.0.0.1:*".into(), - "https://polkadot.js.org".into(), - ]) - }).into(); - - // Override telemetry - if self.no_telemetry { - config.telemetry_endpoints = None; - } else if !self.telemetry_endpoints.is_empty() { - config.telemetry_endpoints = Some( - TelemetryEndpoints::new(self.telemetry_endpoints.clone()).map_err(|e| e.to_string())? - ); - } + let interface = if self.prometheus_external { + Ipv4Addr::UNSPECIFIED + } else { + Ipv4Addr::LOCALHOST + }; - // Override prometheus - if self.no_prometheus { - config.prometheus_config = None; - } else if config.prometheus_config.is_none() { - let prometheus_interface: &str = if self.prometheus_external { "0.0.0.0" } else { "127.0.0.1" }; - config.prometheus_config = Some(PrometheusConfig::new_with_default_registry( - parse_address(&format!("{}:{}", prometheus_interface, 9615), self.prometheus_port)?, - )); - } + Some(PrometheusConfig::new_with_default_registry( + SocketAddr::new(interface.into(), self.prometheus_port.unwrap_or(9615)) + )) + }) + } - config.tracing_targets = self.import_params.tracing_targets.clone().into(); - config.tracing_receiver = self.import_params.tracing_receiver.clone().into(); + fn disable_grandpa(&self) -> Result { + Ok(self.no_grandpa) + } - // Imply forced authoring on --dev - config.force_authoring = self.shared_params.dev || self.force_authoring; + fn rpc_ws_max_connections(&self) -> Result> { + Ok(self.ws_max_connections) + } - config.max_runtime_instances = self.max_runtime_instances.min(256); + fn rpc_cors(&self, is_dev: bool) -> Result>> { + Ok(self + .rpc_cors + .clone() + .unwrap_or_else(|| { + if is_dev { + log::warn!("Running in --dev mode, RPC CORS has been disabled."); + Cors::All + } else { + Cors::List(vec![ + "http://localhost:*".into(), + "http://127.0.0.1:*".into(), + "https://localhost:*".into(), + "https://127.0.0.1:*".into(), + "https://polkadot.js.org".into(), + ]) + } + }) + .into()) + } + + fn rpc_http(&self) -> Result> { + let interface = rpc_interface( + self.rpc_external, + self.unsafe_rpc_external, + self.unsafe_rpc_expose, + self.validator + )?; - Ok(()) + Ok(Some(SocketAddr::new(interface, self.rpc_port.unwrap_or(9933)))) } - /// Run the command that runs the node. - pub fn run( - self, - config: Configuration, - new_light: FNL, - new_full: FNF, - version: &VersionInfo, - ) -> error::Result<()> - where - FNL: FnOnce(Configuration) -> Result, - FNF: FnOnce(Configuration) -> Result, - SL: AbstractService + Unpin, - SF: AbstractService + Unpin, - { - info!("{}", version.name); - info!("✌️ version {}", config.full_version()); - info!("❤️ by {}, {}-{}", version.author, version.copyright_start_year, Local::today().year()); - info!("📋 Chain specification: {}", config.expect_chain_spec().name()); - info!("🏷 Node name: {}", config.name); - info!("👤 Role: {}", config.display_role()); - - match config.role { - Role::Light => run_service_until_exit( - config, - new_light, - ), - _ => run_service_until_exit( - config, - new_full, - ), - } + fn rpc_ws(&self) -> Result> { + let interface = rpc_interface( + self.ws_external, + self.unsafe_ws_external, + self.unsafe_rpc_expose, + self.validator + )?; + + Ok(Some(SocketAddr::new(interface, self.ws_port.unwrap_or(9944)))) } - /// Initialize substrate. This must be done only once. - /// - /// This method: - /// - /// 1. Set the panic handler - /// 2. Raise the FD limit - /// 3. Initialize the logger - pub fn init(&self, version: &VersionInfo) -> error::Result<()> { - self.shared_params.init(version) + fn unsafe_rpc_expose(&self) -> Result { + Ok(self.unsafe_rpc_expose) + } + + fn transaction_pool(&self) -> Result { + Ok(self.pool_config.transaction_pool()) + } + + fn max_runtime_instances(&self) -> Result> { + Ok(self.max_runtime_instances.map(|x| x.min(256))) } } /// Check whether a node name is considered as valid. -pub fn is_node_name_valid(_name: &str) -> Result<(), &str> { +pub fn is_node_name_valid(_name: &str) -> std::result::Result<(), &str> { let name = _name.to_string(); - if name.chars().count() >= NODE_NAME_MAX_LENGTH { + if name.chars().count() >= crate::NODE_NAME_MAX_LENGTH { return Err("Node name too long"); } @@ -512,57 +456,30 @@ pub fn is_node_name_valid(_name: &str) -> Result<(), &str> { Ok(()) } -#[cfg(not(target_os = "unknown"))] -fn input_keystore_password() -> Result { - rpassword::read_password_from_tty(Some("Keystore password: ")) - .map_err(|e| format!("{:?}", e)) -} - -fn generate_node_name() -> String { - let result = loop { - let node_name = Generator::with_naming(Name::Numbered).next().unwrap(); - let count = node_name.chars().count(); - - if count < NODE_NAME_MAX_LENGTH { - break node_name - } - }; - - result -} - -fn parse_address( - address: &str, - port: Option, -) -> Result { - let mut address: SocketAddr = address.parse().map_err( - |_| format!("Invalid address: {}", address) - )?; - if let Some(port) = port { - address.set_port(port); - } - - Ok(address) -} - -fn interface_str( +fn rpc_interface( is_external: bool, is_unsafe_external: bool, + is_unsafe_rpc_expose: bool, is_validator: bool, -) -> Result<&'static str, error::Error> { - if is_external && is_validator { - return Err(error::Error::Input("--rpc-external and --ws-external options shouldn't be \ +) -> Result { + if is_external && is_validator && !is_unsafe_rpc_expose { + return Err(Error::Input( + "--rpc-external and --ws-external options shouldn't be \ used if the node is running as a validator. Use `--unsafe-rpc-external` if you understand \ - the risks. See the options description for more information.".to_owned())); + the risks. See the options description for more information." + .to_owned(), + )); } if is_external || is_unsafe_external { - log::warn!("It isn't safe to expose RPC publicly without a proxy server that filters \ - available set of RPC methods."); + log::warn!( + "It isn't safe to expose RPC publicly without a proxy server that filters \ + available set of RPC methods." + ); - Ok("0.0.0.0") + Ok(Ipv4Addr::UNSPECIFIED.into()) } else { - Ok("127.0.0.1") + Ok(Ipv4Addr::LOCALHOST.into()) } } @@ -574,8 +491,8 @@ enum TelemetryParsingError { impl std::error::Error for TelemetryParsingError {} -impl fmt::Display for TelemetryParsingError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +impl std::fmt::Display for TelemetryParsingError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match &*self { TelemetryParsingError::MissingVerbosity => write!(f, "Verbosity level missing"), TelemetryParsingError::VerbosityParsingError(e) => write!(f, "{}", e), @@ -583,13 +500,15 @@ impl fmt::Display for TelemetryParsingError { } } -fn parse_telemetry_endpoints(s: &str) -> Result<(String, u8), TelemetryParsingError> { +fn parse_telemetry_endpoints(s: &str) -> std::result::Result<(String, u8), TelemetryParsingError> { let pos = s.find(' '); match pos { None => Err(TelemetryParsingError::MissingVerbosity), Some(pos_) => { let url = s[..pos_].to_string(); - let verbosity = s[pos_ + 1..].parse().map_err(TelemetryParsingError::VerbosityParsingError)?; + let verbosity = s[pos_ + 1..] + .parse() + .map_err(TelemetryParsingError::VerbosityParsingError)?; Ok((url, verbosity)) } } @@ -617,7 +536,7 @@ impl From for Option> { } /// Parse cors origins. -fn parse_cors(s: &str) -> Result> { +fn parse_cors(s: &str) -> std::result::Result> { let mut is_all = false; let mut origins = Vec::new(); for part in s.split(',') { @@ -625,29 +544,21 @@ fn parse_cors(s: &str) -> Result> { "all" | "*" => { is_all = true; break; - }, + } other => origins.push(other.to_owned()), } } - Ok(if is_all { Cors::All } else { Cors::List(origins) }) + Ok(if is_all { + Cors::All + } else { + Cors::List(origins) + }) } #[cfg(test)] mod tests { use super::*; - use sc_service::{GenericChainSpec, config::DatabaseConfig}; - - const TEST_VERSION_INFO: &'static VersionInfo = &VersionInfo { - name: "node-test", - version: "0.1.0", - commit: "some_commit", - executable_name: "node-test", - description: "description", - author: "author", - support_url: "http://example.org", - copyright_start_year: 2020, - }; #[test] fn tests_node_name_good() { @@ -663,94 +574,4 @@ mod tests { assert!(is_node_name_valid("www.visit.me").is_err()); assert!(is_node_name_valid("email@domain").is_err()); } - - #[test] - fn keystore_path_is_generated_correctly() { - let chain_spec = GenericChainSpec::from_genesis( - "test", - "test-id", - || (), - Vec::new(), - None, - None, - None, - None::<()>, - ); - - for keystore_path in vec![None, Some("/keystore/path")] { - let args: Vec<&str> = vec![]; - let mut cli = RunCmd::from_iter(args); - cli.keystore_path = keystore_path.clone().map(PathBuf::from); - - let mut config = Configuration::default(); - config.config_dir = Some(PathBuf::from("/test/path")); - config.chain_spec = Some(Box::new(chain_spec.clone())); - let chain_spec = chain_spec.clone(); - cli.update_config(&mut config, move |_| Ok(Box::new(chain_spec)), TEST_VERSION_INFO).unwrap(); - - let expected_path = match keystore_path { - Some(path) => PathBuf::from(path), - None => PathBuf::from("/test/path/chains/test-id/keystore"), - }; - - assert_eq!(expected_path, config.keystore.path().unwrap().to_owned()); - } - } - - #[test] - fn ensure_load_spec_provide_defaults() { - let chain_spec = GenericChainSpec::from_genesis( - "test", - "test-id", - || (), - vec!["/ip4/127.0.0.1/tcp/30333/p2p/QmdSHZLmwEL5Axz5JvWNE2mmxU7qyd7xHBFpyUfktgAdg7".parse().unwrap()], - Some(TelemetryEndpoints::new(vec![("wss://foo/bar".to_string(), 42)]) - .expect("provided url should be valid")), - None, - None, - None::<()>, - ); - - let args: Vec<&str> = vec![]; - let cli = RunCmd::from_iter(args); - - let mut config = Configuration::from_version(TEST_VERSION_INFO); - cli.update_config(&mut config, |_| Ok(Box::new(chain_spec)), TEST_VERSION_INFO).unwrap(); - - assert!(config.chain_spec.is_some()); - assert!(!config.network.boot_nodes.is_empty()); - assert!(config.telemetry_endpoints.is_some()); - } - - #[test] - fn ensure_update_config_for_running_node_provides_defaults() { - let chain_spec = GenericChainSpec::from_genesis( - "test", - "test-id", - || (), - vec![], - None, - None, - None, - None::<()>, - ); - - let args: Vec<&str> = vec![]; - let cli = RunCmd::from_iter(args); - - let mut config = Configuration::from_version(TEST_VERSION_INFO); - cli.init(&TEST_VERSION_INFO).unwrap(); - cli.update_config(&mut config, |_| Ok(Box::new(chain_spec)), TEST_VERSION_INFO).unwrap(); - - assert!(config.config_dir.is_some()); - assert!(config.database.is_some()); - if let Some(DatabaseConfig::Path { ref cache_size, .. }) = config.database { - assert!(cache_size.is_some()); - } else { - panic!("invalid config.database variant"); - } - assert!(!config.name.is_empty()); - assert!(config.network.config_path.is_some()); - assert!(!config.network.listen_addresses.is_empty()); - } } diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs new file mode 100644 index 0000000000000000000000000000000000000000..c73992c59cbdfe52b577211adbce4611ede4a6a6 --- /dev/null +++ b/client/cli/src/config.rs @@ -0,0 +1,500 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Configuration trait for a CLI based on substrate + +use crate::error::Result; +use crate::{ + init_logger, ImportParams, KeystoreParams, NetworkParams, NodeKeyParams, + OffchainWorkerParams, PruningParams, SharedParams, SubstrateCli, +}; +use crate::arg_enums::Database; +use app_dirs::{AppDataType, AppInfo}; +use names::{Generator, Name}; +use sc_service::config::{ + WasmExecutionMethod, Role, OffchainWorkerConfig, + Configuration, DatabaseConfig, ExtTransport, KeystoreConfig, NetworkConfiguration, + NodeKeyConfig, PrometheusConfig, PruningMode, TelemetryEndpoints, TransactionPoolOptions, +}; +use sc_client_api::execution_extensions::ExecutionStrategies; +use sc_service::{ChainSpec, TracingReceiver}; +use std::future::Future; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::pin::Pin; +use std::sync::Arc; + +/// The maximum number of characters for a node name. +pub(crate) const NODE_NAME_MAX_LENGTH: usize = 32; + +/// default sub directory to store network config +pub(crate) const DEFAULT_NETWORK_CONFIG_PATH: &'static str = "network"; + +/// A trait that allows converting an object to a Configuration +pub trait CliConfiguration: Sized { + /// Get the SharedParams for this object + fn shared_params(&self) -> &SharedParams; + + /// Get the ImportParams for this object + fn import_params(&self) -> Option<&ImportParams> { + None + } + + /// Get the PruningParams for this object + fn pruning_params(&self) -> Option<&PruningParams> { + self.import_params().map(|x| &x.pruning_params) + } + + /// Get the KeystoreParams for this object + fn keystore_params(&self) -> Option<&KeystoreParams> { + None + } + + /// Get the NetworkParams for this object + fn network_params(&self) -> Option<&NetworkParams> { + None + } + + /// Get a reference to `OffchainWorkerParams` for this object. + fn offchain_worker_params(&self) -> Option<&OffchainWorkerParams> { + None + } + + /// Get the NodeKeyParams for this object + fn node_key_params(&self) -> Option<&NodeKeyParams> { + self.network_params() + .map(|x| &x.node_key_params) + } + + /// Get the base path of the configuration (if any) + /// + /// By default this is retrieved from `SharedParams`. + fn base_path(&self) -> Result> { + Ok(self.shared_params().base_path()) + } + + /// Returns `true` if the node is for development or not + /// + /// By default this is retrieved from `SharedParams`. + fn is_dev(&self) -> Result { + Ok(self.shared_params().is_dev()) + } + + /// Gets the role + /// + /// By default this is `Role::Full`. + fn role(&self, _is_dev: bool) -> Result { + Ok(Role::Full) + } + + /// Get the transaction pool options + /// + /// By default this is `TransactionPoolOptions::default()`. + fn transaction_pool(&self) -> Result { + Ok(Default::default()) + } + + /// Get the network configuration + /// + /// By default this is retrieved from `NetworkParams` if it is available otherwise it creates + /// a default `NetworkConfiguration` based on `node_name`, `client_id`, `node_key` and + /// `net_config_dir`. + fn network_config( + &self, + chain_spec: &Box, + is_dev: bool, + net_config_dir: PathBuf, + client_id: &str, + node_name: &str, + node_key: NodeKeyConfig, + ) -> Result { + Ok(if let Some(network_params) = self.network_params() { + network_params.network_config( + chain_spec, + is_dev, + Some(net_config_dir), + client_id, + node_name, + node_key, + ) + } else { + NetworkConfiguration::new( + node_name, + client_id, + node_key, + Some(net_config_dir), + ) + }) + } + + /// Get the keystore configuration. + /// + /// Bu default this is retrieved from `KeystoreParams` if it is available. Otherwise it uses + /// `KeystoreConfig::InMemory`. + fn keystore_config(&self, base_path: &PathBuf) -> Result { + self.keystore_params() + .map(|x| x.keystore_config(base_path)) + .unwrap_or(Ok(KeystoreConfig::InMemory)) + } + + /// Get the database cache size. + /// + /// By default this is retrieved from `ImportParams` if it is available. Otherwise its `None`. + fn database_cache_size(&self) -> Result> { + Ok(self.import_params() + .map(|x| x.database_cache_size()) + .unwrap_or(Default::default())) + } + + /// Get the database backend variant. + /// + /// By default this is retrieved from `ImportParams` if it is available. Otherwise its `None`. + fn database(&self) -> Result> { + Ok(self.import_params().map(|x| x.database())) + } + + /// Get the database configuration. + /// + /// By default this is retrieved from `SharedParams` + fn database_config(&self, + base_path: &PathBuf, + cache_size: usize, + database: Database, + ) -> Result { + Ok(self.shared_params().database_config( + base_path, + cache_size, + database, + )) + } + + /// Get the state cache size. + /// + /// By default this is retrieved from `ImportParams` if it is available. Otherwise its `0`. + fn state_cache_size(&self) -> Result { + Ok(self.import_params() + .map(|x| x.state_cache_size()) + .unwrap_or(Default::default())) + } + + /// Get the state cache child ratio (if any). + /// + /// By default this is `None`. + fn state_cache_child_ratio(&self) -> Result> { + Ok(Default::default()) + } + + /// Get the pruning mode. + /// + /// By default this is retrieved from `PruningMode` if it is available. Otherwise its + /// `PruningMode::default()`. + fn pruning(&self, unsafe_pruning: bool, role: &Role) -> Result { + self.pruning_params() + .map(|x| x.pruning(unsafe_pruning, role)) + .unwrap_or(Ok(Default::default())) + } + + /// Get the chain ID (string). + /// + /// By default this is retrieved from `SharedParams`. + fn chain_id(&self, is_dev: bool) -> Result { + Ok(self.shared_params().chain_id(is_dev)) + } + + /// Get the name of the node. + /// + /// By default a random name is generated. + fn node_name(&self) -> Result { + Ok(generate_node_name()) + } + + /// Get the WASM execution method. + /// + /// By default this is retrieved from `ImportParams` if it is available. Otherwise its + /// `WasmExecutionMethod::default()`. + fn wasm_method(&self) -> Result { + Ok(self.import_params() + .map(|x| x.wasm_method()) + .unwrap_or(Default::default())) + } + + /// Get the execution strategies. + /// + /// By default this is retrieved from `ImportParams` if it is available. Otherwise its + /// `ExecutionStrategies::default()`. + fn execution_strategies(&self, is_dev: bool) -> Result { + Ok(self.import_params() + .map(|x| x.execution_strategies(is_dev)) + .unwrap_or(Default::default())) + } + + /// Get the RPC HTTP address (`None` if disabled). + /// + /// By default this is `None`. + fn rpc_http(&self) -> Result> { + Ok(Default::default()) + } + + /// Get the RPC websocket address (`None` if disabled). + /// + /// By default this is `None`. + fn rpc_ws(&self) -> Result> { + Ok(Default::default()) + } + + /// Returns `Ok(true) if potentially unsafe RPC is to be exposed. + /// + /// By default this is `false`. + fn unsafe_rpc_expose(&self) -> Result { + Ok(Default::default()) + } + + /// Get the RPC websockets maximum connections (`None` if unlimited). + /// + /// By default this is `None`. + fn rpc_ws_max_connections(&self) -> Result> { + Ok(Default::default()) + } + + /// Get the RPC cors (`None` if disabled) + /// + /// By default this is `None`. + fn rpc_cors(&self, _is_dev: bool) -> Result>> { + Ok(Some(Vec::new())) + } + + /// Get the prometheus configuration (`None` if disabled) + /// + /// By default this is `None`. + fn prometheus_config(&self) -> Result> { + Ok(Default::default()) + } + + /// Get the telemetry endpoints (if any) + /// + /// By default this is retrieved from the chain spec loaded by `load_spec`. + fn telemetry_endpoints( + &self, + chain_spec: &Box, + ) -> Result> { + Ok(chain_spec.telemetry_endpoints().clone()) + } + + /// Get the telemetry external transport + /// + /// By default this is `None`. + fn telemetry_external_transport(&self) -> Result> { + Ok(Default::default()) + } + + /// Get the default value for heap pages + /// + /// By default this is `None`. + fn default_heap_pages(&self) -> Result> { + Ok(Default::default()) + } + + /// Returns an offchain worker config wrapped in `Ok(_)` + /// + /// By default offchain workers are disabled. + fn offchain_worker(&self, role: &Role) -> Result { + self.offchain_worker_params() + .map(|x| x.offchain_worker(role)) + .unwrap_or_else(|| { Ok(OffchainWorkerConfig::default()) }) + } + + /// Returns `Ok(true)` if authoring should be forced + /// + /// By default this is `false`. + fn force_authoring(&self) -> Result { + Ok(Default::default()) + } + + /// Returns `Ok(true)` if grandpa should be disabled + /// + /// By default this is `false`. + fn disable_grandpa(&self) -> Result { + Ok(Default::default()) + } + + /// Get the development key seed from the current object + /// + /// By default this is `None`. + fn dev_key_seed(&self, _is_dev: bool) -> Result> { + Ok(Default::default()) + } + + /// Get the tracing targets from the current object (if any) + /// + /// By default this is retrieved from `ImportParams` if it is available. Otherwise its + /// `None`. + fn tracing_targets(&self) -> Result> { + Ok(self.import_params() + .map(|x| x.tracing_targets()) + .unwrap_or(Default::default())) + } + + /// Get the TracingReceiver value from the current object + /// + /// By default this is retrieved from `ImportParams` if it is available. Otherwise its + /// `TracingReceiver::default()`. + fn tracing_receiver(&self) -> Result { + Ok(self.import_params() + .map(|x| x.tracing_receiver()) + .unwrap_or(Default::default())) + } + + /// Get the node key from the current object + /// + /// By default this is retrieved from `NodeKeyParams` if it is available. Otherwise its + /// `NodeKeyConfig::default()`. + fn node_key(&self, net_config_dir: &PathBuf) -> Result { + self.node_key_params() + .map(|x| x.node_key(net_config_dir)) + .unwrap_or(Ok(Default::default())) + } + + /// Get maximum runtime instances + /// + /// By default this is `None`. + fn max_runtime_instances(&self) -> Result> { + Ok(Default::default()) + } + + /// Activate or not the automatic announcing of blocks after import + /// + /// By default this is `false`. + fn announce_block(&self) -> Result { + Ok(true) + } + + /// Create a Configuration object from the current object + fn create_configuration( + &self, + cli: &C, + task_executor: Arc + Send>>) + Send + Sync>, + ) -> Result { + let is_dev = self.is_dev()?; + let chain_id = self.chain_id(is_dev)?; + let chain_spec = cli.load_spec(chain_id.as_str())?; + let config_dir = self + .base_path()? + .unwrap_or_else(|| { + app_dirs::get_app_root( + AppDataType::UserData, + &AppInfo { + name: C::executable_name(), + author: C::author(), + }, + ) + .expect("app directories exist on all supported platforms; qed") + }) + .join("chains") + .join(chain_spec.id()); + let net_config_dir = config_dir.join(DEFAULT_NETWORK_CONFIG_PATH); + let client_id = C::client_id(); + let database_cache_size = self.database_cache_size()?.unwrap_or(128); + let database = self.database()?.unwrap_or(Database::RocksDb); + let node_key = self.node_key(&net_config_dir)?; + let role = self.role(is_dev)?; + let max_runtime_instances = self.max_runtime_instances()?.unwrap_or(8); + + let unsafe_pruning = self + .import_params() + .map(|p| p.unsafe_pruning) + .unwrap_or(false); + + Ok(Configuration { + impl_name: C::impl_name(), + impl_version: C::impl_version(), + task_executor, + transaction_pool: self.transaction_pool()?, + network: self.network_config( + &chain_spec, + is_dev, + net_config_dir, + client_id.as_str(), + self.node_name()?.as_str(), + node_key, + )?, + keystore: self.keystore_config(&config_dir)?, + database: self.database_config(&config_dir, database_cache_size, database)?, + state_cache_size: self.state_cache_size()?, + state_cache_child_ratio: self.state_cache_child_ratio()?, + pruning: self.pruning(unsafe_pruning, &role)?, + wasm_method: self.wasm_method()?, + execution_strategies: self.execution_strategies(is_dev)?, + rpc_http: self.rpc_http()?, + rpc_ws: self.rpc_ws()?, + unsafe_rpc_expose: self.unsafe_rpc_expose()?, + rpc_ws_max_connections: self.rpc_ws_max_connections()?, + rpc_cors: self.rpc_cors(is_dev)?, + prometheus_config: self.prometheus_config()?, + telemetry_endpoints: self.telemetry_endpoints(&chain_spec)?, + telemetry_external_transport: self.telemetry_external_transport()?, + default_heap_pages: self.default_heap_pages()?, + offchain_worker: self.offchain_worker(&role)?, + force_authoring: self.force_authoring()?, + disable_grandpa: self.disable_grandpa()?, + dev_key_seed: self.dev_key_seed(is_dev)?, + tracing_targets: self.tracing_targets()?, + tracing_receiver: self.tracing_receiver()?, + chain_spec, + max_runtime_instances, + announce_block: self.announce_block()?, + role, + }) + } + + /// Get the filters for the logging. + /// + /// By default this is retrieved from `SharedParams`. + fn log_filters(&self) -> Result> { + Ok(self.shared_params().log_filters()) + } + + /// Initialize substrate. This must be done only once. + /// + /// This method: + /// + /// 1. Set the panic handler + /// 2. Raise the FD limit + /// 3. Initialize the logger + fn init(&self) -> Result<()> { + let logger_pattern = self.log_filters()?.unwrap_or_default(); + + sp_panic_handler::set(C::support_url(), C::impl_version()); + + fdlimit::raise_fd_limit(); + init_logger(logger_pattern.as_str()); + + Ok(()) + } +} + +/// Generate a valid random name for the node +pub fn generate_node_name() -> String { + loop { + let node_name = Generator::with_naming(Name::Numbered) + .next() + .expect("RNG is available on all supported platforms; qed"); + let count = node_name.chars().count(); + + if count < NODE_NAME_MAX_LENGTH { + return node_name; + } + }; +} diff --git a/client/cli/src/lib.rs b/client/cli/src/lib.rs index ba59b94c8287a97f338b6bf7d8d21e347a05ae35..25b71059b17e62a397bde14bccee91fe95fb62cd 100644 --- a/client/cli/src/lib.rs +++ b/client/cli/src/lib.rs @@ -19,137 +19,175 @@ #![warn(missing_docs)] #![warn(unused_extern_crates)] -mod params; mod arg_enums; -mod error; -mod runtime; mod commands; +mod config; +mod error; +mod params; +mod runner; -pub use sc_service::config::VersionInfo; - -use std::io::Write; - -use regex::Regex; -use structopt::{StructOpt, clap::{self, AppSettings}}; -pub use structopt; -pub use params::*; -pub use commands::*; pub use arg_enums::*; +pub use commands::*; +pub use config::*; pub use error::*; -use log::info; use lazy_static::lazy_static; -pub use crate::runtime::{run_until_exit, run_service_until_exit}; +use log::info; +pub use params::*; +use regex::Regex; +pub use runner::*; +use sc_service::{ChainSpec, Configuration}; +use std::future::Future; +use std::io::Write; +use std::pin::Pin; +use std::sync::Arc; +pub use structopt; +use structopt::{ + clap::{self, AppSettings}, + StructOpt, +}; -/// Helper function used to parse the command line arguments. This is the equivalent of -/// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of -/// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. +/// Substrate client CLI /// -/// To allow running the node without subcommand, tt also sets a few more settings: -/// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. +/// This trait needs to be defined on the root structopt of the application. It will provide the +/// implementation name, version, executable name, description, author, support_url, copyright start +/// year and most importantly: how to load the chain spec. /// -/// Gets the struct from the command line arguments. Print the -/// error message and quit the program in case of failure. -pub fn from_args(version: &VersionInfo) -> T -where - T: StructOpt + Sized, -{ - from_iter::(&mut std::env::args_os(), version) -} +/// StructOpt must not be in scope to use from_args (or the similar methods). This trait provides +/// its own implementation that will fill the necessary field based on the trait's functions. +pub trait SubstrateCli: Sized { + /// Implementation name. + fn impl_name() -> &'static str; + + /// Implementation version. + /// + /// By default this will look like this: 2.0.0-b950f731c-x86_64-linux-gnu where the hash is the + /// short commit hash of the commit of in the Git repository. + fn impl_version() -> &'static str; + + /// Executable file name. + fn executable_name() -> &'static str; + + /// Executable file description. + fn description() -> &'static str; + + /// Executable file author. + fn author() -> &'static str; + + /// Support URL. + fn support_url() -> &'static str; + + /// Copyright starting year (x-current year) + fn copyright_start_year() -> i32; + + /// Chain spec factory + fn load_spec(&self, id: &str) -> std::result::Result, String>; + + /// Helper function used to parse the command line arguments. This is the equivalent of + /// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of + /// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. + /// + /// To allow running the node without subcommand, tt also sets a few more settings: + /// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. + /// + /// Gets the struct from the command line arguments. Print the + /// error message and quit the program in case of failure. + fn from_args() -> Self + where + Self: StructOpt + Sized, + { + ::from_iter(&mut std::env::args_os()) + } -/// Helper function used to parse the command line arguments. This is the equivalent of -/// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of -/// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. -/// -/// To allow running the node without subcommand, tt also sets a few more settings: -/// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. -/// -/// Gets the struct from any iterator such as a `Vec` of your making. -/// Print the error message and quit the program in case of failure. -pub fn from_iter(iter: I, version: &VersionInfo) -> T -where - T: StructOpt + Sized, - I: IntoIterator, - I::Item: Into + Clone, -{ - let app = T::clap(); - - let mut full_version = sc_service::config::full_version_from_strs( - version.version, - version.commit - ); - full_version.push_str("\n"); - - let app = app - .name(version.executable_name) - .author(version.author) - .about(version.description) - .version(full_version.as_str()) - .settings(&[ - AppSettings::GlobalVersion, - AppSettings::ArgsNegateSubcommands, - AppSettings::SubcommandsNegateReqs, - ]); - - T::from_clap(&app.get_matches_from(iter)) -} + /// Helper function used to parse the command line arguments. This is the equivalent of + /// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of + /// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. + /// + /// To allow running the node without subcommand, it also sets a few more settings: + /// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. + /// + /// Gets the struct from any iterator such as a `Vec` of your making. + /// Print the error message and quit the program in case of failure. + fn from_iter(iter: I) -> Self + where + Self: StructOpt + Sized, + I: IntoIterator, + I::Item: Into + Clone, + { + let app = ::clap(); + + let mut full_version = Self::impl_version().to_string(); + full_version.push_str("\n"); + + let app = app + .name(Self::executable_name()) + .author(Self::author()) + .about(Self::description()) + .version(full_version.as_str()) + .settings(&[ + AppSettings::GlobalVersion, + AppSettings::ArgsNegateSubcommands, + AppSettings::SubcommandsNegateReqs, + ]); + + ::from_clap(&app.get_matches_from(iter)) + } -/// Helper function used to parse the command line arguments. This is the equivalent of -/// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of -/// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. -/// -/// To allow running the node without subcommand, tt also sets a few more settings: -/// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. -/// -/// Gets the struct from any iterator such as a `Vec` of your making. -/// Print the error message and quit the program in case of failure. -/// -/// **NOTE:** This method WILL NOT exit when `--help` or `--version` (or short versions) are -/// used. It will return a [`clap::Error`], where the [`kind`] is a -/// [`ErrorKind::HelpDisplayed`] or [`ErrorKind::VersionDisplayed`] respectively. You must call -/// [`Error::exit`] or perform a [`std::process::exit`]. -pub fn try_from_iter(iter: I, version: &VersionInfo) -> clap::Result -where - T: StructOpt + Sized, - I: IntoIterator, - I::Item: Into + Clone, -{ - let app = T::clap(); - - let mut full_version = sc_service::config::full_version_from_strs( - version.version, - version.commit, - ); - full_version.push_str("\n"); - - let app = app - .name(version.executable_name) - .author(version.author) - .about(version.description) - .version(full_version.as_str()); - - let matches = app.get_matches_from_safe(iter)?; - - Ok(T::from_clap(&matches)) -} + /// Helper function used to parse the command line arguments. This is the equivalent of + /// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the name of + /// the application, author, "about" and version. It will also set `AppSettings::GlobalVersion`. + /// + /// To allow running the node without subcommand, it also sets a few more settings: + /// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. + /// + /// Gets the struct from any iterator such as a `Vec` of your making. + /// Print the error message and quit the program in case of failure. + /// + /// **NOTE:** This method WILL NOT exit when `--help` or `--version` (or short versions) are + /// used. It will return a [`clap::Error`], where the [`kind`] is a + /// [`ErrorKind::HelpDisplayed`] or [`ErrorKind::VersionDisplayed`] respectively. You must call + /// [`Error::exit`] or perform a [`std::process::exit`]. + fn try_from_iter(iter: I) -> clap::Result + where + Self: StructOpt + Sized, + I: IntoIterator, + I::Item: Into + Clone, + { + let app = ::clap(); -/// Initialize substrate. This must be done only once. -/// -/// This method: -/// -/// 1. Set the panic handler -/// 2. Raise the FD limit -/// 3. Initialize the logger -pub fn init(logger_pattern: &str, version: &VersionInfo) -> error::Result<()> { - let full_version = sc_service::config::full_version_from_strs( - version.version, - version.commit - ); - sp_panic_handler::set(version.support_url, &full_version); - - fdlimit::raise_fd_limit(); - init_logger(logger_pattern); + let mut full_version = Self::impl_version().to_string(); + full_version.push_str("\n"); - Ok(()) + let app = app + .name(Self::executable_name()) + .author(Self::author()) + .about(Self::description()) + .version(full_version.as_str()); + + let matches = app.get_matches_from_safe(iter)?; + + Ok(::from_clap(&matches)) + } + + /// Returns the client ID: `{impl_name}/v{impl_version}` + fn client_id() -> String { + format!("{}/v{}", Self::impl_name(), Self::impl_version()) + } + + /// Only create a Configuration for the command provided in argument + fn create_configuration( + &self, + command: &T, + task_executor: Arc + Send>>) + Send + Sync>, + ) -> error::Result { + command.create_configuration(self, task_executor) + } + + /// Create a runner for the command provided in argument. This will create a Configuration and + /// a tokio runtime + fn create_runner(&self, command: &T) -> error::Result> { + command.init::()?; + Runner::new(self, command) + } } /// Initialize the logger @@ -177,15 +215,20 @@ pub fn init_logger(pattern: &str) { builder.format(move |buf, record| { let now = time::now(); let timestamp = - time::strftime("%Y-%m-%d %H:%M:%S", &now) - .expect("Error formatting log timestamp"); + time::strftime("%Y-%m-%d %H:%M:%S", &now).expect("Error formatting log timestamp"); let mut output = if log::max_level() <= log::LevelFilter::Info { - format!("{} {}", Colour::Black.bold().paint(timestamp), record.args()) + format!( + "{} {}", + Colour::Black.bold().paint(timestamp), + record.args(), + ) } else { let name = ::std::thread::current() .name() - .map_or_else(Default::default, |x| format!("{}", Colour::Blue.bold().paint(x))); + .map_or_else(Default::default, |x| { + format!("{}", Colour::Blue.bold().paint(x)) + }); let millis = (now.tm_nsec as f32 / 1000000.0).floor() as usize; let timestamp = format!("{}.{}", timestamp, millis); format!( diff --git a/client/cli/src/params/import_params.rs b/client/cli/src/params/import_params.rs index 8d34b706f570246d02d4adcd40cf6dba39796f54..95b04b039a48e9ed5c4124b105cd3508356eca51 100644 --- a/client/cli/src/params/import_params.rs +++ b/client/cli/src/params/import_params.rs @@ -14,16 +14,16 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use structopt::StructOpt; -use sc_service::{Configuration, config::DatabaseConfig}; - -use crate::error; use crate::arg_enums::{ - WasmExecutionMethod, TracingReceiver, ExecutionStrategy, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION, + ExecutionStrategy, TracingReceiver, WasmExecutionMethod, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION, DEFAULT_EXECUTION_IMPORT_BLOCK, DEFAULT_EXECUTION_OFFCHAIN_WORKER, DEFAULT_EXECUTION_OTHER, - DEFAULT_EXECUTION_SYNCING + DEFAULT_EXECUTION_SYNCING, Database, }; use crate::params::PruningParams; +use crate::Result; +use sc_client_api::execution_extensions::ExecutionStrategies; +use sc_service::{PruningMode, Role}; +use structopt::StructOpt; /// Parameters for block import. #[derive(Debug, StructOpt, Clone)] @@ -52,11 +52,21 @@ pub struct ImportParams { #[allow(missing_docs)] #[structopt(flatten)] - pub execution_strategies: ExecutionStrategies, + pub execution_strategies: ExecutionStrategiesParams, + + /// Select database backend to use. + #[structopt( + long = "database", + alias = "db", + value_name = "DB", + case_insensitive = true, + default_value = "RocksDb" + )] + pub database: Database, /// Limit the memory the database cache can use. - #[structopt(long = "db-cache", value_name = "MiB", default_value = "128")] - pub database_cache_size: u32, + #[structopt(long = "db-cache", value_name = "MiB")] + pub database_cache_size: Option, /// Specify the state cache size. #[structopt(long = "state-cache-size", value_name = "Bytes", default_value = "67108864")] @@ -78,25 +88,31 @@ pub struct ImportParams { } impl ImportParams { - /// Put block import CLI params into `config` object. - pub fn update_config( - &self, - mut config: &mut Configuration, - role: &sc_service::Role, - is_dev: bool, - ) -> error::Result<()> { - use sc_client_api::execution_extensions::ExecutionStrategies; - - if let Some(DatabaseConfig::Path { ref mut cache_size, .. }) = config.database { - *cache_size = Some(self.database_cache_size); - } + /// Receiver to process tracing messages. + pub fn tracing_receiver(&self) -> sc_service::TracingReceiver { + self.tracing_receiver.clone().into() + } - config.state_cache_size = self.state_cache_size; + /// Comma separated list of targets for tracing. + pub fn tracing_targets(&self) -> Option { + self.tracing_targets.clone() + } - self.pruning_params.update_config(&mut config, role, self.unsafe_pruning)?; + /// Specify the state cache size. + pub fn state_cache_size(&self) -> usize { + self.state_cache_size + } - config.wasm_method = self.wasm_method.into(); + /// Get the WASM execution method from the parameters + pub fn wasm_method(&self) -> sc_service::config::WasmExecutionMethod { + self.wasm_method.into() + } + /// Get execution strategies for the parameters + pub fn execution_strategies( + &self, + is_dev: bool, + ) -> ExecutionStrategies { let exec = &self.execution_strategies; let exec_all_or = |strat: ExecutionStrategy, default: ExecutionStrategy| { exec.execution.unwrap_or(if strat == default && is_dev { @@ -106,7 +122,7 @@ impl ImportParams { }).into() }; - config.execution_strategies = ExecutionStrategies { + ExecutionStrategies { syncing: exec_all_or(exec.execution_syncing, DEFAULT_EXECUTION_SYNCING), importing: exec_all_or(exec.execution_import_block, DEFAULT_EXECUTION_IMPORT_BLOCK), block_construction: @@ -114,15 +130,28 @@ impl ImportParams { offchain_worker: exec_all_or(exec.execution_offchain_worker, DEFAULT_EXECUTION_OFFCHAIN_WORKER), other: exec_all_or(exec.execution_other, DEFAULT_EXECUTION_OTHER), - }; + } + } + + /// Get the pruning mode from the parameters + pub fn pruning(&self, unsafe_pruning: bool, role: &Role) -> Result { + self.pruning_params.pruning(unsafe_pruning, role) + } - Ok(()) + /// Limit the memory the database cache can use. + pub fn database_cache_size(&self) -> Option { + self.database_cache_size + } + + /// Limit the memory the database cache can use. + pub fn database(&self) -> Database { + self.database } } /// Execution strategies parameters. #[derive(Debug, StructOpt, Clone)] -pub struct ExecutionStrategies { +pub struct ExecutionStrategiesParams { /// The means of execution used when calling into the runtime while syncing blocks. #[structopt( long = "execution-syncing", diff --git a/client/cli/src/params/keystore_params.rs b/client/cli/src/params/keystore_params.rs new file mode 100644 index 0000000000000000000000000000000000000000..c6131c2f64941afa507931e5e090a4389a1e61eb --- /dev/null +++ b/client/cli/src/params/keystore_params.rs @@ -0,0 +1,92 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::error::Result; +use sc_service::config::KeystoreConfig; +use std::fs; +use std::path::PathBuf; +use structopt::StructOpt; + +/// default sub directory for the key store +const DEFAULT_KEYSTORE_CONFIG_PATH: &'static str = "keystore"; + +/// Parameters of the keystore +#[derive(Debug, StructOpt, Clone)] +pub struct KeystoreParams { + /// Specify custom keystore path. + #[structopt(long = "keystore-path", value_name = "PATH", parse(from_os_str))] + pub keystore_path: Option, + + /// Use interactive shell for entering the password used by the keystore. + #[structopt( + long = "password-interactive", + conflicts_with_all = &[ "password", "password-filename" ] + )] + pub password_interactive: bool, + + /// Password used by the keystore. + #[structopt( + long = "password", + conflicts_with_all = &[ "password-interactive", "password-filename" ] + )] + pub password: Option, + + /// File that contains the password used by the keystore. + #[structopt( + long = "password-filename", + value_name = "PATH", + parse(from_os_str), + conflicts_with_all = &[ "password-interactive", "password" ] + )] + pub password_filename: Option, +} + +impl KeystoreParams { + /// Get the keystore configuration for the parameters + pub fn keystore_config(&self, base_path: &PathBuf) -> Result { + let password = if self.password_interactive { + #[cfg(not(target_os = "unknown"))] + { + Some(input_keystore_password()?.into()) + } + #[cfg(target_os = "unknown")] + None + } else if let Some(ref file) = self.password_filename { + Some( + fs::read_to_string(file) + .map_err(|e| format!("{}", e))? + .into(), + ) + } else if let Some(ref password) = self.password { + Some(password.clone().into()) + } else { + None + }; + + let path = self + .keystore_path + .clone() + .unwrap_or(base_path.join(DEFAULT_KEYSTORE_CONFIG_PATH)); + + Ok(KeystoreConfig::Path { path, password }) + } +} + +#[cfg(not(target_os = "unknown"))] +fn input_keystore_password() -> Result { + rpassword::read_password_from_tty(Some("Keystore password: ")) + .map_err(|e| format!("{:?}", e).into()) +} diff --git a/client/cli/src/params/mod.rs b/client/cli/src/params/mod.rs index f684cab336423159c6a18ff492c3d7741c4cfded..af94a0c6f6b3f735108eac9a85c75f6ae74151ec 100644 --- a/client/cli/src/params/mod.rs +++ b/client/cli/src/params/mod.rs @@ -15,21 +15,25 @@ // along with Substrate. If not, see . mod import_params; -mod transaction_pool_params; -mod shared_params; +mod keystore_params; +mod network_params; mod node_key_params; -mod network_configuration_params; mod pruning_params; +mod shared_params; +mod transaction_pool_params; +mod offchain_worker_params; -use std::str::FromStr; use std::fmt::Debug; +use std::str::FromStr; pub use crate::params::import_params::*; -pub use crate::params::transaction_pool_params::*; -pub use crate::params::shared_params::*; +pub use crate::params::keystore_params::*; +pub use crate::params::network_params::*; pub use crate::params::node_key_params::*; -pub use crate::params::network_configuration_params::*; +pub use crate::params::offchain_worker_params::*; pub use crate::params::pruning_params::*; +pub use crate::params::shared_params::*; +pub use crate::params::transaction_pool_params::*; /// Wrapper type of `String` that holds an unsigned integer of arbitrary size, formatted as a decimal. #[derive(Debug, Clone)] diff --git a/client/cli/src/params/network_configuration_params.rs b/client/cli/src/params/network_params.rs similarity index 55% rename from client/cli/src/params/network_configuration_params.rs rename to client/cli/src/params/network_params.rs index b01cdeeb1cbeeea2a406d328d456c6b0ceb0d4f4..d0ec3fb9f3f206a024409610bada46457902a326 100644 --- a/client/cli/src/params/network_configuration_params.rs +++ b/client/cli/src/params/network_params.rs @@ -14,21 +14,18 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use std::path::PathBuf; -use std::iter; -use std::net::Ipv4Addr; -use structopt::StructOpt; +use crate::params::node_key_params::NodeKeyParams; use sc_network::{ - config::{MultiaddrWithPeerId, NonReservedPeerMode, TransportConfig}, multiaddr::Protocol, Multiaddr, + config::{NetworkConfiguration, NodeKeyConfig, NonReservedPeerMode, TransportConfig}, + multiaddr::Protocol, }; -use sc_service::Configuration; - -use crate::error; -use crate::params::node_key_params::NodeKeyParams; +use sc_service::{ChainSpec, config::{Multiaddr, MultiaddrWithPeerId}}; +use std::path::PathBuf; +use structopt::StructOpt; /// Parameters used to create the network configuration. #[derive(Debug, StructOpt, Clone)] -pub struct NetworkConfigurationParams { +pub struct NetworkParams { /// Specify a list of bootnodes. #[structopt(long = "bootnodes", value_name = "ADDR")] pub bootnodes: Vec, @@ -44,22 +41,12 @@ pub struct NetworkConfigurationParams { #[structopt(long = "reserved-only")] pub reserved_only: bool, - /// Specify a list of sentry node public addresses. - #[structopt( - long = "sentry-nodes", - value_name = "ADDR", - conflicts_with_all = &[ "sentry" ] - )] - pub sentry_nodes: Vec, - /// Listen on this multiaddress. #[structopt(long = "listen-addr", value_name = "LISTEN_ADDR")] pub listen_addr: Vec, /// Specify p2p protocol TCP port. - /// - /// Only used if --listen-addr is not specified. - #[structopt(long = "port", value_name = "PORT")] + #[structopt(long = "port", value_name = "PORT", conflicts_with_all = &[ "listen-addr" ])] pub port: Option, /// Forbid connecting to private IPv4 addresses (as specified in @@ -87,65 +74,86 @@ pub struct NetworkConfigurationParams { /// /// This allows downloading announced blocks from multiple peers. Decrease to save /// traffic and risk increased latency. - #[structopt(long = "max-parallel-downloads", value_name = "COUNT", default_value = "5")] + #[structopt( + long = "max-parallel-downloads", + value_name = "COUNT", + default_value = "5" + )] pub max_parallel_downloads: u32, #[allow(missing_docs)] #[structopt(flatten)] pub node_key_params: NodeKeyParams, - /// Experimental feature flag. - #[structopt(long = "use-yamux-flow-control")] - pub use_yamux_flow_control: bool, + /// Disable the yamux flow control. This option will be removed in the future once there is + /// enough confidence that this feature is properly working. + #[structopt(long)] + pub no_yamux_flow_control: bool, + + /// Enable peer discovery on local networks. + /// + /// By default this option is true for `--dev` and false otherwise. + #[structopt(long)] + pub discover_local: bool, + + /// Use the legacy "pre-mainnet-launch" networking protocol. Enable if things seem broken. + /// This option will be removed in the future. + #[structopt(long)] + pub legacy_network_protocol: bool, } -impl NetworkConfigurationParams { +impl NetworkParams { /// Fill the given `NetworkConfiguration` by looking at the cli parameters. - pub fn update_config( + pub fn network_config( &self, - mut config: &mut Configuration, - config_path: PathBuf, - client_id: String, + chain_spec: &Box, is_dev: bool, - ) -> error::Result<()> { - config.network.boot_nodes.extend(self.bootnodes.clone()); - config.network.config_path = Some(config_path.clone()); - config.network.net_config_path = Some(config_path.clone()); - - config.network.reserved_nodes.extend(self.reserved_nodes.clone()); - if self.reserved_only { - config.network.non_reserved_mode = NonReservedPeerMode::Deny; - } - - config.network.listen_addresses.extend(self.listen_addr.iter().cloned()); - if config.network.listen_addresses.is_empty() { - let port = match self.port { - Some(port) => port, - None => 30333, - }; - - config.network.listen_addresses = vec![ - iter::once(Protocol::Ip4(Ipv4Addr::new(0, 0, 0, 0))) - .chain(iter::once(Protocol::Tcp(port))) - .collect() - ]; - } - - config.network.client_version = client_id; - self.node_key_params.update_config(&mut config, Some(&config_path))?; - - config.network.in_peers = self.in_peers; - config.network.out_peers = self.out_peers; - - config.network.transport = TransportConfig::Normal { - enable_mdns: !is_dev && !self.no_mdns, - allow_private_ipv4: !self.no_private_ipv4, - wasm_external_transport: None, - use_yamux_flow_control: self.use_yamux_flow_control, + net_config_path: Option, + client_id: &str, + node_name: &str, + node_key: NodeKeyConfig, + ) -> NetworkConfiguration { + let port = self.port.unwrap_or(30333); + + let listen_addresses = if self.listen_addr.is_empty() { + vec![ + Multiaddr::empty() + .with(Protocol::Ip4([0, 0, 0, 0].into())) + .with(Protocol::Tcp(port)), + ] + } else { + self.listen_addr.clone() }; - config.network.max_parallel_downloads = self.max_parallel_downloads; - - Ok(()) + let mut boot_nodes = chain_spec.boot_nodes().to_vec(); + boot_nodes.extend(self.bootnodes.clone()); + + NetworkConfiguration { + boot_nodes, + net_config_path, + reserved_nodes: self.reserved_nodes.clone(), + non_reserved_mode: if self.reserved_only { + NonReservedPeerMode::Deny + } else { + NonReservedPeerMode::Accept + }, + listen_addresses, + public_addresses: Vec::new(), + notifications_protocols: Vec::new(), + node_key, + node_name: node_name.to_string(), + client_version: client_id.to_string(), + in_peers: self.in_peers, + out_peers: self.out_peers, + transport: TransportConfig::Normal { + enable_mdns: !is_dev && !self.no_mdns, + allow_private_ipv4: !self.no_private_ipv4, + wasm_external_transport: None, + use_yamux_flow_control: !self.no_yamux_flow_control, + }, + max_parallel_downloads: self.max_parallel_downloads, + allow_non_globals_in_dht: self.discover_local || is_dev, + use_new_block_requests_protocol: !self.legacy_network_protocol, + } } } diff --git a/client/cli/src/params/node_key_params.rs b/client/cli/src/params/node_key_params.rs index c55ec8ee692db5715c88414c905f87fc20ca51fa..2913ff2c1035e2f04ae30888e9f352420c40331d 100644 --- a/client/cli/src/params/node_key_params.rs +++ b/client/cli/src/params/node_key_params.rs @@ -14,14 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use std::{path::PathBuf, str::FromStr}; -use structopt::StructOpt; -use sc_service::Configuration; use sc_network::config::NodeKeyConfig; use sp_core::H256; +use std::{path::PathBuf, str::FromStr}; +use structopt::StructOpt; -use crate::error; use crate::arg_enums::NodeKeyType; +use crate::error; /// The file name of the node's Ed25519 secret key inside the chain-specific /// network config directory, if neither `--node-key` nor `--node-key-file` @@ -93,31 +92,23 @@ pub struct NodeKeyParams { impl NodeKeyParams { /// Create a `NodeKeyConfig` from the given `NodeKeyParams` in the context /// of an optional network config storage directory. - pub fn update_config<'a>( - &self, - mut config: &'a mut Configuration, - net_config_path: Option<&PathBuf>, - ) -> error::Result<&'a NodeKeyConfig> { - config.network.node_key = match self.node_key_type { + pub fn node_key(&self, net_config_dir: &PathBuf) -> error::Result { + Ok(match self.node_key_type { NodeKeyType::Ed25519 => { let secret = if let Some(node_key) = self.node_key.as_ref() { parse_ed25519_secret(node_key)? } else { - let path = self.node_key_file.clone() - .or_else(|| net_config_path.map(|d| d.join(NODE_KEY_ED25519_FILE))); + let path = self + .node_key_file + .clone() + .unwrap_or_else(|| net_config_dir.join(NODE_KEY_ED25519_FILE)); - if let Some(path) = path { - sc_network::config::Secret::File(path) - } else { - sc_network::config::Secret::New - } + sc_network::config::Secret::File(path) }; NodeKeyConfig::Ed25519(secret) } - }; - - Ok(&config.network.node_key) + }) } } @@ -128,114 +119,107 @@ fn invalid_node_key(e: impl std::fmt::Display) -> error::Error { /// Parse a Ed25519 secret key from a hex string into a `sc_network::Secret`. fn parse_ed25519_secret(hex: &str) -> error::Result { - H256::from_str(&hex).map_err(invalid_node_key).and_then(|bytes| - sc_network::config::identity::ed25519::SecretKey::from_bytes(bytes) - .map(sc_network::config::Secret::Input) - .map_err(invalid_node_key)) + H256::from_str(&hex) + .map_err(invalid_node_key) + .and_then(|bytes| { + sc_network::config::identity::ed25519::SecretKey::from_bytes(bytes) + .map(sc_network::config::Secret::Input) + .map_err(invalid_node_key) + }) } #[cfg(test)] mod tests { - use sc_network::config::identity::ed25519; use super::*; + use sc_network::config::identity::ed25519; #[test] fn test_node_key_config_input() { - fn secret_input(net_config_dir: Option<&PathBuf>) -> error::Result<()> { + fn secret_input(net_config_dir: &PathBuf) -> error::Result<()> { NodeKeyType::variants().iter().try_for_each(|t| { - let mut config = Configuration::default(); let node_key_type = NodeKeyType::from_str(t).unwrap(); let sk = match node_key_type { - NodeKeyType::Ed25519 => ed25519::SecretKey::generate().as_ref().to_vec() + NodeKeyType::Ed25519 => ed25519::SecretKey::generate().as_ref().to_vec(), }; let params = NodeKeyParams { node_key_type, node_key: Some(format!("{:x}", H256::from_slice(sk.as_ref()))), - node_key_file: None + node_key_file: None, }; - params.update_config(&mut config, net_config_dir).and_then(|c| match c { + params.node_key(net_config_dir).and_then(|c| match c { NodeKeyConfig::Ed25519(sc_network::config::Secret::Input(ref ski)) - if node_key_type == NodeKeyType::Ed25519 && - &sk[..] == ski.as_ref() => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) + if node_key_type == NodeKeyType::Ed25519 && &sk[..] == ski.as_ref() => + { + Ok(()) + } + _ => Err(error::Error::Input("Unexpected node key config".into())), }) }) } - assert!(secret_input(None).is_ok()); - assert!(secret_input(Some(&PathBuf::from_str("x").unwrap())).is_ok()); + assert!(secret_input(&PathBuf::from_str("x").unwrap()).is_ok()); } #[test] fn test_node_key_config_file() { - fn secret_file(net_config_dir: Option<&PathBuf>) -> error::Result<()> { + fn secret_file(net_config_dir: &PathBuf) -> error::Result<()> { NodeKeyType::variants().iter().try_for_each(|t| { - let mut config = Configuration::default(); let node_key_type = NodeKeyType::from_str(t).unwrap(); let tmp = tempfile::Builder::new().prefix("alice").tempdir()?; let file = tmp.path().join(format!("{}_mysecret", t)).to_path_buf(); let params = NodeKeyParams { node_key_type, node_key: None, - node_key_file: Some(file.clone()) + node_key_file: Some(file.clone()), }; - params.update_config(&mut config, net_config_dir).and_then(|c| match c { + params.node_key(net_config_dir).and_then(|c| match c { NodeKeyConfig::Ed25519(sc_network::config::Secret::File(ref f)) - if node_key_type == NodeKeyType::Ed25519 && f == &file => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) + if node_key_type == NodeKeyType::Ed25519 && f == &file => + { + Ok(()) + } + _ => Err(error::Error::Input("Unexpected node key config".into())), }) }) } - assert!(secret_file(None).is_ok()); - assert!(secret_file(Some(&PathBuf::from_str("x").unwrap())).is_ok()); + assert!(secret_file(&PathBuf::from_str("x").unwrap()).is_ok()); } #[test] fn test_node_key_config_default() { fn with_def_params(f: F) -> error::Result<()> where - F: Fn(NodeKeyParams) -> error::Result<()> + F: Fn(NodeKeyParams) -> error::Result<()>, { NodeKeyType::variants().iter().try_for_each(|t| { let node_key_type = NodeKeyType::from_str(t).unwrap(); f(NodeKeyParams { node_key_type, node_key: None, - node_key_file: None + node_key_file: None, }) }) } - fn no_config_dir() -> error::Result<()> { - with_def_params(|params| { - let mut config = Configuration::default(); - let typ = params.node_key_type; - params.update_config(&mut config, None) - .and_then(|c| match c { - NodeKeyConfig::Ed25519(sc_network::config::Secret::New) - if typ == NodeKeyType::Ed25519 => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) - }) - }) - } - fn some_config_dir(net_config_dir: &PathBuf) -> error::Result<()> { with_def_params(|params| { - let mut config = Configuration::default(); let dir = PathBuf::from(net_config_dir.clone()); let typ = params.node_key_type; - params.update_config(&mut config, Some(net_config_dir)) + params + .node_key(net_config_dir) .and_then(move |c| match c { NodeKeyConfig::Ed25519(sc_network::config::Secret::File(ref f)) - if typ == NodeKeyType::Ed25519 && - f == &dir.join(NODE_KEY_ED25519_FILE) => Ok(()), - _ => Err(error::Error::Input("Unexpected node key config".into())) - }) + if typ == NodeKeyType::Ed25519 + && f == &dir.join(NODE_KEY_ED25519_FILE) => + { + Ok(()) + } + _ => Err(error::Error::Input("Unexpected node key config".into())), + }) }) } - assert!(no_config_dir().is_ok()); assert!(some_config_dir(&PathBuf::from_str("x").unwrap()).is_ok()); } } diff --git a/client/cli/src/params/offchain_worker_params.rs b/client/cli/src/params/offchain_worker_params.rs new file mode 100644 index 0000000000000000000000000000000000000000..eb3cc74ad900cd72db2511a305e2ebc733a1a299 --- /dev/null +++ b/client/cli/src/params/offchain_worker_params.rs @@ -0,0 +1,77 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + + +//! Offchain worker related configuration parameters. +//! +//! A subset of configuration parameters which are relevant to +//! the inner working of offchain workers. The usage is solely +//! targeted at handling input parameter parsing providing +//! a reasonable abstraction. + +use structopt::StructOpt; +use sc_service::config::OffchainWorkerConfig; +use sc_network::config::Role; + +use crate::error; +use crate::OffchainWorkerEnabled; + + +/// Offchain worker related parameters. +#[derive(Debug, StructOpt, Clone)] +pub struct OffchainWorkerParams { + /// Should execute offchain workers on every block. + /// + /// By default it's only enabled for nodes that are authoring new blocks. + #[structopt( + long = "offchain-worker", + value_name = "ENABLED", + possible_values = &OffchainWorkerEnabled::variants(), + case_insensitive = true, + default_value = "WhenValidating" + )] + pub enabled: OffchainWorkerEnabled, + + /// Enable Offchain Indexing API, which allows block import to write to Offchain DB. + /// + /// Enables a runtime to write directly to a offchain workers + /// DB during block import. + #[structopt( + long = "enable-offchain-indexing", + value_name = "ENABLE_OFFCHAIN_INDEXING" + )] + pub indexing_enabled: bool, +} + +impl OffchainWorkerParams { + /// Load spec to `Configuration` from `OffchainWorkerParams` and spec factory. + pub fn offchain_worker( + &self, + role: &Role, + ) -> error::Result + { + let enabled = match (&self.enabled, role) { + (OffchainWorkerEnabled::WhenValidating, Role::Authority { .. }) => true, + (OffchainWorkerEnabled::Always, _) => true, + (OffchainWorkerEnabled::Never, _) => false, + (OffchainWorkerEnabled::WhenValidating, _) => false, + }; + + let indexing_enabled = enabled && self.indexing_enabled; + + Ok(OffchainWorkerConfig { enabled, indexing_enabled }) + } +} diff --git a/client/cli/src/params/pruning_params.rs b/client/cli/src/params/pruning_params.rs index 8d069a299ff39b7068fed931b4c71dadc7d763c2..ed8f7ab16858dce49f0b8a78ae2bb60141a47eac 100644 --- a/client/cli/src/params/pruning_params.rs +++ b/client/cli/src/params/pruning_params.rs @@ -14,10 +14,9 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use structopt::StructOpt; -use sc_service::{Configuration, PruningMode}; - use crate::error; +use sc_service::{PruningMode, Role}; +use structopt::StructOpt; /// Parameters to define the pruning mode #[derive(Debug, StructOpt, Clone)] @@ -32,18 +31,13 @@ pub struct PruningParams { } impl PruningParams { - /// Put block pruning CLI params into `config` object. - pub fn update_config( - &self, - mut config: &mut Configuration, - role: &sc_service::Role, - unsafe_pruning: bool, - ) -> error::Result<()> { + /// Get the pruning value from the parameters + pub fn pruning(&self, unsafe_pruning: bool, role: &Role) -> error::Result { // by default we disable pruning if the node is an authority (i.e. // `ArchiveAll`), otherwise we keep state for the last 256 blocks. if the // node is an authority and pruning is enabled explicitly, then we error // unless `unsafe_pruning` is set. - config.pruning = match &self.pruning { + Ok(match &self.pruning { Some(ref s) if s == "archive" => PruningMode::ArchiveAll, None if role.is_network_authority() => PruningMode::ArchiveAll, None => PruningMode::default(), @@ -51,16 +45,15 @@ impl PruningParams { if role.is_network_authority() && !unsafe_pruning { return Err(error::Error::Input( "Validators should run with state pruning disabled (i.e. archive). \ - You can ignore this check with `--unsafe-pruning`.".to_string() + You can ignore this check with `--unsafe-pruning`." + .to_string(), )); } - PruningMode::keep_blocks(s.parse() - .map_err(|_| error::Error::Input("Invalid pruning mode specified".to_string()))? - ) - }, - }; - - Ok(()) + PruningMode::keep_blocks(s.parse().map_err(|_| { + error::Error::Input("Invalid pruning mode specified".to_string()) + })?) + } + }) } } diff --git a/client/cli/src/params/shared_params.rs b/client/cli/src/params/shared_params.rs index 41b9cce8264cc0ce46de7d81661b396f4275266b..d6dd1bd9c16e1ba6f8469eb18830db838b0ef695 100644 --- a/client/cli/src/params/shared_params.rs +++ b/client/cli/src/params/shared_params.rs @@ -14,18 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +use sc_service::config::DatabaseConfig; use std::path::PathBuf; use structopt::StructOpt; -use app_dirs::{AppInfo, AppDataType}; -use sc_service::{ - Configuration, config::DatabaseConfig, ChainSpec, -}; - -use crate::VersionInfo; -use crate::error; - -/// default sub directory to store database -const DEFAULT_DB_CONFIG_PATH : &'static str = "db"; +use crate::arg_enums::Database; /// Shared parameters used by all `CoreParams`. #[derive(Debug, StructOpt, Clone)] @@ -39,7 +31,12 @@ pub struct SharedParams { pub dev: bool, /// Specify custom base path. - #[structopt(long = "base-path", short = "d", value_name = "PATH", parse(from_os_str))] + #[structopt( + long = "base-path", + short = "d", + value_name = "PATH", + parse(from_os_str) + )] pub base_path: Option, /// Sets a custom logging filter. Syntax is =, e.g. -lsync=debug. @@ -51,62 +48,53 @@ pub struct SharedParams { } impl SharedParams { - /// Load spec to `Configuration` from `SharedParams` and spec factory. - pub fn update_config<'a, F>( - &self, - mut config: &'a mut Configuration, - spec_factory: F, - version: &VersionInfo, - ) -> error::Result<&'a dyn ChainSpec> where - F: FnOnce(&str) -> Result, String>, - { - let chain_key = match self.chain { - Some(ref chain) => chain.clone(), - None => if self.dev { "dev".into() } else { "".into() } - }; - let spec = spec_factory(&chain_key)?; - config.network.boot_nodes = spec.boot_nodes().to_vec(); - config.telemetry_endpoints = spec.telemetry_endpoints().clone(); + /// Specify custom base path. + pub fn base_path(&self) -> Option { + self.base_path.clone() + } - config.chain_spec = Some(spec); + /// Specify the development chain. + pub fn is_dev(&self) -> bool { + self.dev + } - if config.config_dir.is_none() { - config.config_dir = Some(base_path(self, version)); + /// Get the chain spec for the parameters provided + pub fn chain_id(&self, is_dev: bool) -> String { + match self.chain { + Some(ref chain) => chain.clone(), + None => { + if is_dev { + "dev".into() + } else { + "".into() + } + } } + } - if config.database.is_none() { - config.database = Some(DatabaseConfig::Path { - path: config - .in_chain_config_dir(DEFAULT_DB_CONFIG_PATH) - .expect("We provided a base_path/config_dir."), - cache_size: None, - }); + /// Get the database configuration object for the parameters provided + pub fn database_config( + &self, + base_path: &PathBuf, + cache_size: usize, + database: Database, + ) -> DatabaseConfig { + match database { + Database::RocksDb => DatabaseConfig::RocksDb { + path: base_path.join("db"), + cache_size, + }, + Database::SubDb => DatabaseConfig::SubDb { + path: base_path.join("subdb"), + }, + Database::ParityDb => DatabaseConfig::ParityDb { + path: base_path.join("paritydb"), + }, } - - Ok(config.expect_chain_spec()) } - /// Initialize substrate. This must be done only once. - /// - /// This method: - /// - /// 1. Set the panic handler - /// 2. Raise the FD limit - /// 3. Initialize the logger - pub fn init(&self, version: &VersionInfo) -> error::Result<()> { - crate::init(self.log.as_ref().map(|v| v.as_ref()).unwrap_or(""), version) + /// Get the filters for the logging + pub fn log_filters(&self) -> Option { + self.log.clone() } } - -fn base_path(cli: &SharedParams, version: &VersionInfo) -> PathBuf { - cli.base_path.clone() - .unwrap_or_else(|| - app_dirs::get_app_root( - AppDataType::UserData, - &AppInfo { - name: version.executable_name, - author: version.author - } - ).expect("app directories exist on all supported platforms; qed") - ) -} diff --git a/client/cli/src/params/transaction_pool_params.rs b/client/cli/src/params/transaction_pool_params.rs index 3468c12243b3fe75d872891aebf12b3df6f0e356..dfcdf9af70537e51586654e4cc72da4a7549f118 100644 --- a/client/cli/src/params/transaction_pool_params.rs +++ b/client/cli/src/params/transaction_pool_params.rs @@ -14,9 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +use sc_service::config::TransactionPoolOptions; use structopt::StructOpt; -use sc_service::Configuration; -use crate::error; /// Parameters used to create the pool configuration. #[derive(Debug, StructOpt, Clone)] @@ -24,6 +23,7 @@ pub struct TransactionPoolParams { /// Maximum number of transactions in the transaction pool. #[structopt(long = "pool-limit", value_name = "COUNT", default_value = "8192")] pub pool_limit: usize, + /// Maximum number of kilobytes of all transactions stored in the pool. #[structopt(long = "pool-kbytes", value_name = "COUNT", default_value = "20480")] pub pool_kbytes: usize, @@ -31,19 +31,18 @@ pub struct TransactionPoolParams { impl TransactionPoolParams { /// Fill the given `PoolConfiguration` by looking at the cli parameters. - pub fn update_config( - &self, - config: &mut Configuration, - ) -> error::Result<()> { + pub fn transaction_pool(&self) -> TransactionPoolOptions { + let mut opts = TransactionPoolOptions::default(); + // ready queue - config.transaction_pool.ready.count = self.pool_limit; - config.transaction_pool.ready.total_bytes = self.pool_kbytes * 1024; + opts.ready.count = self.pool_limit; + opts.ready.total_bytes = self.pool_kbytes * 1024; // future queue let factor = 10; - config.transaction_pool.future.count = self.pool_limit / factor; - config.transaction_pool.future.total_bytes = self.pool_kbytes * 1024 / factor; + opts.future.count = self.pool_limit / factor; + opts.future.total_bytes = self.pool_kbytes * 1024 / factor; - Ok(()) + opts } } diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs new file mode 100644 index 0000000000000000000000000000000000000000..6ebe84f9c55e8139d8a7386ebb2670ddbedfcc69 --- /dev/null +++ b/client/cli/src/runner.rs @@ -0,0 +1,241 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::CliConfiguration; +use crate::Result; +use crate::SubstrateCli; +use crate::Subcommand; +use chrono::prelude::*; +use futures::pin_mut; +use futures::select; +use futures::{future, future::FutureExt, Future}; +use log::info; +use sc_service::{AbstractService, Configuration, Role, ServiceBuilderCommand}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; +use sp_utils::metrics::{TOKIO_THREADS_ALIVE, TOKIO_THREADS_TOTAL}; +use std::fmt::Debug; +use std::marker::PhantomData; +use std::sync::Arc; + +#[cfg(target_family = "unix")] +async fn main(func: F) -> std::result::Result<(), Box> +where + F: Future> + future::FusedFuture, + E: 'static + std::error::Error, +{ + use tokio::signal::unix::{signal, SignalKind}; + + let mut stream_int = signal(SignalKind::interrupt())?; + let mut stream_term = signal(SignalKind::terminate())?; + + let t1 = stream_int.recv().fuse(); + let t2 = stream_term.recv().fuse(); + let t3 = func; + + pin_mut!(t1, t2, t3); + + select! { + _ = t1 => {}, + _ = t2 => {}, + res = t3 => res?, + } + + Ok(()) +} + +#[cfg(not(unix))] +async fn main(func: F) -> std::result::Result<(), Box> +where + F: Future> + future::FusedFuture, + E: 'static + std::error::Error, +{ + use tokio::signal::ctrl_c; + + let t1 = ctrl_c().fuse(); + let t2 = func; + + pin_mut!(t1, t2); + + select! { + _ = t1 => {}, + res = t2 => res?, + } + + Ok(()) +} + +/// Build a tokio runtime with all features +pub fn build_runtime() -> std::result::Result { + tokio::runtime::Builder::new() + .threaded_scheduler() + .on_thread_start(||{ + TOKIO_THREADS_ALIVE.inc(); + TOKIO_THREADS_TOTAL.inc(); + }) + .on_thread_stop(||{ + TOKIO_THREADS_ALIVE.dec(); + }) + .enable_all() + .build() +} + +fn run_until_exit(mut tokio_runtime: tokio::runtime::Runtime, future: FUT) -> Result<()> +where + FUT: Future> + future::Future, + ERR: 'static + std::error::Error, +{ + let f = future.fuse(); + pin_mut!(f); + + tokio_runtime.block_on(main(f)).map_err(|e| e.to_string())?; + + Ok(()) +} + +/// A Substrate CLI runtime that can be used to run a node or a command +pub struct Runner { + config: Configuration, + tokio_runtime: tokio::runtime::Runtime, + phantom: PhantomData, +} + +impl Runner { + /// Create a new runtime with the command provided in argument + pub fn new(cli: &C, command: &T) -> Result> { + let tokio_runtime = build_runtime()?; + + let task_executor = { + let runtime_handle = tokio_runtime.handle().clone(); + Arc::new(move |fut| { + runtime_handle.spawn(fut); + }) + }; + + Ok(Runner { + config: command.create_configuration(cli, task_executor)?, + tokio_runtime, + phantom: PhantomData, + }) + } + + /// A helper function that runs an `AbstractService` with tokio and stops if the process receives + /// the signal `SIGTERM` or `SIGINT`. + pub fn run_node( + self, + new_light: FNL, + new_full: FNF, + runtime_version: sp_version::RuntimeVersion, + ) -> Result<()> where + FNL: FnOnce(Configuration) -> sc_service::error::Result, + FNF: FnOnce(Configuration) -> sc_service::error::Result, + SL: AbstractService + Unpin, + SF: AbstractService + Unpin, + { + info!("{}", C::impl_name()); + info!("✌️ version {}", C::impl_version()); + info!( + "❤️ by {}, {}-{}", + C::author(), + C::copyright_start_year(), + Local::today().year(), + ); + info!("📋 Chain specification: {}", self.config.chain_spec.name()); + info!("🏷 Node name: {}", self.config.network.node_name); + info!("👤 Role: {}", self.config.display_role()); + info!("⛓ Native runtime: {}", runtime_version); + + match self.config.role { + Role::Light => self.run_service_until_exit(new_light), + _ => self.run_service_until_exit(new_full), + } + } + + /// A helper function that runs a future with tokio and stops if the process receives the signal + /// `SIGTERM` or `SIGINT`. + pub fn run_subcommand(self, subcommand: &Subcommand, builder: B) -> Result<()> + where + B: FnOnce(Configuration) -> sc_service::error::Result, + BC: ServiceBuilderCommand + Unpin, + BB: sp_runtime::traits::Block + Debug, + <<::Header as HeaderT>::Number as std::str::FromStr>::Err: Debug, + ::Hash: std::str::FromStr, + { + match subcommand { + Subcommand::BuildSpec(cmd) => cmd.run(self.config), + Subcommand::ExportBlocks(cmd) => { + run_until_exit(self.tokio_runtime, cmd.run(self.config, builder)) + } + Subcommand::ImportBlocks(cmd) => { + run_until_exit(self.tokio_runtime, cmd.run(self.config, builder)) + } + Subcommand::CheckBlock(cmd) => { + run_until_exit(self.tokio_runtime, cmd.run(self.config, builder)) + } + Subcommand::Revert(cmd) => cmd.run(self.config, builder), + Subcommand::PurgeChain(cmd) => cmd.run(self.config), + } + } + + fn run_service_until_exit(mut self, service_builder: F) -> Result<()> + where + F: FnOnce(Configuration) -> std::result::Result, + T: AbstractService + Unpin, + { + let service = service_builder(self.config)?; + + let informant_future = sc_informant::build(&service, sc_informant::OutputFormat::Coloured); + let _informant_handle = self.tokio_runtime.spawn(informant_future); + + // we eagerly drop the service so that the internal exit future is fired, + // but we need to keep holding a reference to the global telemetry guard + // and drop the runtime first. + let _telemetry = service.telemetry(); + + let f = service.fuse(); + pin_mut!(f); + + self.tokio_runtime + .block_on(main(f)) + .map_err(|e| e.to_string())?; + drop(self.tokio_runtime); + + Ok(()) + } + + /// A helper function that runs a command with the configuration of this node + pub fn sync_run(self, runner: impl FnOnce(Configuration) -> Result<()>) -> Result<()> { + runner(self.config) + } + + /// A helper function that runs a future with tokio and stops if the process receives + /// the signal SIGTERM or SIGINT + pub fn async_run(self, runner: impl FnOnce(Configuration) -> FUT) -> Result<()> + where + FUT: Future>, + { + run_until_exit(self.tokio_runtime, runner(self.config)) + } + + /// Get an immutable reference to the node Configuration + pub fn config(&self) -> &Configuration { + &self.config + } + + /// Get a mutable reference to the node Configuration + pub fn config_mut(&mut self) -> &Configuration { + &mut self.config + } +} diff --git a/client/cli/src/runtime.rs b/client/cli/src/runtime.rs deleted file mode 100644 index 5c9bc7508858dfb4d0c896ef15f08504449f1900..0000000000000000000000000000000000000000 --- a/client/cli/src/runtime.rs +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2017-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -use std::sync::Arc; - -use futures::{Future, future, future::FutureExt}; -use futures::select; -use futures::pin_mut; -use sc_service::{AbstractService, Configuration}; -use sp_utils::metrics::{TOKIO_THREADS_ALIVE, TOKIO_THREADS_TOTAL}; -use crate::error; - -#[cfg(target_family = "unix")] -async fn main(func: F) -> Result<(), Box> -where - F: Future> + future::FusedFuture, - E: 'static + std::error::Error, -{ - use tokio::signal::unix::{signal, SignalKind}; - - let mut stream_int = signal(SignalKind::interrupt())?; - let mut stream_term = signal(SignalKind::terminate())?; - - let t1 = stream_int.recv().fuse(); - let t2 = stream_term.recv().fuse(); - let t3 = func; - - pin_mut!(t1, t2, t3); - - select! { - _ = t1 => {}, - _ = t2 => {}, - res = t3 => res?, - } - - Ok(()) -} - -#[cfg(not(unix))] -async fn main(func: F) -> Result<(), Box> -where - F: Future> + future::FusedFuture, - E: 'static + std::error::Error, -{ - use tokio::signal::ctrl_c; - - let t1 = ctrl_c().fuse(); - let t2 = func; - - pin_mut!(t1, t2); - - select! { - _ = t1 => {}, - res = t2 => res?, - } - - Ok(()) -} - -fn build_runtime() -> Result { - tokio::runtime::Builder::new() - .thread_name("main-tokio-") - .threaded_scheduler() - .on_thread_start(||{ - TOKIO_THREADS_ALIVE.inc(); - TOKIO_THREADS_TOTAL.inc(); - }) - .on_thread_stop(||{ - TOKIO_THREADS_ALIVE.dec(); - }) - .enable_all() - .build() -} - -/// A helper function that runs a future with tokio and stops if the process receives the signal -/// SIGTERM or SIGINT -pub fn run_until_exit( - mut config: Configuration, - future_builder: F, -) -> error::Result<()> -where - F: FnOnce(Configuration) -> error::Result, - FUT: Future> + future::Future, - ERR: 'static + std::error::Error, -{ - let mut runtime = build_runtime()?; - - config.task_executor = { - let runtime_handle = runtime.handle().clone(); - Some(Arc::new(move |fut| { runtime_handle.spawn(fut); })) - }; - - let f = future_builder(config)?; - let f = f.fuse(); - pin_mut!(f); - - runtime.block_on(main(f)).map_err(|e| e.to_string())?; - - Ok(()) -} - -/// A helper function that runs an `AbstractService` with tokio and stops if the process receives -/// the signal SIGTERM or SIGINT -pub fn run_service_until_exit( - mut config: Configuration, - service_builder: F, -) -> error::Result<()> -where - F: FnOnce(Configuration) -> Result, - T: AbstractService + Unpin, -{ - let mut runtime = build_runtime()?; - - config.task_executor = { - let runtime_handle = runtime.handle().clone(); - Some(Arc::new(move |fut| { runtime_handle.spawn(fut); })) - }; - - let service = service_builder(config)?; - - let informant_future = sc_informant::build(&service, sc_informant::OutputFormat::Coloured); - let _informant_handle = runtime.spawn(informant_future); - - // we eagerly drop the service so that the internal exit future is fired, - // but we need to keep holding a reference to the global telemetry guard - // and drop the runtime first. - let _telemetry = service.telemetry(); - - let f = service.fuse(); - pin_mut!(f); - - runtime.block_on(main(f)).map_err(|e| e.to_string())?; - drop(runtime); - - Ok(()) -} diff --git a/client/consensus/aura/Cargo.toml b/client/consensus/aura/Cargo.toml index f0fe368acbb037f0b9181263889baa31d3e3f660..82196feac0396ae32567bf9c2173f83d5618f2a4 100644 --- a/client/consensus/aura/Cargo.toml +++ b/client/consensus/aura/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-aura" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "Aura consensus algorithm for substrate" edition = "2018" @@ -8,41 +8,40 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.5", path = "../../../primitives/application-crypto" } -sp-consensus-aura = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/aura" } -sp-block-builder = { version = "2.0.0-alpha.5", path = "../../../primitives/block-builder" } -sc-block-builder = { version = "0.8.0-alpha.5", path = "../../../client/block-builder" } -sc-client = { version = "0.8.0-alpha.5", path = "../../" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../../api" } +sp-application-crypto = { version = "2.0.0-dev", path = "../../../primitives/application-crypto" } +sp-consensus-aura = { version = "0.8.0-dev", path = "../../../primitives/consensus/aura" } +sp-block-builder = { version = "2.0.0-dev", path = "../../../primitives/block-builder" } +sc-block-builder = { version = "0.8.0-dev", path = "../../../client/block-builder" } +sc-client-api = { version = "2.0.0-dev", path = "../../api" } codec = { package = "parity-scale-codec", version = "1.3.0" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } +sp-consensus = { version = "0.8.0-dev", path = "../../../primitives/consensus/common" } derive_more = "0.99.2" futures = "0.3.4" futures-timer = "3.0.1" -sp-inherents = { version = "2.0.0-alpha.5", path = "../../../primitives/inherents" } -sc-keystore = { version = "2.0.0-alpha.5", path = "../../keystore" } +sp-inherents = { version = "2.0.0-dev", path = "../../../primitives/inherents" } +sc-keystore = { version = "2.0.0-dev", path = "../../keystore" } log = "0.4.8" parking_lot = "0.10.0" -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } -sp-io = { version = "2.0.0-alpha.5", path = "../../../primitives/io" } -sp-version = { version = "2.0.0-alpha.5", path = "../../../primitives/version" } -sc-consensus-slots = { version = "0.8.0-alpha.5", path = "../slots" } -sp-api = { version = "2.0.0-alpha.5", path = "../../../primitives/api" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sp-timestamp = { version = "2.0.0-alpha.5", path = "../../../primitives/timestamp" } -sc-telemetry = { version = "2.0.0-alpha.5", path = "../../telemetry" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } +sp-io = { version = "2.0.0-dev", path = "../../../primitives/io" } +sp-version = { version = "2.0.0-dev", path = "../../../primitives/version" } +sc-consensus-slots = { version = "0.8.0-dev", path = "../slots" } +sp-api = { version = "2.0.0-dev", path = "../../../primitives/api" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-timestamp = { version = "2.0.0-dev", path = "../../../primitives/timestamp" } +sc-telemetry = { version = "2.0.0-dev", path = "../../telemetry" } [dev-dependencies] -sp-keyring = { version = "2.0.0-alpha.5", path = "../../../primitives/keyring" } -sc-executor = { version = "0.8.0-alpha.5", path = "../../executor" } -sc-network = { version = "0.8.0-alpha.5", path = "../../network" } +sp-keyring = { version = "2.0.0-dev", path = "../../../primitives/keyring" } +sc-executor = { version = "0.8.0-dev", path = "../../executor" } +sc-network = { version = "0.8.0-dev", path = "../../network" } sc-network-test = { version = "0.8.0-dev", path = "../../network/test" } -sc-service = { version = "0.8.0-alpha.5", path = "../../service" } +sc-service = { version = "0.8.0-dev", default-features = false, path = "../../service" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } env_logger = "0.7.0" tempfile = "3.1.0" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/consensus/aura/src/lib.rs b/client/consensus/aura/src/lib.rs index 56674546d372bb67ed8b44ef0ece086e1febb653..daa181abba5a505c8a417d7b2e448f1bc85820f6 100644 --- a/client/consensus/aura/src/lib.rs +++ b/client/consensus/aura/src/lib.rs @@ -46,8 +46,7 @@ use sp_consensus::{ use sp_consensus::import_queue::{ Verifier, BasicQueue, BoxJustificationImport, BoxFinalityProofImport, }; -use sc_client_api::backend::AuxStore; -use sc_client::BlockOf; +use sc_client_api::{backend::AuxStore, BlockOf}; use sp_blockchain::{ self, Result as CResult, well_known_cache_keys::{self, Id as CacheKeyId}, ProvideCache, HeaderBackend, @@ -299,27 +298,28 @@ impl sc_consensus_slots::SimpleSlotWorker for AuraW fn proposing_remaining_duration( &self, head: &B::Header, - slot_info: &SlotInfo + slot_info: &SlotInfo, ) -> Option { - // never give more than 20 times more lenience. - const BACKOFF_CAP: u64 = 20; - let slot_remaining = self.slot_remaining_duration(slot_info); + let parent_slot = match find_pre_digest::(head) { Err(_) => return Some(slot_remaining), Ok(d) => d, }; - // we allow a lenience of the number of slots since the head of the - // chain was produced, minus 1 (since there is always a difference of at least 1) - // - // linear back-off. - // in normal cases we only attempt to issue blocks up to the end of the slot. - // when the chain has been stalled for a few slots, we give more lenience. - let slot_lenience = slot_info.number.saturating_sub(parent_slot + 1); - let slot_lenience = std::cmp::min(slot_lenience, BACKOFF_CAP); - let slot_lenience = Duration::from_secs(slot_lenience * slot_info.duration); - Some(slot_lenience + slot_remaining) + if let Some(slot_lenience) = + sc_consensus_slots::slot_lenience_exponential(parent_slot, slot_info) + { + debug!(target: "aura", + "No block for {} slots. Applying linear lenience of {}s", + slot_info.number.saturating_sub(parent_slot + 1), + slot_lenience.as_secs(), + ); + + Some(slot_remaining + slot_lenience) + } else { + Some(slot_remaining) + } } } @@ -830,14 +830,14 @@ mod tests { use sc_network::config::ProtocolConfig; use parking_lot::Mutex; use sp_keyring::sr25519::Keyring; - use sc_client::BlockchainEvents; + use sc_client_api::BlockchainEvents; use sp_consensus_aura::sr25519::AuthorityPair; use std::task::Poll; use sc_block_builder::BlockBuilderProvider; type Error = sp_blockchain::Error; - type TestClient = sc_client::Client< + type TestClient = substrate_test_runtime_client::client::Client< substrate_test_runtime_client::Backend, substrate_test_runtime_client::Executor, TestBlock, diff --git a/client/consensus/babe/Cargo.toml b/client/consensus/babe/Cargo.toml index 19c5bf9e588b3d51f933855ddd01e1a80cd6eae9..ddae8f84b7e3ae9edc265df6d3b1f76bcc40e837 100644 --- a/client/consensus/babe/Cargo.toml +++ b/client/consensus/babe/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-babe" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "BABE consensus algorithm for substrate" edition = "2018" @@ -9,33 +9,35 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sc-consensus-babe" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } -sp-consensus-babe = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/babe" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-application-crypto = { version = "2.0.0-alpha.5", path = "../../../primitives/application-crypto" } +sp-consensus-babe = { version = "0.8.0-dev", path = "../../../primitives/consensus/babe" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-application-crypto = { version = "2.0.0-dev", path = "../../../primitives/application-crypto" } num-bigint = "0.2.3" num-rational = "0.2.2" num-traits = "0.2.8" serde = { version = "1.0.104", features = ["derive"] } -sp-version = { version = "2.0.0-alpha.5", path = "../../../primitives/version" } -sp-io = { version = "2.0.0-alpha.5", path = "../../../primitives/io" } -sp-inherents = { version = "2.0.0-alpha.5", path = "../../../primitives/inherents" } -sp-timestamp = { version = "2.0.0-alpha.5", path = "../../../primitives/timestamp" } -sc-telemetry = { version = "2.0.0-alpha.5", path = "../../telemetry" } -sc-keystore = { version = "2.0.0-alpha.5", path = "../../keystore" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../../api" } -sc-client = { version = "0.8.0-alpha.5", path = "../../" } -sc-consensus-epochs = { version = "0.8.0-alpha.5", path = "../epochs" } -sp-api = { version = "2.0.0-alpha.5", path = "../../../primitives/api" } -sp-block-builder = { version = "2.0.0-alpha.5", path = "../../../primitives/block-builder" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } -sp-consensus-vrf = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/vrf" } -sc-consensus-uncles = { version = "0.8.0-alpha.5", path = "../uncles" } -sc-consensus-slots = { version = "0.8.0-alpha.5", path = "../slots" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -fork-tree = { version = "2.0.0-alpha.5", path = "../../../utils/fork-tree" } +sp-version = { version = "2.0.0-dev", path = "../../../primitives/version" } +sp-io = { version = "2.0.0-dev", path = "../../../primitives/io" } +sp-inherents = { version = "2.0.0-dev", path = "../../../primitives/inherents" } +sp-timestamp = { version = "2.0.0-dev", path = "../../../primitives/timestamp" } +sc-telemetry = { version = "2.0.0-dev", path = "../../telemetry" } +sc-keystore = { version = "2.0.0-dev", path = "../../keystore" } +sc-client-api = { version = "2.0.0-dev", path = "../../api" } +sc-consensus-epochs = { version = "0.8.0-dev", path = "../epochs" } +sp-api = { version = "2.0.0-dev", path = "../../../primitives/api" } +sp-block-builder = { version = "2.0.0-dev", path = "../../../primitives/block-builder" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } +sp-consensus = { version = "0.8.0-dev", path = "../../../primitives/consensus/common" } +sp-consensus-vrf = { version = "0.8.0-dev", path = "../../../primitives/consensus/vrf" } +sc-consensus-uncles = { version = "0.8.0-dev", path = "../uncles" } +sc-consensus-slots = { version = "0.8.0-dev", path = "../slots" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +fork-tree = { version = "2.0.0-dev", path = "../../../utils/fork-tree" } futures = "0.3.4" futures-timer = "3.0.1" parking_lot = "0.10.0" @@ -47,18 +49,15 @@ pdqselect = "0.1.0" derive_more = "0.99.2" [dev-dependencies] -sp-keyring = { version = "2.0.0-alpha.5", path = "../../../primitives/keyring" } -sc-executor = { version = "0.8.0-alpha.5", path = "../../executor" } -sc-network = { version = "0.8.0-alpha.5", path = "../../network" } +sp-keyring = { version = "2.0.0-dev", path = "../../../primitives/keyring" } +sc-executor = { version = "0.8.0-dev", path = "../../executor" } +sc-network = { version = "0.8.0-dev", path = "../../network" } sc-network-test = { version = "0.8.0-dev", path = "../../network/test" } -sc-service = { version = "0.8.0-alpha.5", path = "../../service" } +sc-service = { version = "0.8.0-dev", default-features = false, path = "../../service" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } -sc-block-builder = { version = "0.8.0-alpha.5", path = "../../block-builder" } +sc-block-builder = { version = "0.8.0-dev", path = "../../block-builder" } env_logger = "0.7.0" tempfile = "3.1.0" [features] test-helpers = [] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/consensus/babe/rpc/Cargo.toml b/client/consensus/babe/rpc/Cargo.toml index 6bdaeb5ddf0709a336b0bccf6d30698ba810d2dd..e3b27e28dd6c0212a7e276564080ce8d4abbccea 100644 --- a/client/consensus/babe/rpc/Cargo.toml +++ b/client/consensus/babe/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-babe-rpc" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "RPC extensions for the BABE consensus algorithm" edition = "2018" @@ -8,28 +8,28 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sc-consensus-babe = { version = "0.8.0-alpha.5", path = "../" } +sc-consensus-babe = { version = "0.8.0-dev", path = "../" } jsonrpc-core = "14.0.3" jsonrpc-core-client = "14.0.5" jsonrpc-derive = "14.0.3" -sp-consensus-babe = { version = "0.8.0-alpha.5", path = "../../../../primitives/consensus/babe" } +sp-consensus-babe = { version = "0.8.0-dev", path = "../../../../primitives/consensus/babe" } serde = { version = "1.0.104", features=["derive"] } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../../primitives/runtime" } -sc-consensus-epochs = { version = "0.8.0-alpha.5", path = "../../epochs" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-dev", path = "../../../../primitives/runtime" } +sc-consensus-epochs = { version = "0.8.0-dev", path = "../../epochs" } futures = "0.3.4" derive_more = "0.99.2" -sp-api = { version = "2.0.0-alpha.5", path = "../../../../primitives/api" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../../../primitives/consensus/common" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../../primitives/core" } -sc-keystore = { version = "2.0.0-alpha.5", path = "../../../keystore" } +sp-api = { version = "2.0.0-dev", path = "../../../../primitives/api" } +sp-consensus = { version = "0.8.0-dev", path = "../../../../primitives/consensus/common" } +sp-core = { version = "2.0.0-dev", path = "../../../../primitives/core" } +sc-keystore = { version = "2.0.0-dev", path = "../../../keystore" } [dev-dependencies] substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../../test-utils/runtime/client" } -sp-application-crypto = { version = "2.0.0-alpha.5", path = "../../../../primitives/application-crypto" } -sp-keyring = { version = "2.0.0-alpha.5", path = "../../../../primitives/keyring" } +sp-application-crypto = { version = "2.0.0-dev", path = "../../../../primitives/application-crypto" } +sp-keyring = { version = "2.0.0-dev", path = "../../../../primitives/keyring" } tempfile = "3.1.0" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/consensus/babe/rpc/src/lib.rs b/client/consensus/babe/rpc/src/lib.rs index cb78504b1f78882395f5273afbfc5adb2a5c45a3..fa5421761cea818b8ebef9909458764db0155ad7 100644 --- a/client/consensus/babe/rpc/src/lib.rs +++ b/client/consensus/babe/rpc/src/lib.rs @@ -118,14 +118,17 @@ impl BabeApi for BabeRPCHandler for slot_number in epoch_start..epoch_end { let epoch = epoch_data(&shared_epoch, &client, &babe_config, slot_number, &select_chain)?; - if let Some((claim, key)) = authorship::claim_slot(slot_number, &epoch, &babe_config, &keystore) { + if let Some((claim, key)) = authorship::claim_slot(slot_number, &epoch, &keystore) { match claim { PreDigest::Primary { .. } => { claims.entry(key.public()).or_default().primary.push(slot_number); } - PreDigest::Secondary { .. } => { + PreDigest::SecondaryPlain { .. } => { claims.entry(key.public()).or_default().secondary.push(slot_number); } + PreDigest::SecondaryVRF { .. } => { + claims.entry(key.public()).or_default().secondary_vrf.push(slot_number); + }, }; } } @@ -144,6 +147,8 @@ pub struct EpochAuthorship { primary: Vec, /// the array of secondary slots that can be claimed secondary: Vec, + /// The array of secondary VRF slots that can be claimed. + secondary_vrf: Vec, } /// Errors encountered by the RPC @@ -184,7 +189,7 @@ fn epoch_data( &parent.hash(), parent.number().clone(), slot_number, - |slot| babe_config.genesis_epoch(slot), + |slot| Epoch::genesis(&babe_config, slot), ) .map_err(|e| Error::Consensus(ConsensusError::ChainLookup(format!("{:?}", e))))? .ok_or(Error::Consensus(ConsensusError::InvalidAuthoritiesSet)) @@ -236,7 +241,7 @@ mod tests { io.extend_with(BabeApi::to_delegate(handler)); let request = r#"{"jsonrpc":"2.0","method":"babe_epochAuthorship","params": [],"id":1}"#; - let response = r#"{"jsonrpc":"2.0","result":{"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY":{"primary":[0],"secondary":[1,2,4]}},"id":1}"#; + let response = r#"{"jsonrpc":"2.0","result":{"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY":{"primary":[0],"secondary":[1,2,4],"secondary_vrf":[]}},"id":1}"#; assert_eq!(Some(response.into()), io.handle_request_sync(request)); } diff --git a/client/consensus/babe/src/authorship.rs b/client/consensus/babe/src/authorship.rs index 074e582bff252395a18e691211eda944b380c723..56841225ce49c7d9865f3fc9029fabbefa15d8d4 100644 --- a/client/consensus/babe/src/authorship.rs +++ b/client/consensus/babe/src/authorship.rs @@ -19,9 +19,11 @@ use merlin::Transcript; use sp_consensus_babe::{ AuthorityId, BabeAuthorityWeight, BABE_ENGINE_ID, BABE_VRF_PREFIX, - SlotNumber, AuthorityPair, BabeConfiguration + SlotNumber, AuthorityPair, +}; +use sp_consensus_babe::digests::{ + PreDigest, PrimaryPreDigest, SecondaryPlainPreDigest, SecondaryVRFPreDigest, }; -use sp_consensus_babe::digests::{PreDigest, PrimaryPreDigest, SecondaryPreDigest}; use sp_consensus_vrf::schnorrkel::{VRFOutput, VRFProof}; use sp_core::{U256, blake2_256}; use codec::Encode; @@ -47,14 +49,44 @@ pub(super) fn calculate_primary_threshold( authorities[authority_index].1 as f64 / authorities.iter().map(|(_, weight)| weight).sum::() as f64; - let calc = || { - let p = BigRational::from_float(1f64 - (1f64 - c).powf(theta))?; - let numer = p.numer().to_biguint()?; - let denom = p.denom().to_biguint()?; - ((BigUint::one() << 128) * numer / denom).to_u128() - }; - - calc().unwrap_or(u128::max_value()) + assert!(theta > 0.0, "authority with weight 0."); + + // NOTE: in the equation `p = 1 - (1 - c)^theta` the value of `p` is always + // capped by `c`. For all pratical purposes `c` should always be set to a + // value < 0.5, as such in the computations below we should never be near + // edge cases like `0.999999`. + + let p = BigRational::from_float(1f64 - (1f64 - c).powf(theta)).expect( + "returns None when the given value is not finite; \ + c is a configuration parameter defined in (0, 1]; \ + theta must be > 0 if the given authority's weight is > 0; \ + theta represents the validator's relative weight defined in (0, 1]; \ + powf will always return values in (0, 1] given both the \ + base and exponent are in that domain; \ + qed.", + ); + + let numer = p.numer().to_biguint().expect( + "returns None when the given value is negative; \ + p is defined as `1 - n` where n is defined in (0, 1]; \ + p must be a value in [0, 1); \ + qed." + ); + + let denom = p.denom().to_biguint().expect( + "returns None when the given value is negative; \ + p is defined as `1 - n` where n is defined in (0, 1]; \ + p must be a value in [0, 1); \ + qed." + ); + + ((BigUint::one() << 128) * numer / denom).to_u128().expect( + "returns None if the underlying value cannot be represented with 128 bits; \ + we start with 2^128 which is one more than can be represented with 128 bits; \ + we multiple by p which is defined in [0, 1); \ + the result must be lower than 2^128 by at least one and thus representable with 128 bits; \ + qed.", + ) } /// Returns true if the given VRF output is lower than the given threshold, @@ -105,10 +137,12 @@ pub(super) fn make_transcript( /// to propose. fn claim_secondary_slot( slot_number: SlotNumber, - authorities: &[(AuthorityId, BabeAuthorityWeight)], + epoch: &Epoch, keystore: &KeyStorePtr, - randomness: [u8; 32], + author_secondary_vrf: bool, ) -> Option<(PreDigest, AuthorityPair)> { + let Epoch { authorities, randomness, epoch_index, .. } = epoch; + if authorities.is_empty() { return None; } @@ -116,7 +150,7 @@ fn claim_secondary_slot( let expected_author = super::authorship::secondary_slot_author( slot_number, authorities, - randomness, + *randomness, )?; let keystore = keystore.read(); @@ -128,10 +162,27 @@ fn claim_secondary_slot( }) { if pair.public() == *expected_author { - let pre_digest = PreDigest::Secondary(SecondaryPreDigest { - slot_number, - authority_index: authority_index as u32, - }); + let pre_digest = if author_secondary_vrf { + let transcript = super::authorship::make_transcript( + randomness, + slot_number, + *epoch_index, + ); + + let s = get_keypair(&pair).vrf_sign(transcript); + + PreDigest::SecondaryVRF(SecondaryVRFPreDigest { + slot_number, + vrf_output: VRFOutput(s.0.to_output()), + vrf_proof: VRFProof(s.1), + authority_index: authority_index as u32, + }) + } else { + PreDigest::SecondaryPlain(SecondaryPlainPreDigest { + slot_number, + authority_index: authority_index as u32, + }) + }; return Some((pre_digest, pair)); } @@ -147,17 +198,18 @@ fn claim_secondary_slot( pub fn claim_slot( slot_number: SlotNumber, epoch: &Epoch, - config: &BabeConfiguration, keystore: &KeyStorePtr, ) -> Option<(PreDigest, AuthorityPair)> { - claim_primary_slot(slot_number, epoch, config.c, keystore) + claim_primary_slot(slot_number, epoch, epoch.config.c, keystore) .or_else(|| { - if config.secondary_slots { + if epoch.config.allowed_slots.is_secondary_plain_slots_allowed() || + epoch.config.allowed_slots.is_secondary_vrf_slots_allowed() + { claim_secondary_slot( slot_number, - &epoch.authorities, + &epoch, keystore, - epoch.randomness, + epoch.config.allowed_slots.is_secondary_vrf_slots_allowed(), ) } else { None diff --git a/client/consensus/babe/src/aux_schema.rs b/client/consensus/babe/src/aux_schema.rs index 6f69e65940c05d60bfef77e839af4887c8910347..2a3f23981dc3026288259f5c90cf4da17c96b625 100644 --- a/client/consensus/babe/src/aux_schema.rs +++ b/client/consensus/babe/src/aux_schema.rs @@ -24,13 +24,13 @@ use codec::{Decode, Encode}; use sc_client_api::backend::AuxStore; use sp_blockchain::{Result as ClientResult, Error as ClientError}; use sp_runtime::traits::Block as BlockT; -use sp_consensus_babe::BabeBlockWeight; +use sp_consensus_babe::{BabeBlockWeight, BabeGenesisConfiguration}; use sc_consensus_epochs::{EpochChangesFor, SharedEpochChanges, migration::EpochChangesForV0}; -use crate::Epoch; +use crate::{Epoch, migration::EpochV0}; const BABE_EPOCH_CHANGES_VERSION: &[u8] = b"babe_epoch_changes_version"; const BABE_EPOCH_CHANGES_KEY: &[u8] = b"babe_epoch_changes"; -const BABE_EPOCH_CHANGES_CURRENT_VERSION: u32 = 1; +const BABE_EPOCH_CHANGES_CURRENT_VERSION: u32 = 2; fn block_weight_key(block_hash: H) -> Vec { (b"block_weight", block_hash).encode() @@ -53,14 +53,19 @@ fn load_decode(backend: &B, key: &[u8]) -> ClientResult> /// Load or initialize persistent epoch change data from backend. pub(crate) fn load_epoch_changes( backend: &B, + config: &BabeGenesisConfiguration, ) -> ClientResult> { let version = load_decode::<_, u32>(backend, BABE_EPOCH_CHANGES_VERSION)?; let maybe_epoch_changes = match version { - None => load_decode::<_, EpochChangesForV0>( + None => load_decode::<_, EpochChangesForV0>( backend, BABE_EPOCH_CHANGES_KEY, - )?.map(|v0| v0.migrate()), + )?.map(|v0| v0.migrate().map(|_, _, epoch| epoch.migrate(config))), + Some(1) => load_decode::<_, EpochChangesFor>( + backend, + BABE_EPOCH_CHANGES_KEY, + )?.map(|v1| v1.map(|_, _, epoch| epoch.migrate(config))), Some(BABE_EPOCH_CHANGES_CURRENT_VERSION) => load_decode::<_, EpochChangesFor>( backend, BABE_EPOCH_CHANGES_KEY, @@ -74,7 +79,7 @@ pub(crate) fn load_epoch_changes( let epoch_changes = Arc::new(Mutex::new(maybe_epoch_changes.unwrap_or_else(|| { info!(target: "babe", - "Creating empty BABE epoch changes on what appears to be first startup." + "👶 Creating empty BABE epoch changes on what appears to be first startup." ); EpochChangesFor::::default() }))); @@ -131,18 +136,19 @@ pub(crate) fn load_block_weight( #[cfg(test)] mod test { use super::*; - use crate::Epoch; + use crate::migration::EpochV0; use fork_tree::ForkTree; use substrate_test_runtime_client; use sp_core::H256; use sp_runtime::traits::NumberFor; + use sp_consensus_babe::{AllowedSlots, BabeGenesisConfiguration}; use sc_consensus_epochs::{PersistedEpoch, PersistedEpochHeader, EpochHeader}; use sp_consensus::Error as ConsensusError; use sc_network_test::Block as TestBlock; #[test] fn load_decode_from_v0_epoch_changes() { - let epoch = Epoch { + let epoch = EpochV0 { start_slot: 0, authorities: vec![], randomness: [0; 32], @@ -160,7 +166,7 @@ mod test { client.insert_aux( &[(BABE_EPOCH_CHANGES_KEY, - &EpochChangesForV0::::from_raw(v0_tree).encode()[..])], + &EpochChangesForV0::::from_raw(v0_tree).encode()[..])], &[], ).unwrap(); @@ -169,7 +175,16 @@ mod test { None, ); - let epoch_changes = load_epoch_changes::(&client).unwrap(); + let epoch_changes = load_epoch_changes::( + &client, &BabeGenesisConfiguration { + slot_duration: 10, + epoch_length: 4, + c: (3, 10), + genesis_authorities: Vec::new(), + randomness: Default::default(), + allowed_slots: AllowedSlots::PrimaryAndSecondaryPlainSlots, + }, + ).unwrap(); assert!( epoch_changes.lock() @@ -192,7 +207,7 @@ mod test { assert_eq!( load_decode::<_, u32>(&client, BABE_EPOCH_CHANGES_VERSION).unwrap(), - Some(1), + Some(2), ); } } diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index 5365aae2aa6cb30793a41ec9d4f7a02d3df6bc9a..028fe8e5e346f0395884d5dc63b5cd5257532db5 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -49,6 +49,11 @@ //! //! `blake2_256(epoch_randomness ++ slot_number) % authorities_len`. //! +//! The secondary slots supports either a `SecondaryPlain` or `SecondaryVRF` +//! variant. Comparing with `SecondaryPlain` variant, the `SecondaryVRF` variant +//! generates an additional VRF output. The output is not included in beacon +//! randomness, but can be consumed by parachains. +//! //! The fork choice rule is weight-based, where weight equals the number of //! primary blocks in the chain. We will pick the heaviest chain (more primary //! blocks) and will go with the longest one in case of a tie. @@ -59,11 +64,13 @@ #![forbid(unsafe_code)] #![warn(missing_docs)] pub use sp_consensus_babe::{ - BabeApi, ConsensusLog, BABE_ENGINE_ID, SlotNumber, BabeConfiguration, + BabeApi, ConsensusLog, BABE_ENGINE_ID, SlotNumber, + BabeEpochConfiguration, BabeGenesisConfiguration, AuthorityId, AuthorityPair, AuthoritySignature, BabeAuthorityWeight, VRF_OUTPUT_LENGTH, digests::{ - CompatibleDigestItem, NextEpochDescriptor, PreDigest, PrimaryPreDigest, SecondaryPreDigest, + CompatibleDigestItem, NextEpochDescriptor, NextConfigDescriptor, PreDigest, + PrimaryPreDigest, SecondaryPlainPreDigest, }, }; pub use sp_consensus::SyncOracle; @@ -101,7 +108,7 @@ use sc_client_api::{ use sp_block_builder::BlockBuilder as BlockBuilderApi; use futures::prelude::*; -use log::{warn, debug, info, trace}; +use log::{debug, info, log, trace, warn}; use sc_consensus_slots::{ SlotWorker, SlotInfo, SlotCompatible, StorageChanges, CheckedHeader, check_equivocation, }; @@ -118,36 +125,43 @@ use sp_api::ApiExt; mod aux_schema; mod verification; +mod migration; pub mod authorship; #[cfg(test)] mod tests; /// BABE epoch information -#[derive(Decode, Encode, Default, PartialEq, Eq, Clone, Debug)] +#[derive(Decode, Encode, PartialEq, Eq, Clone, Debug)] pub struct Epoch { - /// The epoch index + /// The epoch index. pub epoch_index: u64, - /// The starting slot of the epoch, + /// The starting slot of the epoch. pub start_slot: SlotNumber, - /// The duration of this epoch + /// The duration of this epoch. pub duration: SlotNumber, - /// The authorities and their weights + /// The authorities and their weights. pub authorities: Vec<(AuthorityId, BabeAuthorityWeight)>, - /// Randomness for this epoch + /// Randomness for this epoch. pub randomness: [u8; VRF_OUTPUT_LENGTH], + /// Configuration of the epoch. + pub config: BabeEpochConfiguration, } impl EpochT for Epoch { - type NextEpochDescriptor = NextEpochDescriptor; + type NextEpochDescriptor = (NextEpochDescriptor, BabeEpochConfiguration); type SlotNumber = SlotNumber; - fn increment(&self, descriptor: NextEpochDescriptor) -> Epoch { + fn increment( + &self, + (descriptor, config): (NextEpochDescriptor, BabeEpochConfiguration) + ) -> Epoch { Epoch { epoch_index: self.epoch_index + 1, start_slot: self.start_slot + self.duration, duration: self.duration, authorities: descriptor.authorities, randomness: descriptor.randomness, + config, } } @@ -160,6 +174,27 @@ impl EpochT for Epoch { } } +impl Epoch { + /// Create the genesis epoch (epoch #0). This is defined to start at the slot of + /// the first block, so that has to be provided. + pub fn genesis( + genesis_config: &BabeGenesisConfiguration, + slot_number: SlotNumber + ) -> Epoch { + Epoch { + epoch_index: 0, + start_slot: slot_number, + duration: genesis_config.epoch_length, + authorities: genesis_config.genesis_authorities.clone(), + randomness: genesis_config.randomness.clone(), + config: BabeEpochConfiguration { + c: genesis_config.c, + allowed_slots: genesis_config.allowed_slots, + }, + } + } +} + #[derive(derive_more::Display, Debug)] enum Error { #[display(fmt = "Multiple BABE pre-runtime digests, rejecting!")] @@ -168,6 +203,8 @@ enum Error { NoPreRuntimeDigest, #[display(fmt = "Multiple BABE epoch change digests, rejecting!")] MultipleEpochChangeDigests, + #[display(fmt = "Multiple BABE config change digests, rejecting!")] + MultipleConfigChangeDigests, #[display(fmt = "Could not extract timestamp and slot: {:?}", _0)] Extraction(sp_consensus::Error), #[display(fmt = "Could not fetch epoch at {:?}", _0)] @@ -200,6 +237,8 @@ enum Error { FetchParentHeader(sp_blockchain::Error), #[display(fmt = "Expected epoch change to happen at {:?}, s{}", _0, _1)] ExpectedEpochChange(B::Hash, u64), + #[display(fmt = "Unexpected config change")] + UnexpectedConfigChange, #[display(fmt = "Unexpected epoch change")] UnexpectedEpochChange, #[display(fmt = "Parent block of {} has no associated weight", _0)] @@ -222,16 +261,6 @@ fn babe_err(error: Error) -> Error { error } -macro_rules! babe_info { - ($($i: expr),+) => { - { - info!(target: "babe", $($i),+); - format!($($i),+) - } - }; -} - - /// Intermediate value passed to block importer. pub struct BabeIntermediate { /// The epoch descriptor. @@ -246,7 +275,7 @@ pub static INTERMEDIATE_KEY: &[u8] = b"babe1"; // and `super::babe::Config` can be eliminated. // https://github.com/paritytech/substrate/issues/2434 #[derive(Clone)] -pub struct Config(sc_consensus_slots::SlotDuration); +pub struct Config(sc_consensus_slots::SlotDuration); impl Config { /// Either fetch the slot duration from disk or compute it from the genesis @@ -255,7 +284,26 @@ impl Config { C: AuxStore + ProvideRuntimeApi, C::Api: BabeApi, { trace!(target: "babe", "Getting slot duration"); - match sc_consensus_slots::SlotDuration::get_or_compute(client, |a, b| a.configuration(b)).map(Self) { + match sc_consensus_slots::SlotDuration::get_or_compute(client, |a, b| { + let has_api_v1 = a.has_api_with::, _>( + &b, |v| v == 1, + )?; + let has_api_v2 = a.has_api_with::, _>( + &b, |v| v == 2, + )?; + + if has_api_v1 { + #[allow(deprecated)] { + Ok(a.configuration_before_version_2(b)?.into()) + } + } else if has_api_v2 { + a.configuration(b) + } else { + Err(sp_blockchain::Error::VersionInvalid( + "Unsupported or invalid BabeApi version".to_string() + )) + } + }).map(Self) { Ok(s) => Ok(s), Err(s) => { warn!(target: "babe", "Failed to get slot duration"); @@ -263,24 +311,12 @@ impl Config { } } } - - /// Create the genesis epoch (epoch #0). This is defined to start at the slot of - /// the first block, so that has to be provided. - pub fn genesis_epoch(&self, slot_number: SlotNumber) -> Epoch { - Epoch { - epoch_index: 0, - start_slot: slot_number, - duration: self.epoch_length, - authorities: self.genesis_authorities.clone(), - randomness: self.randomness.clone(), - } - } } impl std::ops::Deref for Config { - type Target = BabeConfiguration; + type Target = BabeGenesisConfiguration; - fn deref(&self) -> &BabeConfiguration { + fn deref(&self) -> &BabeGenesisConfiguration { &*self.0 } } @@ -368,7 +404,7 @@ pub fn start_babe(BabeParams { &inherent_data_providers, )?; - babe_info!("👶 Starting BABE Authorship worker"); + info!(target: "babe", "👶 Starting BABE Authorship worker"); Ok(sc_consensus_slots::start_slot_worker( config.0, select_chain, @@ -438,7 +474,7 @@ impl sc_consensus_slots::SimpleSlotWorker for BabeWork fn authorities_len(&self, epoch_descriptor: &Self::EpochData) -> Option { self.epoch_changes.lock() - .viable_epoch(&epoch_descriptor, |slot| self.config.genesis_epoch(slot)) + .viable_epoch(&epoch_descriptor, |slot| Epoch::genesis(&self.config, slot)) .map(|epoch| epoch.as_ref().authorities.len()) } @@ -453,9 +489,8 @@ impl sc_consensus_slots::SimpleSlotWorker for BabeWork slot_number, self.epoch_changes.lock().viable_epoch( &epoch_descriptor, - |slot| self.config.genesis_epoch(slot) + |slot| Epoch::genesis(&self.config, slot) )?.as_ref(), - &*self.config, &self.keystore, ); @@ -520,38 +555,28 @@ impl sc_consensus_slots::SimpleSlotWorker for BabeWork fn proposing_remaining_duration( &self, head: &B::Header, - slot_info: &SlotInfo + slot_info: &SlotInfo, ) -> Option { - // never give more than 2^this times the lenience. - const BACKOFF_CAP: u64 = 8; - - // how many slots it takes before we double the lenience. - const BACKOFF_STEP: u64 = 2; - let slot_remaining = self.slot_remaining_duration(slot_info); + let parent_slot = match find_pre_digest::(head) { Err(_) => return Some(slot_remaining), Ok(d) => d.slot_number(), }; - // we allow a lenience of the number of slots since the head of the - // chain was produced, minus 1 (since there is always a difference of at least 1) - // - // exponential back-off. - // in normal cases we only attempt to issue blocks up to the end of the slot. - // when the chain has been stalled for a few slots, we give more lenience. - let slot_lenience = slot_info.number.saturating_sub(parent_slot + 1); - - let slot_lenience = std::cmp::min(slot_lenience, BACKOFF_CAP); - let slot_duration = slot_info.duration << (slot_lenience / BACKOFF_STEP); + if let Some(slot_lenience) = + sc_consensus_slots::slot_lenience_exponential(parent_slot, slot_info) + { + debug!(target: "babe", + "No block for {} slots. Applying exponential lenience of {}s", + slot_info.number.saturating_sub(parent_slot + 1), + slot_lenience.as_secs(), + ); - if slot_lenience >= 1 { - debug!(target: "babe", "No block for {} slots. Applying 2^({}/{}) lenience", - slot_lenience, slot_lenience, BACKOFF_STEP); + Some(slot_remaining + slot_lenience) + } else { + Some(slot_remaining) } - - let slot_lenience = Duration::from_secs(slot_duration); - Some(slot_lenience + slot_remaining) } } @@ -582,7 +607,7 @@ fn find_pre_digest(header: &B::Header) -> Result> // genesis block doesn't contain a pre digest so let's generate a // dummy one to not break any invariants in the rest of the code if header.number().is_zero() { - return Ok(PreDigest::Secondary(SecondaryPreDigest { + return Ok(PreDigest::SecondaryPlain(SecondaryPlainPreDigest { slot_number: 0, authority_index: 0, })); @@ -619,6 +644,24 @@ fn find_next_epoch_digest(header: &B::Header) Ok(epoch_digest) } +/// Extract the BABE config change digest from the given header, if it exists. +fn find_next_config_digest(header: &B::Header) + -> Result, Error> + where DigestItemFor: CompatibleDigestItem, +{ + let mut config_digest: Option<_> = None; + for log in header.digest().logs() { + trace!(target: "babe", "Checking log {:?}, looking for epoch change digest.", log); + let log = log.try_to::(OpaqueDigestItemId::Consensus(&BABE_ENGINE_ID)); + match (log, config_digest.is_some()) { + (Some(ConsensusLog::NextConfigData(_)), true) => return Err(babe_err(Error::MultipleConfigChangeDigests)), + (Some(ConsensusLog::NextConfigData(config)), false) => config_digest = Some(config), + _ => trace!(target: "babe", "Ignoring digest not meant for us"), + } + } + + Ok(config_digest) +} #[derive(Default, Clone)] struct TimeSource(Arc, Vec<(Instant, u64)>)>>); @@ -746,7 +789,7 @@ impl Verifier for BabeVerifier where .ok_or_else(|| Error::::FetchEpoch(parent_hash))?; let viable_epoch = epoch_changes.viable_epoch( &epoch_descriptor, - |slot| self.config.genesis_epoch(slot) + |slot| Epoch::genesis(&self.config, slot) ).ok_or_else(|| Error::::FetchEpoch(parent_hash))?; // We add one to the current slot to allow for some small drift. @@ -756,7 +799,6 @@ impl Verifier for BabeVerifier where pre_digest: Some(pre_digest.clone()), slot_now: slot_now + 1, epoch: viable_epoch.as_ref(), - config: &self.config, }; match verification::check_header::(v_params)? { @@ -978,19 +1020,32 @@ impl BlockImport for BabeBlockImport(&block.header) .map_err(|e| ConsensusError::ClientImport(e.to_string()))?; + let next_config_digest = find_next_config_digest::(&block.header) + .map_err(|e| ConsensusError::ClientImport(e.to_string()))?; - match (first_in_epoch, next_epoch_digest.is_some()) { - (true, true) => {}, - (false, false) => {}, - (true, false) => { + match (first_in_epoch, next_epoch_digest.is_some(), next_config_digest.is_some()) { + (true, true, _) => {}, + (false, false, false) => {}, + (false, false, true) => { + return Err( + ConsensusError::ClientImport( + babe_err(Error::::UnexpectedConfigChange).into(), + ) + ) + }, + (true, false, _) => { return Err( ConsensusError::ClientImport( babe_err(Error::::ExpectedEpochChange(hash, slot_number)).into(), ) - ); + ) }, - (false, true) => { - return Err(ConsensusError::ClientImport(Error::::UnexpectedEpochChange.into())); + (false, true, _) => { + return Err( + ConsensusError::ClientImport( + babe_err(Error::::UnexpectedEpochChange).into(), + ) + ) }, } @@ -1005,20 +1060,38 @@ impl BlockImport for BabeBlockImport::FetchEpoch(parent_hash).into()) })?; - babe_info!("👶 New epoch {} launching at block {} (block slot {} >= start slot {}).", - viable_epoch.as_ref().epoch_index, - hash, - slot_number, - viable_epoch.as_ref().start_slot); + let epoch_config = next_config_digest.map(Into::into).unwrap_or_else( + || viable_epoch.as_ref().config.clone() + ); + + // restrict info logging during initial sync to avoid spam + let log_level = if block.origin == BlockOrigin::NetworkInitialSync { + log::Level::Debug + } else { + log::Level::Info + }; + + log!(target: "babe", + log_level, + "👶 New epoch {} launching at block {} (block slot {} >= start slot {}).", + viable_epoch.as_ref().epoch_index, + hash, + slot_number, + viable_epoch.as_ref().start_slot, + ); - let next_epoch = viable_epoch.increment(next_epoch_descriptor); + let next_epoch = viable_epoch.increment((next_epoch_descriptor, epoch_config)); - babe_info!("👶 Next epoch starts at slot {}", next_epoch.as_ref().start_slot); + log!(target: "babe", + log_level, + "👶 Next epoch starts at slot {}", + next_epoch.as_ref().start_slot, + ); // prune the tree of epochs not part of the finalized chain or // that are not live anymore, and then track the given epoch change @@ -1158,7 +1231,7 @@ pub fn block_import( ) -> ClientResult<(BabeBlockImport, BabeLink)> where Client: AuxStore + HeaderBackend + HeaderMetadata, { - let epoch_changes = aux_schema::load_epoch_changes::(&*client)?; + let epoch_changes = aux_schema::load_epoch_changes::(&*client, &config)?; let link = BabeLink { epoch_changes: epoch_changes.clone(), time_source: Default::default(), @@ -1251,13 +1324,12 @@ pub mod test_helpers { &parent.hash(), parent.number().clone(), slot_number, - |slot| link.config.genesis_epoch(slot), + |slot| Epoch::genesis(&link.config, slot), ).unwrap().unwrap(); authorship::claim_slot( slot_number, &epoch, - &link.config, keystore, ).map(|(digest, _)| digest) } diff --git a/client/consensus/babe/src/migration.rs b/client/consensus/babe/src/migration.rs new file mode 100644 index 0000000000000000000000000000000000000000..2a5a8749cc3c1f790b6a774ce32e5f392ba38ae0 --- /dev/null +++ b/client/consensus/babe/src/migration.rs @@ -0,0 +1,64 @@ +use codec::{Encode, Decode}; +use sc_consensus_epochs::Epoch as EpochT; +use crate::{ + Epoch, SlotNumber, AuthorityId, BabeAuthorityWeight, BabeGenesisConfiguration, + BabeEpochConfiguration, VRF_OUTPUT_LENGTH, NextEpochDescriptor, +}; + +/// BABE epoch information, version 0. +#[derive(Decode, Encode, PartialEq, Eq, Clone, Debug)] +pub struct EpochV0 { + /// The epoch index. + pub epoch_index: u64, + /// The starting slot of the epoch. + pub start_slot: SlotNumber, + /// The duration of this epoch. + pub duration: SlotNumber, + /// The authorities and their weights. + pub authorities: Vec<(AuthorityId, BabeAuthorityWeight)>, + /// Randomness for this epoch. + pub randomness: [u8; VRF_OUTPUT_LENGTH], +} + +impl EpochT for EpochV0 { + type NextEpochDescriptor = NextEpochDescriptor; + type SlotNumber = SlotNumber; + + fn increment( + &self, + descriptor: NextEpochDescriptor + ) -> EpochV0 { + EpochV0 { + epoch_index: self.epoch_index + 1, + start_slot: self.start_slot + self.duration, + duration: self.duration, + authorities: descriptor.authorities, + randomness: descriptor.randomness, + } + } + + fn start_slot(&self) -> SlotNumber { + self.start_slot + } + + fn end_slot(&self) -> SlotNumber { + self.start_slot + self.duration + } +} + +impl EpochV0 { + /// Migrate the sturct to current epoch version. + pub fn migrate(self, config: &BabeGenesisConfiguration) -> Epoch { + Epoch { + epoch_index: self.epoch_index, + start_slot: self.start_slot, + duration: self.duration, + authorities: self.authorities, + randomness: self.randomness, + config: BabeEpochConfiguration { + c: config.c, + allowed_slots: config.allowed_slots, + }, + } + } +} diff --git a/client/consensus/babe/src/tests.rs b/client/consensus/babe/src/tests.rs index 20b924669d6147816c0fbf9c0c8e85937b17c1af..89514906be413c0cbd761b5c43dc932fe4c82572 100644 --- a/client/consensus/babe/src/tests.rs +++ b/client/consensus/babe/src/tests.rs @@ -22,7 +22,7 @@ use super::*; use authorship::claim_slot; -use sp_consensus_babe::{AuthorityPair, SlotNumber}; +use sp_consensus_babe::{AuthorityPair, SlotNumber, AllowedSlots}; use sc_block_builder::{BlockBuilder, BlockBuilderProvider}; use sp_consensus::{ NoNetwork as DummyOracle, Proposal, RecordProof, @@ -40,7 +40,7 @@ type Item = DigestItem; type Error = sp_blockchain::Error; -type TestClient = sc_client::Client< +type TestClient = substrate_test_runtime_client::client::Client< substrate_test_runtime_client::Backend, substrate_test_runtime_client::Executor, TestBlock, @@ -127,7 +127,7 @@ impl DummyProposer { &self.parent_hash, self.parent_number, this_slot, - |slot| self.factory.config.genesis_epoch(slot), + |slot| Epoch::genesis(&self.factory.config, slot), ) .expect("client has data to find epoch") .expect("can compute epoch for baked block"); @@ -505,28 +505,32 @@ fn can_author_block() { randomness: [0; 32], epoch_index: 1, duration: 100, + config: BabeEpochConfiguration { + c: (3, 10), + allowed_slots: AllowedSlots::PrimaryAndSecondaryPlainSlots, + }, }; - let mut config = crate::BabeConfiguration { + let mut config = crate::BabeGenesisConfiguration { slot_duration: 1000, epoch_length: 100, c: (3, 10), genesis_authorities: Vec::new(), randomness: [0; 32], - secondary_slots: true, + allowed_slots: AllowedSlots::PrimaryAndSecondaryPlainSlots, }; // with secondary slots enabled it should never be empty - match claim_slot(i, &epoch, &config, &keystore) { + match claim_slot(i, &epoch, &keystore) { None => i += 1, Some(s) => debug!(target: "babe", "Authored block {:?}", s.0), } // otherwise with only vrf-based primary slots we might need to try a couple // of times. - config.secondary_slots = false; + config.allowed_slots = AllowedSlots::PrimarySlots; loop { - match claim_slot(i, &epoch, &config, &keystore) { + match claim_slot(i, &epoch, &keystore) { None => i += 1, Some(s) => { debug!(target: "babe", "Authored block {:?}", s.0); @@ -553,7 +557,7 @@ fn propose_and_import_block( let pre_digest = sp_runtime::generic::Digest { logs: vec![ Item::babe_pre_digest( - PreDigest::Secondary(SecondaryPreDigest { + PreDigest::SecondaryPlain(SecondaryPlainPreDigest { authority_index: 0, slot_number, }), @@ -632,7 +636,7 @@ fn importing_block_one_sets_genesis_epoch() { &mut block_import, ); - let genesis_epoch = data.link.config.genesis_epoch(999); + let genesis_epoch = Epoch::genesis(&data.link.config, 999); let epoch_changes = data.link.epoch_changes.lock(); let epoch_for_second_block = epoch_changes.epoch_data_for_child_of( @@ -640,7 +644,7 @@ fn importing_block_one_sets_genesis_epoch() { &block_hash, 1, 1000, - |slot| data.link.config.genesis_epoch(slot), + |slot| Epoch::genesis(&data.link.config, slot), ).unwrap().unwrap(); assert_eq!(epoch_for_second_block, genesis_epoch); diff --git a/client/consensus/babe/src/verification.rs b/client/consensus/babe/src/verification.rs index 2fd37280b3b369bdb1e73895dc9ae683030eaa23..1b89bbc643fc1b1a19e0296ac1049d1312b2de75 100644 --- a/client/consensus/babe/src/verification.rs +++ b/client/consensus/babe/src/verification.rs @@ -19,7 +19,8 @@ use sp_runtime::{traits::Header, traits::DigestItemFor}; use sp_core::{Pair, Public}; use sp_consensus_babe::{AuthoritySignature, SlotNumber, AuthorityPair, AuthorityId}; use sp_consensus_babe::digests::{ - PreDigest, PrimaryPreDigest, SecondaryPreDigest, CompatibleDigestItem + PreDigest, PrimaryPreDigest, SecondaryPlainPreDigest, SecondaryVRFPreDigest, + CompatibleDigestItem }; use sc_consensus_slots::CheckedHeader; use log::{debug, trace}; @@ -28,18 +29,16 @@ use super::authorship::{make_transcript, calculate_primary_threshold, check_prim /// BABE verification parameters pub(super) struct VerificationParams<'a, B: 'a + BlockT> { - /// the header being verified. + /// The header being verified. pub(super) header: B::Header, - /// the pre-digest of the header being verified. this is optional - if prior + /// The pre-digest of the header being verified. this is optional - if prior /// verification code had to read it, it can be included here to avoid duplicate /// work. pub(super) pre_digest: Option, - /// the slot number of the current time. + /// The slot number of the current time. pub(super) slot_now: SlotNumber, - /// epoch descriptor of the epoch this block _should_ be under, if it's valid. + /// Epoch descriptor of the epoch this block _should_ be under, if it's valid. pub(super) epoch: &'a Epoch, - /// genesis config of this BABE chain. - pub(super) config: &'a super::Config, } /// Check a header has been signed by the right key. If the slot is too far in @@ -63,7 +62,6 @@ pub(super) fn check_header( pre_digest, slot_now, epoch, - config, } = params; let authorities = &epoch.authorities; @@ -102,13 +100,21 @@ pub(super) fn check_header( primary, sig, &epoch, - config.c, + epoch.config.c, )?; }, - PreDigest::Secondary(secondary) if config.secondary_slots => { - debug!(target: "babe", "Verifying Secondary block"); - - check_secondary_header::( + PreDigest::SecondaryPlain(secondary) if epoch.config.allowed_slots.is_secondary_plain_slots_allowed() => { + debug!(target: "babe", "Verifying Secondary plain block"); + check_secondary_plain_header::( + pre_hash, + secondary, + sig, + &epoch, + )?; + }, + PreDigest::SecondaryVRF(secondary) if epoch.config.allowed_slots.is_secondary_vrf_slots_allowed() => { + debug!(target: "babe", "Verifying Secondary VRF block"); + check_secondary_vrf_header::( pre_hash, secondary, sig, @@ -182,9 +188,9 @@ fn check_primary_header( /// properly signed by the expected authority, which we have a deterministic way /// of computing. Additionally, the weight of this block must stay the same /// compared to its parent since it is a secondary block. -fn check_secondary_header( +fn check_secondary_plain_header( pre_hash: B::Hash, - pre_digest: &SecondaryPreDigest, + pre_digest: &SecondaryPlainPreDigest, signature: AuthoritySignature, epoch: &Epoch, ) -> Result<(), Error> { @@ -208,3 +214,43 @@ fn check_secondary_header( Err(Error::BadSignature(pre_hash)) } } + +/// Check a secondary VRF slot proposal header. +fn check_secondary_vrf_header( + pre_hash: B::Hash, + pre_digest: &SecondaryVRFPreDigest, + signature: AuthoritySignature, + epoch: &Epoch, +) -> Result<(), Error> { + // check the signature is valid under the expected authority and + // chain state. + let expected_author = secondary_slot_author( + pre_digest.slot_number, + &epoch.authorities, + epoch.randomness, + ).ok_or_else(|| Error::NoSecondaryAuthorExpected)?; + + let author = &epoch.authorities[pre_digest.authority_index as usize].0; + + if expected_author != author { + return Err(Error::InvalidAuthor(expected_author.clone(), author.clone())); + } + + if AuthorityPair::verify(&signature, pre_hash.as_ref(), author) { + let transcript = make_transcript( + &epoch.randomness, + pre_digest.slot_number, + epoch.epoch_index, + ); + + schnorrkel::PublicKey::from_bytes(author.as_slice()).and_then(|p| { + p.vrf_verify(transcript, &pre_digest.vrf_output, &pre_digest.vrf_proof) + }).map_err(|s| { + babe_err(Error::VRFVerificationFailed(s)) + })?; + + Ok(()) + } else { + Err(Error::BadSignature(pre_hash)) + } +} diff --git a/client/consensus/common/Cargo.toml b/client/consensus/common/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..e8854faa078c0770ca65e414a705c7f763dcd594 --- /dev/null +++ b/client/consensus/common/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "sc-consensus" +version = "0.8.0-dev" +authors = ["Parity Technologies "] +edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +sc-client-api = { version = "2.0.0-dev", path = "../../api" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-consensus = { version = "0.8.0-dev", path = "../../../primitives/consensus/common" } diff --git a/client/consensus/common/src/lib.rs b/client/consensus/common/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..9bfe7e56d4f3498f68747dba6207d155f592b949 --- /dev/null +++ b/client/consensus/common/src/lib.rs @@ -0,0 +1,19 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . +//! Collection of consensus specific imlementations +mod longest_chain; + +pub use longest_chain::LongestChain; \ No newline at end of file diff --git a/client/consensus/common/src/longest_chain.rs b/client/consensus/common/src/longest_chain.rs new file mode 100644 index 0000000000000000000000000000000000000000..981dbad0f607029671b7ad858f2327578fa24e94 --- /dev/null +++ b/client/consensus/common/src/longest_chain.rs @@ -0,0 +1,101 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . +//! Longest chain implementation + +use std::sync::Arc; +use std::marker::PhantomData; +use sc_client_api::backend; +use sp_consensus::{SelectChain, Error as ConsensusError}; +use sp_blockchain::{Backend, HeaderBackend}; +use sp_runtime::{ + traits::{NumberFor, Block as BlockT}, + generic::BlockId, +}; + +/// Implement Longest Chain Select implementation +/// where 'longest' is defined as the highest number of blocks +pub struct LongestChain { + backend: Arc, + _phantom: PhantomData +} + +impl Clone for LongestChain { + fn clone(&self) -> Self { + let backend = self.backend.clone(); + LongestChain { + backend, + _phantom: Default::default() + } + } +} + +impl LongestChain + where + B: backend::Backend, + Block: BlockT, +{ + /// Instantiate a new LongestChain for Backend B + pub fn new(backend: Arc) -> Self { + LongestChain { + backend, + _phantom: Default::default() + } + } + + fn best_block_header(&self) -> sp_blockchain::Result<::Header> { + let info = self.backend.blockchain().info(); + let import_lock = self.backend.get_import_lock(); + let best_hash = self.backend + .blockchain() + .best_containing(info.best_hash, None, import_lock)? + .unwrap_or(info.best_hash); + + Ok(self.backend.blockchain().header(BlockId::Hash(best_hash))? + .expect("given block hash was fetched from block in db; qed")) + } + + fn leaves(&self) -> Result::Hash>, sp_blockchain::Error> { + self.backend.blockchain().leaves() + } +} + +impl SelectChain for LongestChain + where + B: backend::Backend, + Block: BlockT, +{ + + fn leaves(&self) -> Result::Hash>, ConsensusError> { + LongestChain::leaves(self) + .map_err(|e| ConsensusError::ChainLookup(e.to_string()).into()) + } + + fn best_chain(&self) -> Result<::Header, ConsensusError> + { + LongestChain::best_block_header(&self) + .map_err(|e| ConsensusError::ChainLookup(e.to_string()).into()) + } + + fn finality_target( + &self, + target_hash: Block::Hash, + maybe_max_number: Option> + ) -> Result, ConsensusError> { + let import_lock = self.backend.get_import_lock(); + self.backend.blockchain().best_containing(target_hash, maybe_max_number, import_lock) + .map_err(|e| ConsensusError::ChainLookup(e.to_string()).into()) + } +} \ No newline at end of file diff --git a/client/consensus/epochs/Cargo.toml b/client/consensus/epochs/Cargo.toml index 3496141ec712bdc4c8daa043e10faaecb13d8b7e..de14b5c6be617ff01038abbaeb5c833aa81d05d8 100644 --- a/client/consensus/epochs/Cargo.toml +++ b/client/consensus/epochs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-epochs" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "Generic epochs-based utilities for consensus" edition = "2018" @@ -8,13 +8,13 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } parking_lot = "0.10.0" -fork-tree = { version = "2.0.0-alpha.5", path = "../../../utils/fork-tree" } -sp-runtime = { path = "../../../primitives/runtime" , version = "2.0.0-alpha.5"} -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } -sc-client-api = { path = "../../api" , version = "2.0.0-alpha.5"} - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +fork-tree = { version = "2.0.0-dev", path = "../../../utils/fork-tree" } +sp-runtime = { path = "../../../primitives/runtime" , version = "2.0.0-dev"} +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } +sc-client-api = { path = "../../api" , version = "2.0.0-dev"} diff --git a/client/consensus/epochs/src/lib.rs b/client/consensus/epochs/src/lib.rs index 001c172b3490573dea6a0dcfbefc428cf79c13aa..acb07dd668a3c4f3b2c7a2350451d9ac93008926 100644 --- a/client/consensus/epochs/src/lib.rs +++ b/client/consensus/epochs/src/lib.rs @@ -335,6 +335,55 @@ impl EpochChanges where self.inner.rebalance() } + /// Map the epoch changes from one storing data to a different one. + pub fn map(self, mut f: F) -> EpochChanges where + B: Epoch, + F: FnMut(&Hash, &Number, E) -> B, + { + EpochChanges { + inner: self.inner.map(&mut |_, _, header| { + match header { + PersistedEpochHeader::Genesis(epoch_0, epoch_1) => { + PersistedEpochHeader::Genesis( + EpochHeader { + start_slot: epoch_0.start_slot, + end_slot: epoch_0.end_slot, + }, + EpochHeader { + start_slot: epoch_1.start_slot, + end_slot: epoch_1.end_slot, + }, + ) + }, + PersistedEpochHeader::Regular(epoch_n) => { + PersistedEpochHeader::Regular( + EpochHeader { + start_slot: epoch_n.start_slot, + end_slot: epoch_n.end_slot, + }, + ) + }, + } + }), + epochs: self.epochs.into_iter().map(|((hash, number), epoch)| { + let bepoch = match epoch { + PersistedEpoch::Genesis(epoch_0, epoch_1) => { + PersistedEpoch::Genesis( + f(&hash, &number, epoch_0), + f(&hash, &number, epoch_1), + ) + }, + PersistedEpoch::Regular(epoch_n) => { + PersistedEpoch::Regular( + f(&hash, &number, epoch_n) + ) + }, + }; + ((hash, number), bepoch) + }).collect(), + } + } + /// Prune out finalized epochs, except for the ancestor of the finalized /// block. The given slot should be the slot number at which the finalized /// block was authored. diff --git a/client/consensus/manual-seal/Cargo.toml b/client/consensus/manual-seal/Cargo.toml index b7e5f7b0b1a9ff7c3f14303fa8c0065aa572048d..807c370edf332cc6602b28bccf38efe6a73c3bed 100644 --- a/client/consensus/manual-seal/Cargo.toml +++ b/client/consensus/manual-seal/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-manual-seal" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "Manual sealing engine for Substrate" edition = "2018" @@ -8,6 +8,9 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] derive_more = "0.99.2" futures = "0.3.4" @@ -19,22 +22,18 @@ parking_lot = "0.10.0" serde = { version = "1.0", features=["derive"] } assert_matches = "1.3.0" -sc-client = { path = "../../../client" , version = "0.8.0-alpha.5"} -sc-client-api = { path = "../../../client/api" , version = "2.0.0-alpha.5"} -sc-transaction-pool = { path = "../../transaction-pool" , version = "2.0.0-alpha.5"} -sp-blockchain = { path = "../../../primitives/blockchain" , version = "2.0.0-alpha.5"} -sp-consensus = { package = "sp-consensus", path = "../../../primitives/consensus/common" , version = "0.8.0-alpha.5"} -sp-inherents = { path = "../../../primitives/inherents" , version = "2.0.0-alpha.5"} -sp-runtime = { path = "../../../primitives/runtime" , version = "2.0.0-alpha.5"} -sp-transaction-pool = { path = "../../../primitives/transaction-pool" , version = "2.0.0-alpha.5"} +sc-client-api = { path = "../../../client/api" , version = "2.0.0-dev"} +sc-transaction-pool = { path = "../../transaction-pool" , version = "2.0.0-dev"} +sp-blockchain = { path = "../../../primitives/blockchain" , version = "2.0.0-dev"} +sp-consensus = { package = "sp-consensus", path = "../../../primitives/consensus/common" , version = "0.8.0-dev"} +sp-inherents = { path = "../../../primitives/inherents" , version = "2.0.0-dev"} +sp-runtime = { path = "../../../primitives/runtime" , version = "2.0.0-dev"} +sp-transaction-pool = { path = "../../../primitives/transaction-pool" , version = "2.0.0-dev"} [dev-dependencies] -sc-basic-authorship = { path = "../../basic-authorship" , version = "0.8.0-alpha.5"} +sc-basic-authorship = { path = "../../basic-authorship" , version = "0.8.0-dev"} substrate-test-runtime-client = { path = "../../../test-utils/runtime/client" , version = "2.0.0-dev"} substrate-test-runtime-transaction-pool = { path = "../../../test-utils/runtime/transaction-pool" , version = "2.0.0-dev"} tokio = { version = "0.2", features = ["rt-core", "macros"] } env_logger = "0.7.0" tempfile = "3.1.0" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/consensus/manual-seal/src/finalize_block.rs b/client/consensus/manual-seal/src/finalize_block.rs index b3b60e223805b857c23602ae718a2347c8cb3291..5780a25f97256331eb1d11b2370dd838ec112ac7 100644 --- a/client/consensus/manual-seal/src/finalize_block.rs +++ b/client/consensus/manual-seal/src/finalize_block.rs @@ -23,35 +23,40 @@ use sp_runtime::{ generic::BlockId, }; use std::sync::Arc; -use sc_client_api::backend::Backend as ClientBackend; +use sc_client_api::backend::{Backend as ClientBackend, Finalizer}; +use std::marker::PhantomData; /// params for block finalization. -pub struct FinalizeBlockParams { +pub struct FinalizeBlockParams { /// hash of the block pub hash: ::Hash, /// sender to report errors/success to the rpc. pub sender: rpc::Sender<()>, /// finalization justification pub justification: Option, - /// client backend - pub backend: Arc, + /// Finalizer trait object. + pub finalizer: Arc, + /// phantom type to pin the Backend type + pub _phantom: PhantomData, } + /// finalizes a block in the backend with the given params. -pub async fn finalize_block(params: FinalizeBlockParams) +pub async fn finalize_block(params: FinalizeBlockParams) where B: BlockT, + F: Finalizer, CB: ClientBackend, { let FinalizeBlockParams { hash, mut sender, justification, - backend: back_end, + finalizer, .. } = params; - match back_end.finalize_block(BlockId::Hash(hash), justification) { + match finalizer.finalize_block(BlockId::Hash(hash), justification, true) { Err(e) => { log::warn!("Failed to finalize block {:?}", e); rpc::send_result(&mut sender, Err(e.into())) diff --git a/client/consensus/manual-seal/src/lib.rs b/client/consensus/manual-seal/src/lib.rs index f3a0ca887fd005ed34ade733ed13d9e8e68898b9..687d072aaa0586f3906e893d93a1d4f0f5f60305 100644 --- a/client/consensus/manual-seal/src/lib.rs +++ b/client/consensus/manual-seal/src/lib.rs @@ -17,66 +17,32 @@ //! A manual sealing engine: the engine listens for rpc calls to seal blocks and create forks. //! This is suitable for a testing environment. +use futures::prelude::*; use sp_consensus::{ - self, BlockImport, Environment, Proposer, BlockCheckParams, - ForkChoiceStrategy, BlockImportParams, BlockOrigin, - ImportResult, SelectChain, - import_queue::{ - BasicQueue, - CacheKeyId, - Verifier, - BoxBlockImport, - }, + Environment, Proposer, ForkChoiceStrategy, BlockImportParams, BlockOrigin, SelectChain, + import_queue::{BasicQueue, CacheKeyId, Verifier, BoxBlockImport}, }; +use sp_blockchain::HeaderBackend; use sp_inherents::InherentDataProviders; use sp_runtime::{traits::Block as BlockT, Justification}; -use sc_client_api::backend::Backend as ClientBackend; -use futures::prelude::*; +use sc_client_api::backend::{Backend as ClientBackend, Finalizer}; use sc_transaction_pool::txpool; -use std::collections::HashMap; -use std::sync::Arc; +use std::{sync::Arc, marker::PhantomData}; -pub mod rpc; mod error; mod finalize_block; mod seal_new_block; -use finalize_block::{finalize_block, FinalizeBlockParams}; -use seal_new_block::{seal_new_block, SealBlockParams}; -pub use error::Error; -pub use rpc::{EngineCommand, CreatedBlock}; - -/// The synchronous block-import worker of the engine. -pub struct ManualSealBlockImport { - inner: I, -} - -impl From for ManualSealBlockImport { - fn from(i: I) -> Self { - ManualSealBlockImport { inner: i } - } -} - -impl BlockImport for ManualSealBlockImport - where - B: BlockT, - I: BlockImport, -{ - type Error = I::Error; - type Transaction = (); - - fn check_block(&mut self, block: BlockCheckParams) -> Result - { - self.inner.check_block(block) - } +pub mod rpc; - fn import_block( - &mut self, - block: BlockImportParams, - cache: HashMap>, - ) -> Result { - self.inner.import_block(block, cache) - } -} +use self::{ + finalize_block::{finalize_block, FinalizeBlockParams}, + seal_new_block::{seal_new_block, SealBlockParams}, +}; +pub use self::{ + error::Error, + rpc::{EngineCommand, CreatedBlock}, +}; +use sc_client_api::{TransactionFor, Backend}; /// The verifier for the manual seal engine; instantly finalizes. struct ManualSealVerifier; @@ -92,7 +58,7 @@ impl Verifier for ManualSealVerifier { let mut import_params = BlockImportParams::new(origin, header); import_params.justification = justification; import_params.body = body; - import_params.finalized = true; + import_params.finalized = false; import_params.fork_choice = Some(ForkChoiceStrategy::LongestChain); Ok((import_params, None)) @@ -100,37 +66,43 @@ impl Verifier for ManualSealVerifier { } /// Instantiate the import queue for the manual seal consensus engine. -pub fn import_queue(block_import: BoxBlockImport) -> BasicQueue +pub fn import_queue( + block_import: BoxBlockImport> +) -> BasicQueue> + where + Block: BlockT, + B: Backend + 'static, { BasicQueue::new( ManualSealVerifier, - block_import, + Box::new(block_import), None, None, ) } /// Creates the background authorship task for the manual seal engine. -pub async fn run_manual_seal( +pub async fn run_manual_seal( mut block_import: BoxBlockImport, mut env: E, - backend: Arc, + client: Arc, pool: Arc>, - mut seal_block_channel: S, - select_chain: C, + mut commands_stream: S, + select_chain: SC, inherent_data_providers: InherentDataProviders, ) where + A: txpool::ChainApi::Hash> + 'static, B: BlockT + 'static, + C: HeaderBackend + Finalizer + 'static, CB: ClientBackend + 'static, E: Environment + 'static, E::Error: std::fmt::Display, >::Error: std::fmt::Display, - A: txpool::ChainApi::Hash> + 'static, S: Stream::Hash>> + Unpin + 'static, - C: SelectChain + 'static, + SC: SelectChain + 'static, { - while let Some(command) = seal_block_channel.next().await { + while let Some(command) = commands_stream.next().await { match command { EngineCommand::SealNewBlock { create_empty, @@ -149,7 +121,7 @@ pub async fn run_manual_seal( block_import: &mut block_import, inherent_data_provider: &inherent_data_providers, pool: pool.clone(), - backend: backend.clone(), + client: client.clone(), } ).await; } @@ -159,7 +131,8 @@ pub async fn run_manual_seal( hash, sender, justification, - backend: backend.clone(), + finalizer: client.clone(), + _phantom: PhantomData, } ).await } @@ -170,26 +143,28 @@ pub async fn run_manual_seal( /// runs the background authorship task for the instant seal engine. /// instant-seal creates a new block for every transaction imported into /// the transaction pool. -pub async fn run_instant_seal( +pub async fn run_instant_seal( block_import: BoxBlockImport, env: E, - backend: Arc, + client: Arc, pool: Arc>, - select_chain: C, + select_chain: SC, inherent_data_providers: InherentDataProviders, ) where A: txpool::ChainApi::Hash> + 'static, B: BlockT + 'static, + C: HeaderBackend + Finalizer + 'static, CB: ClientBackend + 'static, E: Environment + 'static, E::Error: std::fmt::Display, >::Error: std::fmt::Display, - C: SelectChain + 'static + SC: SelectChain + 'static { // instant-seal creates blocks as soon as transactions are imported // into the transaction pool. - let seal_block_channel = pool.validated_pool().import_notification_stream() + let commands_stream = pool.validated_pool() + .import_notification_stream() .map(|_| { EngineCommand::SealNewBlock { create_empty: false, @@ -202,9 +177,9 @@ pub async fn run_instant_seal( run_manual_seal( block_import, env, - backend, + client, pool, - seal_block_channel, + commands_stream, select_chain, inherent_data_providers, ).await @@ -226,9 +201,7 @@ mod tests { use substrate_test_runtime_transaction_pool::{TestApi, uxt}; use sp_transaction_pool::{TransactionPool, MaintainedTransactionPool, TransactionSource}; use sp_runtime::generic::BlockId; - use sp_blockchain::HeaderBackend; use sp_consensus::ImportedAux; - use sc_client::LongestChain; use sp_inherents::InherentDataProviders; use sc_basic_authorship::ProposerFactory; @@ -241,11 +214,10 @@ mod tests { #[tokio::test] async fn instant_seal() { let builder = TestClientBuilder::new(); - let backend = builder.backend(); - let client = Arc::new(builder.build()); - let select_chain = LongestChain::new(backend.clone()); + let (client, select_chain) = builder.build_with_longest_chain(); + let client = Arc::new(client); let inherent_data_providers = InherentDataProviders::new(); - let pool = Arc::new(BasicPool::new(Options::default(), api()).0); + let pool = Arc::new(BasicPool::new(Options::default(), api(), None).0); let env = ProposerFactory::new( client.clone(), pool.clone() @@ -268,7 +240,7 @@ mod tests { let future = run_manual_seal( Box::new(client.clone()), env, - backend.clone(), + client.clone(), pool.pool().clone(), stream, select_chain, @@ -300,17 +272,16 @@ mod tests { } ); // assert that there's a new block in the db. - assert!(backend.blockchain().header(BlockId::Number(1)).unwrap().is_some()) + assert!(client.header(&BlockId::Number(1)).unwrap().is_some()) } #[tokio::test] async fn manual_seal_and_finalization() { let builder = TestClientBuilder::new(); - let backend = builder.backend(); - let client = Arc::new(builder.build()); - let select_chain = LongestChain::new(backend.clone()); + let (client, select_chain) = builder.build_with_longest_chain(); + let client = Arc::new(client); let inherent_data_providers = InherentDataProviders::new(); - let pool = Arc::new(BasicPool::new(Options::default(), api()).0); + let pool = Arc::new(BasicPool::new(Options::default(), api(), None).0); let env = ProposerFactory::new( client.clone(), pool.clone() @@ -320,7 +291,7 @@ mod tests { let future = run_manual_seal( Box::new(client.clone()), env, - backend.clone(), + client.clone(), pool.pool().clone(), stream, select_chain, @@ -360,7 +331,7 @@ mod tests { } ); // assert that there's a new block in the db. - let header = backend.blockchain().header(BlockId::Number(1)).unwrap().unwrap(); + let header = client.header(&BlockId::Number(1)).unwrap().unwrap(); let (tx, rx) = futures::channel::oneshot::channel(); sink.send(EngineCommand::FinalizeBlock { sender: Some(tx), @@ -374,12 +345,11 @@ mod tests { #[tokio::test] async fn manual_seal_fork_blocks() { let builder = TestClientBuilder::new(); - let backend = builder.backend(); - let client = Arc::new(builder.build()); - let select_chain = LongestChain::new(backend.clone()); + let (client, select_chain) = builder.build_with_longest_chain(); + let client = Arc::new(client); let inherent_data_providers = InherentDataProviders::new(); let pool_api = api(); - let pool = Arc::new(BasicPool::new(Options::default(), pool_api.clone()).0); + let pool = Arc::new(BasicPool::new(Options::default(), pool_api.clone(), None).0); let env = ProposerFactory::new( client.clone(), pool.clone(), @@ -389,7 +359,7 @@ mod tests { let future = run_manual_seal( Box::new(client.clone()), env, - backend.clone(), + client.clone(), pool.pool().clone(), stream, select_chain, @@ -431,12 +401,12 @@ mod tests { } ); // assert that there's a new block in the db. - assert!(backend.blockchain().header(BlockId::Number(0)).unwrap().is_some()); + assert!(client.header(&BlockId::Number(0)).unwrap().is_some()); assert!(pool.submit_one(&BlockId::Number(1), SOURCE, uxt(Alice, 1)).await.is_ok()); pool.maintain(sp_transaction_pool::ChainEvent::NewBlock { id: BlockId::Number(1), - header: backend.blockchain().header(BlockId::Number(1)).expect("db error").expect("imported above"), + header: client.header(&BlockId::Number(1)).expect("db error").expect("imported above"), is_new_best: true, retracted: vec![], }).await; @@ -452,7 +422,7 @@ mod tests { rx1.await.expect("should be no error receiving"), Ok(_) ); - assert!(backend.blockchain().header(BlockId::Number(1)).unwrap().is_some()); + assert!(client.header(&BlockId::Number(1)).unwrap().is_some()); pool_api.increment_nonce(Alice.into()); assert!(pool.submit_one(&BlockId::Number(2), SOURCE, uxt(Alice, 2)).await.is_ok()); @@ -465,6 +435,6 @@ mod tests { }).await.is_ok()); let imported = rx2.await.unwrap().unwrap(); // assert that fork block is in the db - assert!(backend.blockchain().header(BlockId::Hash(imported.hash)).unwrap().is_some()) + assert!(client.header(&BlockId::Hash(imported.hash)).unwrap().is_some()) } } diff --git a/client/consensus/manual-seal/src/seal_new_block.rs b/client/consensus/manual-seal/src/seal_new_block.rs index 39d73e16ab74fdadde2e3a597fa2ef71615cc9fa..88b58ef4cc2b39cbbe3af1209e54408390e074ef 100644 --- a/client/consensus/manual-seal/src/seal_new_block.rs +++ b/client/consensus/manual-seal/src/seal_new_block.rs @@ -33,7 +33,6 @@ use sp_consensus::{ import_queue::BoxBlockImport, }; use sp_blockchain::HeaderBackend; -use sc_client_api::backend::Backend as ClientBackend; use std::collections::HashMap; use std::time::Duration; use sp_inherents::InherentDataProviders; @@ -42,7 +41,7 @@ use sp_inherents::InherentDataProviders; const MAX_PROPOSAL_DURATION: u64 = 10; /// params for sealing a new block -pub struct SealBlockParams<'a, B: BlockT, C, CB, E, T, P: txpool::ChainApi> { +pub struct SealBlockParams<'a, B: BlockT, SC, HB, E, T, P: txpool::ChainApi> { /// if true, empty blocks(without extrinsics) will be created. /// otherwise, will return Error::EmptyTransactionPool. pub create_empty: bool, @@ -54,12 +53,12 @@ pub struct SealBlockParams<'a, B: BlockT, C, CB, E, T, P: txpool::ChainApi> { pub sender: rpc::Sender::Hash>>, /// transaction pool pub pool: Arc>, - /// client backend - pub backend: Arc, + /// header backend + pub client: Arc, /// Environment trait object for creating a proposer pub env: &'a mut E, /// SelectChain object - pub select_chain: &'a C, + pub select_chain: &'a SC, /// block import object pub block_import: &'a mut BoxBlockImport, /// inherent data provider @@ -67,24 +66,24 @@ pub struct SealBlockParams<'a, B: BlockT, C, CB, E, T, P: txpool::ChainApi> { } /// seals a new block with the given params -pub async fn seal_new_block( +pub async fn seal_new_block( SealBlockParams { create_empty, finalize, pool, parent_hash, - backend: back_end, + client, select_chain, block_import, env, inherent_data_provider, mut sender, .. - }: SealBlockParams<'_, B, SC, CB, E, T, P> + }: SealBlockParams<'_, B, SC, HB, E, T, P> ) where B: BlockT, - CB: ClientBackend, + HB: HeaderBackend, E: Environment, >::Error: std::fmt::Display, >::Error: std::fmt::Display, @@ -101,7 +100,7 @@ pub async fn seal_new_block( // or fetch the best_block. let header = match parent_hash { Some(hash) => { - match back_end.blockchain().header(BlockId::Hash(hash))? { + match client.header(BlockId::Hash(hash))? { Some(header) => header, None => return Err(Error::BlockNotFound(format!("{}", hash))), } diff --git a/client/consensus/pow/Cargo.toml b/client/consensus/pow/Cargo.toml index c7832baae0ee4a2a27a35b88c4ec34f31e0854e5..0c1cc9c12cd5f61068ed16d7e980c7f03feb1260 100644 --- a/client/consensus/pow/Cargo.toml +++ b/client/consensus/pow/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-pow" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "PoW consensus algorithm for substrate" edition = "2018" @@ -8,21 +8,21 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.5", path = "../../../primitives/api" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../../api" } -sp-block-builder = { version = "2.0.0-alpha.5", path = "../../../primitives/block-builder" } -sp-inherents = { version = "2.0.0-alpha.5", path = "../../../primitives/inherents" } -sp-consensus-pow = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/pow" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-dev", path = "../../../primitives/api" } +sc-client-api = { version = "2.0.0-dev", path = "../../api" } +sp-block-builder = { version = "2.0.0-dev", path = "../../../primitives/block-builder" } +sp-inherents = { version = "2.0.0-dev", path = "../../../primitives/inherents" } +sp-consensus-pow = { version = "0.8.0-dev", path = "../../../primitives/consensus/pow" } +sp-consensus = { version = "0.8.0-dev", path = "../../../primitives/consensus/common" } log = "0.4.8" futures = { version = "0.3.1", features = ["compat"] } -sp-timestamp = { version = "2.0.0-alpha.5", path = "../../../primitives/timestamp" } +sp-timestamp = { version = "2.0.0-dev", path = "../../../primitives/timestamp" } derive_more = "0.99.2" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/consensus/pow/src/lib.rs b/client/consensus/pow/src/lib.rs index de41ea7bd2356f02f5857344b70eae335273a540..e0149b475b7132542578fa55544a72cd8bd0a52b 100644 --- a/client/consensus/pow/src/lib.rs +++ b/client/consensus/pow/src/lib.rs @@ -49,7 +49,7 @@ use sp_consensus::{ SelectChain, Error as ConsensusError, CanAuthorWith, RecordProof, BlockImport, BlockCheckParams, ImportResult, }; -use sp_consensus::import_queue::{BoxBlockImport, BasicQueue, Verifier}; +use sp_consensus::import_queue::{BoxBlockImport, BasicQueue, Verifier, BoxJustificationImport, BoxFinalityProofImport}; use codec::{Encode, Decode}; use sc_client_api; use log::*; @@ -457,6 +457,8 @@ pub type PowImportQueue = BasicQueue; /// Import queue for PoW engine. pub fn import_queue( block_import: BoxBlockImport, + justification_import: Option>, + finality_proof_import: Option>, algorithm: Algorithm, inherent_data_providers: InherentDataProviders, ) -> Result< @@ -474,8 +476,8 @@ pub fn import_queue( Ok(BasicQueue::new( verifier, block_import, - None, - None + justification_import, + finality_proof_import )) } diff --git a/client/consensus/slots/Cargo.toml b/client/consensus/slots/Cargo.toml index bf973ef47a893bbd45a6bb47b03a9f5afbfd99df..d39d5382557ddc84c6a3f3a1d232ea191c0d0eb8 100644 --- a/client/consensus/slots/Cargo.toml +++ b/client/consensus/slots/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-slots" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "Generic slots-based utilities for consensus" edition = "2018" @@ -9,17 +9,20 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../../api" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../../primitives/state-machine" } -sp-api = { version = "2.0.0-alpha.5", path = "../../../primitives/api" } -sc-telemetry = { version = "2.0.0-alpha.5", path = "../../telemetry" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } -sp-inherents = { version = "2.0.0-alpha.5", path = "../../../primitives/inherents" } +sc-client-api = { version = "2.0.0-dev", path = "../../api" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-state-machine = { version = "0.8.0-dev", path = "../../../primitives/state-machine" } +sp-api = { version = "2.0.0-dev", path = "../../../primitives/api" } +sc-telemetry = { version = "2.0.0-dev", path = "../../telemetry" } +sp-consensus = { version = "0.8.0-dev", path = "../../../primitives/consensus/common" } +sp-inherents = { version = "2.0.0-dev", path = "../../../primitives/inherents" } futures = "0.3.4" futures-timer = "3.0.1" parking_lot = "0.10.0" @@ -27,6 +30,3 @@ log = "0.4.8" [dev-dependencies] substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/consensus/slots/src/aux_schema.rs b/client/consensus/slots/src/aux_schema.rs index df4772a8e92a16be328b0b9d9d61d7501b1e21d0..d54190ca0715890272ede7319cbc1cf21aacf296 100644 --- a/client/consensus/slots/src/aux_schema.rs +++ b/client/consensus/slots/src/aux_schema.rs @@ -85,8 +85,8 @@ pub fn check_equivocation( P: Clone + Encode + Decode + PartialEq, { // We don't check equivocations for old headers out of our capacity. - if slot_now - slot > MAX_SLOT_CAPACITY { - return Ok(None) + if slot_now.saturating_sub(slot) > MAX_SLOT_CAPACITY { + return Ok(None); } // Key for this slot. @@ -102,6 +102,11 @@ pub fn check_equivocation( let first_saved_slot = load_decode::<_, u64>(backend, &slot_header_start[..])? .unwrap_or(slot); + if slot_now < first_saved_slot { + // The code below assumes that slots will be visited sequentially. + return Ok(None); + } + for (prev_header, prev_signer) in headers_with_sig.iter() { // A proof of equivocation consists of two headers: // 1) signed by the same voter, @@ -114,7 +119,7 @@ pub fn check_equivocation( snd_header: header.clone(), })); } else { - // We don't need to continue in case of duplicated header, + // We don't need to continue in case of duplicated header, // since it's already saved and a possible equivocation // would have been detected before. return Ok(None) diff --git a/client/consensus/slots/src/lib.rs b/client/consensus/slots/src/lib.rs index d0f1f6ec4bf9ea1f91cbe48974981df92d3ac918..611e0fbb7b8053675c7b385fccbe03747eaa00a4 100644 --- a/client/consensus/slots/src/lib.rs +++ b/client/consensus/slots/src/lib.rs @@ -466,7 +466,7 @@ impl SlotDuration { cb(client.runtime_api(), &BlockId::number(Zero::zero()))?; info!( - "Loaded block-time = {:?} milliseconds from genesis on first-launch", + "⏱ Loaded block-time = {:?} milliseconds from genesis on first-launch", genesis_slot_duration ); @@ -483,3 +483,120 @@ impl SlotDuration { self.0.clone() } } + +/// Calculate a slot duration lenience based on the number of missed slots from current +/// to parent. If the number of skipped slots is greated than 0 this method will apply +/// an exponential backoff of at most `2^7 * slot_duration`, if no slots were skipped +/// this method will return `None.` +pub fn slot_lenience_exponential(parent_slot: u64, slot_info: &SlotInfo) -> Option { + // never give more than 2^this times the lenience. + const BACKOFF_CAP: u64 = 7; + + // how many slots it takes before we double the lenience. + const BACKOFF_STEP: u64 = 2; + + // we allow a lenience of the number of slots since the head of the + // chain was produced, minus 1 (since there is always a difference of at least 1) + // + // exponential back-off. + // in normal cases we only attempt to issue blocks up to the end of the slot. + // when the chain has been stalled for a few slots, we give more lenience. + let skipped_slots = slot_info.number.saturating_sub(parent_slot + 1); + + if skipped_slots == 0 { + None + } else { + let slot_lenience = skipped_slots / BACKOFF_STEP; + let slot_lenience = std::cmp::min(slot_lenience, BACKOFF_CAP); + let slot_lenience = 1 << slot_lenience; + Some(Duration::from_millis(slot_lenience * slot_info.duration)) + } +} + +/// Calculate a slot duration lenience based on the number of missed slots from current +/// to parent. If the number of skipped slots is greated than 0 this method will apply +/// a linear backoff of at most `20 * slot_duration`, if no slots were skipped +/// this method will return `None.` +pub fn slot_lenience_linear(parent_slot: u64, slot_info: &SlotInfo) -> Option { + // never give more than 20 times more lenience. + const BACKOFF_CAP: u64 = 20; + + // we allow a lenience of the number of slots since the head of the + // chain was produced, minus 1 (since there is always a difference of at least 1) + // + // linear back-off. + // in normal cases we only attempt to issue blocks up to the end of the slot. + // when the chain has been stalled for a few slots, we give more lenience. + let skipped_slots = slot_info.number.saturating_sub(parent_slot + 1); + + if skipped_slots == 0 { + None + } else { + let slot_lenience = std::cmp::min(skipped_slots, BACKOFF_CAP); + Some(Duration::from_millis(slot_lenience * slot_info.duration)) + } +} + +#[cfg(test)] +mod test { + use std::time::{Duration, Instant}; + + const SLOT_DURATION: Duration = Duration::from_millis(6000); + + fn slot(n: u64) -> super::slots::SlotInfo { + super::slots::SlotInfo { + number: n, + last_number: n - 1, + duration: SLOT_DURATION.as_millis() as u64, + timestamp: Default::default(), + inherent_data: Default::default(), + ends_at: Instant::now(), + } + } + + #[test] + fn linear_slot_lenience() { + // if no slots are skipped there should be no lenience + assert_eq!(super::slot_lenience_linear(1, &slot(2)), None); + + // otherwise the lenience is incremented linearly with + // the number of skipped slots. + for n in 3..=22 { + assert_eq!( + super::slot_lenience_linear(1, &slot(n)), + Some(SLOT_DURATION * (n - 2) as u32), + ); + } + + // but we cap it to a maximum of 20 slots + assert_eq!( + super::slot_lenience_linear(1, &slot(23)), + Some(SLOT_DURATION * 20), + ); + } + + #[test] + fn exponential_slot_lenience() { + // if no slots are skipped there should be no lenience + assert_eq!(super::slot_lenience_exponential(1, &slot(2)), None); + + // otherwise the lenience is incremented exponentially every two slots + for n in 3..=17 { + assert_eq!( + super::slot_lenience_exponential(1, &slot(n)), + Some(SLOT_DURATION * 2u32.pow((n / 2 - 1) as u32)), + ); + } + + // but we cap it to a maximum of 14 slots + assert_eq!( + super::slot_lenience_exponential(1, &slot(18)), + Some(SLOT_DURATION * 2u32.pow(7)), + ); + + assert_eq!( + super::slot_lenience_exponential(1, &slot(19)), + Some(SLOT_DURATION * 2u32.pow(7)), + ); + } +} diff --git a/client/consensus/uncles/Cargo.toml b/client/consensus/uncles/Cargo.toml index 7e8014199baa5241935cc198cce100bba9677cfa..019c933a20e0cc24ef3cbd0e284afce66dd379cc 100644 --- a/client/consensus/uncles/Cargo.toml +++ b/client/consensus/uncles/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-consensus-uncles" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "Generic uncle inclusion utilities for consensus" edition = "2018" @@ -8,14 +8,14 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" -[dependencies] -sc-client-api = { version = "2.0.0-alpha.5", path = "../../api" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sp-authorship = { version = "2.0.0-alpha.5", path = "../../../primitives/authorship" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } -sp-inherents = { version = "2.0.0-alpha.5", path = "../../../primitives/inherents" } -log = "0.4.8" - [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +sc-client-api = { version = "2.0.0-dev", path = "../../api" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-authorship = { version = "2.0.0-dev", path = "../../../primitives/authorship" } +sp-consensus = { version = "0.8.0-dev", path = "../../../primitives/consensus/common" } +sp-inherents = { version = "2.0.0-dev", path = "../../../primitives/inherents" } +log = "0.4.8" diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml index 9308d4ee74adf169deb8faff092fd81f26cfbdd9..2df1836850cd7957145198f55afc889ed2a3ac15 100644 --- a/client/db/Cargo.toml +++ b/client/db/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-client-db" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,32 +8,36 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Client backend that uses RocksDB database as storage." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] parking_lot = "0.10.0" log = "0.4.8" -rand = "0.7" kvdb = "0.5.0" kvdb-rocksdb = { version = "0.7", optional = true } kvdb-memorydb = "0.5.0" linked-hash-map = "0.5.2" hash-db = "0.15.2" -parity-util-mem = { version = "0.6.0", default-features = false, features = ["std"] } +parity-util-mem = { version = "0.6.1", default-features = false, features = ["std"] } codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } +blake2-rfc = "0.2.18" -sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } -sc-client = { version = "0.8.0-alpha.5", path = "../" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../primitives/state-machine" } -sc-executor = { version = "0.8.0-alpha.5", path = "../executor" } -sc-state-db = { version = "0.8.0-alpha.5", path = "../state-db" } -sp-trie = { version = "2.0.0-alpha.5", path = "../../primitives/trie" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/common" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.8.0-alpha.5", path = "../../utils/prometheus" } +sc-client-api = { version = "2.0.0-dev", path = "../api" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } +sp-state-machine = { version = "0.8.0-dev", path = "../../primitives/state-machine" } +sc-executor = { version = "0.8.0-dev", path = "../executor" } +sc-state-db = { version = "0.8.0-dev", path = "../state-db" } +sp-trie = { version = "2.0.0-dev", path = "../../primitives/trie" } +sp-consensus = { version = "0.8.0-dev", path = "../../primitives/consensus/common" } +sp-blockchain = { version = "2.0.0-dev", path = "../../primitives/blockchain" } +sp-database = { version = "2.0.0-dev", path = "../../primitives/database" } +parity-db = { version = "0.1.2", optional = true } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.8.0-dev", path = "../../utils/prometheus" } [dev-dependencies] -sp-keyring = { version = "2.0.0-alpha.5", path = "../../primitives/keyring" } +sp-keyring = { version = "2.0.0-dev", path = "../../primitives/keyring" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } env_logger = "0.7.0" quickcheck = "0.9" @@ -43,6 +47,3 @@ tempfile = "3" [features] default = [] test-helpers = [] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 02b30a085af2b9fddf3e22671d3f3ce0c469ec0a..9d6f595498bd0e3f5a4f287090b25485142c0bb0 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -17,10 +17,8 @@ //! State backend that's useful for benchmarking use std::sync::Arc; -use std::path::PathBuf; use std::cell::{Cell, RefCell}; use std::collections::HashMap; -use rand::Rng; use hash_db::{Prefix, Hasher}; use sp_trie::{MemoryDB, prefixed_key}; @@ -29,12 +27,14 @@ use sp_runtime::traits::{Block as BlockT, HashFor}; use sp_runtime::Storage; use sp_state_machine::{DBValue, backend::Backend as StateBackend}; use kvdb::{KeyValueDB, DBTransaction}; -use kvdb_rocksdb::{Database, DatabaseConfig}; +use crate::storage_cache::{CachingState, SharedCache, new_shared_cache}; type DbState = sp_state_machine::TrieBackend< Arc>>, HashFor >; +type State = CachingState, B>; + struct StorageDb { db: Arc, _block: std::marker::PhantomData, @@ -50,44 +50,36 @@ impl sp_state_machine::Storage> for StorageDb { - path: PathBuf, root: Cell, genesis_root: B::Hash, - state: RefCell>>, + state: RefCell>>, db: Cell>>, genesis: HashMap, (Vec, i32)>, record: Cell>>, - cache_size_mb: Option, + shared_cache: SharedCache, // shared cache is always empty } impl BenchmarkingState { /// Create a new instance that creates a database in a temporary dir. - pub fn new(genesis: Storage, cache_size_mb: Option) -> Result { - let temp_dir = PathBuf::from(std::env::temp_dir()); - let name: String = rand::thread_rng().sample_iter(&rand::distributions::Alphanumeric).take(10).collect(); - let path = temp_dir.join(&name); - + pub fn new(genesis: Storage, _cache_size_mb: Option) -> Result { let mut root = B::Hash::default(); let mut mdb = MemoryDB::>::default(); sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); - std::fs::create_dir(&path).map_err(|_| String::from("Error creating temp dir"))?; let mut state = BenchmarkingState { state: RefCell::new(None), db: Cell::new(None), - path, root: Cell::new(root), genesis: Default::default(), genesis_root: Default::default(), record: Default::default(), - cache_size_mb, + shared_cache: new_shared_cache(0, (1, 10)), }; state.reopen()?; - let child_delta = genesis.children.into_iter().map(|(storage_key, child_content)| ( - storage_key, + let child_delta = genesis.children_default.into_iter().map(|(_storage_key, child_content)| ( + child_content.child_info, child_content.data.into_iter().map(|(k, v)| (k, Some(v))), - child_content.child_info )); let (root, transaction): (B::Hash, _) = state.state.borrow_mut().as_mut().unwrap().full_storage_root( genesis.top.into_iter().map(|(k, v)| (k, Some(v))), @@ -96,41 +88,25 @@ impl BenchmarkingState { state.genesis = transaction.clone().drain(); state.genesis_root = root.clone(); state.commit(root, transaction)?; + state.record.take(); Ok(state) } fn reopen(&self) -> Result<(), String> { *self.state.borrow_mut() = None; - self.db.set(None); - let mut db_config = DatabaseConfig::with_columns(1); - if let Some(size) = &self.cache_size_mb { - db_config.memory_budget.insert(0, *size); - } - let path = self.path.to_str() - .ok_or_else(|| String::from("Invalid database path"))?; - let db = Arc::new(Database::open(&db_config, &path).map_err(|e| format!("Error opening database: {:?}", e))?); + let db = match self.db.take() { + Some(db) => db, + None => Arc::new(::kvdb_memorydb::create(1)), + }; self.db.set(Some(db.clone())); let storage_db = Arc::new(StorageDb:: { db, _block: Default::default() }); - *self.state.borrow_mut() = Some(DbState::::new(storage_db, self.root.get())); + *self.state.borrow_mut() = Some(State::new( + DbState::::new(storage_db, self.root.get()), + self.shared_cache.clone(), + None + )); Ok(()) } - - fn kill(&self) -> Result<(), String> { - self.db.set(None); - *self.state.borrow_mut() = None; - let mut root = B::Hash::default(); - let mut mdb = MemoryDB::>::default(); - sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); - self.root.set(root); - - std::fs::remove_dir_all(&self.path).map_err(|_| "Error removing database dir".into()) - } -} - -impl Drop for BenchmarkingState { - fn drop(&mut self) { - self.kill().ok(); - } } fn state_err() -> String { @@ -152,11 +128,10 @@ impl StateBackend> for BenchmarkingState { fn child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result>, Self::Error> { - self.state.borrow().as_ref().ok_or_else(state_err)?.child_storage(storage_key, child_info, key) + self.state.borrow().as_ref().ok_or_else(state_err)?.child_storage(child_info, key) } fn exists_storage(&self, key: &[u8]) -> Result { @@ -165,11 +140,10 @@ impl StateBackend> for BenchmarkingState { fn exists_child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result { - self.state.borrow().as_ref().ok_or_else(state_err)?.exists_child_storage(storage_key, child_info, key) + self.state.borrow().as_ref().ok_or_else(state_err)?.exists_child_storage(child_info, key) } fn next_storage_key(&self, key: &[u8]) -> Result>, Self::Error> { @@ -178,11 +152,10 @@ impl StateBackend> for BenchmarkingState { fn next_child_storage_key( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result>, Self::Error> { - self.state.borrow().as_ref().ok_or_else(state_err)?.next_child_storage_key(storage_key, child_info, key) + self.state.borrow().as_ref().ok_or_else(state_err)?.next_child_storage_key(child_info, key) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { @@ -199,24 +172,22 @@ impl StateBackend> for BenchmarkingState { fn for_keys_in_child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, f: F, ) { if let Some(ref state) = *self.state.borrow() { - state.for_keys_in_child_storage(storage_key, child_info, f) + state.for_keys_in_child_storage(child_info, f) } } fn for_child_keys_with_prefix( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], f: F, ) { if let Some(ref state) = *self.state.borrow() { - state.for_child_keys_with_prefix(storage_key, child_info, prefix, f) + state.for_child_keys_with_prefix(child_info, prefix, f) } } @@ -228,13 +199,12 @@ impl StateBackend> for BenchmarkingState { fn child_storage_root( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, delta: I, ) -> (B::Hash, bool, Self::Transaction) where I: IntoIterator, Option>)>, { - self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_storage_root(storage_key, child_info, delta)) + self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_storage_root(child_info, delta)) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -247,11 +217,10 @@ impl StateBackend> for BenchmarkingState { fn child_keys( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], ) -> Vec> { - self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_keys(storage_key, child_info, prefix)) + self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_keys(child_info, prefix)) } fn as_trie_backend(&mut self) @@ -278,6 +247,7 @@ impl StateBackend> for BenchmarkingState { self.record.set(keys); db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?; self.root.set(storage_root); + self.db.set(Some(db)) } else { return Err("Trying to commit to a closed db".into()) } @@ -296,11 +266,9 @@ impl StateBackend> for BenchmarkingState { } } db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?; + self.db.set(Some(db)); } - self.db.set(None); - *self.state.borrow_mut() = None; - self.root.set(self.genesis_root.clone()); self.reopen()?; Ok(()) @@ -317,6 +285,6 @@ impl StateBackend> for BenchmarkingState { impl std::fmt::Debug for BenchmarkingState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "DB at {:?}", self.path) + write!(f, "Bench DB") } } diff --git a/client/db/src/cache/list_storage.rs b/client/db/src/cache/list_storage.rs index 606090ee1401dc89d25e1d005824a6fa083cba39..07cd9fb866359b377ce7bc35f97efcb208c3adf5 100644 --- a/client/db/src/cache/list_storage.rs +++ b/client/db/src/cache/list_storage.rs @@ -18,17 +18,17 @@ use std::sync::Arc; -use kvdb::{KeyValueDB, DBTransaction}; - use sp_blockchain::{Error as ClientError, Result as ClientResult}; use codec::{Encode, Decode}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; -use crate::utils::{self, db_err, meta_keys}; +use sp_database::{Database, Transaction}; +use crate::utils::{self, meta_keys}; use crate::cache::{CacheItemT, ComplexBlockId}; use crate::cache::list_cache::{CommitOperation, Fork}; use crate::cache::list_entry::{Entry, StorageEntry}; +use crate::DbHash; /// Single list-cache metadata. #[derive(Debug)] @@ -97,19 +97,19 @@ pub struct DbColumns { pub struct DbStorage { name: Vec, meta_key: Vec, - db: Arc, + db: Arc>, columns: DbColumns, } impl DbStorage { /// Create new database-backed list cache storage. - pub fn new(name: Vec, db: Arc, columns: DbColumns) -> Self { + pub fn new(name: Vec, db: Arc>, columns: DbColumns) -> Self { let meta_key = meta::key(&name); DbStorage { name, meta_key, db, columns } } /// Get reference to the database. - pub fn db(&self) -> &Arc { &self.db } + pub fn db(&self) -> &Arc> { &self.db } /// Get reference to the database columns. pub fn columns(&self) -> &DbColumns { &self.columns } @@ -135,49 +135,45 @@ impl Storage for DbStorage { } fn read_meta(&self) -> ClientResult> { - self.db.get(self.columns.meta, &self.meta_key) - .map_err(db_err) - .and_then(|meta| match meta { - Some(meta) => meta::decode(&*meta), - None => Ok(Metadata { - finalized: None, - unfinalized: Vec::new(), - }), + match self.db.get(self.columns.meta, &self.meta_key) { + Some(meta) => meta::decode(&*meta), + None => Ok(Metadata { + finalized: None, + unfinalized: Vec::new(), }) + } } fn read_entry(&self, at: &ComplexBlockId) -> ClientResult>> { - self.db.get(self.columns.cache, &self.encode_block_id(at)) - .map_err(db_err) - .and_then(|entry| match entry { - Some(entry) => StorageEntry::::decode(&mut &entry[..]) - .map_err(|_| ClientError::Backend("Failed to decode cache entry".into())) - .map(Some), - None => Ok(None), - }) + match self.db.get(self.columns.cache, &self.encode_block_id(at)) { + Some(entry) => StorageEntry::::decode(&mut &entry[..]) + .map_err(|_| ClientError::Backend("Failed to decode cache entry".into())) + .map(Some), + None => Ok(None), + } } } /// Database-backed list cache storage transaction. pub struct DbStorageTransaction<'a> { storage: &'a DbStorage, - tx: &'a mut DBTransaction, + tx: &'a mut Transaction, } impl<'a> DbStorageTransaction<'a> { /// Create new database transaction. - pub fn new(storage: &'a DbStorage, tx: &'a mut DBTransaction) -> Self { + pub fn new(storage: &'a DbStorage, tx: &'a mut Transaction) -> Self { DbStorageTransaction { storage, tx } } } impl<'a, Block: BlockT, T: CacheItemT> StorageTransaction for DbStorageTransaction<'a> { fn insert_storage_entry(&mut self, at: &ComplexBlockId, entry: &StorageEntry) { - self.tx.put(self.storage.columns.cache, &self.storage.encode_block_id(at), &entry.encode()); + self.tx.set_from_vec(self.storage.columns.cache, &self.storage.encode_block_id(at), entry.encode()); } fn remove_storage_entry(&mut self, at: &ComplexBlockId) { - self.tx.delete(self.storage.columns.cache, &self.storage.encode_block_id(at)); + self.tx.remove(self.storage.columns.cache, &self.storage.encode_block_id(at)); } fn update_meta( @@ -186,10 +182,10 @@ impl<'a, Block: BlockT, T: CacheItemT> StorageTransaction for DbStorag unfinalized: &[Fork], operation: &CommitOperation, ) { - self.tx.put( + self.tx.set_from_vec( self.storage.columns.meta, &self.storage.meta_key, - &meta::encode(best_finalized_entry, unfinalized, operation)); + meta::encode(best_finalized_entry, unfinalized, operation)); } } diff --git a/client/db/src/cache/mod.rs b/client/db/src/cache/mod.rs index 8fd1adc094ae4e241c7f0e73e8ffedecd60a7cd4..8d3e1f358b3229603dbe1659b526cc0ba797c9c3 100644 --- a/client/db/src/cache/mod.rs +++ b/client/db/src/cache/mod.rs @@ -19,14 +19,14 @@ use std::{sync::Arc, collections::{HashMap, hash_map::Entry}}; use parking_lot::RwLock; -use kvdb::{KeyValueDB, DBTransaction}; - use sc_client_api::blockchain::{well_known_cache_keys::{self, Id as CacheKeyId}, Cache as BlockchainCache}; use sp_blockchain::Result as ClientResult; +use sp_database::{Database, Transaction}; use codec::{Encode, Decode}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero}; -use crate::utils::{self, COLUMN_META, db_err}; +use crate::utils::{self, COLUMN_META}; +use crate::DbHash; use self::list_cache::{ListCache, PruningStrategy}; @@ -78,7 +78,7 @@ impl CacheItemT for T where T: Clone + Decode + Encode + PartialEq {} /// Database-backed blockchain data cache. pub struct DbCache { cache_at: HashMap, self::list_storage::DbStorage>>, - db: Arc, + db: Arc>, key_lookup_column: u32, header_column: u32, cache_column: u32, @@ -89,7 +89,7 @@ pub struct DbCache { impl DbCache { /// Create new cache. pub fn new( - db: Arc, + db: Arc>, key_lookup_column: u32, header_column: u32, cache_column: u32, @@ -113,7 +113,7 @@ impl DbCache { } /// Begin cache transaction. - pub fn transaction<'a>(&'a mut self, tx: &'a mut DBTransaction) -> DbCacheTransaction<'a, Block> { + pub fn transaction<'a>(&'a mut self, tx: &'a mut Transaction) -> DbCacheTransaction<'a, Block> { DbCacheTransaction { cache: self, tx, @@ -125,7 +125,7 @@ impl DbCache { /// Begin cache transaction with given ops. pub fn transaction_with_ops<'a>( &'a mut self, - tx: &'a mut DBTransaction, + tx: &'a mut Transaction, ops: DbCacheTransactionOps, ) -> DbCacheTransaction<'a, Block> { DbCacheTransaction { @@ -169,7 +169,7 @@ impl DbCache { fn get_cache_helper<'a, Block: BlockT>( cache_at: &'a mut HashMap, self::list_storage::DbStorage>>, name: CacheKeyId, - db: &Arc, + db: &Arc>, key_lookup: u32, header: u32, cache: u32, @@ -215,7 +215,7 @@ impl DbCacheTransactionOps { /// Database-backed blockchain data cache transaction valid for single block import. pub struct DbCacheTransaction<'a, Block: BlockT> { cache: &'a mut DbCache, - tx: &'a mut DBTransaction, + tx: &'a mut Transaction, cache_at_ops: HashMap>>, best_finalized_block: Option>, } @@ -328,7 +328,7 @@ impl BlockchainCache for DbCacheSync { let genesis_hash = cache.genesis_hash; let cache_contents = vec![(*key, data)].into_iter().collect(); let db = cache.db.clone(); - let mut dbtx = DBTransaction::new(); + let mut dbtx = Transaction::new(); let tx = cache.transaction(&mut dbtx); let tx = tx.on_block_insert( ComplexBlockId::new(Default::default(), Zero::zero()), @@ -337,7 +337,7 @@ impl BlockchainCache for DbCacheSync { EntryType::Genesis, )?; let tx_ops = tx.into_ops(); - db.write(dbtx).map_err(db_err)?; + db.commit(dbtx); cache.commit(tx_ops)?; Ok(()) } diff --git a/client/db/src/changes_tries_storage.rs b/client/db/src/changes_tries_storage.rs index a28cd604fe3633d952dfd10a94ea8df01c4bc975..985251f403d6263dce0b8b0f25685c3d513fa9ac 100644 --- a/client/db/src/changes_tries_storage.rs +++ b/client/db/src/changes_tries_storage.rs @@ -19,7 +19,6 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use hash_db::Prefix; -use kvdb::{KeyValueDB, DBTransaction}; use codec::{Decode, Encode}; use parking_lot::RwLock; use sp_blockchain::{Error as ClientError, Result as ClientResult}; @@ -27,12 +26,15 @@ use sp_trie::MemoryDB; use sc_client_api::backend::PrunableStateChangesTrieStorage; use sp_blockchain::{well_known_cache_keys, Cache as BlockchainCache}; use sp_core::{ChangesTrieConfiguration, ChangesTrieConfigurationRange, convert_hash}; +use sp_core::storage::PrefixedStorageKey; +use sp_database::Transaction; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, HashFor, NumberFor, One, Zero, CheckedSub, }; use sp_runtime::generic::{BlockId, DigestItem, ChangesTrieSignal}; -use sp_state_machine::{DBValue, ChangesTrieBuildCache, ChangesTrieCacheAction}; -use crate::utils::{self, Meta, meta_keys, db_err}; +use sp_state_machine::{ChangesTrieBuildCache, ChangesTrieCacheAction}; +use crate::{Database, DbHash}; +use crate::utils::{self, Meta, meta_keys}; use crate::cache::{ DbCacheSync, DbCache, DbCacheTransactionOps, ComplexBlockId, EntryType as CacheEntryType, @@ -76,7 +78,7 @@ impl From> for DbChangesTrieStorageT /// Stores all tries in separate DB column. /// Lock order: meta, tries_meta, cache, build_cache. pub struct DbChangesTrieStorage { - db: Arc, + db: Arc>, meta_column: u32, changes_tries_column: u32, key_lookup_column: u32, @@ -111,7 +113,7 @@ struct ChangesTriesMeta { impl DbChangesTrieStorage { /// Create new changes trie storage. pub fn new( - db: Arc, + db: Arc>, meta_column: u32, changes_tries_column: u32, key_lookup_column: u32, @@ -149,7 +151,7 @@ impl DbChangesTrieStorage { /// Commit new changes trie. pub fn commit( &self, - tx: &mut DBTransaction, + tx: &mut Transaction, mut changes_trie: MemoryDB>, parent_block: ComplexBlockId, block: ComplexBlockId, @@ -160,7 +162,7 @@ impl DbChangesTrieStorage { ) -> ClientResult> { // insert changes trie, associated with block, into DB for (key, (val, _)) in changes_trie.drain() { - tx.put(self.changes_tries_column, key.as_ref(), &val); + tx.set(self.changes_tries_column, key.as_ref(), &val); } // if configuration has not been changed AND block is not finalized => nothing to do here @@ -205,7 +207,7 @@ impl DbChangesTrieStorage { /// Called when block is finalized. pub fn finalize( &self, - tx: &mut DBTransaction, + tx: &mut Transaction, parent_block_hash: Block::Hash, block_hash: Block::Hash, block_num: NumberFor, @@ -254,7 +256,7 @@ impl DbChangesTrieStorage { /// When block is reverted. pub fn revert( &self, - tx: &mut DBTransaction, + tx: &mut Transaction, block: &ComplexBlockId, ) -> ClientResult> { Ok(self.cache.0.write().transaction(tx) @@ -280,7 +282,7 @@ impl DbChangesTrieStorage { /// Prune obsolete changes tries. fn prune( &self, - tx: &mut DBTransaction, + tx: &mut Transaction, block_hash: Block::Hash, block_num: NumberFor, new_header: Option<&Block::Header>, @@ -313,7 +315,7 @@ impl DbChangesTrieStorage { hash: convert_hash(&block_hash), number: block_num, }, - |node| tx.delete(self.changes_tries_column, node.as_ref()), + |node| tx.remove(self.changes_tries_column, node.as_ref()), ); next_digest_range_start = end + One::one(); @@ -481,23 +483,22 @@ where fn with_cached_changed_keys( &self, root: &Block::Hash, - functor: &mut dyn FnMut(&HashMap>, HashSet>>), + functor: &mut dyn FnMut(&HashMap, HashSet>>), ) -> bool { self.build_cache.read().with_changed_keys(root, functor) } - fn get(&self, key: &Block::Hash, _prefix: Prefix) -> Result, String> { - self.db.get(self.changes_tries_column, key.as_ref()) - .map_err(|err| format!("{}", err)) + fn get(&self, key: &Block::Hash, _prefix: Prefix) -> Result>, String> { + Ok(self.db.get(self.changes_tries_column, key.as_ref())) } } /// Read changes tries metadata from database. fn read_tries_meta( - db: &dyn KeyValueDB, + db: &dyn Database, meta_column: u32, ) -> ClientResult> { - match db.get(meta_column, meta_keys::CHANGES_TRIES_META).map_err(db_err)? { + match db.get(meta_column, meta_keys::CHANGES_TRIES_META) { Some(h) => match Decode::decode(&mut &h[..]) { Ok(h) => Ok(h), Err(err) => Err(ClientError::Backend(format!("Error decoding changes tries metadata: {}", err))), @@ -511,11 +512,11 @@ fn read_tries_meta( /// Write changes tries metadata from database. fn write_tries_meta( - tx: &mut DBTransaction, + tx: &mut Transaction, meta_column: u32, meta: &ChangesTriesMeta, ) { - tx.put(meta_column, meta_keys::CHANGES_TRIES_META, &meta.encode()); + tx.set_from_vec(meta_column, meta_keys::CHANGES_TRIES_META, meta.encode()); } #[cfg(test)] @@ -707,7 +708,7 @@ mod tests { let finalize_block = |number| { let header = backend.blockchain().header(BlockId::Number(number)).unwrap().unwrap(); - let mut tx = DBTransaction::new(); + let mut tx = Transaction::new(); let cache_ops = backend.changes_tries_storage.finalize( &mut tx, *header.parent_hash(), @@ -716,7 +717,7 @@ mod tests { None, None, ).unwrap(); - backend.storage.db.write(tx).unwrap(); + backend.storage.db.commit(tx); backend.changes_tries_storage.post_commit(Some(cache_ops)); }; diff --git a/client/db/src/children.rs b/client/db/src/children.rs index 2ef67de6a83e1116d20483982ff284ecbd5adf08..3916321f17286313a5f1bfc1d0ee44a3648b03ee 100644 --- a/client/db/src/children.rs +++ b/client/db/src/children.rs @@ -16,23 +16,21 @@ //! Functionality for reading and storing children hashes from db. -use kvdb::{KeyValueDB, DBTransaction}; use codec::{Encode, Decode}; use sp_blockchain; use std::hash::Hash; +use sp_database::{Database, Transaction}; +use crate::DbHash; /// Returns the hashes of the children blocks of the block with `parent_hash`. pub fn read_children< K: Eq + Hash + Clone + Encode + Decode, V: Eq + Hash + Clone + Encode + Decode, ->(db: &dyn KeyValueDB, column: u32, prefix: &[u8], parent_hash: K) -> sp_blockchain::Result> { +>(db: &dyn Database, column: u32, prefix: &[u8], parent_hash: K) -> sp_blockchain::Result> { let mut buf = prefix.to_vec(); parent_hash.using_encoded(|s| buf.extend(s)); - let raw_val_opt = match db.get(column, &buf[..]) { - Ok(raw_val_opt) => raw_val_opt, - Err(_) => return Err(sp_blockchain::Error::Backend("Error reading value from database".into())), - }; + let raw_val_opt = db.get(column, &buf[..]); let raw_val = match raw_val_opt { Some(val) => val, @@ -53,7 +51,7 @@ pub fn write_children< K: Eq + Hash + Clone + Encode + Decode, V: Eq + Hash + Clone + Encode + Decode, >( - tx: &mut DBTransaction, + tx: &mut Transaction, column: u32, prefix: &[u8], parent_hash: K, @@ -61,34 +59,35 @@ pub fn write_children< ) { let mut key = prefix.to_vec(); parent_hash.using_encoded(|s| key.extend(s)); - tx.put_vec(column, &key[..], children_hashes.encode()); + tx.set_from_vec(column, &key[..], children_hashes.encode()); } /// Prepare transaction to remove the children of `parent_hash`. pub fn remove_children< K: Eq + Hash + Clone + Encode + Decode, >( - tx: &mut DBTransaction, + tx: &mut Transaction, column: u32, prefix: &[u8], parent_hash: K, ) { let mut key = prefix.to_vec(); parent_hash.using_encoded(|s| key.extend(s)); - tx.delete(column, &key[..]); + tx.remove(column, &key); } #[cfg(test)] mod tests { use super::*; + use std::sync::Arc; #[test] fn children_write_read_remove() { const PREFIX: &[u8] = b"children"; - let db = ::kvdb_memorydb::create(1); + let db = Arc::new(sp_database::MemDb::default()); - let mut tx = DBTransaction::new(); + let mut tx = Transaction::new(); let mut children1 = Vec::new(); children1.push(1_3); @@ -100,19 +99,19 @@ mod tests { children2.push(1_6); write_children(&mut tx, 0, PREFIX, 1_2, children2); - db.write(tx.clone()).expect("(2) Committing transaction failed"); + db.commit(tx.clone()); - let r1: Vec = read_children(&db, 0, PREFIX, 1_1).expect("(1) Getting r1 failed"); - let r2: Vec = read_children(&db, 0, PREFIX, 1_2).expect("(1) Getting r2 failed"); + let r1: Vec = read_children(&*db, 0, PREFIX, 1_1).expect("(1) Getting r1 failed"); + let r2: Vec = read_children(&*db, 0, PREFIX, 1_2).expect("(1) Getting r2 failed"); assert_eq!(r1, vec![1_3, 1_5]); assert_eq!(r2, vec![1_4, 1_6]); remove_children(&mut tx, 0, PREFIX, 1_2); - db.write(tx).expect("(2) Committing transaction failed"); + db.commit(tx); - let r1: Vec = read_children(&db, 0, PREFIX, 1_1).expect("(2) Getting r1 failed"); - let r2: Vec = read_children(&db, 0, PREFIX, 1_2).expect("(2) Getting r2 failed"); + let r1: Vec = read_children(&*db, 0, PREFIX, 1_1).expect("(2) Getting r1 failed"); + let r2: Vec = read_children(&*db, 0, PREFIX, 1_2).expect("(2) Getting r2 failed"); assert_eq!(r1, vec![1_3, 1_5]); assert_eq!(r2.len(), 0); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index f1cb5f9a793c015e61b473b78465bcb4156be05f..b9ca63b11715ebbee9d7fd7998987166a2d2c157 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Client backend that uses RocksDB database as storage. +//! Client backend that is backed by a database. //! //! # Canonicality vs. Finality //! @@ -40,16 +40,21 @@ mod storage_cache; mod upgrade; mod utils; mod stats; +#[cfg(feature = "parity-db")] +mod parity_db; +#[cfg(feature = "subdb")] +mod subdb; use std::sync::Arc; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::io; use std::collections::HashMap; + use sc_client_api::{ - ForkBlocks, UsageInfo, MemoryInfo, BadBlocks, IoInfo, MemorySize, CloneableSpawn, - execution_extensions::ExecutionExtensions, + UsageInfo, MemoryInfo, IoInfo, MemorySize, backend::{NewBlockState, PrunableStateChangesTrieStorage}, + leaves::{LeafSet, FinalizationDisplaced}, }; use sp_blockchain::{ Result as ClientResult, Error as ClientError, @@ -57,42 +62,36 @@ use sp_blockchain::{ }; use codec::{Decode, Encode}; use hash_db::Prefix; -use kvdb::{KeyValueDB, DBTransaction}; use sp_trie::{MemoryDB, PrefixedMemoryDB, prefixed_key}; +use sp_database::Transaction; use parking_lot::RwLock; -use sp_core::{ChangesTrieConfiguration, traits::CodeExecutor}; +use sp_core::ChangesTrieConfiguration; +use sp_core::offchain::storage::{OffchainOverlayedChange, OffchainOverlayedChanges}; use sp_core::storage::{well_known_keys, ChildInfo}; -use sp_runtime::{ - generic::BlockId, Justification, Storage, - BuildStorage, -}; +use sp_runtime::{generic::BlockId, Justification, Storage}; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, NumberFor, Zero, One, SaturatedConversion, HashFor, }; -use sc_executor::RuntimeInfo; use sp_state_machine::{ DBValue, ChangesTrieTransaction, ChangesTrieCacheAction, UsageInfo as StateUsageInfo, StorageCollection, ChildStorageCollection, backend::Backend as StateBackend, StateMachineStats, }; -use crate::utils::{DatabaseType, Meta, db_err, meta_keys, read_db, read_meta}; +use crate::utils::{DatabaseType, Meta, meta_keys, read_db, read_meta}; use crate::changes_tries_storage::{DbChangesTrieStorage, DbChangesTrieStorageTransaction}; -use sc_client::leaves::{LeafSet, FinalizationDisplaced}; use sc_state_db::StateDb; use sp_blockchain::{CachedHeaderMetadata, HeaderMetadata, HeaderMetadataCache}; use crate::storage_cache::{CachingState, SyncingCachingState, SharedCache, new_shared_cache}; use crate::stats::StateUsageStats; use log::{trace, debug, warn}; + +// Re-export the Database trait so that one can pass an implementation of it. +pub use sp_database::Database; pub use sc_state_db::PruningMode; -use prometheus_endpoint::Registry; #[cfg(any(feature = "kvdb-rocksdb", test))] pub use bench::BenchmarkingState; -#[cfg(feature = "test-helpers")] -use sc_client::in_mem::Backend as InMemoryBackend; - -const CANONICALIZATION_DELAY: u64 = 4096; const MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR: u32 = 32768; /// Default value for storage cache child ratio. @@ -103,8 +102,9 @@ pub type DbState = sp_state_machine::TrieBackend< Arc>>, HashFor >; -/// Re-export the KVDB trait so that one can pass an implementation of it. -pub use kvdb; +const DB_HASH_LEN: usize = 32; +/// Hash type that this backend uses for the database. +pub type DbHash = [u8; DB_HASH_LEN]; /// A reference tracking state. /// @@ -155,11 +155,10 @@ impl StateBackend> for RefTrackingState { fn child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result>, Self::Error> { - self.state.child_storage(storage_key, child_info, key) + self.state.child_storage(child_info, key) } fn exists_storage(&self, key: &[u8]) -> Result { @@ -168,11 +167,10 @@ impl StateBackend> for RefTrackingState { fn exists_child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result { - self.state.exists_child_storage(storage_key, child_info, key) + self.state.exists_child_storage(child_info, key) } fn next_storage_key(&self, key: &[u8]) -> Result>, Self::Error> { @@ -181,11 +179,10 @@ impl StateBackend> for RefTrackingState { fn next_child_storage_key( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result>, Self::Error> { - self.state.next_child_storage_key(storage_key, child_info, key) + self.state.next_child_storage_key(child_info, key) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { @@ -198,21 +195,19 @@ impl StateBackend> for RefTrackingState { fn for_keys_in_child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, f: F, ) { - self.state.for_keys_in_child_storage(storage_key, child_info, f) + self.state.for_keys_in_child_storage(child_info, f) } fn for_child_keys_with_prefix( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], f: F, ) { - self.state.for_child_keys_with_prefix(storage_key, child_info, prefix, f) + self.state.for_child_keys_with_prefix(child_info, prefix, f) } fn storage_root(&self, delta: I) -> (B::Hash, Self::Transaction) @@ -224,14 +219,13 @@ impl StateBackend> for RefTrackingState { fn child_storage_root( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, delta: I, ) -> (B::Hash, bool, Self::Transaction) where I: IntoIterator, Option>)>, { - self.state.child_storage_root(storage_key, child_info, delta) + self.state.child_storage_root(child_info, delta) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -244,11 +238,10 @@ impl StateBackend> for RefTrackingState { fn child_keys( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], ) -> Vec> { - self.state.child_keys(storage_key, child_info, prefix) + self.state.child_keys(child_info, prefix) } fn as_trie_backend(&mut self) @@ -279,58 +272,49 @@ pub struct DatabaseSettings { } /// Where to find the database.. +#[derive(Clone)] pub enum DatabaseSettingsSrc { - /// Load a database from a given path. Recommended for most uses. - Path { + /// Load a RocksDB database from a given path. Recommended for most uses. + RocksDb { + /// Path to the database. + path: PathBuf, + /// Cache size in MiB. + cache_size: usize, + }, + + /// Load a ParityDb database from a given path. + ParityDb { + /// Path to the database. + path: PathBuf, + }, + + /// Load a Subdb database from a given path. + SubDb { /// Path to the database. path: PathBuf, - /// Cache size in bytes. If `None` default is used. - cache_size: Option, }, /// Use a custom already-open database. - Custom(Arc), + Custom(Arc>), } -/// Create an instance of db-backed client. -pub fn new_client( - settings: DatabaseSettings, - executor: E, - genesis_storage: &dyn BuildStorage, - fork_blocks: ForkBlocks, - bad_blocks: BadBlocks, - execution_extensions: ExecutionExtensions, - spawn_handle: Box, - prometheus_registry: Option, -) -> Result<( - sc_client::Client< - Backend, - sc_client::LocalCallExecutor, E>, - Block, - RA, - >, - Arc>, - ), - sp_blockchain::Error, -> - where - Block: BlockT, - E: CodeExecutor + RuntimeInfo, -{ - let backend = Arc::new(Backend::new(settings, CANONICALIZATION_DELAY)?); - let executor = sc_client::LocalCallExecutor::new(backend.clone(), executor, spawn_handle); - Ok(( - sc_client::Client::new( - backend.clone(), - executor, - genesis_storage, - fork_blocks, - bad_blocks, - execution_extensions, - prometheus_registry, - )?, - backend, - )) +impl DatabaseSettingsSrc { + /// Return dabase path for databases that are on the disk. + pub fn path(&self) -> Option<&Path> { + match self { + DatabaseSettingsSrc::RocksDb { path, .. } => Some(path.as_path()), + DatabaseSettingsSrc::ParityDb { path, .. } => Some(path.as_path()), + DatabaseSettingsSrc::SubDb { path, .. } => Some(path.as_path()), + DatabaseSettingsSrc::Custom(_) => None, + } + } + /// Check if database supports internal ref counting for state data. + pub fn supports_ref_counting(&self) -> bool { + match self { + DatabaseSettingsSrc::ParityDb { .. } => true, + _ => false, + } + } } pub(crate) mod columns { @@ -357,26 +341,26 @@ struct PendingBlock { } // wrapper that implements trait required for state_db -struct StateMetaDb<'a>(&'a dyn KeyValueDB); +struct StateMetaDb<'a>(&'a dyn Database); impl<'a> sc_state_db::MetaDb for StateMetaDb<'a> { type Error = io::Error; fn get_meta(&self, key: &[u8]) -> Result>, Self::Error> { - self.0.get(columns::STATE_META, key).map(|r| r.map(|v| v.to_vec())) + Ok(self.0.get(columns::STATE_META, key)) } } /// Block database pub struct BlockchainDb { - db: Arc, + db: Arc>, meta: Arc, Block::Hash>>>, leaves: RwLock>>, header_metadata_cache: HeaderMetadataCache, } impl BlockchainDb { - fn new(db: Arc) -> ClientResult { + fn new(db: Arc>) -> ClientResult { let meta = read_meta::(&*db, columns::HEADER)?; let leaves = LeafSet::read_from_db(&*db, columns::META, meta_keys::LEAF_PREFIX)?; Ok(BlockchainDb { @@ -412,14 +396,14 @@ impl BlockchainDb { } } -impl sc_client::blockchain::HeaderBackend for BlockchainDb { +impl sc_client_api::blockchain::HeaderBackend for BlockchainDb { fn header(&self, id: BlockId) -> ClientResult> { utils::read_header(&*self.db, columns::KEY_LOOKUP, columns::HEADER, id) } - fn info(&self) -> sc_client::blockchain::Info { + fn info(&self) -> sc_client_api::blockchain::Info { let meta = self.meta.read(); - sc_client::blockchain::Info { + sc_client_api::blockchain::Info { best_hash: meta.best_hash, best_number: meta.best_number, genesis_hash: meta.genesis_hash, @@ -429,7 +413,7 @@ impl sc_client::blockchain::HeaderBackend for BlockchainDb } } - fn status(&self, id: BlockId) -> ClientResult { + fn status(&self, id: BlockId) -> ClientResult { let exists = match id { BlockId::Hash(_) => read_db( &*self.db, @@ -440,8 +424,8 @@ impl sc_client::blockchain::HeaderBackend for BlockchainDb BlockId::Number(n) => n <= self.meta.read().best_number, }; match exists { - true => Ok(sc_client::blockchain::BlockStatus::InChain), - false => Ok(sc_client::blockchain::BlockStatus::Unknown), + true => Ok(sc_client_api::blockchain::BlockStatus::InChain), + false => Ok(sc_client_api::blockchain::BlockStatus::Unknown), } } @@ -457,7 +441,7 @@ impl sc_client::blockchain::HeaderBackend for BlockchainDb } } -impl sc_client::blockchain::Backend for BlockchainDb { +impl sc_client_api::blockchain::Backend for BlockchainDb { fn body(&self, id: BlockId) -> ClientResult>> { match read_db(&*self.db, columns::KEY_LOOKUP, columns::BODY, id)? { Some(body) => match Decode::decode(&mut &body[..]) { @@ -486,7 +470,7 @@ impl sc_client::blockchain::Backend for BlockchainDb Option>> { + fn cache(&self) -> Option>> { None } @@ -499,8 +483,8 @@ impl sc_client::blockchain::Backend for BlockchainDb sc_client::blockchain::ProvideCache for BlockchainDb { - fn cache(&self) -> Option>> { +impl sc_client_api::blockchain::ProvideCache for BlockchainDb { + fn cache(&self) -> Option>> { None } } @@ -536,6 +520,7 @@ pub struct BlockImportOperation { db_updates: PrefixedMemoryDB>, storage_updates: StorageCollection, child_storage_updates: ChildStorageCollection, + offchain_storage_updates: OffchainOverlayedChanges, changes_trie_updates: MemoryDB>, changes_trie_build_cache_update: Option>>, changes_trie_config_update: Option>, @@ -547,11 +532,20 @@ pub struct BlockImportOperation { } impl BlockImportOperation { - fn apply_aux(&mut self, transaction: &mut DBTransaction) { + fn apply_offchain(&mut self, transaction: &mut Transaction) { + for (key, value_operation) in self.offchain_storage_updates.drain() { + match value_operation { + OffchainOverlayedChange::SetValue(val) => transaction.set_from_vec(columns::OFFCHAIN, &key, val), + OffchainOverlayedChange::Remove => transaction.remove(columns::OFFCHAIN, &key), + } + } + } + + fn apply_aux(&mut self, transaction: &mut Transaction) { for (key, maybe_val) in self.aux_ops.drain(..) { match maybe_val { - Some(val) => transaction.put_vec(columns::AUX, &key, val), - None => transaction.delete(columns::AUX, &key), + Some(val) => transaction.set_from_vec(columns::AUX, &key, val), + None => transaction.remove(columns::AUX, &key), } } } @@ -602,16 +596,10 @@ impl sc_client_api::backend::BlockImportOperation for Bloc return Err(sp_blockchain::Error::GenesisInvalid.into()); } - for child_key in storage.children.keys() { - if !well_known_keys::is_child_storage_key(&child_key) { - return Err(sp_blockchain::Error::GenesisInvalid.into()); - } - } - - let child_delta = storage.children.into_iter().map(|(storage_key, child_content)| ( - storage_key, - child_content.data.into_iter().map(|(k, v)| (k, Some(v))), child_content.child_info), - ); + let child_delta = storage.children_default.into_iter().map(|(_storage_key, child_content)|( + child_content.child_info, + child_content.data.into_iter().map(|(k, v)| (k, Some(v))), + )); let mut changes_trie_config: Option = None; let (root, transaction) = self.old_state.full_storage_root( @@ -659,6 +647,14 @@ impl sc_client_api::backend::BlockImportOperation for Bloc Ok(()) } + fn update_offchain_storage( + &mut self, + offchain_update: OffchainOverlayedChanges, + ) -> ClientResult<()> { + self.offchain_storage_updates = offchain_update; + Ok(()) + } + fn mark_finalized( &mut self, block: BlockId, @@ -676,15 +672,20 @@ impl sc_client_api::backend::BlockImportOperation for Bloc } struct StorageDb { - pub db: Arc, + pub db: Arc>, pub state_db: StateDb>, + prefix_keys: bool, } impl sp_state_machine::Storage> for StorageDb { fn get(&self, key: &Block::Hash, prefix: Prefix) -> Result, String> { - let key = prefixed_key::>(key, prefix); - self.state_db.get(&key, self) - .map_err(|e| format!("Database backend error: {:?}", e)) + if self.prefix_keys { + let key = prefixed_key::>(key, prefix); + self.state_db.get(&key, self) + } else { + self.state_db.get(key.as_ref(), self) + } + .map_err(|e| format!("Database backend error: {:?}", e)) } } @@ -693,7 +694,7 @@ impl sc_state_db::NodeDb for StorageDb { type Key = [u8]; fn get(&self, key: &[u8]) -> Result>, Self::Error> { - self.db.get(columns::STATE, key).map(|r| r.map(|v| v.to_vec())) + Ok(self.db.get(columns::STATE, key)) } } @@ -776,13 +777,14 @@ impl Backend { /// The pruning window is how old a block must be before the state is pruned. pub fn new(config: DatabaseSettings, canonicalization_delay: u64) -> ClientResult { let db = crate::utils::open_database::(&config, DatabaseType::Full)?; - Self::from_kvdb(db as Arc<_>, canonicalization_delay, &config) + Self::from_database(db as Arc<_>, canonicalization_delay, &config) } /// Create new memory-backed client backend for tests. #[cfg(any(test, feature = "test-helpers"))] pub fn new_test(keep_blocks: u32, canonicalization_delay: u64) -> Self { - let db = Arc::new(kvdb_memorydb::create(crate::utils::NUM_COLUMNS)); + let db = kvdb_memorydb::create(crate::utils::NUM_COLUMNS); + let db = sp_database::as_database(db); let db_setting = DatabaseSettings { state_cache_size: 16777216, state_cache_child_ratio: Some((50, 100)), @@ -793,8 +795,8 @@ impl Backend { Self::new(db_setting, canonicalization_delay).expect("failed to create test-db") } - fn from_kvdb( - db: Arc, + fn from_database( + db: Arc>, canonicalization_delay: u64, config: &DatabaseSettings, ) -> ClientResult { @@ -804,11 +806,15 @@ impl Backend { let map_e = |e: sc_state_db::Error| sp_blockchain::Error::from( format!("State database error: {:?}", e) ); - let state_db: StateDb<_, _> = StateDb::new(config.pruning.clone(), &StateMetaDb(&*db)) - .map_err(map_e)?; + let state_db: StateDb<_, _> = StateDb::new( + config.pruning.clone(), + !config.source.supports_ref_counting(), + &StateMetaDb(&*db), + ).map_err(map_e)?; let storage_db = StorageDb { db: db.clone(), state_db, + prefix_keys: !config.source.supports_ref_counting(), }; let offchain_storage = offchain::LocalStorage::new(db.clone()); let changes_tries_storage = DbChangesTrieStorage::new( @@ -843,61 +849,6 @@ impl Backend { }) } - /// Returns in-memory blockchain that contains the same set of blocks as self. - #[cfg(feature = "test-helpers")] - pub fn as_in_memory(&self) -> InMemoryBackend { - use sc_client_api::backend::{Backend as ClientBackend, BlockImportOperation}; - use sc_client::blockchain::Backend as BlockchainBackend; - - let inmem = InMemoryBackend::::new(); - - // get all headers hashes && sort them by number (could be duplicate) - let mut headers: Vec<(NumberFor, Block::Hash, Block::Header)> = Vec::new(); - for (_, header) in self.blockchain.db.iter(columns::HEADER) { - let header = Block::Header::decode(&mut &header[..]).unwrap(); - let hash = header.hash(); - let number = *header.number(); - let pos = headers.binary_search_by(|item| item.0.cmp(&number)); - match pos { - Ok(pos) => headers.insert(pos, (number, hash, header)), - Err(pos) => headers.insert(pos, (number, hash, header)), - } - } - - // insert all other headers + bodies + justifications - let info = self.blockchain.info(); - for (number, hash, header) in headers { - let id = BlockId::Hash(hash); - let justification = self.blockchain.justification(id).unwrap(); - let body = self.blockchain.body(id).unwrap(); - let state = self.state_at(id).unwrap().pairs(); - - let new_block_state = if number.is_zero() { - NewBlockState::Final - } else if hash == info.best_hash { - NewBlockState::Best - } else { - NewBlockState::Normal - }; - let mut op = inmem.begin_operation().unwrap(); - op.set_block_data(header, body, justification, new_block_state).unwrap(); - op.update_db_storage(vec![(None, state.into_iter().map(|(k, v)| (k, Some(v))).collect())]) - .unwrap(); - inmem.commit_operation(op).unwrap(); - } - - // and now finalize the best block we have - inmem.finalize_block(BlockId::Hash(info.finalized_hash), None).unwrap(); - - inmem - } - - /// Returns total number of blocks (headers) in the block DB. - #[cfg(feature = "test-helpers")] - pub fn blocks_count(&self) -> u64 { - self.blockchain.db.iter(columns::HEADER).count() as u64 - } - /// Handle setting head within a transaction. `route_to` should be the last /// block that existed in the database. `best_to` should be the best block /// to be set. @@ -907,7 +858,7 @@ impl Backend { /// to be best, `route_to` should equal to `best_to`. fn set_head_with_transaction( &self, - transaction: &mut DBTransaction, + transaction: &mut Transaction, route_to: Block::Hash, best_to: (NumberFor, Block::Hash), ) -> ClientResult<(Vec, Vec)> { @@ -957,7 +908,7 @@ impl Backend { } let lookup_key = utils::number_and_hash_to_lookup_key(best_to.0, &best_to.1)?; - transaction.put(columns::META, meta_keys::BEST_BLOCK, &lookup_key); + transaction.set_from_vec(columns::META, meta_keys::BEST_BLOCK, lookup_key); utils::insert_number_to_key_mapping( transaction, columns::KEY_LOOKUP, @@ -984,7 +935,7 @@ impl Backend { fn finalize_block_with_transaction( &self, - transaction: &mut DBTransaction, + transaction: &mut Transaction, hash: &Block::Hash, header: &Block::Header, last_finalized: Option, @@ -1005,10 +956,10 @@ impl Backend { )?; if let Some(justification) = justification { - transaction.put( + transaction.set_from_vec( columns::JUSTIFICATION, &utils::number_and_hash_to_lookup_key(number, hash)?, - &justification.encode(), + justification.encode(), ); } Ok((*hash, number, false, true)) @@ -1017,7 +968,7 @@ impl Backend { // performs forced canonicalization with a delay after importing a non-finalized block. fn force_delayed_canonicalize( &self, - transaction: &mut DBTransaction, + transaction: &mut Transaction, hash: Block::Hash, number: NumberFor, ) @@ -1034,7 +985,7 @@ impl Backend { let hash = if new_canonical == number_u64 { hash } else { - ::sc_client::blockchain::HeaderBackend::hash(&self.blockchain, new_canonical.saturated_into())? + ::sc_client_api::blockchain::HeaderBackend::hash(&self.blockchain, new_canonical.saturated_into())? .expect("existence of block with number `new_canonical` \ implies existence of blocks with all numbers before it; qed") }; @@ -1051,10 +1002,11 @@ impl Backend { fn try_commit_operation(&self, mut operation: BlockImportOperation) -> ClientResult<()> { - let mut transaction = DBTransaction::new(); + let mut transaction = Transaction::new(); let mut finalization_displaced_leaves = None; operation.apply_aux(&mut transaction); + operation.apply_offchain(&mut transaction); let mut meta_updates = Vec::with_capacity(operation.finalized_blocks.len()); let mut last_finalized_hash = self.blockchain.meta.read().finalized_hash; @@ -1103,17 +1055,17 @@ impl Backend { header_metadata, ); - transaction.put(columns::HEADER, &lookup_key, &pending_block.header.encode()); + transaction.set_from_vec(columns::HEADER, &lookup_key, pending_block.header.encode()); if let Some(body) = &pending_block.body { - transaction.put(columns::BODY, &lookup_key, &body.encode()); + transaction.set_from_vec(columns::BODY, &lookup_key, body.encode()); } if let Some(justification) = pending_block.justification { - transaction.put(columns::JUSTIFICATION, &lookup_key, &justification.encode()); + transaction.set_from_vec(columns::JUSTIFICATION, &lookup_key, justification.encode()); } if number.is_zero() { - transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, &lookup_key); - transaction.put(columns::META, meta_keys::GENESIS_HASH, hash.as_ref()); + transaction.set_from_vec(columns::META, meta_keys::FINALIZED_BLOCK, lookup_key); + transaction.set(columns::META, meta_keys::GENESIS_HASH, hash.as_ref()); // for tests, because config is set from within the reset_storage if operation.changes_trie_config_update.is_none() { @@ -1127,17 +1079,32 @@ impl Backend { let mut bytes: u64 = 0; let mut removal: u64 = 0; let mut bytes_removal: u64 = 0; - for (key, (val, rc)) in operation.db_updates.drain() { + for (mut key, (val, rc)) in operation.db_updates.drain() { + if !self.storage.prefix_keys { + // Strip prefix + key.drain(0 .. key.len() - DB_HASH_LEN); + }; if rc > 0 { ops += 1; bytes += key.len() as u64 + val.len() as u64; - - changeset.inserted.push((key, val.to_vec())); + if rc == 1 { + changeset.inserted.push((key, val.to_vec())); + } else { + changeset.inserted.push((key.clone(), val.to_vec())); + for _ in 0 .. rc - 1 { + changeset.inserted.push((key.clone(), Default::default())); + } + } } else if rc < 0 { removal += 1; bytes_removal += key.len() as u64; - - changeset.deleted.push(key); + if rc == -1 { + changeset.deleted.push(key); + } else { + for _ in 0 .. -rc { + changeset.deleted.push(key.clone()); + } + } } } self.state_usage.tally_writes_nodes(ops, bytes); @@ -1242,7 +1209,7 @@ impl Backend { }; let cache_update = if let Some(set_head) = operation.set_head { - if let Some(header) = sc_client::blockchain::HeaderBackend::header(&self.blockchain, set_head)? { + if let Some(header) = sc_client_api::blockchain::HeaderBackend::header(&self.blockchain, set_head)? { let number = header.number(); let hash = header.hash(); @@ -1260,31 +1227,17 @@ impl Backend { None }; - let write_result = self.storage.db.write(transaction).map_err(db_err); + self.storage.db.commit(transaction); if let Some(( number, hash, enacted, retracted, - displaced_leaf, + _displaced_leaf, is_best, mut cache, )) = imported { - if let Err(e) = write_result { - let mut leaves = self.blockchain.leaves.write(); - let mut undo = leaves.undo(); - if let Some(displaced_leaf) = displaced_leaf { - undo.undo_import(displaced_leaf); - } - - if let Some(finalization_displaced) = finalization_displaced_leaves { - undo.undo_finalization(finalization_displaced); - } - - return Err(e) - } - cache.sync_cache( &enacted, &retracted, @@ -1317,7 +1270,7 @@ impl Backend { // was not a child of the last finalized block. fn note_finalized( &self, - transaction: &mut DBTransaction, + transaction: &mut Transaction, is_inserted: bool, f_header: &Block::Header, f_hash: Block::Hash, @@ -1328,7 +1281,7 @@ impl Backend { if self.storage.state_db.best_canonical().map(|c| f_num.saturated_into::() > c).unwrap_or(true) { let lookup_key = utils::number_and_hash_to_lookup_key(f_num, f_hash.clone())?; - transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, &lookup_key); + transaction.set_from_vec(columns::META, meta_keys::FINALIZED_BLOCK, lookup_key); let commit = self.storage.state_db.canonicalize_block(&f_hash) .map_err(|e: sc_state_db::Error| sp_blockchain::Error::from(format!("State database error: {:?}", e)))?; @@ -1357,18 +1310,18 @@ impl Backend { } } -fn apply_state_commit(transaction: &mut DBTransaction, commit: sc_state_db::CommitSet>) { +fn apply_state_commit(transaction: &mut Transaction, commit: sc_state_db::CommitSet>) { for (key, val) in commit.data.inserted.into_iter() { - transaction.put(columns::STATE, &key[..], &val); + transaction.set_from_vec(columns::STATE, &key[..], val); } for key in commit.data.deleted.into_iter() { - transaction.delete(columns::STATE, &key[..]); + transaction.remove(columns::STATE, &key[..]); } for (key, val) in commit.meta.inserted.into_iter() { - transaction.put(columns::STATE_META, &key[..], &val); + transaction.set_from_vec(columns::STATE_META, &key[..], val); } for key in commit.meta.deleted.into_iter() { - transaction.delete(columns::STATE_META, &key[..]); + transaction.remove(columns::STATE_META, &key[..]); } } @@ -1380,19 +1333,19 @@ impl sc_client_api::backend::AuxStore for Backend where Block: Blo I: IntoIterator, D: IntoIterator, >(&self, insert: I, delete: D) -> ClientResult<()> { - let mut transaction = DBTransaction::new(); + let mut transaction = Transaction::new(); for (k, v) in insert { - transaction.put(columns::AUX, k, v); + transaction.set(columns::AUX, k, v); } for k in delete { - transaction.delete(columns::AUX, k); + transaction.remove(columns::AUX, k); } - self.storage.db.write(transaction).map_err(db_err)?; + self.storage.db.commit(transaction); Ok(()) } fn get_aux(&self, key: &[u8]) -> ClientResult>> { - Ok(self.storage.db.get(columns::AUX, key).map(|r| r.map(|v| v.to_vec())).map_err(db_err)?) + Ok(self.storage.db.get(columns::AUX, key)) } } @@ -1412,6 +1365,7 @@ impl sc_client_api::backend::Backend for Backend { db_updates: PrefixedMemoryDB::default(), storage_updates: Default::default(), child_storage_updates: Default::default(), + offchain_storage_updates: Default::default(), changes_trie_config_update: None, changes_trie_updates: MemoryDB::default(), changes_trie_build_cache_update: None, @@ -1453,36 +1407,24 @@ impl sc_client_api::backend::Backend for Backend { fn finalize_block(&self, block: BlockId, justification: Option) -> ClientResult<()> { - let mut transaction = DBTransaction::new(); + let mut transaction = Transaction::new(); let hash = self.blockchain.expect_block_hash_from_id(&block)?; let header = self.blockchain.expect_header(block)?; let mut displaced = None; - let commit = |displaced| { - let mut changes_trie_cache_ops = None; - let (hash, number, is_best, is_finalized) = self.finalize_block_with_transaction( - &mut transaction, - &hash, - &header, - None, - justification, - &mut changes_trie_cache_ops, - displaced, - )?; - self.storage.db.write(transaction).map_err(db_err)?; - self.blockchain.update_meta(hash, number, is_best, is_finalized); - self.changes_tries_storage.post_commit(changes_trie_cache_ops); - Ok(()) - }; - match commit(&mut displaced) { - Ok(()) => self.storage.state_db.apply_pending(), - e @ Err(_) => { - self.storage.state_db.revert_pending(); - if let Some(displaced) = displaced { - self.blockchain.leaves.write().undo().undo_finalization(displaced); - } - return e; - } - } + + let mut changes_trie_cache_ops = None; + let (hash, number, is_best, is_finalized) = self.finalize_block_with_transaction( + &mut transaction, + &hash, + &header, + None, + justification, + &mut changes_trie_cache_ops, + &mut displaced, + )?; + self.storage.db.commit(transaction); + self.blockchain.update_meta(hash, number, is_best, is_finalized); + self.changes_tries_storage.post_commit(changes_trie_cache_ops); Ok(()) } @@ -1497,11 +1439,12 @@ impl sc_client_api::backend::Backend for Backend { fn usage_info(&self) -> Option { let (io_stats, state_stats) = self.io_stats.take_or_else(|| ( - self.storage.db.io_stats(kvdb::IoStatsKind::SincePrevious), + // TODO: implement DB stats and cache size retrieval + kvdb::IoStats::empty(), self.state_usage.take(), ) ); - let database_cache = MemorySize::from_bytes(parity_util_mem::malloc_size(&*self.storage.db)); + let database_cache = MemorySize::from_bytes(0); let state_cache = MemorySize::from_bytes( (*&self.shared_cache).lock().used_storage_cache_size(), ); @@ -1547,7 +1490,7 @@ impl sc_client_api::backend::Backend for Backend { if best_number.is_zero() { return Ok(c.saturated_into::>()) } - let mut transaction = DBTransaction::new(); + let mut transaction = Transaction::new(); match self.storage.state_db.revert_one() { Some(commit) => { apply_state_commit(&mut transaction, commit); @@ -1571,13 +1514,13 @@ impl sc_client_api::backend::Backend for Backend { removed_number, ), )?; - transaction.put(columns::META, meta_keys::BEST_BLOCK, &key); if update_finalized { - transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, &key); + transaction.set_from_vec(columns::META, meta_keys::FINALIZED_BLOCK, key.clone()); } - transaction.delete(columns::KEY_LOOKUP, removed.hash().as_ref()); + transaction.set_from_vec(columns::META, meta_keys::BEST_BLOCK, key); + transaction.remove(columns::KEY_LOOKUP, removed.hash().as_ref()); children::remove_children(&mut transaction, columns::META, meta_keys::CHILDREN_PREFIX, best_hash); - self.storage.db.write(transaction).map_err(db_err)?; + self.storage.db.commit(transaction); self.changes_tries_storage.post_commit(Some(changes_trie_cache_ops)); self.blockchain.update_meta(best_hash, best_number, true, update_finalized); } @@ -1591,12 +1534,12 @@ impl sc_client_api::backend::Backend for Backend { let reverted = revert_blocks()?; let revert_leaves = || -> ClientResult<()> { - let mut transaction = DBTransaction::new(); + let mut transaction = Transaction::new(); let mut leaves = self.blockchain.leaves.write(); leaves.revert(best_hash, best_number); leaves.prepare_transaction(&mut transaction, columns::META, meta_keys::LEAF_PREFIX); - self.storage.db.write(transaction).map_err(db_err)?; + self.storage.db.commit(transaction); Ok(()) }; @@ -1611,7 +1554,7 @@ impl sc_client_api::backend::Backend for Backend { } fn state_at(&self, block: BlockId) -> ClientResult { - use sc_client::blockchain::HeaderBackend as BcHeaderBackend; + use sc_client_api::blockchain::HeaderBackend as BcHeaderBackend; // special case for genesis initialization match block { @@ -1712,7 +1655,7 @@ pub(crate) mod tests { use crate::columns; use sp_core::H256; use sc_client_api::backend::{Backend as BTrait, BlockImportOperation as Op}; - use sc_client::blockchain::Backend as BLBTrait; + use sc_client_api::blockchain::Backend as BLBTrait; use sp_runtime::testing::{Header, Block as RawBlock, ExtrinsicWrapper}; use sp_runtime::traits::{Hash, BlakeTwo256}; use sp_runtime::generic::DigestItem; @@ -1858,7 +1801,7 @@ pub(crate) mod tests { op.reset_storage(Storage { top: storage.iter().cloned().collect(), - children: Default::default(), + children_default: Default::default(), }).unwrap(); op.set_block_data( header.clone(), @@ -1944,7 +1887,7 @@ pub(crate) mod tests { op.reset_storage(Storage { top: storage.iter().cloned().collect(), - children: Default::default(), + children_default: Default::default(), }).unwrap(); key = op.db_updates.insert(EMPTY_PREFIX, b"hello"); @@ -1959,7 +1902,7 @@ pub(crate) mod tests { assert_eq!(backend.storage.db.get( columns::STATE, &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) - ).unwrap().unwrap(), &b"hello"[..]); + ).unwrap(), &b"hello"[..]); hash }; @@ -1996,7 +1939,7 @@ pub(crate) mod tests { assert_eq!(backend.storage.db.get( columns::STATE, &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) - ).unwrap().unwrap(), &b"hello"[..]); + ).unwrap(), &b"hello"[..]); hash }; @@ -2034,7 +1977,7 @@ pub(crate) mod tests { assert!(backend.storage.db.get( columns::STATE, &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) - ).unwrap().is_some()); + ).is_some()); hash }; @@ -2068,7 +2011,7 @@ pub(crate) mod tests { assert!(backend.storage.db.get( columns::STATE, &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) - ).unwrap().is_none()); + ).is_none()); } backend.finalize_block(BlockId::Number(1), None).unwrap(); @@ -2077,7 +2020,7 @@ pub(crate) mod tests { assert!(backend.storage.db.get( columns::STATE, &sp_trie::prefixed_key::(&key, EMPTY_PREFIX) - ).unwrap().is_none()); + ).is_none()); } #[test] @@ -2291,7 +2234,7 @@ pub(crate) mod tests { #[test] fn test_finalize_block_with_justification() { - use sc_client::blockchain::{Backend as BlockChainBackend}; + use sc_client_api::blockchain::{Backend as BlockChainBackend}; let backend = Backend::::new_test(10, 10); diff --git a/client/db/src/light.rs b/client/db/src/light.rs index 808a209f527c170afdbb0e9091c7233a74dbd248..edc7f8fc552dd90d513e339eb848eb469171eb31 100644 --- a/client/db/src/light.rs +++ b/client/db/src/light.rs @@ -20,26 +20,26 @@ use std::{sync::Arc, collections::HashMap}; use std::convert::TryInto; use parking_lot::RwLock; -use kvdb::{KeyValueDB, DBTransaction}; - -use sc_client_api::{backend::{AuxStore, NewBlockState}, UsageInfo}; -use sc_client::blockchain::{ - BlockStatus, Cache as BlockchainCache,Info as BlockchainInfo, +use sc_client_api::{ + cht, backend::{AuxStore, NewBlockState}, UsageInfo, + blockchain::{ + BlockStatus, Cache as BlockchainCache, Info as BlockchainInfo, + }, + Storage }; -use sc_client::cht; use sp_blockchain::{ CachedHeaderMetadata, HeaderMetadata, HeaderMetadataCache, Error as ClientError, Result as ClientResult, HeaderBackend as BlockchainHeaderBackend, well_known_cache_keys, }; -use sc_client::light::blockchain::Storage as LightBlockchainStorage; +use sp_database::{Database, Transaction}; use codec::{Decode, Encode}; use sp_runtime::generic::{DigestItem, BlockId}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, One, NumberFor, HashFor}; use crate::cache::{DbCacheSync, DbCache, ComplexBlockId, EntryType as CacheEntryType}; -use crate::utils::{self, meta_keys, DatabaseType, Meta, db_err, read_db, block_id_to_lookup_key, read_meta}; -use crate::{DatabaseSettings, FrozenForDuration}; +use crate::utils::{self, meta_keys, DatabaseType, Meta, read_db, block_id_to_lookup_key, read_meta}; +use crate::{DatabaseSettings, FrozenForDuration, DbHash}; use log::{trace, warn, debug}; pub(crate) mod columns { @@ -59,7 +59,7 @@ const CHANGES_TRIE_CHT_PREFIX: u8 = 1; /// Light blockchain storage. Stores most recent headers + CHTs for older headers. /// Locks order: meta, cache. pub struct LightStorage { - db: Arc, + db: Arc>, meta: RwLock, Block::Hash>>, cache: Arc>, header_metadata_cache: HeaderMetadataCache, @@ -78,14 +78,11 @@ impl LightStorage { /// Create new memory-backed `LightStorage` for tests. #[cfg(any(test, feature = "test-helpers"))] pub fn new_test() -> Self { - use utils::NUM_COLUMNS; - - let db = Arc::new(::kvdb_memorydb::create(NUM_COLUMNS)); - + let db = Arc::new(sp_database::MemDb::default()); Self::from_kvdb(db as Arc<_>).expect("failed to create test-db") } - fn from_kvdb(db: Arc) -> ClientResult { + fn from_kvdb(db: Arc>) -> ClientResult { let meta = read_meta::(&*db, columns::HEADER)?; let cache = DbCache::new( db.clone(), @@ -230,7 +227,7 @@ impl LightStorage { /// to be best, `route_to` should equal to `best_to`. fn set_head_with_transaction( &self, - transaction: &mut DBTransaction, + transaction: &mut Transaction, route_to: Block::Hash, best_to: (NumberFor, Block::Hash), ) -> ClientResult<()> { @@ -266,7 +263,7 @@ impl LightStorage { } } - transaction.put(columns::META, meta_keys::BEST_BLOCK, &lookup_key); + transaction.set_from_vec(columns::META, meta_keys::BEST_BLOCK, lookup_key); utils::insert_number_to_key_mapping( transaction, columns::KEY_LOOKUP, @@ -280,7 +277,7 @@ impl LightStorage { // Note that a block is finalized. Only call with child of last finalized block. fn note_finalized( &self, - transaction: &mut DBTransaction, + transaction: &mut Transaction, header: &Block::Header, hash: Block::Hash, ) -> ClientResult<()> { @@ -293,7 +290,7 @@ impl LightStorage { } let lookup_key = utils::number_and_hash_to_lookup_key(header.number().clone(), hash)?; - transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, &lookup_key); + transaction.set_from_vec(columns::META, meta_keys::FINALIZED_BLOCK, lookup_key); // build new CHT(s) if required if let Some(new_cht_number) = cht::is_build_required(cht::size(), *header.number()) { @@ -309,7 +306,7 @@ impl LightStorage { let new_header_cht_root = cht::compute_root::, _>( cht::size(), new_cht_number, cht_range.map(|num| self.hash(num)) )?; - transaction.put( + transaction.set( columns::CHT, &cht_key(HEADER_CHT_PREFIX, new_cht_start)?, new_header_cht_root.as_ref() @@ -327,7 +324,7 @@ impl LightStorage { cht::size(), new_cht_number, cht_range .map(|num| self.changes_trie_root(BlockId::Number(num))) )?; - transaction.put( + transaction.set( columns::CHT, &cht_key(CHANGES_TRIE_CHT_PREFIX, new_cht_start)?, new_changes_trie_cht_root.as_ref() @@ -350,7 +347,7 @@ impl LightStorage { prune_block, hash )?; - transaction.delete(columns::HEADER, &lookup_key); + transaction.remove(columns::HEADER, &lookup_key); } prune_block += One::one(); } @@ -377,7 +374,7 @@ impl LightStorage { } let cht_start = cht::start_number(cht_size, cht_number); - self.db.get(columns::CHT, &cht_key(cht_type, cht_start)?).map_err(db_err)? + self.db.get(columns::CHT, &cht_key(cht_type, cht_start)?) .ok_or_else(no_cht_for_block) .and_then(|hash| Block::Hash::decode(&mut &*hash).map_err(|_| no_cht_for_block())) .map(Some) @@ -394,22 +391,23 @@ impl AuxStore for LightStorage I: IntoIterator, D: IntoIterator, >(&self, insert: I, delete: D) -> ClientResult<()> { - let mut transaction = DBTransaction::new(); + let mut transaction = Transaction::new(); for (k, v) in insert { - transaction.put(columns::AUX, k, v); + transaction.set(columns::AUX, k, v); } for k in delete { - transaction.delete(columns::AUX, k); + transaction.remove(columns::AUX, k); } - self.db.write(transaction).map_err(db_err) + self.db.commit(transaction); + Ok(()) } fn get_aux(&self, key: &[u8]) -> ClientResult>> { - self.db.get(columns::AUX, key).map(|r| r.map(|v| v.to_vec())).map_err(db_err) + Ok(self.db.get(columns::AUX, key)) } } -impl LightBlockchainStorage for LightStorage +impl Storage for LightStorage where Block: BlockT, { fn import_header( @@ -419,7 +417,7 @@ impl LightBlockchainStorage for LightStorage leaf_state: NewBlockState, aux_ops: Vec<(Vec, Option>)>, ) -> ClientResult<()> { - let mut transaction = DBTransaction::new(); + let mut transaction = Transaction::new(); let hash = header.hash(); let number = *header.number(); @@ -427,8 +425,8 @@ impl LightBlockchainStorage for LightStorage for (key, maybe_val) in aux_ops { match maybe_val { - Some(val) => transaction.put_vec(columns::AUX, &key, val), - None => transaction.delete(columns::AUX, &key), + Some(val) => transaction.set_from_vec(columns::AUX, &key, val), + None => transaction.remove(columns::AUX, &key), } } @@ -445,7 +443,7 @@ impl LightBlockchainStorage for LightStorage number, hash, )?; - transaction.put(columns::HEADER, &lookup_key, &header.encode()); + transaction.set_from_vec(columns::HEADER, &lookup_key, header.encode()); let header_metadata = CachedHeaderMetadata::from(&header); self.header_metadata_cache.insert_header_metadata( @@ -456,7 +454,7 @@ impl LightBlockchainStorage for LightStorage let is_genesis = number.is_zero(); if is_genesis { self.cache.0.write().set_genesis_hash(hash); - transaction.put(columns::META, meta_keys::GENESIS_HASH, hash.as_ref()); + transaction.set(columns::META, meta_keys::GENESIS_HASH, hash.as_ref()); } let finalized = match leaf_state { @@ -493,7 +491,7 @@ impl LightBlockchainStorage for LightStorage debug!("Light DB Commit {:?} ({})", hash, number); - self.db.write(transaction).map_err(db_err)?; + self.db.commit(transaction); cache.commit(cache_ops) .expect("only fails if cache with given name isn't loaded yet;\ cache is already loaded because there are cache_ops; qed"); @@ -509,9 +507,9 @@ impl LightBlockchainStorage for LightStorage let hash = header.hash(); let number = header.number(); - let mut transaction = DBTransaction::new(); + let mut transaction = Transaction::new(); self.set_head_with_transaction(&mut transaction, hash.clone(), (number.clone(), hash.clone()))?; - self.db.write(transaction).map_err(db_err)?; + self.db.commit(transaction); self.update_meta(hash, header.number().clone(), true, false); Ok(()) } else { @@ -537,7 +535,7 @@ impl LightBlockchainStorage for LightStorage fn finalize_header(&self, id: BlockId) -> ClientResult<()> { if let Some(header) = self.header(id)? { - let mut transaction = DBTransaction::new(); + let mut transaction = Transaction::new(); let hash = header.hash(); let number = *header.number(); self.note_finalized(&mut transaction, &header, hash.clone())?; @@ -550,7 +548,7 @@ impl LightBlockchainStorage for LightStorage )? .into_ops(); - self.db.write(transaction).map_err(db_err)?; + self.db.commit(transaction); cache.commit(cache_ops) .expect("only fails if cache with given name isn't loaded yet;\ cache is already loaded because there are cache_ops; qed"); @@ -575,8 +573,9 @@ impl LightBlockchainStorage for LightStorage fn usage_info(&self) -> Option { use sc_client_api::{MemoryInfo, IoInfo, MemorySize}; - let database_cache = MemorySize::from_bytes(parity_util_mem::malloc_size(&*self.db)); - let io_stats = self.io_stats.take_or_else(|| self.db.io_stats(kvdb::IoStatsKind::SincePrevious)); + // TODO: reimplement IO stats + let database_cache = MemorySize::from_bytes(0); + let io_stats = self.io_stats.take_or_else(|| kvdb::IoStats::empty()); Some(UsageInfo { memory: MemoryInfo { @@ -616,7 +615,7 @@ fn cht_key>(cht_type: u8, block: N) -> ClientResult<[u8; 5]> { #[cfg(test)] pub(crate) mod tests { - use sc_client::cht; + use sc_client_api::cht; use sp_core::ChangesTrieConfiguration; use sp_runtime::generic::{DigestItem, ChangesTrieSignal}; use sp_runtime::testing::{H256 as Hash, Header, Block as RawBlock, ExtrinsicWrapper}; @@ -732,21 +731,25 @@ pub(crate) mod tests { #[test] fn import_header_works() { - let db = LightStorage::new_test(); + let raw_db = Arc::new(sp_database::MemDb::default()); + let db = LightStorage::from_kvdb(raw_db.clone()).unwrap(); let genesis_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0)); - assert_eq!(db.db.iter(columns::HEADER).count(), 1); - assert_eq!(db.db.iter(columns::KEY_LOOKUP).count(), 2); + assert_eq!(raw_db.count(columns::HEADER), 1); + assert_eq!(raw_db.count(columns::KEY_LOOKUP), 2); let _ = insert_block(&db, HashMap::new(), || default_header(&genesis_hash, 1)); - assert_eq!(db.db.iter(columns::HEADER).count(), 2); - assert_eq!(db.db.iter(columns::KEY_LOOKUP).count(), 4); + assert_eq!(raw_db.count(columns::HEADER), 2); + assert_eq!(raw_db.count(columns::KEY_LOOKUP), 4); } #[test] fn finalized_ancient_headers_are_replaced_with_cht() { - fn insert_headers Header>(header_producer: F) -> LightStorage { - let db = LightStorage::new_test(); + fn insert_headers Header>(header_producer: F) -> + (Arc>, LightStorage) + { + let raw_db = Arc::new(sp_database::MemDb::default()); + let db = LightStorage::from_kvdb(raw_db.clone()).unwrap(); let cht_size: u64 = cht::size(); let ucht_size: usize = cht_size as _; @@ -758,8 +761,8 @@ pub(crate) mod tests { for number in 0..cht::size() { prev_hash = insert_block(&db, HashMap::new(), || header_producer(&prev_hash, 1 + number)); } - assert_eq!(db.db.iter(columns::HEADER).count(), 1 + ucht_size); - assert_eq!(db.db.iter(columns::CHT).count(), 0); + assert_eq!(raw_db.count(columns::HEADER), 1 + ucht_size); + assert_eq!(raw_db.count(columns::CHT), 0); // insert next SIZE blocks && ensure that nothing is pruned for number in 0..(cht_size as _) { @@ -769,8 +772,8 @@ pub(crate) mod tests { || header_producer(&prev_hash, 1 + cht_size + number), ); } - assert_eq!(db.db.iter(columns::HEADER).count(), 1 + ucht_size + ucht_size); - assert_eq!(db.db.iter(columns::CHT).count(), 0); + assert_eq!(raw_db.count(columns::HEADER), 1 + ucht_size + ucht_size); + assert_eq!(raw_db.count(columns::CHT), 0); // insert block #{2 * cht::size() + 1} && check that new CHT is created + headers of this CHT are pruned // nothing is yet finalized, so nothing is pruned. @@ -779,23 +782,23 @@ pub(crate) mod tests { HashMap::new(), || header_producer(&prev_hash, 1 + cht_size + cht_size), ); - assert_eq!(db.db.iter(columns::HEADER).count(), 2 + ucht_size + ucht_size); - assert_eq!(db.db.iter(columns::CHT).count(), 0); + assert_eq!(raw_db.count(columns::HEADER), 2 + ucht_size + ucht_size); + assert_eq!(raw_db.count(columns::CHT), 0); // now finalize the block. for i in (0..(ucht_size + ucht_size)).map(|i| i + 1) { db.finalize_header(BlockId::Number(i as _)).unwrap(); } db.finalize_header(BlockId::Hash(prev_hash)).unwrap(); - db + (raw_db, db) } // when headers are created without changes tries roots - let db = insert_headers(default_header); + let (raw_db, db) = insert_headers(default_header); let cht_size: u64 = cht::size(); - assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht_size + 1) as usize); - assert_eq!(db.db.iter(columns::KEY_LOOKUP).count(), (2 * (1 + cht_size + 1)) as usize); - assert_eq!(db.db.iter(columns::CHT).count(), 1); + assert_eq!(raw_db.count(columns::HEADER), (1 + cht_size + 1) as usize); + assert_eq!(raw_db.count(columns::KEY_LOOKUP), (2 * (1 + cht_size + 1)) as usize); + assert_eq!(raw_db.count(columns::CHT), 1); assert!((0..cht_size as _).all(|i| db.header(BlockId::Number(1 + i)).unwrap().is_none())); assert!(db.header_cht_root(cht_size, cht_size / 2).unwrap().is_some()); assert!(db.header_cht_root(cht_size, cht_size + cht_size / 2).unwrap().is_none()); @@ -803,9 +806,9 @@ pub(crate) mod tests { assert!(db.changes_trie_cht_root(cht_size, cht_size + cht_size / 2).unwrap().is_none()); // when headers are created with changes tries roots - let db = insert_headers(header_with_changes_trie); - assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht_size + 1) as usize); - assert_eq!(db.db.iter(columns::CHT).count(), 2); + let (raw_db, db) = insert_headers(header_with_changes_trie); + assert_eq!(raw_db.count(columns::HEADER), (1 + cht_size + 1) as usize); + assert_eq!(raw_db.count(columns::CHT), 2); assert!((0..cht_size as _).all(|i| db.header(BlockId::Number(1 + i)).unwrap().is_none())); assert!(db.header_cht_root(cht_size, cht_size / 2).unwrap().is_some()); assert!(db.header_cht_root(cht_size, cht_size + cht_size / 2).unwrap().is_none()); @@ -963,7 +966,7 @@ pub(crate) mod tests { fn run_checks(db: &LightStorage, max: u64, checks: &[(u64, Option>)]) { for (at, expected) in checks.iter().take_while(|(at, _)| *at <= max) { - let actual = get_authorities(db.cache(), BlockId::Number(*at)); + let actual = authorities(db.cache(), BlockId::Number(*at)); assert_eq!(*expected, actual); } } @@ -978,7 +981,7 @@ pub(crate) mod tests { map } - fn get_authorities(cache: &dyn BlockchainCache, at: BlockId) -> Option> { + fn authorities(cache: &dyn BlockchainCache, at: BlockId) -> Option> { cache.get_at(&well_known_cache_keys::AUTHORITIES, &at).unwrap_or(None) .and_then(|(_, _, val)| Decode::decode(&mut &val[..]).ok()) } @@ -1026,9 +1029,9 @@ pub(crate) mod tests { // ... -> B2(1) -> B2_1(1) -> B2_2(2) // => the cache ignores all writes before best finalized block let hash2_1 = insert_non_best_block(&db, make_authorities(vec![auth1()]), || default_header(&hash2, 3)); - assert_eq!(None, get_authorities(db.cache(), BlockId::Hash(hash2_1))); + assert_eq!(None, authorities(db.cache(), BlockId::Hash(hash2_1))); let hash2_2 = insert_non_best_block(&db, make_authorities(vec![auth1(), auth2()]), || default_header(&hash2_1, 4)); - assert_eq!(None, get_authorities(db.cache(), BlockId::Hash(hash2_2))); + assert_eq!(None, authorities(db.cache(), BlockId::Hash(hash2_2))); } let (hash7, hash8, hash6_1, hash6_2, hash6_1_1, hash6_1_2) = { @@ -1040,55 +1043,55 @@ pub(crate) mod tests { let hash7 = insert_block(&db, make_authorities(vec![auth3()]), || default_header(&hash6, 7)); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); let hash8 = insert_block(&db, make_authorities(vec![auth3()]), || default_header(&hash7, 8)); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); let hash6_1 = insert_block(&db, make_authorities(vec![auth4()]), || default_header(&hash6, 7)); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); let hash6_1_1 = insert_non_best_block(&db, make_authorities(vec![auth5()]), || default_header(&hash6_1, 8)); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); let hash6_1_2 = insert_non_best_block(&db, make_authorities(vec![auth6()]), || default_header(&hash6_1, 8)); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()])); let hash6_2 = insert_block(&db, make_authorities(vec![auth4()]), || default_header(&hash6_1, 8)); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()])); (hash7, hash8, hash6_1, hash6_2, hash6_1_1, hash6_1_2) }; @@ -1097,27 +1100,27 @@ pub(crate) mod tests { // finalize block hash6_1 db.finalize_header(BlockId::Hash(hash6_1)).unwrap(); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), None); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash8)), None); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), None); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), None); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()])); // finalize block hash6_2 db.finalize_header(BlockId::Hash(hash6_2)).unwrap(); assert_eq!( - get_authorities(db.cache(), BlockId::Hash(hash6)), + authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]), ); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash7)), None); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash8)), None); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_1)), None); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_1_2)), None); - assert_eq!(get_authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), None); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), None); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), None); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_2)), None); + assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()])); } } diff --git a/client/db/src/offchain.rs b/client/db/src/offchain.rs index 3d0f8c6795a940d246835826817fe2661bec74ec..8c58d5f42c391293268268eedd60859df0e54766 100644 --- a/client/db/src/offchain.rs +++ b/client/db/src/offchain.rs @@ -21,14 +21,13 @@ use std::{ sync::Arc, }; -use crate::columns; -use kvdb::KeyValueDB; +use crate::{columns, Database, DbHash, Transaction}; use parking_lot::Mutex; /// Offchain local storage #[derive(Clone)] pub struct LocalStorage { - db: Arc, + db: Arc>, locks: Arc, Arc>>>>, } @@ -43,12 +42,13 @@ impl LocalStorage { /// Create new offchain storage for tests (backed by memorydb) #[cfg(any(test, feature = "test-helpers"))] pub fn new_test() -> Self { - let db = Arc::new(kvdb_memorydb::create(crate::utils::NUM_COLUMNS)); + let db = kvdb_memorydb::create(crate::utils::NUM_COLUMNS); + let db = sp_database::as_database(db); Self::new(db as _) } /// Create offchain local storage with given `KeyValueDB` backend. - pub fn new(db: Arc) -> Self { + pub fn new(db: Arc>) -> Self { Self { db, locks: Default::default(), @@ -59,20 +59,15 @@ impl LocalStorage { impl sp_core::offchain::OffchainStorage for LocalStorage { fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { let key: Vec = prefix.iter().chain(key).cloned().collect(); - let mut tx = self.db.transaction(); - tx.put(columns::OFFCHAIN, &key, value); + let mut tx = Transaction::new(); + tx.set(columns::OFFCHAIN, &key, value); - if let Err(e) = self.db.write(tx) { - log::warn!("Error writing to the offchain DB: {:?}", e); - } + self.db.commit(tx); } fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { let key: Vec = prefix.iter().chain(key).cloned().collect(); self.db.get(columns::OFFCHAIN, &key) - .ok() - .and_then(|x| x) - .map(|v| v.to_vec()) } fn compare_and_set( @@ -91,9 +86,7 @@ impl sp_core::offchain::OffchainStorage for LocalStorage { let is_set; { let _key_guard = key_lock.lock(); - let val = self.db.get(columns::OFFCHAIN, &key) - .ok() - .and_then(|x| x); + let val = self.db.get(columns::OFFCHAIN, &key); is_set = val.as_ref().map(|x| &**x) == old_value; if is_set { diff --git a/client/db/src/parity_db.rs b/client/db/src/parity_db.rs new file mode 100644 index 0000000000000000000000000000000000000000..7333f70e25f36ae0418e43d5fd4e8ebff8fff496 --- /dev/null +++ b/client/db/src/parity_db.rs @@ -0,0 +1,63 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +/// A `Database` adapter for parity-db. + +use sp_database::{Database, Change, Transaction, ColumnId}; +use crate::utils::NUM_COLUMNS; +use crate::columns; + +struct DbAdapter(parity_db::Db); + +fn handle_err(result: parity_db::Result) -> T { + match result { + Ok(r) => r, + Err(e) => { + panic!("Critical database eror: {:?}", e); + } + } +} + +/// Wrap RocksDb database into a trait object that implements `sp_database::Database` +pub fn open(path: &std::path::Path) -> parity_db::Result>> { + let mut config = parity_db::Options::with_columns(path, NUM_COLUMNS as u8); + let mut state_col = &mut config.columns[columns::STATE as usize]; + state_col.ref_counted = true; + state_col.preimage = true; + state_col.uniform = true; + let db = parity_db::Db::open(&config)?; + Ok(std::sync::Arc::new(DbAdapter(db))) +} + +impl Database for DbAdapter { + fn commit(&self, transaction: Transaction) { + handle_err(self.0.commit(transaction.0.into_iter().map(|change| + match change { + Change::Set(col, key, value) => (col as u8, key, Some(value)), + Change::Remove(col, key) => (col as u8, key, None), + _ => unimplemented!(), + })) + ); + } + + fn get(&self, col: ColumnId, key: &[u8]) -> Option> { + handle_err(self.0.get(col as u8, key)) + } + + fn lookup(&self, _hash: &H) -> Option> { + unimplemented!(); + } +} diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index 63268992632410934d0518039882240dee328c0d..66ac74afa4f2a1d00458c58eeeda86e0b19e0f51 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -542,11 +542,10 @@ impl>, B: BlockT> StateBackend> for Cachin fn child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result>, Self::Error> { - let key = (storage_key.to_vec(), key.to_vec()); + let key = (child_info.storage_key().to_vec(), key.to_vec()); let local_cache = self.cache.local_cache.upgradable_read(); if let Some(entry) = local_cache.child_storage.get(&key).cloned() { trace!("Found in local cache: {:?}", key); @@ -564,7 +563,7 @@ impl>, B: BlockT> StateBackend> for Cachin } } trace!("Cache miss: {:?}", key); - let value = self.state.child_storage(storage_key, child_info, &key.1[..])?; + let value = self.state.child_storage(child_info, &key.1[..])?; // just pass it through the usage counter let value = self.usage.tally_child_key_read(&key, value, false); @@ -579,20 +578,18 @@ impl>, B: BlockT> StateBackend> for Cachin fn exists_child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result { - self.state.exists_child_storage(storage_key, child_info, key) + self.state.exists_child_storage(child_info, key) } fn for_keys_in_child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, f: F, ) { - self.state.for_keys_in_child_storage(storage_key, child_info, f) + self.state.for_keys_in_child_storage(child_info, f) } fn next_storage_key(&self, key: &[u8]) -> Result>, Self::Error> { @@ -601,11 +598,10 @@ impl>, B: BlockT> StateBackend> for Cachin fn next_child_storage_key( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result>, Self::Error> { - self.state.next_child_storage_key(storage_key, child_info, key) + self.state.next_child_storage_key(child_info, key) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { @@ -618,12 +614,11 @@ impl>, B: BlockT> StateBackend> for Cachin fn for_child_keys_with_prefix( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], f: F, ) { - self.state.for_child_keys_with_prefix(storage_key, child_info, prefix, f) + self.state.for_child_keys_with_prefix(child_info, prefix, f) } fn storage_root(&self, delta: I) -> (B::Hash, Self::Transaction) @@ -635,14 +630,13 @@ impl>, B: BlockT> StateBackend> for Cachin fn child_storage_root( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, delta: I, ) -> (B::Hash, bool, Self::Transaction) where I: IntoIterator, Option>)>, { - self.state.child_storage_root(storage_key, child_info, delta) + self.state.child_storage_root(child_info, delta) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -655,11 +649,10 @@ impl>, B: BlockT> StateBackend> for Cachin fn child_keys( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], ) -> Vec> { - self.state.child_keys(storage_key, child_info, prefix) + self.state.child_keys(child_info, prefix) } fn as_trie_backend(&mut self) -> Option<&TrieBackend>> { @@ -758,11 +751,10 @@ impl>, B: BlockT> StateBackend> for Syncin fn child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result>, Self::Error> { - self.caching_state().child_storage(storage_key, child_info, key) + self.caching_state().child_storage(child_info, key) } fn exists_storage(&self, key: &[u8]) -> Result { @@ -771,20 +763,18 @@ impl>, B: BlockT> StateBackend> for Syncin fn exists_child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result { - self.caching_state().exists_child_storage(storage_key, child_info, key) + self.caching_state().exists_child_storage(child_info, key) } fn for_keys_in_child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, f: F, ) { - self.caching_state().for_keys_in_child_storage(storage_key, child_info, f) + self.caching_state().for_keys_in_child_storage(child_info, f) } fn next_storage_key(&self, key: &[u8]) -> Result>, Self::Error> { @@ -793,11 +783,10 @@ impl>, B: BlockT> StateBackend> for Syncin fn next_child_storage_key( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result>, Self::Error> { - self.caching_state().next_child_storage_key(storage_key, child_info, key) + self.caching_state().next_child_storage_key(child_info, key) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { @@ -810,12 +799,11 @@ impl>, B: BlockT> StateBackend> for Syncin fn for_child_keys_with_prefix( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], f: F, ) { - self.caching_state().for_child_keys_with_prefix(storage_key, child_info, prefix, f) + self.caching_state().for_child_keys_with_prefix(child_info, prefix, f) } fn storage_root(&self, delta: I) -> (B::Hash, Self::Transaction) @@ -827,14 +815,13 @@ impl>, B: BlockT> StateBackend> for Syncin fn child_storage_root( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, delta: I, ) -> (B::Hash, bool, Self::Transaction) where I: IntoIterator, Option>)>, { - self.caching_state().child_storage_root(storage_key, child_info, delta) + self.caching_state().child_storage_root(child_info, delta) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -847,11 +834,10 @@ impl>, B: BlockT> StateBackend> for Syncin fn child_keys( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], ) -> Vec> { - self.caching_state().child_keys(storage_key, child_info, prefix) + self.caching_state().child_keys(child_info, prefix) } fn as_trie_backend(&mut self) -> Option<&TrieBackend>> { diff --git a/client/db/src/subdb.rs b/client/db/src/subdb.rs new file mode 100644 index 0000000000000000000000000000000000000000..2e436aa2c92c870dfe60117ec50b61c6213094f3 --- /dev/null +++ b/client/db/src/subdb.rs @@ -0,0 +1,87 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +/// A `Database` adapter for subdb. + +use sp_database::{self, ColumnId}; +use parking_lot::RwLock; +use blake2_rfc::blake2b::blake2b; +use codec::Encode; +use subdb::{Database, KeyType}; + +/// A database hidden behind an RwLock, so that it implements Send + Sync. +/// +/// Construct by creating a `Database` and then using `.into()`. +pub struct DbAdapter(RwLock>); + +/// Wrap RocksDb database into a trait object that implements `sp_database::Database` +pub fn open( + path: &std::path::Path, + _num_columns: u32, +) -> Result>, subdb::Error> { + let db = subdb::Options::from_path(path.into()).open()?; + Ok(std::sync::Arc::new(DbAdapter(RwLock::new(db)))) +} + +impl sp_database::Database for DbAdapter { + fn get(&self, col: ColumnId, key: &[u8]) -> Option> { + let mut hash = H::default(); + (col, key).using_encoded(|d| + hash.as_mut().copy_from_slice(blake2b(32, &[], d).as_bytes()) + ); + self.0.read().get(&hash) + } + + fn with_get(&self, col: ColumnId, key: &[u8], f: &mut dyn FnMut(&[u8])) { + let mut hash = H::default(); + (col, key).using_encoded(|d| + hash.as_mut().copy_from_slice(blake2b(32, &[], d).as_bytes()) + ); + let _ = self.0.read().get_ref(&hash).map(|d| f(d.as_ref())); + } + + fn set(&self, col: ColumnId, key: &[u8], value: &[u8]) { + let mut hash = H::default(); + (col, key).using_encoded(|d| + hash.as_mut().copy_from_slice(blake2b(32, &[], d).as_bytes()) + ); + self.0.write().insert(&value, &hash); + } + + fn remove(&self, col: ColumnId, key: &[u8]) { + let mut hash = H::default(); + (col, key).using_encoded(|d| + hash.as_mut().copy_from_slice(blake2b(32, &[], d).as_bytes()) + ); + let _ = self.0.write().remove(&hash); + } + + fn lookup(&self, hash: &H) -> Option> { + self.0.read().get(hash) + } + + fn with_lookup(&self, hash: &H, f: &mut dyn FnMut(&[u8])) { + let _ = self.0.read().get_ref(hash).map(|d| f(d.as_ref())); + } + + fn store(&self, hash: &H, preimage: &[u8]) { + self.0.write().insert(preimage, hash); + } + + fn release(&self, hash: &H) { + let _ = self.0.write().remove(hash); + } +} diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index 971acf8456b6463fe315768f16e2f8cf3af8254b..95592d071f777db07cf15e8e2765225f0756bfb2 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -19,18 +19,9 @@ use std::fs; use std::io::{Read, Write, ErrorKind}; use std::path::{Path, PathBuf}; -use std::sync::Arc; -use codec::Encode; -use kvdb_rocksdb::{Database, DatabaseConfig}; -use parking_lot::RwLock; -use sp_blockchain::{well_known_cache_keys, Cache}; -use sp_core::ChangesTrieConfiguration; use sp_runtime::traits::Block as BlockT; -use crate::{ - cache::{ComplexBlockId, DbCache, DbCacheSync}, - utils::{DatabaseType, check_database_type, db_err, read_genesis_hash}, -}; +use crate::utils::DatabaseType; /// Version file name. const VERSION_FILE_NAME: &'static str = "db_version"; @@ -38,69 +29,21 @@ const VERSION_FILE_NAME: &'static str = "db_version"; /// Current db version. const CURRENT_VERSION: u32 = 1; -/// Number of columns in v0. -const V0_NUM_COLUMNS: u32 = 10; - /// Upgrade database to current version. -pub fn upgrade_db(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> { - let db_version = current_version(db_path)?; - match db_version { - 0 => migrate_0_to_1::(db_path, db_type)?, - 1 => (), - _ => Err(sp_blockchain::Error::Backend(format!("Future database version: {}", db_version)))?, +pub fn upgrade_db(db_path: &Path, _db_type: DatabaseType) -> sp_blockchain::Result<()> { + let is_empty = db_path.read_dir().map_or(true, |mut d| d.next().is_none()); + if !is_empty { + let db_version = current_version(db_path)?; + match db_version { + 0 => Err(sp_blockchain::Error::Backend(format!("Unsupported database version: {}", db_version)))?, + 1 => (), + _ => Err(sp_blockchain::Error::Backend(format!("Future database version: {}", db_version)))?, + } } update_version(db_path) } -/// Migration from version0 to version1: -/// 1) the number of columns has changed from 10 to 11; -/// 2) changes tries configuration are now cached. -fn migrate_0_to_1(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> { - { - let db = open_database(db_path, db_type, V0_NUM_COLUMNS)?; - db.add_column().map_err(db_err)?; - db.flush().map_err(db_err)?; - } - - let db = open_database(db_path, db_type, V0_NUM_COLUMNS + 1)?; - - const V0_FULL_KEY_LOOKUP_COLUMN: u32 = 3; - const V0_FULL_HEADER_COLUMN: u32 = 4; - const V0_FULL_CACHE_COLUMN: u32 = 10; // that's the column we have just added - const V0_LIGHT_KEY_LOOKUP_COLUMN: u32 = 1; - const V0_LIGHT_HEADER_COLUMN: u32 = 2; - const V0_LIGHT_CACHE_COLUMN: u32 = 3; - - let (key_lookup_column, header_column, cache_column) = match db_type { - DatabaseType::Full => ( - V0_FULL_KEY_LOOKUP_COLUMN, - V0_FULL_HEADER_COLUMN, - V0_FULL_CACHE_COLUMN, - ), - DatabaseType::Light => ( - V0_LIGHT_KEY_LOOKUP_COLUMN, - V0_LIGHT_HEADER_COLUMN, - V0_LIGHT_CACHE_COLUMN, - ), - }; - - let genesis_hash: Option = read_genesis_hash(&db)?; - if let Some(genesis_hash) = genesis_hash { - let cache: DbCacheSync = DbCacheSync(RwLock::new(DbCache::new( - Arc::new(db), - key_lookup_column, - header_column, - cache_column, - genesis_hash, - ComplexBlockId::new(genesis_hash, 0.into()), - ))); - let changes_trie_config: Option = None; - cache.initialize(&well_known_cache_keys::CHANGES_TRIE_CONFIG, changes_trie_config.encode())?; - } - - Ok(()) -} /// Reads current database version from the file at given path. /// If the file does not exist returns 0. @@ -118,14 +61,9 @@ fn current_version(path: &Path) -> sp_blockchain::Result { } } -/// Opens database of given type with given number of columns. -fn open_database(db_path: &Path, db_type: DatabaseType, db_columns: u32) -> sp_blockchain::Result { - let db_path = db_path.to_str() - .ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?; - let db_cfg = DatabaseConfig::with_columns(db_columns); - let db = Database::open(&db_cfg, db_path).map_err(db_err)?; - check_database_type(&db, db_type)?; - Ok(db) +/// Maps database error to client error +fn db_err(err: std::io::Error) -> sp_blockchain::Error { + sp_blockchain::Error::Backend(format!("{}", err)) } /// Writes current database version to the file. @@ -152,8 +90,6 @@ mod tests { use super::*; fn create_db(db_path: &Path, version: Option) { - let db_cfg = DatabaseConfig::with_columns(V0_NUM_COLUMNS); - Database::open(&db_cfg, db_path.to_str().unwrap()).unwrap(); if let Some(version) = version { fs::create_dir_all(db_path).unwrap(); let mut file = fs::File::create(version_file_path(db_path)).unwrap(); @@ -166,7 +102,7 @@ mod tests { state_cache_size: 0, state_cache_child_ratio: None, pruning: PruningMode::ArchiveAll, - source: DatabaseSettingsSrc::Path { path: db_path.to_owned(), cache_size: None }, + source: DatabaseSettingsSrc::RocksDb { path: db_path.to_owned(), cache_size: 128 }, }, DatabaseType::Full).map(|_| ()) } @@ -184,15 +120,4 @@ mod tests { open_database(db_dir.path()).unwrap(); assert_eq!(current_version(db_dir.path()).unwrap(), CURRENT_VERSION); } - - #[test] - fn upgrade_from_0_to_1_works() { - for version_from_file in &[None, Some(0)] { - let db_dir = tempfile::TempDir::new().unwrap(); - let db_path = db_dir.path(); - create_db(db_path, *version_from_file); - open_database(db_path).unwrap(); - assert_eq!(current_version(db_path).unwrap(), CURRENT_VERSION); - } - } } diff --git a/client/db/src/utils.rs b/client/db/src/utils.rs index f26714eb5a7dbf9edae0fcac6a850e877478fd0e..d40abcab6693c6b0cb8cce5820256ae95bccd4ab 100644 --- a/client/db/src/utils.rs +++ b/client/db/src/utils.rs @@ -18,21 +18,19 @@ //! full and light storages. use std::sync::Arc; -use std::{io, convert::TryInto}; +use std::convert::TryInto; -use kvdb::{KeyValueDB, DBTransaction}; -#[cfg(any(feature = "kvdb-rocksdb", test))] -use kvdb_rocksdb::{Database, DatabaseConfig}; use log::debug; use codec::Decode; use sp_trie::DBValue; +use sp_database::Transaction; use sp_runtime::generic::BlockId; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, Zero, UniqueSaturatedFrom, UniqueSaturatedInto, }; -use crate::{DatabaseSettings, DatabaseSettingsSrc}; +use crate::{DatabaseSettings, DatabaseSettingsSrc, Database, DbHash}; /// Number of columns in the db. Must be the same for both full && light dbs. /// Otherwise RocksDb will fail to open database && check its type. @@ -136,35 +134,35 @@ pub fn lookup_key_to_number(key: &[u8]) -> sp_blockchain::Result where /// Delete number to hash mapping in DB transaction. pub fn remove_number_to_key_mapping>( - transaction: &mut DBTransaction, + transaction: &mut Transaction, key_lookup_col: u32, number: N, ) -> sp_blockchain::Result<()> { - transaction.delete(key_lookup_col, number_index_key(number)?.as_ref()); + transaction.remove(key_lookup_col, number_index_key(number)?.as_ref()); Ok(()) } /// Remove key mappings. pub fn remove_key_mappings, H: AsRef<[u8]>>( - transaction: &mut DBTransaction, + transaction: &mut Transaction, key_lookup_col: u32, number: N, hash: H, ) -> sp_blockchain::Result<()> { remove_number_to_key_mapping(transaction, key_lookup_col, number)?; - transaction.delete(key_lookup_col, hash.as_ref()); + transaction.remove(key_lookup_col, hash.as_ref()); Ok(()) } /// Place a number mapping into the database. This maps number to current perceived /// block hash at that position. pub fn insert_number_to_key_mapping + Clone, H: AsRef<[u8]>>( - transaction: &mut DBTransaction, + transaction: &mut Transaction, key_lookup_col: u32, number: N, hash: H, ) -> sp_blockchain::Result<()> { - transaction.put_vec( + transaction.set_from_vec( key_lookup_col, number_index_key(number.clone())?.as_ref(), number_and_hash_to_lookup_key(number, hash)?, @@ -174,12 +172,12 @@ pub fn insert_number_to_key_mapping + Clone, H: AsRef<[u8]>>( /// Insert a hash to key mapping in the database. pub fn insert_hash_to_key_mapping, H: AsRef<[u8]> + Clone>( - transaction: &mut DBTransaction, + transaction: &mut Transaction, key_lookup_col: u32, number: N, hash: H, ) -> sp_blockchain::Result<()> { - transaction.put_vec( + transaction.set_from_vec( key_lookup_col, hash.clone().as_ref(), number_and_hash_to_lookup_key(number, hash)?, @@ -191,68 +189,79 @@ pub fn insert_hash_to_key_mapping, H: AsRef<[u8]> + Clone>( /// block lookup key is the DB-key header, block and justification are stored under. /// looks up lookup key by hash from DB as necessary. pub fn block_id_to_lookup_key( - db: &dyn KeyValueDB, + db: &dyn Database, key_lookup_col: u32, id: BlockId ) -> Result>, sp_blockchain::Error> where Block: BlockT, ::sp_runtime::traits::NumberFor: UniqueSaturatedFrom + UniqueSaturatedInto, { - let res = match id { + Ok(match id { BlockId::Number(n) => db.get( key_lookup_col, number_index_key(n)?.as_ref(), ), - BlockId::Hash(h) => db.get(key_lookup_col, h.as_ref()), - }; - - res.map_err(db_err) -} - -/// Maps database error to client error -pub fn db_err(err: io::Error) -> sp_blockchain::Error { - sp_blockchain::Error::Backend(format!("{}", err)) + BlockId::Hash(h) => db.get(key_lookup_col, h.as_ref()) + }) } -/// Open RocksDB database. +/// Opens the configured database. pub fn open_database( config: &DatabaseSettings, db_type: DatabaseType, -) -> sp_blockchain::Result> { - let db: Arc = match &config.source { +) -> sp_blockchain::Result>> { + let db: Arc> = match &config.source { #[cfg(any(feature = "kvdb-rocksdb", test))] - DatabaseSettingsSrc::Path { path, cache_size } => { + DatabaseSettingsSrc::RocksDb { path, cache_size } => { // first upgrade database to required version crate::upgrade::upgrade_db::(&path, db_type)?; // and now open database assuming that it has the latest version - let mut db_config = DatabaseConfig::with_columns(NUM_COLUMNS); - - if let Some(cache_size) = cache_size { - let state_col_budget = (*cache_size as f64 * 0.9) as usize; - let other_col_budget = (cache_size - state_col_budget) / (NUM_COLUMNS as usize - 1); - - let mut memory_budget = std::collections::HashMap::new(); - for i in 0..NUM_COLUMNS { - if i == crate::columns::STATE { - memory_budget.insert(i, state_col_budget); - } else { - memory_budget.insert(i, other_col_budget); - } - } - - db_config.memory_budget = memory_budget; - } + let mut db_config = kvdb_rocksdb::DatabaseConfig::with_columns(NUM_COLUMNS); + let state_col_budget = (*cache_size as f64 * 0.9) as usize; + let other_col_budget = (cache_size - state_col_budget) / (NUM_COLUMNS as usize - 1); + let mut memory_budget = std::collections::HashMap::new(); let path = path.to_str() .ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?; - Arc::new(Database::open(&db_config, &path).map_err(db_err)?) + + for i in 0..NUM_COLUMNS { + if i == crate::columns::STATE { + memory_budget.insert(i, state_col_budget); + } else { + memory_budget.insert(i, other_col_budget); + } + } + + db_config.memory_budget = memory_budget; + + log::trace!( + target: "db", + "Open RocksDB database at {}, state column budget: {} MiB, others({}) column cache: {} MiB", + path, + state_col_budget, + NUM_COLUMNS, + other_col_budget, + ); + + let db = kvdb_rocksdb::Database::open(&db_config, &path) + .map_err(|err| sp_blockchain::Error::Backend(format!("{}", err)))?; + sp_database::as_database(db) }, - #[cfg(not(any(feature = "kvdb-rocksdb", test)))] - DatabaseSettingsSrc::Path { .. } => { - let msg = "Try to open RocksDB database with RocksDB disabled".into(); - return Err(sp_blockchain::Error::Backend(msg)); + #[cfg(feature = "subdb")] + DatabaseSettingsSrc::SubDb { path } => { + crate::subdb::open(&path, NUM_COLUMNS) + .map_err(|e| sp_blockchain::Error::Backend(format!("{:?}", e)))? + }, + #[cfg(feature = "parity-db")] + DatabaseSettingsSrc::ParityDb { path } => { + crate::parity_db::open(&path) + .map_err(|e| sp_blockchain::Error::Backend(format!("{:?}", e)))? }, DatabaseSettingsSrc::Custom(db) => db.clone(), + _ => { + let msg = "Trying to open a unsupported database".into(); + return Err(sp_blockchain::Error::Backend(msg)); + }, }; check_database_type(&*db, db_type)?; @@ -261,8 +270,8 @@ pub fn open_database( } /// Check database type. -pub fn check_database_type(db: &dyn KeyValueDB, db_type: DatabaseType) -> sp_blockchain::Result<()> { - match db.get(COLUMN_META, meta_keys::TYPE).map_err(db_err)? { +pub fn check_database_type(db: &dyn Database, db_type: DatabaseType) -> sp_blockchain::Result<()> { + match db.get(COLUMN_META, meta_keys::TYPE) { Some(stored_type) => { if db_type.as_str().as_bytes() != &*stored_type { return Err(sp_blockchain::Error::Backend( @@ -270,9 +279,9 @@ pub fn check_database_type(db: &dyn KeyValueDB, db_type: DatabaseType) -> sp_blo } }, None => { - let mut transaction = DBTransaction::new(); - transaction.put(COLUMN_META, meta_keys::TYPE, db_type.as_str().as_bytes()); - db.write(transaction).map_err(db_err)?; + let mut transaction = Transaction::new(); + transaction.set(COLUMN_META, meta_keys::TYPE, db_type.as_str().as_bytes()); + db.commit(transaction) }, } @@ -281,7 +290,7 @@ pub fn check_database_type(db: &dyn KeyValueDB, db_type: DatabaseType) -> sp_blo /// Read database column entry for the given block. pub fn read_db( - db: &dyn KeyValueDB, + db: &dyn Database, col_index: u32, col: u32, id: BlockId @@ -290,14 +299,14 @@ pub fn read_db( Block: BlockT, { block_id_to_lookup_key(db, col_index, id).and_then(|key| match key { - Some(key) => db.get(col, key.as_ref()).map_err(db_err), + Some(key) => Ok(db.get(col, key.as_ref())), None => Ok(None), }) } /// Read a header from the database. pub fn read_header( - db: &dyn KeyValueDB, + db: &dyn Database, col_index: u32, col: u32, id: BlockId, @@ -315,7 +324,7 @@ pub fn read_header( /// Required header from the database. pub fn require_header( - db: &dyn KeyValueDB, + db: &dyn Database, col_index: u32, col: u32, id: BlockId, @@ -327,7 +336,7 @@ pub fn require_header( } /// Read meta from the database. -pub fn read_meta(db: &dyn KeyValueDB, col_header: u32) -> Result< +pub fn read_meta(db: &dyn Database, col_header: u32) -> Result< Meta<<::Header as HeaderT>::Number, Block::Hash>, sp_blockchain::Error, > @@ -346,11 +355,10 @@ pub fn read_meta(db: &dyn KeyValueDB, col_header: u32) -> Result< }; let load_meta_block = |desc, key| -> Result<_, sp_blockchain::Error> { - if let Some(Some(header)) = db.get(COLUMN_META, key).and_then(|id| - match id { - Some(id) => db.get(col_header, &id).map(|h| h.map(|b| Block::Header::decode(&mut &b[..]).ok())), - None => Ok(None), - }).map_err(db_err)? + if let Some(Some(header)) = match db.get(COLUMN_META, key) { + Some(id) => db.get(col_header, &id).map(|b| Block::Header::decode(&mut &b[..]).ok()), + None => None, + } { let hash = header.hash(); debug!("DB Opened blockchain db, fetched {} = {:?} ({})", desc, hash, header.number()); @@ -373,8 +381,8 @@ pub fn read_meta(db: &dyn KeyValueDB, col_header: u32) -> Result< } /// Read genesis hash from database. -pub fn read_genesis_hash(db: &dyn KeyValueDB) -> sp_blockchain::Result> { - match db.get(COLUMN_META, meta_keys::GENESIS_HASH).map_err(db_err)? { +pub fn read_genesis_hash(db: &dyn Database) -> sp_blockchain::Result> { + match db.get(COLUMN_META, meta_keys::GENESIS_HASH) { Some(h) => match Decode::decode(&mut &h[..]) { Ok(h) => Ok(Some(h)), Err(err) => Err(sp_blockchain::Error::Backend( diff --git a/client/executor/Cargo.toml b/client/executor/Cargo.toml index cae0d56d8ed5eea8fef0e77c92eec6c2d5d9a61b..ef1b24ea7112615f21cc48c6569e2adc54cf8d40 100644 --- a/client/executor/Cargo.toml +++ b/client/executor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-executor" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,24 +9,28 @@ repository = "https://github.com/paritytech/substrate/" description = "A crate that provides means of executing/dispatching calls into the runtime." documentation = "https://docs.rs/sc-executor" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] derive_more = "0.99.2" codec = { package = "parity-scale-codec", version = "1.3.0" } -sp-io = { version = "2.0.0-alpha.5", path = "../../primitives/io" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-trie = { version = "2.0.0-alpha.5", path = "../../primitives/trie" } -sp-serializer = { version = "2.0.0-alpha.5", path = "../../primitives/serializer" } -sp-version = { version = "2.0.0-alpha.5", path = "../../primitives/version" } -sp-panic-handler = { version = "2.0.0-alpha.5", path = "../../primitives/panic-handler" } +sp-io = { version = "2.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-trie = { version = "2.0.0-dev", path = "../../primitives/trie" } +sp-serializer = { version = "2.0.0-dev", path = "../../primitives/serializer" } +sp-version = { version = "2.0.0-dev", path = "../../primitives/version" } +sp-panic-handler = { version = "2.0.0-dev", path = "../../primitives/panic-handler" } wasmi = "0.6.2" parity-wasm = "0.41.0" lazy_static = "1.4.0" -sp-wasm-interface = { version = "2.0.0-alpha.5", path = "../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0-alpha.5", path = "../../primitives/runtime-interface" } -sp-externalities = { version = "0.8.0-alpha.5", path = "../../primitives/externalities" } -sc-executor-common = { version = "0.8.0-alpha.5", path = "common" } -sc-executor-wasmi = { version = "0.8.0-alpha.5", path = "wasmi" } -sc-executor-wasmtime = { version = "0.8.0-alpha.5", path = "wasmtime", optional = true } +sp-api = { version = "2.0.0-dev", path = "../../primitives/api" } +sp-wasm-interface = { version = "2.0.0-dev", path = "../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-dev", path = "../../primitives/runtime-interface" } +sp-externalities = { version = "0.8.0-dev", path = "../../primitives/externalities" } +sc-executor-common = { version = "0.8.0-dev", path = "common" } +sc-executor-wasmi = { version = "0.8.0-dev", path = "wasmi" } +sc-executor-wasmtime = { version = "0.8.0-dev", path = "wasmtime", optional = true } parking_lot = "0.10.0" log = "0.4.8" libsecp256k1 = "0.3.4" @@ -37,9 +41,9 @@ wabt = "0.9.2" hex-literal = "0.2.1" sc-runtime-test = { version = "2.0.0-dev", path = "runtime-test" } substrate-test-runtime = { version = "2.0.0-dev", path = "../../test-utils/runtime" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../primitives/state-machine" } +sp-state-machine = { version = "0.8.0-dev", path = "../../primitives/state-machine" } test-case = "0.3.3" -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } [features] default = [ "std" ] @@ -52,6 +56,3 @@ wasmtime = [ wasmi-errno = [ "wasmi/errno" ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/executor/common/Cargo.toml b/client/executor/common/Cargo.toml index 1f52b959d00b1fc72d56b112267a4d70eb0d1b90..c27ed8db0ae885bc22ef5377d975164e83bc57a9 100644 --- a/client/executor/common/Cargo.toml +++ b/client/executor/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-executor-common" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,19 +9,20 @@ repository = "https://github.com/paritytech/substrate/" description = "A set of common definitions that are needed for defining execution engines." documentation = "https://docs.rs/sc-executor-common/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] log = "0.4.8" derive_more = "0.99.2" +parity-wasm = "0.41.0" codec = { package = "parity-scale-codec", version = "1.3.0" } wasmi = "0.6.2" -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-allocator = { version = "2.0.0-alpha.5", path = "../../../primitives/allocator" } -sp-wasm-interface = { version = "2.0.0-alpha.5", path = "../../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime-interface" } -sp-serializer = { version = "2.0.0-alpha.5", path = "../../../primitives/serializer" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-allocator = { version = "2.0.0-dev", path = "../../../primitives/allocator" } +sp-wasm-interface = { version = "2.0.0-dev", path = "../../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-dev", path = "../../../primitives/runtime-interface" } +sp-serializer = { version = "2.0.0-dev", path = "../../../primitives/serializer" } [features] default = [] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/executor/common/src/lib.rs b/client/executor/common/src/lib.rs index cc515dcf9dab9a8f852f2d9b2dc1f3e2af40773f..7f3864e6152fb2a00122bc6e6c2d2e9c2a4102e8 100644 --- a/client/executor/common/src/lib.rs +++ b/client/executor/common/src/lib.rs @@ -18,6 +18,7 @@ #![warn(missing_docs)] -pub mod sandbox; pub mod error; +pub mod sandbox; +pub mod util; pub mod wasm_runtime; diff --git a/client/executor/common/src/util.rs b/client/executor/common/src/util.rs new file mode 100644 index 0000000000000000000000000000000000000000..149db13bc0768a92408f8b1af1a05d518a8ae97d --- /dev/null +++ b/client/executor/common/src/util.rs @@ -0,0 +1,138 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! A set of utilities for resetting a wasm instance to its initial state. + +use crate::error::{self, Error}; +use std::mem; +use parity_wasm::elements::{deserialize_buffer, DataSegment, Instruction, Module as RawModule}; + +/// A bunch of information collected from a WebAssembly module. +pub struct WasmModuleInfo { + raw_module: RawModule, +} + +impl WasmModuleInfo { + /// Create `WasmModuleInfo` from the given wasm code. + /// + /// Returns `None` if the wasm code cannot be deserialized. + pub fn new(wasm_code: &[u8]) -> Option { + let raw_module: RawModule = deserialize_buffer(wasm_code).ok()?; + Some(Self { raw_module }) + } + + /// Extract the data segments from the given wasm code. + /// + /// Returns `Err` if the given wasm code cannot be deserialized. + fn data_segments(&self) -> Vec { + self.raw_module + .data_section() + .map(|ds| ds.entries()) + .unwrap_or(&[]) + .to_vec() + } + + /// The number of globals defined in locally in this module. + pub fn declared_globals_count(&self) -> u32 { + self.raw_module + .global_section() + .map(|gs| gs.entries().len() as u32) + .unwrap_or(0) + } + + /// The number of imports of globals. + pub fn imported_globals_count(&self) -> u32 { + self.raw_module + .import_section() + .map(|is| is.globals() as u32) + .unwrap_or(0) + } +} + +/// This is a snapshot of data segments specialzied for a particular instantiation. +/// +/// Note that this assumes that no mutable globals are used. +#[derive(Clone)] +pub struct DataSegmentsSnapshot { + /// The list of data segments represented by (offset, contents). + data_segments: Vec<(u32, Vec)>, +} + +impl DataSegmentsSnapshot { + /// Create a snapshot from the data segments from the module. + pub fn take(module: &WasmModuleInfo) -> error::Result { + let data_segments = module + .data_segments() + .into_iter() + .map(|mut segment| { + // Just replace contents of the segment since the segments will be discarded later + // anyway. + let contents = mem::replace(segment.value_mut(), vec![]); + + let init_expr = match segment.offset() { + Some(offset) => offset.code(), + // Return if the segment is passive + None => return Err(Error::from("Shared memory is not supported".to_string())), + }; + + // [op, End] + if init_expr.len() != 2 { + return Err(Error::from( + "initializer expression can have only up to 2 expressions in wasm 1.0" + .to_string(), + )); + } + let offset = match &init_expr[0] { + Instruction::I32Const(v) => *v as u32, + Instruction::GetGlobal(_) => { + // In a valid wasm file, initializer expressions can only refer imported + // globals. + // + // At the moment of writing the Substrate Runtime Interface does not provide + // any globals. There is nothing that prevents us from supporting this + // if/when we gain those. + return Err(Error::from( + "Imported globals are not supported yet".to_string(), + )); + } + insn => { + return Err(Error::from(format!( + "{:?} is not supported as initializer expression in wasm 1.0", + insn + ))) + } + }; + + Ok((offset, contents)) + }) + .collect::>>()?; + + Ok(Self { data_segments }) + } + + /// Apply the given snapshot to a linear memory. + /// + /// Linear memory interface is represented by a closure `memory_set`. + pub fn apply( + &self, + mut memory_set: impl FnMut(u32, &[u8]) -> Result<(), E>, + ) -> Result<(), E> { + for (offset, contents) in &self.data_segments { + memory_set(*offset, contents)?; + } + Ok(()) + } +} diff --git a/client/executor/runtime-test/Cargo.toml b/client/executor/runtime-test/Cargo.toml index e50061f4f24bb71639a5204cf326d6be7d397e6f..b8ae805192d9408c474febd8d6d266df2d81b7b2 100644 --- a/client/executor/runtime-test/Cargo.toml +++ b/client/executor/runtime-test/Cargo.toml @@ -9,13 +9,16 @@ publish = false homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/io" } -sp-sandbox = { version = "0.8.0-alpha.5", default-features = false, path = "../../../primitives/sandbox" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/runtime" } -sp-allocator = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/allocator" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/io" } +sp-sandbox = { version = "0.8.0-dev", default-features = false, path = "../../../primitives/sandbox" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/runtime" } +sp-allocator = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/allocator" } [build-dependencies] wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } @@ -28,6 +31,3 @@ std = [ "sp-std/std", "sp-allocator/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/executor/runtime-test/src/lib.rs b/client/executor/runtime-test/src/lib.rs index 38a16ae39ea012806d12d90f1a920e622eec4ad0..15a4177048a408b3b7f308f9d4ea63049f1b0769 100644 --- a/client/executor/runtime-test/src/lib.rs +++ b/client/executor/runtime-test/src/lib.rs @@ -183,6 +183,12 @@ sp_core::wasm_export_functions! { } } + + fn test_offchain_index_set() { + sp_io::offchain_index::set(b"k", b"v"); + } + + fn test_offchain_local_storage() -> bool { let kind = sp_core::offchain::StorageKind::PERSISTENT; assert_eq!(sp_io::offchain::local_storage_get(kind, b"test"), None); diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 72055b77884a651d3b6c3d9dd951b206033e5885..2e62e0601574d542abd0a1652c6242bbdc90f640 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -45,7 +45,6 @@ fn call_in_wasm( execution_method, Some(1024), HostFunctions::host_functions(), - true, 8, ); executor.call_in_wasm( @@ -54,6 +53,7 @@ fn call_in_wasm( function, call_data, ext, + sp_core::traits::MissingHostFunctions::Allow, ) } @@ -186,7 +186,7 @@ fn storage_should_work(wasm_method: WasmExecutionMethod) { b"foo".to_vec() => b"bar".to_vec(), b"baz".to_vec() => b"bar".to_vec() ], - children: map![], + children_default: map![], }); assert_eq!(ext, expected); } @@ -220,7 +220,7 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) { b"aab".to_vec() => b"2".to_vec(), b"bbb".to_vec() => b"5".to_vec() ], - children: map![], + children_default: map![], }); assert_eq!(expected, ext); } @@ -450,6 +450,28 @@ fn ordered_trie_root_should_work(wasm_method: WasmExecutionMethod) { ); } +#[test_case(WasmExecutionMethod::Interpreted)] +#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] +fn offchain_index(wasm_method: WasmExecutionMethod) { + let mut ext = TestExternalities::default(); + let (offchain, _state) = testing::TestOffchainExt::new(); + ext.register_extension(OffchainExt::new(offchain)); + call_in_wasm( + "test_offchain_index_set", + &[0], + wasm_method, + &mut ext.ext(), + ).unwrap(); + + use sp_core::offchain::storage::OffchainOverlayedChange; + assert_eq!( + ext.ext() + .get_offchain_storage_changes() + .get(sp_core::offchain::STORAGE_PREFIX, b"k"), + Some(OffchainOverlayedChange::SetValue(b"v".to_vec())) + ); +} + #[test_case(WasmExecutionMethod::Interpreted)] #[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] fn offchain_local_storage_should_work(wasm_method: WasmExecutionMethod) { @@ -511,7 +533,6 @@ fn should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) { wasm_method, Some(17), // `17` is the initial number of pages compiled into the binary. HostFunctions::host_functions(), - true, 8, ); executor.call_in_wasm( @@ -520,6 +541,7 @@ fn should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) { "test_exhaust_heap", &[0], &mut ext.ext(), + sp_core::traits::MissingHostFunctions::Allow, ).unwrap(); } diff --git a/client/executor/src/lib.rs b/client/executor/src/lib.rs index c3b41bd1990081693110f57d9a1b221a4e3e48f0..3d7db630f04ac74e34538627a2a657af18fceb71 100644 --- a/client/executor/src/lib.rs +++ b/client/executor/src/lib.rs @@ -77,7 +77,6 @@ mod tests { WasmExecutionMethod::Interpreted, Some(8), sp_io::SubstrateHostFunctions::host_functions(), - true, 8, ); let res = executor.call_in_wasm( @@ -86,6 +85,7 @@ mod tests { "test_empty_return", &[], &mut ext, + sp_core::traits::MissingHostFunctions::Allow, ).unwrap(); assert_eq!(res, vec![0u8; 0]); } diff --git a/client/executor/src/native_executor.rs b/client/executor/src/native_executor.rs index 778bc808004bb2470b528c802bd14c1d0fe1bc51..b859b544a3a9b95c44c56f0cd5920a8fd34cab14 100644 --- a/client/executor/src/native_executor.rs +++ b/client/executor/src/native_executor.rs @@ -20,7 +20,9 @@ use crate::{ }; use sp_version::{NativeVersion, RuntimeVersion}; use codec::{Decode, Encode}; -use sp_core::{NativeOrEncoded, traits::{CodeExecutor, Externalities, RuntimeCode}}; +use sp_core::{ + NativeOrEncoded, traits::{CodeExecutor, Externalities, RuntimeCode, MissingHostFunctions}, +}; use log::trace; use std::{result, panic::{UnwindSafe, AssertUnwindSafe}, sync::Arc}; use sp_wasm_interface::{HostFunctions, Function}; @@ -83,8 +85,6 @@ pub struct WasmExecutor { host_functions: Arc>, /// WASM runtime cache. cache: Arc, - /// Allow missing function imports. - allow_missing_func_imports: bool, /// The size of the instances cache. max_runtime_instances: usize, } @@ -102,7 +102,6 @@ impl WasmExecutor { method: WasmExecutionMethod, default_heap_pages: Option, host_functions: Vec<&'static dyn Function>, - allow_missing_func_imports: bool, max_runtime_instances: usize, ) -> Self { WasmExecutor { @@ -110,7 +109,6 @@ impl WasmExecutor { default_heap_pages: default_heap_pages.unwrap_or(DEFAULT_HEAP_PAGES), host_functions: Arc::new(host_functions), cache: Arc::new(RuntimeCache::new(max_runtime_instances)), - allow_missing_func_imports, max_runtime_instances, } } @@ -132,6 +130,7 @@ impl WasmExecutor { &self, runtime_code: &RuntimeCode, ext: &mut dyn Externalities, + allow_missing_host_functions: bool, f: F, ) -> Result where F: FnOnce( @@ -146,7 +145,7 @@ impl WasmExecutor { self.method, self.default_heap_pages, &*self.host_functions, - self.allow_missing_func_imports, + allow_missing_host_functions, |instance, version, ext| { let instance = AssertUnwindSafe(instance); let ext = AssertUnwindSafe(ext); @@ -167,7 +166,10 @@ impl sp_core::traits::CallInWasm for WasmExecutor { method: &str, call_data: &[u8], ext: &mut dyn Externalities, + missing_host_functions: MissingHostFunctions, ) -> std::result::Result, String> { + let allow_missing_host_functions = missing_host_functions.allowed(); + if let Some(hash) = code_hash { let code = RuntimeCode { code_fetcher: &sp_core::traits::WrappedRuntimeCode(wasm_code.into()), @@ -175,7 +177,7 @@ impl sp_core::traits::CallInWasm for WasmExecutor { heap_pages: None, }; - self.with_instance(&code, ext, |instance, _, mut ext| { + self.with_instance(&code, ext, allow_missing_host_functions, |instance, _, mut ext| { with_externalities_safe( &mut **ext, move || instance.call(method, call_data), @@ -187,7 +189,7 @@ impl sp_core::traits::CallInWasm for WasmExecutor { self.default_heap_pages, &wasm_code, self.host_functions.to_vec(), - self.allow_missing_func_imports, + allow_missing_host_functions, ) .map_err(|e| format!("Failed to create module: {:?}", e))?; @@ -240,7 +242,6 @@ impl NativeExecutor { fallback_method, default_heap_pages, host_functions, - false, max_runtime_instances, ); @@ -265,8 +266,9 @@ impl RuntimeInfo for NativeExecutor { self.wasm.with_instance( runtime_code, ext, + false, |_instance, version, _ext| - Ok(version.cloned().ok_or_else(|| Error::ApiError("Unknown version".into()))) + Ok(version.cloned().ok_or_else(|| Error::ApiError("Unknown version".into()))), ) } } @@ -290,6 +292,7 @@ impl CodeExecutor for NativeExecutor { let result = self.wasm.with_instance( runtime_code, ext, + false, |instance, onchain_version, mut ext| { let onchain_version = onchain_version.ok_or_else( || Error::ApiError("Unknown version".into()) @@ -372,8 +375,9 @@ impl sp_core::traits::CallInWasm for NativeExecutor< method: &str, call_data: &[u8], ext: &mut dyn Externalities, + missing_host_functions: MissingHostFunctions, ) -> std::result::Result, String> { - self.wasm.call_in_wasm(wasm_blob, code_hash, method, call_data, ext) + self.wasm.call_in_wasm(wasm_blob, code_hash, method, call_data, ext, missing_host_functions) } } diff --git a/client/executor/src/wasm_runtime.rs b/client/executor/src/wasm_runtime.rs index e6c16453f365fb560f577bcda600edbbb586845a..87a08f714dc93ea6ad070e7f157f202b315dd1e5 100644 --- a/client/executor/src/wasm_runtime.rs +++ b/client/executor/src/wasm_runtime.rs @@ -40,6 +40,12 @@ pub enum WasmExecutionMethod { Compiled, } +impl Default for WasmExecutionMethod { + fn default() -> WasmExecutionMethod { + WasmExecutionMethod::Interpreted + } +} + /// A Wasm runtime object along with its cached runtime version. struct VersionedRuntime { /// Runtime code hash. @@ -281,6 +287,25 @@ pub fn create_wasm_runtime_with_code( } } +fn decode_version(version: &[u8]) -> Result { + let v: RuntimeVersion = sp_api::OldRuntimeVersion::decode(&mut &version[..]) + .map_err(|_| + WasmError::Instantiation( + "failed to decode \"Core_version\" result using old runtime version".into(), + ) + )?.into(); + + let core_api_id = sp_core::hashing::blake2_64(b"Core"); + if v.has_api_with(&core_api_id, |v| v >= 3) { + sp_api::RuntimeVersion::decode(&mut &version[..]) + .map_err(|_| + WasmError::Instantiation("failed to decode \"Core_version\" result".into()) + ) + } else { + Ok(v) + } +} + fn create_versioned_wasm_runtime( code: &[u8], code_hash: Vec, @@ -315,10 +340,7 @@ fn create_versioned_wasm_runtime( ).map_err(|_| WasmError::Instantiation("panic in call to get runtime version".into()))? }; let version = match version_result { - Ok(version) => Some(RuntimeVersion::decode(&mut version.as_slice()) - .map_err(|_| - WasmError::Instantiation("failed to decode \"Core_version\" result".into()) - )?), + Ok(version) => Some(decode_version(&version)?), Err(_) => None, }; #[cfg(not(target_os = "unknown"))] @@ -344,7 +366,11 @@ fn create_versioned_wasm_runtime( #[cfg(test)] mod tests { + use super::*; use sp_wasm_interface::HostFunctions; + use sp_api::{Core, RuntimeApiInfo}; + use substrate_test_runtime::Block; + use codec::Encode; #[test] fn host_functions_are_equal() { @@ -353,4 +379,49 @@ mod tests { let equal = &host_functions[..] == &host_functions[..]; assert!(equal, "Host functions are not equal"); } + + #[test] + fn old_runtime_version_decodes() { + let old_runtime_version = sp_api::OldRuntimeVersion { + spec_name: "test".into(), + impl_name: "test".into(), + authoring_version: 1, + spec_version: 1, + impl_version: 1, + apis: sp_api::create_apis_vec!([(Core::::ID, 1)]), + }; + + let version = decode_version(&old_runtime_version.encode()).unwrap(); + assert_eq!(1, version.transaction_version); + } + + #[test] + fn old_runtime_version_decodes_fails_with_version_3() { + let old_runtime_version = sp_api::OldRuntimeVersion { + spec_name: "test".into(), + impl_name: "test".into(), + authoring_version: 1, + spec_version: 1, + impl_version: 1, + apis: sp_api::create_apis_vec!([(Core::::ID, 3)]), + }; + + decode_version(&old_runtime_version.encode()).unwrap_err(); + } + + #[test] + fn new_runtime_version_decodes() { + let old_runtime_version = sp_api::RuntimeVersion { + spec_name: "test".into(), + impl_name: "test".into(), + authoring_version: 1, + spec_version: 1, + impl_version: 1, + apis: sp_api::create_apis_vec!([(Core::::ID, 3)]), + transaction_version: 3, + }; + + let version = decode_version(&old_runtime_version.encode()).unwrap(); + assert_eq!(3, version.transaction_version); + } } diff --git a/client/executor/wasmi/Cargo.toml b/client/executor/wasmi/Cargo.toml index ea8637b9e2830972b19639440546cf486b606c5d..3180eebb4f56ce76a1601e4b734a32be4c5fae81 100644 --- a/client/executor/wasmi/Cargo.toml +++ b/client/executor/wasmi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-executor-wasmi" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,16 +9,15 @@ repository = "https://github.com/paritytech/substrate/" description = "This crate provides an implementation of `WasmRuntime` that is baked by wasmi." documentation = "https://docs.rs/sc-executor-wasmi" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] log = "0.4.8" wasmi = "0.6.2" -parity-wasm = "0.41.0" codec = { package = "parity-scale-codec", version = "1.3.0" } -sc-executor-common = { version = "0.8.0-alpha.5", path = "../common" } -sp-wasm-interface = { version = "2.0.0-alpha.5", path = "../../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime-interface" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-allocator = { version = "2.0.0-alpha.5", path = "../../../primitives/allocator" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sc-executor-common = { version = "0.8.0-dev", path = "../common" } +sp-wasm-interface = { version = "2.0.0-dev", path = "../../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-dev", path = "../../../primitives/runtime-interface" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-allocator = { version = "2.0.0-dev", path = "../../../primitives/allocator" } diff --git a/client/executor/wasmi/src/lib.rs b/client/executor/wasmi/src/lib.rs index 6348c2413357f03f9de161d4ff537cf8a77e818d..e4b4aca40967d982fe342985f829bf2c4c190e39 100644 --- a/client/executor/wasmi/src/lib.rs +++ b/client/executor/wasmi/src/lib.rs @@ -16,21 +16,25 @@ //! This crate provides an implementation of `WasmModule` that is baked by wasmi. -use sc_executor_common::{error::{Error, WasmError}, sandbox}; -use std::{str, mem, cell::RefCell, sync::Arc}; +use std::{str, cell::RefCell, sync::Arc}; use wasmi::{ Module, ModuleInstance, MemoryInstance, MemoryRef, TableRef, ImportsBuilder, ModuleRef, - memory_units::Pages, RuntimeValue::{I32, I64, self}, + memory_units::Pages, + RuntimeValue::{I32, I64, self}, }; use codec::{Encode, Decode}; use sp_core::sandbox as sandbox_primitives; use log::{error, trace, debug}; -use parity_wasm::elements::{deserialize_buffer, DataSegment, Instruction, Module as RawModule}; use sp_wasm_interface::{ FunctionContext, Pointer, WordSize, Sandbox, MemoryId, Result as WResult, Function, }; use sp_runtime_interface::unpack_ptr_and_len; use sc_executor_common::wasm_runtime::{WasmModule, WasmInstance}; +use sc_executor_common::{ + error::{Error, WasmError}, + sandbox, +}; +use sc_executor_common::util::{DataSegmentsSnapshot, WasmModuleInfo}; struct FunctionExecutor<'a> { sandbox_store: sandbox::Store, @@ -530,52 +534,14 @@ fn instantiate_module( /// /// It is used for restoring the state of the module after execution. #[derive(Clone)] -struct StateSnapshot { - /// The offset and the content of the memory segments that should be used to restore the snapshot - data_segments: Vec<(u32, Vec)>, +struct GlobalValsSnapshot { /// The list of all global mutable variables of the module in their sequential order. global_mut_values: Vec, } -impl StateSnapshot { +impl GlobalValsSnapshot { // Returns `None` if instance is not valid. - fn take( - module_instance: &ModuleRef, - data_segments: Vec, - ) -> Option { - let prepared_segments = data_segments - .into_iter() - .map(|mut segment| { - // Just replace contents of the segment since the segments will be discarded later - // anyway. - let contents = mem::replace(segment.value_mut(), vec![]); - - let init_expr = match segment.offset() { - Some(offset) => offset.code(), - // Return if the segment is passive - None => return None - }; - - // [op, End] - if init_expr.len() != 2 { - return None; - } - let offset = match init_expr[0] { - Instruction::I32Const(v) => v as u32, - Instruction::GetGlobal(idx) => { - let global_val = module_instance.globals().get(idx as usize)?.get(); - match global_val { - RuntimeValue::I32(v) => v as u32, - _ => return None, - } - } - _ => return None, - }; - - Some((offset, contents)) - }) - .collect::>>()?; - + fn take(module_instance: &ModuleRef) -> Self { // Collect all values of mutable globals. let global_mut_values = module_instance .globals() @@ -583,42 +549,27 @@ impl StateSnapshot { .filter(|g| g.is_mutable()) .map(|g| g.get()) .collect(); - - Some(Self { - data_segments: prepared_segments, - global_mut_values, - }) + Self { global_mut_values } } /// Reset the runtime instance to the initial version by restoring /// the preserved memory and globals. /// /// Returns `Err` if applying the snapshot is failed. - fn apply(&self, instance: &ModuleRef, memory: &MemoryRef) -> Result<(), WasmError> { - // First, erase the memory and copy the data segments into it. - memory - .erase() - .map_err(|e| WasmError::ErasingFailed(e.to_string()))?; - for (offset, contents) in &self.data_segments { - memory - .set(*offset, contents) - .map_err(|_| WasmError::ApplySnapshotFailed)?; - } - - // Second, restore the values of mutable globals. + fn apply(&self, instance: &ModuleRef) -> Result<(), WasmError> { for (global_ref, global_val) in instance .globals() .iter() .filter(|g| g.is_mutable()) .zip(self.global_mut_values.iter()) - { - // the instance should be the same as used for preserving and - // we iterate the same way it as we do it for preserving values that means that the - // types should be the same and all the values are mutable. So no error is expected/ - global_ref - .set(*global_val) - .map_err(|_| WasmError::ApplySnapshotFailed)?; - } + { + // the instance should be the same as used for preserving and + // we iterate the same way it as we do it for preserving values that means that the + // types should be the same and all the values are mutable. So no error is expected/ + global_ref + .set(*global_val) + .map_err(|_| WasmError::ApplySnapshotFailed)?; + } Ok(()) } } @@ -634,8 +585,9 @@ pub struct WasmiRuntime { allow_missing_func_imports: bool, /// Numer of heap pages this runtime uses. heap_pages: u64, - /// Data segments created for each new instance. - data_segments: Vec, + + global_vals_snapshot: GlobalValsSnapshot, + data_segments_snapshot: DataSegmentsSnapshot, } impl WasmModule for WasmiRuntime { @@ -648,19 +600,11 @@ impl WasmModule for WasmiRuntime { self.allow_missing_func_imports, ).map_err(|e| WasmError::Instantiation(e.to_string()))?; - // Take state snapshot before executing anything. - let state_snapshot = StateSnapshot::take(&instance, self.data_segments.clone()) - .expect( - "`take` returns `Err` if the module is not valid; - we already loaded module above, thus the `Module` is proven to be valid at this point; - qed - ", - ); - Ok(Box::new(WasmiInstance { instance, memory, - state_snapshot, + global_vals_snapshot: self.global_vals_snapshot.clone(), + data_segments_snapshot: self.data_segments_snapshot.clone(), host_functions: self.host_functions.clone(), allow_missing_func_imports: self.allow_missing_func_imports, missing_functions, @@ -682,10 +626,29 @@ pub fn create_runtime( // // A return of this error actually indicates that there is a problem in logic, since // we just loaded and validated the `module` above. - let data_segments = extract_data_segments(&code)?; + let (data_segments_snapshot, global_vals_snapshot) = { + let (instance, _, _) = instantiate_module( + heap_pages as usize, + &module, + &host_functions, + allow_missing_func_imports, + ) + .map_err(|e| WasmError::Instantiation(e.to_string()))?; + + let data_segments_snapshot = DataSegmentsSnapshot::take( + &WasmModuleInfo::new(code) + .ok_or_else(|| WasmError::Other("cannot deserialize module".to_string()))?, + ) + .map_err(|e| WasmError::Other(e.to_string()))?; + let global_vals_snapshot = GlobalValsSnapshot::take(&instance); + + (data_segments_snapshot, global_vals_snapshot) + }; + Ok(WasmiRuntime { module, - data_segments, + data_segments_snapshot, + global_vals_snapshot, host_functions: Arc::new(host_functions), allow_missing_func_imports, heap_pages, @@ -698,12 +661,14 @@ pub struct WasmiInstance { instance: ModuleRef, /// The memory instance of used by the wasm module. memory: MemoryRef, - /// The snapshot of the instance's state taken just after the instantiation. - state_snapshot: StateSnapshot, + /// The snapshot of global variable values just after instantiation. + global_vals_snapshot: GlobalValsSnapshot, + /// The snapshot of data segments. + data_segments_snapshot: DataSegmentsSnapshot, /// The host functions registered for this instance. host_functions: Arc>, /// Enable stub generation for functions that are not available in `host_functions`. - /// These stubs will error when the wasm blob tries to call them. + /// These stubs will error when the wasm blob trie to call them. allow_missing_func_imports: bool, /// List of missing functions detected during function resolution missing_functions: Vec, @@ -713,19 +678,26 @@ pub struct WasmiInstance { unsafe impl Send for WasmiInstance {} impl WasmInstance for WasmiInstance { - fn call( - &self, - method: &str, - data: &[u8], - ) -> Result, Error> { - self.state_snapshot.apply(&self.instance, &self.memory) - .map_err(|e| { - // Snapshot restoration failed. This is pretty unexpected since this can happen - // if some invariant is broken or if the system is under extreme memory pressure - // (so erasing fails). - error!(target: "wasm-executor", "snapshot restoration failed: {}", e); - e - })?; + fn call(&self, method: &str, data: &[u8]) -> Result, Error> { + // We reuse a single wasm instance for multiple calls and a previous call (if any) + // altered the state. Therefore, we need to restore the instance to original state. + + // First, zero initialize the linear memory. + self.memory.erase().map_err(|e| { + // Snapshot restoration failed. This is pretty unexpected since this can happen + // if some invariant is broken or if the system is under extreme memory pressure + // (so erasing fails). + error!(target: "wasm-executor", "snapshot restoration failed: {}", e); + WasmError::ErasingFailed(e.to_string()) + })?; + + // Second, reapply data segments into the linear memory. + self.data_segments_snapshot + .apply(|offset, contents| self.memory.set(offset, contents))?; + + // Third, restore the global variables to their initial values. + self.global_vals_snapshot.apply(&self.instance)?; + call_in_wasm_module( &self.instance, &self.memory, @@ -750,18 +722,3 @@ impl WasmInstance for WasmiInstance { } } } - -/// Extract the data segments from the given wasm code. -/// -/// Returns `Err` if the given wasm code cannot be deserialized. -fn extract_data_segments(wasm_code: &[u8]) -> Result, WasmError> { - let raw_module: RawModule = deserialize_buffer(wasm_code) - .map_err(|_| WasmError::CantDeserializeWasm)?; - - let segments = raw_module - .data_section() - .map(|ds| ds.entries()) - .unwrap_or(&[]) - .to_vec(); - Ok(segments) -} diff --git a/client/executor/wasmtime/Cargo.toml b/client/executor/wasmtime/Cargo.toml index 11f99e7876bbc57dae3b3ec792a3efe30161d476..5b1c3841410b3f03a1e4235a2c5df3b0eb714b27 100644 --- a/client/executor/wasmtime/Cargo.toml +++ b/client/executor/wasmtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-executor-wasmtime" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,20 +8,24 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Defines a `WasmRuntime` that uses the Wasmtime JIT to execute." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] log = "0.4.8" scoped-tls = "1.0" parity-wasm = "0.41.0" codec = { package = "parity-scale-codec", version = "1.3.0" } -sc-executor-common = { version = "0.8.0-alpha.5", path = "../common" } -sp-wasm-interface = { version = "2.0.0-alpha.5", path = "../../../primitives/wasm-interface" } -sp-runtime-interface = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime-interface" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-allocator = { version = "2.0.0-alpha.5", path = "../../../primitives/allocator" } +sc-executor-common = { version = "0.8.0-dev", path = "../common" } +sp-wasm-interface = { version = "2.0.0-dev", path = "../../../primitives/wasm-interface" } +sp-runtime-interface = { version = "2.0.0-dev", path = "../../../primitives/runtime-interface" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-allocator = { version = "2.0.0-dev", path = "../../../primitives/allocator" } wasmtime = { package = "substrate-wasmtime", version = "0.13.0-threadsafe.1" } +wasmtime_runtime = { package = "substrate-wasmtime-runtime", version = "0.13.0-threadsafe.1" } +wasmtime-environ = "0.12.0" +cranelift-wasm = "0.59.0" +cranelift-codegen = "0.59.0" [dev-dependencies] assert_matches = "1.3.0" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/executor/wasmtime/src/instance_wrapper.rs b/client/executor/wasmtime/src/instance_wrapper.rs index 159746801a52aa68e45c63b4f060a1804415a2f8..469668802f18649c80b59715783a3ad784970396 100644 --- a/client/executor/wasmtime/src/instance_wrapper.rs +++ b/client/executor/wasmtime/src/instance_wrapper.rs @@ -20,11 +20,55 @@ use crate::util; use crate::imports::Imports; -use sc_executor_common::error::{Error, Result}; +use std::{slice, marker}; +use sc_executor_common::{ + error::{Error, Result}, + util::{WasmModuleInfo, DataSegmentsSnapshot}, +}; use sp_wasm_interface::{Pointer, WordSize, Value}; -use std::slice; -use std::marker; -use wasmtime::{Instance, Module, Memory, Table, Val}; +use wasmtime::{Store, Instance, Module, Memory, Table, Val}; + +mod globals_snapshot; + +pub use globals_snapshot::GlobalsSnapshot; + +pub struct ModuleWrapper { + imported_globals_count: u32, + globals_count: u32, + module: Module, + data_segments_snapshot: DataSegmentsSnapshot, +} + +impl ModuleWrapper { + pub fn new(store: &Store, code: &[u8]) -> Result { + let module = Module::new(&store, code) + .map_err(|e| Error::from(format!("cannot create module: {}", e)))?; + + let module_info = WasmModuleInfo::new(code) + .ok_or_else(|| Error::from("cannot deserialize module".to_string()))?; + let declared_globals_count = module_info.declared_globals_count(); + let imported_globals_count = module_info.imported_globals_count(); + let globals_count = imported_globals_count + declared_globals_count; + + let data_segments_snapshot = DataSegmentsSnapshot::take(&module_info) + .map_err(|e| Error::from(format!("cannot take data segments snapshot: {}", e)))?; + + Ok(Self { + module, + imported_globals_count, + globals_count, + data_segments_snapshot, + }) + } + + pub fn module(&self) -> &Module { + &self.module + } + + pub fn data_segments_snapshot(&self) -> &DataSegmentsSnapshot { + &self.data_segments_snapshot + } +} /// Wrap the given WebAssembly Instance of a wasm module with Substrate-runtime. /// @@ -32,6 +76,8 @@ use wasmtime::{Instance, Module, Memory, Table, Val}; /// routines. pub struct InstanceWrapper { instance: Instance, + globals_count: u32, + imported_globals_count: u32, // The memory instance of the `instance`. // // It is important to make sure that we don't make any copies of this to make it easier to proof @@ -44,8 +90,8 @@ pub struct InstanceWrapper { impl InstanceWrapper { /// Create a new instance wrapper from the given wasm module. - pub fn new(module: &Module, imports: &Imports, heap_pages: u32) -> Result { - let instance = Instance::new(module, &imports.externs) + pub fn new(module_wrapper: &ModuleWrapper, imports: &Imports, heap_pages: u32) -> Result { + let instance = Instance::new(&module_wrapper.module, &imports.externs) .map_err(|e| Error::from(format!("cannot instantiate: {}", e)))?; let memory = match imports.memory_import_index { @@ -66,8 +112,10 @@ impl InstanceWrapper { Ok(Self { table: get_table(&instance), - memory, instance, + globals_count: module_wrapper.globals_count, + imported_globals_count: module_wrapper.imported_globals_count, + memory, _not_send_nor_sync: marker::PhantomData, }) } diff --git a/client/executor/wasmtime/src/instance_wrapper/globals_snapshot.rs b/client/executor/wasmtime/src/instance_wrapper/globals_snapshot.rs new file mode 100644 index 0000000000000000000000000000000000000000..a6ab3fed604c43c8569703aba4c79a4484775383 --- /dev/null +++ b/client/executor/wasmtime/src/instance_wrapper/globals_snapshot.rs @@ -0,0 +1,130 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use super::InstanceWrapper; +use sc_executor_common::{ + error::{Error, Result}, +}; +use sp_wasm_interface::Value; +use cranelift_codegen::ir; +use cranelift_wasm::GlobalIndex; + +/// A snapshot of a global variables values. This snapshot can be used later for restoring the +/// values to the preserved state. +/// +/// Technically, a snapshot stores only values of mutable global variables. This is because +/// immutable global variables always have the same values. +pub struct GlobalsSnapshot { + handle: wasmtime_runtime::InstanceHandle, + preserved_mut_globals: Vec<(*mut wasmtime_runtime::VMGlobalDefinition, Value)>, +} + +impl GlobalsSnapshot { + /// Take a snapshot of global variables for a given instance. + pub fn take(instance_wrapper: &InstanceWrapper) -> Result { + // EVIL: + // Usage of an undocumented function. + let handle = instance_wrapper.instance.handle().clone(); + + let mut preserved_mut_globals = vec![]; + + for global_idx in instance_wrapper.imported_globals_count..instance_wrapper.globals_count { + let (def, global) = match handle.lookup_by_declaration( + &wasmtime_environ::Export::Global(GlobalIndex::from_u32(global_idx)), + ) { + wasmtime_runtime::Export::Global { + definition, global, .. + } => (definition, global), + _ => unreachable!("only globals can be returned for a global request"), + }; + + // skip immutable globals. + if !global.mutability { + continue; + } + + let value = unsafe { + // Safety of this function solely depends on the correctness of the reference and + // the type information of the global. + read_global(def, global.ty)? + }; + preserved_mut_globals.push((def, value)); + } + + Ok(Self { + preserved_mut_globals, + handle, + }) + } + + /// Apply the snapshot to the given instance. + /// + /// This instance must be the same that was used for creation of this snapshot. + pub fn apply(&self, instance_wrapper: &InstanceWrapper) -> Result<()> { + if instance_wrapper.instance.handle() != &self.handle { + return Err(Error::from("unexpected instance handle".to_string())); + } + + for (def, value) in &self.preserved_mut_globals { + unsafe { + // The following writes are safe if the precondition that this is the same instance + // this snapshot was created with: + // + // 1. These pointers must be still not-NULL and allocated. + // 2. The set of global variables is fixed for the lifetime of the same instance. + // 3. We obviously assume that the wasmtime references are correct in the first place. + // 4. We write the data with the same type it was read in the first place. + write_global(*def, *value)?; + } + } + Ok(()) + } +} + +unsafe fn read_global( + def: *const wasmtime_runtime::VMGlobalDefinition, + ty: ir::Type, +) -> Result { + let def = def + .as_ref() + .ok_or_else(|| Error::from("wasmtime global reference is null during read".to_string()))?; + let val = match ty { + ir::types::I32 => Value::I32(*def.as_i32()), + ir::types::I64 => Value::I64(*def.as_i64()), + ir::types::F32 => Value::F32(*def.as_u32()), + ir::types::F64 => Value::F64(*def.as_u64()), + _ => { + return Err(Error::from(format!( + "unsupported global variable type: {}", + ty + ))) + } + }; + Ok(val) +} + +unsafe fn write_global(def: *mut wasmtime_runtime::VMGlobalDefinition, value: Value) -> Result<()> { + let def = def + .as_mut() + .ok_or_else(|| Error::from("wasmtime global reference is null during write".to_string()))?; + match value { + Value::I32(v) => *def.as_i32_mut() = v, + Value::I64(v) => *def.as_i64_mut() = v, + Value::F32(v) => *def.as_u32_mut() = v, + Value::F64(v) => *def.as_u64_mut() = v, + } + Ok(()) +} diff --git a/client/executor/wasmtime/src/runtime.rs b/client/executor/wasmtime/src/runtime.rs index 02acd33e69a628705675105e8644bad177bfc15a..0289188ba11fce00d104d21268541167dd68f54a 100644 --- a/client/executor/wasmtime/src/runtime.rs +++ b/client/executor/wasmtime/src/runtime.rs @@ -15,14 +15,14 @@ // along with Substrate. If not, see . //! Defines the compiled Wasm runtime that uses Wasmtime internally. -use std::rc::Rc; -use std::sync::Arc; use crate::host::HostState; use crate::imports::{Imports, resolve_imports}; -use crate::instance_wrapper::InstanceWrapper; +use crate::instance_wrapper::{ModuleWrapper, InstanceWrapper, GlobalsSnapshot}; use crate::state_holder; +use std::rc::Rc; +use std::sync::Arc; use sc_executor_common::{ error::{Error, Result, WasmError}, wasm_runtime::{WasmModule, WasmInstance}, @@ -30,12 +30,12 @@ use sc_executor_common::{ use sp_allocator::FreeingBumpHeapAllocator; use sp_runtime_interface::unpack_ptr_and_len; use sp_wasm_interface::{Function, Pointer, WordSize, Value}; -use wasmtime::{Config, Engine, Module, Store}; +use wasmtime::{Config, Engine, Store}; /// A `WasmModule` implementation using wasmtime to compile the runtime module to machine code /// and execute the compiled code. pub struct WasmtimeRuntime { - module: Arc, + module_wrapper: Arc, heap_pages: u32, allow_missing_func_imports: bool, host_functions: Vec<&'static dyn Function>, @@ -46,16 +46,24 @@ impl WasmModule for WasmtimeRuntime { // Scan all imports, find the matching host functions, and create stubs that adapt arguments // and results. let imports = resolve_imports( - &self.module, + self.module_wrapper.module(), &self.host_functions, self.heap_pages, self.allow_missing_func_imports, )?; + let instance_wrapper = + InstanceWrapper::new(&self.module_wrapper, &imports, self.heap_pages)?; + let heap_base = instance_wrapper.extract_heap_base()?; + let globals_snapshot = GlobalsSnapshot::take(&instance_wrapper)?; + Ok(Box::new(WasmtimeInstance { - module: self.module.clone(), + instance_wrapper: Rc::new(instance_wrapper), + module_wrapper: Arc::clone(&self.module_wrapper), imports, + globals_snapshot, heap_pages: self.heap_pages, + heap_base, })) } } @@ -63,9 +71,12 @@ impl WasmModule for WasmtimeRuntime { /// A `WasmInstance` implementation that reuses compiled module and spawns instances /// to execute the compiled code. pub struct WasmtimeInstance { - module: Arc, + module_wrapper: Arc, + instance_wrapper: Rc, + globals_snapshot: GlobalsSnapshot, imports: Imports, heap_pages: u32, + heap_base: u32, } // This is safe because `WasmtimeInstance` does not leak reference to `self.imports` @@ -74,23 +85,32 @@ unsafe impl Send for WasmtimeInstance {} impl WasmInstance for WasmtimeInstance { fn call(&self, method: &str, data: &[u8]) -> Result> { - // TODO: reuse the instance and reset globals after call - // https://github.com/paritytech/substrate/issues/5141 - let instance = Rc::new(InstanceWrapper::new(&self.module, &self.imports, self.heap_pages)?); - call_method( - instance, - method, + let entrypoint = self.instance_wrapper.resolve_entrypoint(method)?; + let allocator = FreeingBumpHeapAllocator::new(self.heap_base); + + self.module_wrapper + .data_segments_snapshot() + .apply(|offset, contents| { + self.instance_wrapper + .write_memory_from(Pointer::new(offset), contents) + })?; + + self.globals_snapshot.apply(&*self.instance_wrapper)?; + + perform_call( data, + Rc::clone(&self.instance_wrapper), + entrypoint, + allocator, ) } fn get_global_const(&self, name: &str) -> Result> { - let instance = InstanceWrapper::new(&self.module, &self.imports, self.heap_pages)?; + let instance = InstanceWrapper::new(&self.module_wrapper, &self.imports, self.heap_pages)?; instance.get_global_val(name) } } - /// Create a new `WasmtimeRuntime` given the code. This function performs translation from Wasm to /// machine code, which can be computationally heavy. pub fn create_runtime( @@ -105,30 +125,18 @@ pub fn create_runtime( let engine = Engine::new(&config); let store = Store::new(&engine); - let module = Module::new(&store, code) + + let module_wrapper = ModuleWrapper::new(&store, code) .map_err(|e| WasmError::Other(format!("cannot create module: {}", e)))?; Ok(WasmtimeRuntime { - module: Arc::new(module), + module_wrapper: Arc::new(module_wrapper), heap_pages: heap_pages as u32, allow_missing_func_imports, host_functions, }) } -/// Call a function inside a precompiled Wasm module. -fn call_method( - instance_wrapper: Rc, - method: &str, - data: &[u8], -) -> Result> { - let entrypoint = instance_wrapper.resolve_entrypoint(method)?; - let heap_base = instance_wrapper.extract_heap_base()?; - let allocator = FreeingBumpHeapAllocator::new(heap_base); - - perform_call(data, instance_wrapper, entrypoint, allocator) -} - fn perform_call( data: &[u8], instance_wrapper: Rc, diff --git a/client/finality-grandpa/Cargo.toml b/client/finality-grandpa/Cargo.toml index b684c814d1b93607534c2b73fdc74aa4a99eda8b..a634595ed3ed6405c91cace78f4f2672d25c4517 100644 --- a/client/finality-grandpa/Cargo.toml +++ b/client/finality-grandpa/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-finality-grandpa" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,9 +9,12 @@ repository = "https://github.com/paritytech/substrate/" description = "Integration of the GRANDPA finality gadget into substrate." documentation = "https://docs.rs/sc-finality-grandpa" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -fork-tree = { version = "2.0.0-alpha.5", path = "../../utils/fork-tree" } +fork-tree = { version = "2.0.0-dev", path = "../../utils/fork-tree" } futures = "0.3.4" futures-timer = "3.0.1" log = "0.4.8" @@ -19,40 +22,37 @@ parking_lot = "0.10.0" rand = "0.7.2" assert_matches = "1.3.0" parity-scale-codec = { version = "1.3.0", features = ["derive"] } -sp-arithmetic = { version = "2.0.0-alpha.5", path = "../../primitives/arithmetic" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } -sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/common" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } -sc-telemetry = { version = "2.0.0-alpha.5", path = "../telemetry" } -sc-keystore = { version = "2.0.0-alpha.5", path = "../keystore" } +sp-arithmetic = { version = "2.0.0-dev", path = "../../primitives/arithmetic" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-dev", path = "../../primitives/utils" } +sp-consensus = { version = "0.8.0-dev", path = "../../primitives/consensus/common" } +sc-consensus = { version = "0.8.0-dev", path = "../../client/consensus/common" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-api = { version = "2.0.0-dev", path = "../../primitives/api" } +sc-telemetry = { version = "2.0.0-dev", path = "../telemetry" } +sc-keystore = { version = "2.0.0-dev", path = "../keystore" } serde_json = "1.0.41" -sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } -sc-client = { version = "0.8.0-alpha.5", path = "../" } -sp-inherents = { version = "2.0.0-alpha.5", path = "../../primitives/inherents" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } -sc-network = { version = "0.8.0-alpha.5", path = "../network" } -sc-network-gossip = { version = "0.8.0-alpha.5", path = "../network-gossip" } -sp-finality-tracker = { version = "2.0.0-alpha.5", path = "../../primitives/finality-tracker" } -sp-finality-grandpa = { version = "2.0.0-alpha.5", path = "../../primitives/finality-grandpa" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-alpha.5"} -sc-block-builder = { version = "0.8.0-alpha.5", path = "../block-builder" } +sc-client-api = { version = "2.0.0-dev", path = "../api" } +sp-inherents = { version = "2.0.0-dev", path = "../../primitives/inherents" } +sp-blockchain = { version = "2.0.0-dev", path = "../../primitives/blockchain" } +sc-network = { version = "0.8.0-dev", path = "../network" } +sc-network-gossip = { version = "0.8.0-dev", path = "../network-gossip" } +sp-finality-tracker = { version = "2.0.0-dev", path = "../../primitives/finality-tracker" } +sp-finality-grandpa = { version = "2.0.0-dev", path = "../../primitives/finality-grandpa" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-dev"} +sc-block-builder = { version = "0.8.0-dev", path = "../block-builder" } finality-grandpa = { version = "0.11.2", features = ["derive-codec"] } pin-project = "0.4.6" [dev-dependencies] finality-grandpa = { version = "0.11.2", features = ["derive-codec", "test-helpers"] } -sc-network = { version = "0.8.0-alpha.5", path = "../network" } +sc-network = { version = "0.8.0-dev", path = "../network" } sc-network-test = { version = "0.8.0-dev", path = "../network/test" } -sp-keyring = { version = "2.0.0-alpha.5", path = "../../primitives/keyring" } +sp-keyring = { version = "2.0.0-dev", path = "../../primitives/keyring" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } -sp-consensus-babe = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/babe" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../primitives/state-machine" } +sp-consensus-babe = { version = "0.8.0-dev", path = "../../primitives/consensus/babe" } +sp-state-machine = { version = "0.8.0-dev", path = "../../primitives/state-machine" } env_logger = "0.7.0" tokio = { version = "0.2", features = ["rt-core"] } tempfile = "3.1.0" -sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sp-api = { version = "2.0.0-dev", path = "../../primitives/api" } diff --git a/client/finality-grandpa/src/authorities.rs b/client/finality-grandpa/src/authorities.rs index 4709216ebef2c745148c2da207a93ea5cfd0ef0e..fe3f2dd19eb1ce5764508fe5ff86e5919d35a2e7 100644 --- a/client/finality-grandpa/src/authorities.rs +++ b/client/finality-grandpa/src/authorities.rs @@ -20,7 +20,7 @@ use fork_tree::ForkTree; use parking_lot::RwLock; use finality_grandpa::voter_set::VoterSet; use parity_scale_codec::{Encode, Decode}; -use log::{debug, info}; +use log::debug; use sc_telemetry::{telemetry, CONSENSUS_INFO}; use sp_finality_grandpa::{AuthorityId, AuthorityList}; @@ -250,6 +250,7 @@ where best_hash: H, best_number: N, is_descendent_of: &F, + initial_sync: bool, ) -> Result, E> where F: Fn(&H, &H) -> Result, { @@ -262,8 +263,10 @@ where // check if the given best block is in the same branch as the block that signaled the change. if is_descendent_of(&change.canon_hash, &best_hash)? { // apply this change: make the set canonical - info!(target: "afg", "👴 Applying authority set change forced at block #{:?}", - change.canon_height); + afg_log!(initial_sync, + "👴 Applying authority set change forced at block #{:?}", + change.canon_height, + ); telemetry!(CONSENSUS_INFO; "afg.applying_forced_authority_set_change"; "block" => ?change.canon_height ); @@ -305,6 +308,7 @@ where finalized_hash: H, finalized_number: N, is_descendent_of: &F, + initial_sync: bool, ) -> Result, fork_tree::Error> where F: Fn(&H, &H) -> Result, E: std::error::Error, @@ -328,8 +332,10 @@ where self.pending_forced_changes.clear(); if let Some(change) = change { - info!(target: "afg", "👴 Applying authority set change scheduled at block #{:?}", - change.canon_height); + afg_log!(initial_sync, + "👴 Applying authority set change scheduled at block #{:?}", + change.canon_height, + ); telemetry!(CONSENSUS_INFO; "afg.applying_scheduled_authority_set_change"; "block" => ?change.canon_height ); @@ -599,11 +605,16 @@ mod tests { ); // finalizing "hash_c" won't enact the change signaled at "hash_a" but it will prune out "hash_b" - let status = authorities.apply_standard_changes("hash_c", 11, &is_descendent_of(|base, hash| match (*base, *hash) { - ("hash_a", "hash_c") => true, - ("hash_b", "hash_c") => false, - _ => unreachable!(), - })).unwrap(); + let status = authorities.apply_standard_changes( + "hash_c", + 11, + &is_descendent_of(|base, hash| match (*base, *hash) { + ("hash_a", "hash_c") => true, + ("hash_b", "hash_c") => false, + _ => unreachable!(), + }), + false, + ).unwrap(); assert!(status.changed); assert_eq!(status.new_set_block, None); @@ -613,10 +624,15 @@ mod tests { ); // finalizing "hash_d" will enact the change signaled at "hash_a" - let status = authorities.apply_standard_changes("hash_d", 15, &is_descendent_of(|base, hash| match (*base, *hash) { - ("hash_a", "hash_d") => true, - _ => unreachable!(), - })).unwrap(); + let status = authorities.apply_standard_changes( + "hash_d", + 15, + &is_descendent_of(|base, hash| match (*base, *hash) { + ("hash_a", "hash_d") => true, + _ => unreachable!(), + }), + false, + ).unwrap(); assert!(status.changed); assert_eq!(status.new_set_block, Some(("hash_d", 15))); @@ -671,12 +687,18 @@ mod tests { }); // trying to finalize past `change_c` without finalizing `change_a` first - match authorities.apply_standard_changes("hash_d", 40, &is_descendent_of) { - Err(fork_tree::Error::UnfinalizedAncestor) => {}, - _ => unreachable!(), - } + assert!(matches!( + authorities.apply_standard_changes("hash_d", 40, &is_descendent_of, false), + Err(fork_tree::Error::UnfinalizedAncestor) + )); + + let status = authorities.apply_standard_changes( + "hash_b", + 15, + &is_descendent_of, + false, + ).unwrap(); - let status = authorities.apply_standard_changes("hash_b", 15, &is_descendent_of).unwrap(); assert!(status.changed); assert_eq!(status.new_set_block, Some(("hash_b", 15))); @@ -684,7 +706,13 @@ mod tests { assert_eq!(authorities.set_id, 1); // after finalizing `change_a` it should be possible to finalize `change_c` - let status = authorities.apply_standard_changes("hash_d", 40, &is_descendent_of).unwrap(); + let status = authorities.apply_standard_changes( + "hash_d", + 40, + &is_descendent_of, + false, + ).unwrap(); + assert!(status.changed); assert_eq!(status.new_set_block, Some(("hash_d", 40))); @@ -817,20 +845,30 @@ mod tests { assert!(authorities.add_pending_change(change_c, &is_descendent_of_a).is_err()); // too early. - assert!(authorities.apply_forced_changes("hash_a10", 10, &static_is_descendent_of(true)).unwrap().is_none()); + assert!( + authorities.apply_forced_changes("hash_a10", 10, &static_is_descendent_of(true), false) + .unwrap() + .is_none() + ); // too late. - assert!(authorities.apply_forced_changes("hash_a16", 16, &static_is_descendent_of(true)).unwrap().is_none()); + assert!( + authorities.apply_forced_changes("hash_a16", 16, &static_is_descendent_of(true), false) + .unwrap() + .is_none() + ); // on time -- chooses the right change. assert_eq!( - authorities.apply_forced_changes("hash_a15", 15, &is_descendent_of_a).unwrap().unwrap(), + authorities.apply_forced_changes("hash_a15", 15, &is_descendent_of_a, false) + .unwrap() + .unwrap(), (42, AuthoritySet { current_authorities: set_a, set_id: 1, pending_standard_changes: ForkTree::new(), pending_forced_changes: Vec::new(), - }) + }), ); } } diff --git a/client/finality-grandpa/src/aux_schema.rs b/client/finality-grandpa/src/aux_schema.rs index 525a4a99bab5861e57b6fc25b49bbcaccaea7220..fe652f52fe2067e728263efe0acd302d2673be2b 100644 --- a/client/finality-grandpa/src/aux_schema.rs +++ b/client/finality-grandpa/src/aux_schema.rs @@ -330,7 +330,7 @@ pub(crate) fn load_persistent( } // genesis. - info!(target: "afg", "Loading GRANDPA authority set \ + info!(target: "afg", "👴 Loading GRANDPA authority set \ from genesis on what appears to be first startup."); let genesis_authorities = genesis_authorities()?; diff --git a/client/finality-grandpa/src/environment.rs b/client/finality-grandpa/src/environment.rs index dec9492482a5e78ba6e06a752ea29fa9d644acd0..cab212333c73e7af7063f7453a30e70136be475a 100644 --- a/client/finality-grandpa/src/environment.rs +++ b/client/finality-grandpa/src/environment.rs @@ -20,7 +20,7 @@ use std::pin::Pin; use std::sync::Arc; use std::time::Duration; -use log::{debug, warn, info}; +use log::{debug, warn}; use parity_scale_codec::{Decode, Encode}; use futures::prelude::*; use futures_timer::Delay; @@ -28,8 +28,7 @@ use parking_lot::RwLock; use sp_blockchain::{HeaderBackend, Error as ClientError, HeaderMetadata}; use std::marker::PhantomData; -use sc_client_api::{backend::Backend, utils::is_descendent_of}; -use sc_client::apply_aux; +use sc_client_api::{backend::{Backend, apply_aux}, utils::is_descendent_of}; use finality_grandpa::{ BlockNumberOps, Equivocation, Error as GrandpaError, round::State as RoundState, voter, voter_set::VoterSet, @@ -911,6 +910,7 @@ where hash, number, (round, commit).into(), + false, ) } @@ -970,6 +970,7 @@ pub(crate) fn finalize_block( hash: Block::Hash, number: NumberFor, justification_or_commit: JustificationOrCommit, + initial_sync: bool, ) -> Result<(), CommandOrError>> where Block: BlockT, BE: Backend, @@ -1011,6 +1012,7 @@ pub(crate) fn finalize_block( hash, number, &is_descendent_of::(&*client, None), + initial_sync, ).map_err(|e| Error::Safety(e.to_string()))?; // check if this is this is the first finalization of some consensus changes @@ -1091,9 +1093,15 @@ pub(crate) fn finalize_block( let (new_id, set_ref) = authority_set.current(); if set_ref.len() > 16 { - info!("👴 Applying GRANDPA set change to new set with {} authorities", set_ref.len()); + afg_log!(initial_sync, + "👴 Applying GRANDPA set change to new set with {} authorities", + set_ref.len(), + ); } else { - info!("👴 Applying GRANDPA set change to new set {:?}", set_ref); + afg_log!(initial_sync, + "👴 Applying GRANDPA set change to new set {:?}", + set_ref, + ); } telemetry!(CONSENSUS_INFO; "afg.generating_new_authority_set"; diff --git a/client/finality-grandpa/src/finality_proof.rs b/client/finality-grandpa/src/finality_proof.rs index 2c85839b5e3640ca19398767dffb42ea4a3afa05..4035854a380b03d064b067b57b17ca8967bcf1b0 100644 --- a/client/finality-grandpa/src/finality_proof.rs +++ b/client/finality-grandpa/src/finality_proof.rs @@ -596,7 +596,7 @@ impl ProvableJustification for GrandpaJustificatio pub(crate) mod tests { use substrate_test_runtime_client::runtime::{Block, Header, H256}; use sc_client_api::NewBlockState; - use substrate_test_runtime_client::sc_client::in_mem::Blockchain as InMemoryBlockchain; + use sc_client_api::in_mem::Blockchain as InMemoryBlockchain; use super::*; use sp_core::crypto::Public; diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index faf319364162203a685bcf18c519297932d3a3f0..c1e32dfa6cc9e3847c18030f6f88c666d182a611 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -16,7 +16,7 @@ use std::{sync::Arc, collections::HashMap}; -use log::{debug, trace, info}; +use log::{debug, trace}; use parity_scale_codec::Encode; use parking_lot::RwLockWriteGuard; @@ -27,7 +27,7 @@ use sp_api::{TransactionFor}; use sp_consensus::{ BlockImport, Error as ConsensusError, - BlockCheckParams, BlockImportParams, ImportResult, JustificationImport, + BlockCheckParams, BlockImportParams, BlockOrigin, ImportResult, JustificationImport, SelectChain, }; use sp_finality_grandpa::{ConsensusLog, ScheduledChange, SetId, GRANDPA_ENGINE_ID}; @@ -128,7 +128,12 @@ impl JustificationImport number: NumberFor, justification: Justification, ) -> Result<(), Self::Error> { - GrandpaBlockImport::import_justification(self, hash, number, justification, false) + // this justification was requested by the sync service, therefore we + // are not sure if it should enact a change or not. it could have been a + // request made as part of initial sync but that means the justification + // wasn't part of the block and was requested asynchronously, probably + // makes sense to log in that case. + GrandpaBlockImport::import_justification(self, hash, number, justification, false, false) } } @@ -250,6 +255,7 @@ where &self, block: &mut BlockImportParams>, hash: Block::Hash, + initial_sync: bool, ) -> Result, ConsensusError> { // when we update the authorities, we need to hold the lock // until the block is written to prevent a race if we need to restore @@ -324,7 +330,9 @@ where } let applied_changes = { - let forced_change_set = guard.as_mut().apply_forced_changes(hash, number, &is_descendent_of) + let forced_change_set = guard + .as_mut() + .apply_forced_changes(hash, number, &is_descendent_of, initial_sync) .map_err(|e| ConsensusError::ClientImport(e.to_string())) .map_err(ConsensusError::from)?; @@ -419,7 +427,10 @@ impl BlockImport Err(e) => return Err(ConsensusError::ClientImport(e.to_string()).into()), } - let pending_changes = self.make_authorities_changes(&mut block, hash)?; + // on initial sync we will restrict logging under info to avoid spam. + let initial_sync = block.origin == BlockOrigin::NetworkInitialSync; + + let pending_changes = self.make_authorities_changes(&mut block, hash, initial_sync)?; // we don't want to finalize on `inner.import_block` let mut justification = block.justification.take(); @@ -492,7 +503,15 @@ impl BlockImport match justification { Some(justification) => { - self.import_justification(hash, number, justification, needs_justification).unwrap_or_else(|err| { + let import_res = self.import_justification( + hash, + number, + justification, + needs_justification, + initial_sync, + ); + + import_res.unwrap_or_else(|err| { if needs_justification || enacts_consensus_change { debug!(target: "afg", "Imported block #{} that enacts authority set change with \ invalid justification: {:?}, requesting justification from peers.", number, err); @@ -604,6 +623,7 @@ where number: NumberFor, justification: Justification, enacts_change: bool, + initial_sync: bool, ) -> Result<(), ConsensusError> { let justification = GrandpaJustification::decode_and_verify_finalizes( &justification, @@ -625,12 +645,17 @@ where hash, number, justification.into(), + initial_sync, ); match result { Err(CommandOrError::VoterCommand(command)) => { - info!(target: "afg", "👴 Imported justification for block #{} that triggers \ - command {}, signaling voter.", number, command); + afg_log!(initial_sync, + "👴 Imported justification for block #{} that triggers \ + command {}, signaling voter.", + number, + command, + ); // send the command to the voter let _ = self.send_voter_commands.unbounded_send(command); diff --git a/client/finality-grandpa/src/justification.rs b/client/finality-grandpa/src/justification.rs index 084c0042ab19a08fe5f17e31aeffd1820958fcb0..ebce90f2c18a87905a68855a8f69c55af817651e 100644 --- a/client/finality-grandpa/src/justification.rs +++ b/client/finality-grandpa/src/justification.rs @@ -88,7 +88,7 @@ impl GrandpaJustification { /// Decode a GRANDPA justification and validate the commit and the votes' /// ancestry proofs finalize the given block. - pub(crate) fn decode_and_verify_finalizes( + pub fn decode_and_verify_finalizes( encoded: &[u8], finalized_target: (Block::Hash, NumberFor), set_id: u64, diff --git a/client/finality-grandpa/src/lib.rs b/client/finality-grandpa/src/lib.rs index 800fe3442ea66776eb57745611f29f02ec622a40..6fab89ac68acb1f03ed742df06724b62ffb2d54a 100644 --- a/client/finality-grandpa/src/lib.rs +++ b/client/finality-grandpa/src/lib.rs @@ -84,6 +84,23 @@ use std::time::Duration; use std::pin::Pin; use std::task::{Poll, Context}; +// utility logging macro that takes as first argument a conditional to +// decide whether to log under debug or info level (useful to restrict +// logging under initial sync). +macro_rules! afg_log { + ($condition:expr, $($msg: expr),+ $(,)?) => { + { + let log_level = if $condition { + log::Level::Debug + } else { + log::Level::Info + }; + + log::log!(target: "afg", log_level, $($msg),+); + } + }; +} + mod authorities; mod aux_schema; mod communication; diff --git a/client/finality-grandpa/src/light_import.rs b/client/finality-grandpa/src/light_import.rs index 276f5d0f28d7ab1c9c3d376bc23ce4f55f03b1a7..dd80dd82743c0854e976dcf89a94324d2ecd6383 100644 --- a/client/finality-grandpa/src/light_import.rs +++ b/client/finality-grandpa/src/light_import.rs @@ -567,7 +567,7 @@ pub mod tests { use sp_consensus::{ForkChoiceStrategy, BlockImport}; use sp_finality_grandpa::AuthorityId; use sp_core::{H256, crypto::Public}; - use substrate_test_runtime_client::sc_client::in_mem::Blockchain as InMemoryAuxStore; + use sc_client_api::in_mem::Blockchain as InMemoryAuxStore; use substrate_test_runtime_client::runtime::{Block, Header}; use crate::tests::TestApi; use crate::finality_proof::tests::TestJustification; diff --git a/client/finality-grandpa/src/observer.rs b/client/finality-grandpa/src/observer.rs index fbe19a0716aab09edf57a3097d33827c40857c6d..1e6c8ddf18820e4de9e56e26c15d53491c286889 100644 --- a/client/finality-grandpa/src/observer.rs +++ b/client/finality-grandpa/src/observer.rs @@ -124,6 +124,7 @@ fn grandpa_observer( finalized_hash, finalized_number, (round, commit).into(), + false, ) { Ok(_) => {}, Err(e) => return future::err(e), diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index d7d1d1e48d3a0f0f051b1c4a35f4a044ea8ae2a7..2821737c4d47153026e545375b09c3201096d51c 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -27,7 +27,6 @@ use parking_lot::Mutex; use futures_timer::Delay; use tokio::runtime::{Runtime, Handle}; use sp_keyring::Ed25519Keyring; -use sc_client::LongestChain; use sc_client_api::backend::TransactionFor; use sp_blockchain::Result; use sp_api::{ApiRef, StorageProof, ProvideRuntimeApi}; @@ -50,6 +49,7 @@ use finality_proof::{ }; use consensus_changes::ConsensusChanges; use sc_block_builder::BlockBuilderProvider; +use sc_consensus::LongestChain; type PeerData = Mutex< diff --git a/client/informant/Cargo.toml b/client/informant/Cargo.toml index b60886d6dea6cfbb7d4b9151f2a2498b81be4404..f8e6ca85f95b8347934df26254b9999294efe879 100644 --- a/client/informant/Cargo.toml +++ b/client/informant/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-informant" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "Substrate informant." edition = "2018" @@ -8,17 +8,17 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] ansi_term = "0.12.1" futures = "0.3.4" log = "0.4.8" -parity-util-mem = { version = "0.6.0", default-features = false, features = ["primitive-types"] } +parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } wasm-timer = "0.2" -sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } -sc-network = { version = "0.8.0-alpha.5", path = "../network" } -sc-service = { version = "0.8.0-alpha.5", default-features = false, path = "../service" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sc-client-api = { version = "2.0.0-dev", path = "../api" } +sc-network = { version = "0.8.0-dev", path = "../network" } +sc-service = { version = "0.8.0-dev", default-features = false, path = "../service" } +sp-blockchain = { version = "2.0.0-dev", path = "../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } diff --git a/client/informant/src/lib.rs b/client/informant/src/lib.rs index 66d5ed41fb5d418a2bae0952dc263cdee6ff8c84..090282a982092db6ec98e504b546268dd272bb56 100644 --- a/client/informant/src/lib.rs +++ b/client/informant/src/lib.rs @@ -17,7 +17,7 @@ //! Console informant. Prints sync progress and block events. Runs on the calling thread. use ansi_term::Colour; -use sc_client_api::BlockchainEvents; +use sc_client_api::{BlockchainEvents, UsageProvider}; use futures::prelude::*; use log::{info, warn, trace}; use sp_runtime::traits::Header; diff --git a/client/keystore/Cargo.toml b/client/keystore/Cargo.toml index af9cdf81bfe9a472f12dd4284001b95934ce0bd7..f3419f55e1dc3207db64bcee0544d65568f114e1 100644 --- a/client/keystore/Cargo.toml +++ b/client/keystore/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-keystore" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,11 +9,14 @@ repository = "https://github.com/paritytech/substrate/" description = "Keystore (and session key management) for ed25519 based chains like Polkadot." documentation = "https://docs.rs/sc-keystore" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] derive_more = "0.99.2" -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-application-crypto = { version = "2.0.0-alpha.5", path = "../../primitives/application-crypto" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-application-crypto = { version = "2.0.0-dev", path = "../../primitives/application-crypto" } hex = "0.4.0" rand = "0.7.2" serde_json = "1.0.41" @@ -22,6 +25,3 @@ parking_lot = "0.10.0" [dev-dependencies] tempfile = "3.1.0" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/network-gossip/Cargo.toml b/client/network-gossip/Cargo.toml index ad2eb2ae0e391f31ce4551c29001f1a2d20ef922..c6714375fe3fdad55ca7b5a8662baf3227231ffc 100644 --- a/client/network-gossip/Cargo.toml +++ b/client/network-gossip/Cargo.toml @@ -1,7 +1,7 @@ [package] description = "Gossiping for the Substrate network protocol" name = "sc-network-gossip" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" @@ -9,20 +9,21 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sc-network-gossip" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] futures = "0.3.4" futures-timer = "3.0.1" -libp2p = { version = "0.16.2", default-features = false, features = ["libp2p-websocket"] } +libp2p = { version = "0.18.1", default-features = false, features = ["websocket"] } log = "0.4.8" lru = "0.4.3" -sc-network = { version = "0.8.0-alpha.5", path = "../network" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } -sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } +sc-network = { version = "0.8.0-dev", path = "../network" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-dev", path = "../../primitives/utils" } wasm-timer = "0.2" [dev-dependencies] +async-std = "1.5" substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index 6a00b3d5a181ab812aea45007c42096535946a0f..26e49fce8a820e4d5525154fce74d81cac1887d4 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -21,9 +21,16 @@ use sc_network::{Event, ReputationChange}; use futures::prelude::*; use libp2p::PeerId; +use log::trace; use sp_runtime::{traits::Block as BlockT, ConsensusEngineId}; -use std::{borrow::Cow, pin::Pin, sync::Arc, task::{Context, Poll}}; -use sp_utils::mpsc::TracingUnboundedReceiver; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver}; +use std::{ + borrow::Cow, + collections::{HashMap, hash_map::Entry}, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; /// Wraps around an implementation of the `Network` crate and provides gossiping capabilities on /// top of it. @@ -31,8 +38,12 @@ pub struct GossipEngine { state_machine: ConsensusGossip, network: Box + Send>, periodic_maintenance_interval: futures_timer::Delay, - network_event_stream: Pin + Send>>, engine_id: ConsensusEngineId, + + /// Incoming events from the network. + network_event_stream: Pin + Send>>, + /// Outgoing events to the consumer. + message_sinks: HashMap>>, } impl Unpin for GossipEngine {} @@ -40,26 +51,24 @@ impl Unpin for GossipEngine {} impl GossipEngine { /// Create a new instance. pub fn new + Send + Clone + 'static>( - mut network: N, + network: N, engine_id: ConsensusEngineId, protocol_name: impl Into>, validator: Arc>, ) -> Self where B: 'static { - let mut state_machine = ConsensusGossip::new(); - // We grab the event stream before registering the notifications protocol, otherwise we // might miss events. let network_event_stream = network.event_stream(); - network.register_notifications_protocol(engine_id, protocol_name.into()); - state_machine.register_validator(&mut network, engine_id, validator); GossipEngine { - state_machine, + state_machine: ConsensusGossip::new(validator, engine_id), network: Box::new(network), periodic_maintenance_interval: futures_timer::Delay::new(PERIODIC_MAINTENANCE_INTERVAL), - network_event_stream, engine_id, + + network_event_stream, + message_sinks: HashMap::new(), } } @@ -77,7 +86,7 @@ impl GossipEngine { topic: B::Hash, message: Vec, ) { - self.state_machine.register_message(topic, self.engine_id, message); + self.state_machine.register_message(topic, message); } /// Broadcast all messages with given topic. @@ -89,7 +98,15 @@ impl GossipEngine { pub fn messages_for(&mut self, topic: B::Hash) -> TracingUnboundedReceiver { - self.state_machine.messages_for(self.engine_id, topic) + let (tx, rx) = tracing_unbounded("mpsc_gossip_messages_for"); + + for notification in self.state_machine.messages_for(topic) { + tx.unbounded_send(notification).expect("receiver known to be live; qed"); + } + + self.message_sinks.entry(topic).or_default().push(tx); + + rx } /// Send all messages with given topic to a peer. @@ -99,7 +116,7 @@ impl GossipEngine { topic: B::Hash, force: bool ) { - self.state_machine.send_topic(&mut *self.network, who, topic, self.engine_id, force) + self.state_machine.send_topic(&mut *self.network, who, topic, force) } /// Multicast a message to all peers. @@ -109,14 +126,14 @@ impl GossipEngine { message: Vec, force: bool, ) { - self.state_machine.multicast(&mut *self.network, topic, self.engine_id, message, force) + self.state_machine.multicast(&mut *self.network, topic, message, force) } /// Send addressed message to the given peers. The message is not kept or multicast /// later on. pub fn send_message(&mut self, who: Vec, data: Vec) { for who in &who { - self.state_machine.send_message(&mut *self.network, who, self.engine_id, data.clone()); + self.state_machine.send_message(&mut *self.network, who, data.clone()); } } @@ -151,16 +168,40 @@ impl Future for GossipEngine { this.state_machine.peer_disconnected(&mut *this.network, remote); }, Event::NotificationsReceived { remote, messages } => { - let engine_id = this.engine_id.clone(); - this.state_machine.on_incoming( + let messages = messages.into_iter().filter_map(|(engine, data)| { + if engine == this.engine_id { + Some(data.to_vec()) + } else { + None + } + }).collect(); + + let to_forward = this.state_machine.on_incoming( &mut *this.network, remote, - messages.into_iter() - .filter_map(|(engine, data)| if engine == engine_id { - Some((engine, data.to_vec())) - } else { None }) - .collect() + messages, ); + + for (topic, notification) in to_forward { + if let Entry::Occupied(mut entry) = this.message_sinks.entry(topic) { + trace!( + target: "gossip", + "Pushing consensus message to sinks for {}.", topic, + ); + entry.get_mut().retain(move |sink| { + if let Err(e) = sink.unbounded_send(notification.clone()) { + trace!( + target: "gossip", + "Error broadcasting message notification: {:?}", e, + ); + } + !sink.is_closed() + }); + if entry.get().is_empty() { + entry.remove_entry(); + } + } + } }, Event::Dht(_) => {} } @@ -173,6 +214,11 @@ impl Future for GossipEngine { while let Poll::Ready(()) = this.periodic_maintenance_interval.poll_unpin(cx) { this.periodic_maintenance_interval.reset(PERIODIC_MAINTENANCE_INTERVAL); this.state_machine.tick(&mut *this.network); + + this.message_sinks.retain(|_, sinks| { + sinks.retain(|sink| !sink.is_closed()); + !sinks.is_empty() + }); } Poll::Pending @@ -181,23 +227,34 @@ impl Future for GossipEngine { #[cfg(test)] mod tests { - use super::*; + use async_std::task::spawn; use crate::{ValidationResult, ValidatorContext}; + use futures::{channel::mpsc::{channel, Sender}, executor::block_on_stream}; + use sc_network::ObservedRole; + use sp_runtime::{testing::H256, traits::{Block as BlockT}}; + use std::sync::{Arc, Mutex}; use substrate_test_runtime_client::runtime::Block; + use super::*; - struct TestNetwork {} + #[derive(Clone, Default)] + struct TestNetwork { + inner: Arc>, + } - impl Network for Arc { + #[derive(Clone, Default)] + struct TestNetworkInner { + event_senders: Vec>, + } + + impl Network for TestNetwork { fn event_stream(&self) -> Pin + Send>> { - let (_tx, rx) = futures::channel::mpsc::channel(0); + let (tx, rx) = channel(100); + self.inner.lock().unwrap().event_senders.push(tx); - // Return rx and drop tx. Thus the given channel will yield `Poll::Ready(None)` on first - // poll. Box::pin(rx) } fn report_peer(&self, _: PeerId, _: ReputationChange) { - unimplemented!(); } fn disconnect_peer(&self, _: PeerId) { @@ -215,16 +272,15 @@ mod tests { } } - struct TestValidator {} - - impl Validator for TestValidator { + struct AllowAll; + impl Validator for AllowAll { fn validate( &self, - _: &mut dyn ValidatorContext, - _: &PeerId, - _: &[u8] - ) -> ValidationResult { - unimplemented!(); + _context: &mut dyn ValidatorContext, + _sender: &PeerId, + _data: &[u8], + ) -> ValidationResult { + ValidationResult::ProcessAndKeep(H256::default()) } } @@ -234,13 +290,17 @@ mod tests { /// See https://github.com/paritytech/substrate/issues/5000 for details. #[test] fn returns_when_network_event_stream_closes() { + let network = TestNetwork::default(); let mut gossip_engine = GossipEngine::::new( - Arc::new(TestNetwork{}), + network.clone(), [1, 2, 3, 4], "my_protocol".as_bytes(), - Arc::new(TestValidator{}), + Arc::new(AllowAll{}), ); + // Drop network event stream sender side. + drop(network.inner.lock().unwrap().event_senders.pop()); + futures::executor::block_on(futures::future::poll_fn(move |ctx| { if let Poll::Pending = gossip_engine.poll_unpin(ctx) { panic!( @@ -251,4 +311,72 @@ mod tests { Poll::Ready(()) })) } + + #[test] + fn keeps_multiple_subscribers_per_topic_updated_with_both_old_and_new_messages() { + let topic = H256::default(); + let engine_id = [1, 2, 3, 4]; + let remote_peer = PeerId::random(); + let network = TestNetwork::default(); + + let mut gossip_engine = GossipEngine::::new( + network.clone(), + engine_id.clone(), + "my_protocol".as_bytes(), + Arc::new(AllowAll{}), + ); + + let mut event_sender = network.inner.lock() + .unwrap() + .event_senders + .pop() + .unwrap(); + + // Register the remote peer. + event_sender.start_send( + Event::NotificationStreamOpened { + remote: remote_peer.clone(), + engine_id: engine_id.clone(), + role: ObservedRole::Authority, + } + ).unwrap(); + + let messages = vec![vec![1], vec![2]]; + let events = messages.iter().cloned().map(|m| { + Event::NotificationsReceived { + remote: remote_peer.clone(), + messages: vec![(engine_id, m.into())] + } + }).collect::>(); + + // Send first event before subscribing. + event_sender.start_send(events[0].clone()).unwrap(); + + let mut subscribers = vec![]; + for _ in 0..2 { + subscribers.push(gossip_engine.messages_for(topic)); + } + + // Send second event after subscribing. + event_sender.start_send(events[1].clone()).unwrap(); + + spawn(gossip_engine); + + let mut subscribers = subscribers.into_iter() + .map(|s| block_on_stream(s)) + .collect::>(); + + // Expect each subscriber to receive both events. + for message in messages { + for subscriber in subscribers.iter_mut() { + assert_eq!( + subscriber.next(), + Some(TopicNotification { + message: message.clone(), + sender: Some(remote_peer.clone()), + }), + ); + } + } + } } diff --git a/client/network-gossip/src/lib.rs b/client/network-gossip/src/lib.rs index 4e4d32366f29d4c907561fc844e107edc60b8627..42aeca86cb275a9651e9b04c1a092d58790122dd 100644 --- a/client/network-gossip/src/lib.rs +++ b/client/network-gossip/src/lib.rs @@ -99,7 +99,7 @@ pub trait Network { impl Network for Arc> { fn event_stream(&self) -> Pin + Send>> { - Box::pin(NetworkService::event_stream(self)) + Box::pin(NetworkService::event_stream(self, "network-gossip")) } fn report_peer(&self, peer_id: PeerId, reputation: ReputationChange) { diff --git a/client/network-gossip/src/state_machine.rs b/client/network-gossip/src/state_machine.rs index c846534488bf5a94760688089b8ecb1917cace62..433457afe748e422c78c0a2d5bde7c2af54c173c 100644 --- a/client/network-gossip/src/state_machine.rs +++ b/client/network-gossip/src/state_machine.rs @@ -16,15 +16,14 @@ use crate::{Network, MessageIntent, Validator, ValidatorContext, ValidationResult}; -use std::collections::{HashMap, HashSet, hash_map::Entry}; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::iter; use std::time; -use log::trace; +use log::{error, trace}; use lru::LruCache; use libp2p::PeerId; use sp_runtime::traits::{Block as BlockT, Hash, HashFor}; -use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver}; use sp_runtime::ConsensusEngineId; use sc_network::ObservedRole; use wasm_timer::Instant; @@ -42,20 +41,14 @@ mod rep { pub const GOSSIP_SUCCESS: Rep = Rep::new(1 << 4, "Successfull gossip"); /// Reputation change when a peer sends us a gossip message that we already knew about. pub const DUPLICATE_GOSSIP: Rep = Rep::new(-(1 << 2), "Duplicate gossip"); - /// Reputation change when a peer sends us a gossip message for an unknown engine, whatever that - /// means. - pub const UNKNOWN_GOSSIP: Rep = Rep::new(-(1 << 6), "Unknown gossip message engine id"); - /// Reputation change when a peer sends a message from a topic it isn't registered on. - pub const UNREGISTERED_TOPIC: Rep = Rep::new(-(1 << 10), "Unregistered gossip message topic"); } struct PeerConsensus { known_messages: HashSet, - role: ObservedRole, } /// Topic stream message with sender. -#[derive(Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub struct TopicNotification { /// Message data. pub message: Vec, @@ -66,7 +59,6 @@ pub struct TopicNotification { struct MessageEntry { message_hash: B::Hash, topic: B::Hash, - engine_id: ConsensusEngineId, message: Vec, sender: Option, } @@ -75,7 +67,6 @@ struct MessageEntry { struct NetworkContext<'g, 'p, B: BlockT> { gossip: &'g mut ConsensusGossip, network: &'p mut dyn Network, - engine_id: ConsensusEngineId, } impl<'g, 'p, B: BlockT> ValidatorContext for NetworkContext<'g, 'p, B> { @@ -89,7 +80,6 @@ impl<'g, 'p, B: BlockT> ValidatorContext for NetworkContext<'g, 'p, B> { self.gossip.multicast( self.network, topic, - self.engine_id.clone(), message, force, ); @@ -97,40 +87,30 @@ impl<'g, 'p, B: BlockT> ValidatorContext for NetworkContext<'g, 'p, B> { /// Send addressed message to a peer. fn send_message(&mut self, who: &PeerId, message: Vec) { - self.network.write_notification(who.clone(), self.engine_id, message); + self.network.write_notification(who.clone(), self.gossip.engine_id, message); } /// Send all messages with given topic to a peer. fn send_topic(&mut self, who: &PeerId, topic: B::Hash, force: bool) { - self.gossip.send_topic(self.network, who, topic, self.engine_id, force); + self.gossip.send_topic(self.network, who, topic, force); } } fn propagate<'a, B: BlockT, I>( network: &mut dyn Network, + engine_id: ConsensusEngineId, messages: I, intent: MessageIntent, peers: &mut HashMap>, - validators: &HashMap>>, + validator: &Arc>, ) // (msg_hash, topic, message) - where I: Clone + IntoIterator)>, + where I: Clone + IntoIterator)>, { - let mut check_fns = HashMap::new(); - let mut message_allowed = move |who: &PeerId, intent: MessageIntent, topic: &B::Hash, engine_id: ConsensusEngineId, message: &Vec| { - let check_fn = match check_fns.entry(engine_id) { - Entry::Occupied(entry) => entry.into_mut(), - Entry::Vacant(vacant) => match validators.get(&engine_id) { - None => return false, // treat all messages with no validator as not allowed - Some(validator) => vacant.insert(validator.message_allowed()), - } - }; - - (check_fn)(who, intent, topic, &message) - }; + let mut message_allowed = validator.message_allowed(); for (id, ref mut peer) in peers.iter_mut() { - for (message_hash, topic, engine_id, message) in messages.clone() { + for (message_hash, topic, message) in messages.clone() { let intent = match intent { MessageIntent::Broadcast { .. } => if peer.known_messages.contains(&message_hash) { @@ -149,7 +129,7 @@ fn propagate<'a, B: BlockT, I>( other => other, }; - if !message_allowed(id, intent, &topic, engine_id, &message) { + if !message_allowed(id, intent, &topic, &message) { continue; } @@ -164,45 +144,26 @@ fn propagate<'a, B: BlockT, I>( /// Consensus network protocol handler. Manages statements and candidate requests. pub struct ConsensusGossip { peers: HashMap>, - live_message_sinks: HashMap<(ConsensusEngineId, B::Hash), Vec>>, messages: Vec>, known_messages: LruCache, - validators: HashMap>>, + engine_id: ConsensusEngineId, + validator: Arc>, next_broadcast: Instant, } impl ConsensusGossip { - /// Create a new instance. - pub fn new() -> Self { + /// Create a new instance using the given validator. + pub fn new(validator: Arc>, engine_id: ConsensusEngineId) -> Self { ConsensusGossip { peers: HashMap::new(), - live_message_sinks: HashMap::new(), messages: Default::default(), known_messages: LruCache::new(KNOWN_MESSAGES_CACHE_SIZE), - validators: Default::default(), + engine_id, + validator, next_broadcast: Instant::now() + REBROADCAST_INTERVAL, } } - /// Register message validator for a message type. - pub fn register_validator( - &mut self, - network: &mut dyn Network, - engine_id: ConsensusEngineId, - validator: Arc> - ) { - self.register_validator_internal(engine_id, validator.clone()); - let peers: Vec<_> = self.peers.iter().map(|(id, peer)| (id.clone(), peer.role.clone())).collect(); - for (id, role) in peers { - let mut context = NetworkContext { gossip: self, network, engine_id: engine_id.clone() }; - validator.new_peer(&mut context, &id, role); - } - } - - fn register_validator_internal(&mut self, engine_id: ConsensusEngineId, validator: Arc>) { - self.validators.insert(engine_id, validator.clone()); - } - /// Handle new connected peer. pub fn new_peer(&mut self, network: &mut dyn Network, who: PeerId, role: ObservedRole) { // light nodes are not valid targets for consensus gossip messages @@ -213,19 +174,17 @@ impl ConsensusGossip { trace!(target:"gossip", "Registering {:?} {}", role, who); self.peers.insert(who.clone(), PeerConsensus { known_messages: HashSet::new(), - role: role.clone(), }); - for (engine_id, v) in self.validators.clone() { - let mut context = NetworkContext { gossip: self, network, engine_id: engine_id.clone() }; - v.new_peer(&mut context, &who, role.clone()); - } + + let validator = self.validator.clone(); + let mut context = NetworkContext { gossip: self, network }; + validator.new_peer(&mut context, &who, role.clone()); } fn register_message_hashed( &mut self, message_hash: B::Hash, topic: B::Hash, - engine_id: ConsensusEngineId, message: Vec, sender: Option, ) { @@ -233,7 +192,6 @@ impl ConsensusGossip { self.messages.push(MessageEntry { message_hash, topic, - engine_id, message, sender, }); @@ -248,19 +206,17 @@ impl ConsensusGossip { pub fn register_message( &mut self, topic: B::Hash, - engine_id: ConsensusEngineId, message: Vec, ) { let message_hash = HashFor::::hash(&message[..]); - self.register_message_hashed(message_hash, topic, engine_id, message, None); + self.register_message_hashed(message_hash, topic, message, None); } /// Call when a peer has been disconnected to stop tracking gossip status. pub fn peer_disconnected(&mut self, network: &mut dyn Network, who: PeerId) { - for (engine_id, v) in self.validators.clone() { - let mut context = NetworkContext { gossip: self, network, engine_id: engine_id.clone() }; - v.peer_disconnected(&mut context, &who); - } + let validator = self.validator.clone(); + let mut context = NetworkContext { gossip: self, network }; + validator.peer_disconnected(&mut context, &who); self.peers.remove(&who); } @@ -276,8 +232,8 @@ impl ConsensusGossip { /// Rebroadcast all messages to all peers. fn rebroadcast(&mut self, network: &mut dyn Network) { let messages = self.messages.iter() - .map(|entry| (&entry.message_hash, &entry.topic, entry.engine_id, &entry.message)); - propagate(network, messages, MessageIntent::PeriodicRebroadcast, &mut self.peers, &self.validators); + .map(|entry| (&entry.message_hash, &entry.topic, &entry.message)); + propagate(network, self.engine_id, messages, MessageIntent::PeriodicRebroadcast, &mut self.peers, &self.validator); } /// Broadcast all messages with given topic. @@ -285,40 +241,21 @@ impl ConsensusGossip { let messages = self.messages.iter() .filter_map(|entry| if entry.topic == topic { - Some((&entry.message_hash, &entry.topic, entry.engine_id, &entry.message)) + Some((&entry.message_hash, &entry.topic, &entry.message)) } else { None } ); let intent = if force { MessageIntent::ForcedBroadcast } else { MessageIntent::Broadcast }; - propagate(network, messages, intent, &mut self.peers, &self.validators); + propagate(network, self.engine_id, messages, intent, &mut self.peers, &self.validator); } /// Prune old or no longer relevant consensus messages. Provide a predicate /// for pruning, which returns `false` when the items with a given topic should be pruned. pub fn collect_garbage(&mut self) { - self.live_message_sinks.retain(|_, sinks| { - sinks.retain(|sink| !sink.is_closed()); - !sinks.is_empty() - }); - let known_messages = &mut self.known_messages; let before = self.messages.len(); - let validators = &self.validators; - - let mut check_fns = HashMap::new(); - let mut message_expired = move |entry: &MessageEntry| { - let engine_id = entry.engine_id; - let check_fn = match check_fns.entry(engine_id) { - Entry::Occupied(entry) => entry.into_mut(), - Entry::Vacant(vacant) => match validators.get(&engine_id) { - None => return true, // treat all messages with no validator as expired - Some(validator) => vacant.insert(validator.message_expired()), - } - }; - - (check_fn)(entry.topic, &entry.message) - }; - self.messages.retain(|entry| !message_expired(entry)); + let mut message_expired = self.validator.message_expired(); + self.messages.retain(|entry| !message_expired(entry.topic, &entry.message)); trace!(target: "gossip", "Cleaned up {} stale messages, {} left ({} known)", before - self.messages.len(), @@ -331,40 +268,29 @@ impl ConsensusGossip { } } - /// Get data of valid, incoming messages for a topic (but might have expired meanwhile) - pub fn messages_for(&mut self, engine_id: ConsensusEngineId, topic: B::Hash) - -> TracingUnboundedReceiver - { - let (tx, rx) = tracing_unbounded("mpsc_gossip_messages_for"); - for entry in self.messages.iter_mut() - .filter(|e| e.topic == topic && e.engine_id == engine_id) - { - tx.unbounded_send(TopicNotification { - message: entry.message.clone(), - sender: entry.sender.clone(), - }) - .expect("receiver known to be live; qed"); - } - - self.live_message_sinks.entry((engine_id, topic)).or_default().push(tx); - - rx + /// Get valid messages received in the past for a topic (might have expired meanwhile). + pub fn messages_for(&mut self, topic: B::Hash) -> impl Iterator + '_ { + self.messages.iter().filter(move |e| e.topic == topic).map(|entry| TopicNotification { + message: entry.message.clone(), + sender: entry.sender.clone(), + }) } - /// Handle an incoming message for topic by who via protocol. Discard message if topic already - /// known, the message is old, its source peers isn't a registered peer or the connection to - /// them is broken. + /// Register incoming messages and return the ones that are new and valid (according to a gossip + /// validator) and should thus be forwarded to the upper layers. pub fn on_incoming( &mut self, network: &mut dyn Network, who: PeerId, - messages: Vec<(ConsensusEngineId, Vec)>, - ) { + messages: Vec>, + ) -> Vec<(B::Hash, TopicNotification)> { + let mut to_forward = vec![]; + if !messages.is_empty() { trace!(target: "gossip", "Received {} messages from peer {}", messages.len(), who); } - for (engine_id, message) in messages { + for message in messages { let message_hash = HashFor::::hash(&message[..]); if self.known_messages.contains(&message_hash) { @@ -374,55 +300,47 @@ impl ConsensusGossip { } // validate the message - let validation = self.validators.get(&engine_id) - .cloned() - .map(|v| { - let mut context = NetworkContext { gossip: self, network, engine_id }; - v.validate(&mut context, &who, &message) - }); + let validation = { + let validator = self.validator.clone(); + let mut context = NetworkContext { gossip: self, network }; + validator.validate(&mut context, &who, &message) + }; + + let (topic, keep) = match validation { + ValidationResult::ProcessAndKeep(topic) => (topic, true), + ValidationResult::ProcessAndDiscard(topic) => (topic, false), + ValidationResult::Discard => { + trace!(target:"gossip", "Discard message from peer {}", who); + continue; + }, + }; - let validation_result = match validation { - Some(ValidationResult::ProcessAndKeep(topic)) => Some((topic, true)), - Some(ValidationResult::ProcessAndDiscard(topic)) => Some((topic, false)), - Some(ValidationResult::Discard) => None, + let peer = match self.peers.get_mut(&who) { + Some(peer) => peer, None => { - trace!(target:"gossip", "Unknown message engine id {:?} from {}", engine_id, who); - network.report_peer(who.clone(), rep::UNKNOWN_GOSSIP); - network.disconnect_peer(who.clone()); + error!(target:"gossip", "Got message from unregistered peer {}", who); continue; } }; - if let Some((topic, keep)) = validation_result { - network.report_peer(who.clone(), rep::GOSSIP_SUCCESS); - if let Some(ref mut peer) = self.peers.get_mut(&who) { - peer.known_messages.insert(message_hash); - if let Entry::Occupied(mut entry) = self.live_message_sinks.entry((engine_id, topic)) { - trace!(target: "gossip", "Pushing consensus message to sinks for {}.", topic); - entry.get_mut().retain(|sink| { - if let Err(e) = sink.unbounded_send(TopicNotification { - message: message.clone(), - sender: Some(who.clone()) - }) { - trace!(target: "gossip", "Error broadcasting message notification: {:?}", e); - } - !sink.is_closed() - }); - if entry.get().is_empty() { - entry.remove_entry(); - } - } - if keep { - self.register_message_hashed(message_hash, topic, engine_id, message, Some(who.clone())); - } - } else { - trace!(target:"gossip", "Ignored statement from unregistered peer {}", who); - network.report_peer(who.clone(), rep::UNREGISTERED_TOPIC); - } - } else { - trace!(target:"gossip", "Discard message from peer {}", who); + network.report_peer(who.clone(), rep::GOSSIP_SUCCESS); + peer.known_messages.insert(message_hash); + to_forward.push((topic, TopicNotification { + message: message.clone(), + sender: Some(who.clone()) + })); + + if keep { + self.register_message_hashed( + message_hash, + topic, + message, + Some(who.clone()), + ); } } + + to_forward } /// Send all messages with given topic to a peer. @@ -431,17 +349,12 @@ impl ConsensusGossip { network: &mut dyn Network, who: &PeerId, topic: B::Hash, - engine_id: ConsensusEngineId, force: bool ) { - let validator = self.validators.get(&engine_id); - let mut message_allowed = match validator { - None => return, // treat all messages with no validator as not allowed - Some(validator) => validator.message_allowed(), - }; + let mut message_allowed = self.validator.message_allowed(); if let Some(ref mut peer) = self.peers.get_mut(who) { - for entry in self.messages.iter().filter(|m| m.topic == topic && m.engine_id == engine_id) { + for entry in self.messages.iter().filter(|m| m.topic == topic) { let intent = if force { MessageIntent::ForcedBroadcast } else { @@ -459,7 +372,7 @@ impl ConsensusGossip { peer.known_messages.insert(entry.message_hash.clone()); trace!(target: "gossip", "Sending topic message to {}: {:?}", who, entry.message); - network.write_notification(who.clone(), engine_id, entry.message.clone()); + network.write_notification(who.clone(), self.engine_id, entry.message.clone()); } } } @@ -469,14 +382,13 @@ impl ConsensusGossip { &mut self, network: &mut dyn Network, topic: B::Hash, - engine_id: ConsensusEngineId, message: Vec, force: bool, ) { let message_hash = HashFor::::hash(&message); - self.register_message_hashed(message_hash, topic, engine_id, message.clone(), None); + self.register_message_hashed(message_hash, topic, message.clone(), None); let intent = if force { MessageIntent::ForcedBroadcast } else { MessageIntent::Broadcast }; - propagate(network, iter::once((&message_hash, &topic, engine_id, &message)), intent, &mut self.peers, &self.validators); + propagate(network, self.engine_id, iter::once((&message_hash, &topic, &message)), intent, &mut self.peers, &self.validator); } /// Send addressed message to a peer. The message is not kept or multicast @@ -485,7 +397,6 @@ impl ConsensusGossip { &mut self, network: &mut dyn Network, who: &PeerId, - engine_id: ConsensusEngineId, message: Vec, ) { let peer = match self.peers.get_mut(who) { @@ -498,16 +409,16 @@ impl ConsensusGossip { trace!(target: "gossip", "Sending direct to {}: {:?}", who, message); peer.known_messages.insert(message_hash); - network.write_notification(who.clone(), engine_id, message); + network.write_notification(who.clone(), self.engine_id, message); } } #[cfg(test)] mod tests { - use std::sync::Arc; + use futures::prelude::*; + use sc_network::{Event, ReputationChange}; use sp_runtime::testing::{H256, Block as RawBlock, ExtrinsicWrapper}; - use futures::executor::block_on_stream; - + use std::{borrow::Cow, pin::Pin, sync::{Arc, Mutex}}; use super::*; type Block = RawBlock>; @@ -518,7 +429,6 @@ mod tests { $consensus.messages.push(MessageEntry { message_hash: $hash, topic: $topic, - engine_id: [0, 0, 0, 0], message: $m, sender: None, }); @@ -538,6 +448,52 @@ mod tests { } } + struct DiscardAll; + impl Validator for DiscardAll{ + fn validate( + &self, + _context: &mut dyn ValidatorContext, + _sender: &PeerId, + _data: &[u8], + ) -> ValidationResult { + ValidationResult::Discard + } + } + + #[derive(Clone, Default)] + struct NoOpNetwork { + inner: Arc>, + } + + #[derive(Clone, Default)] + struct NoOpNetworkInner { + peer_reports: Vec<(PeerId, ReputationChange)>, + } + + impl Network for NoOpNetwork { + fn event_stream(&self) -> Pin + Send>> { + unimplemented!(); + } + + fn report_peer(&self, peer_id: PeerId, reputation_change: ReputationChange) { + self.inner.lock().unwrap().peer_reports.push((peer_id, reputation_change)); + } + + fn disconnect_peer(&self, _: PeerId) { + unimplemented!(); + } + + fn write_notification(&self, _: PeerId, _: ConsensusEngineId, _: Vec) { + unimplemented!(); + } + + fn register_notifications_protocol(&self, _: ConsensusEngineId, _: Cow<'static, [u8]>) {} + + fn announce(&self, _: B::Hash, _: Vec) { + unimplemented!(); + } + } + #[test] fn collects_garbage() { struct AllowOne; @@ -562,7 +518,7 @@ mod tests { let prev_hash = H256::random(); let best_hash = H256::random(); - let mut consensus = ConsensusGossip::::new(); + let mut consensus = ConsensusGossip::::new(Arc::new(AllowAll), [0, 0, 0, 0]); let m1_hash = H256::random(); let m2_hash = H256::random(); let m1 = vec![1, 2, 3]; @@ -573,13 +529,11 @@ mod tests { consensus.known_messages.put(m1_hash, ()); consensus.known_messages.put(m2_hash, ()); - let test_engine_id = Default::default(); - consensus.register_validator_internal(test_engine_id, Arc::new(AllowAll)); consensus.collect_garbage(); assert_eq!(consensus.messages.len(), 2); assert_eq!(consensus.known_messages.len(), 2); - consensus.register_validator_internal(test_engine_id, Arc::new(AllowOne)); + consensus.validator = Arc::new(AllowOne); // m2 is expired consensus.collect_garbage(); @@ -590,116 +544,84 @@ mod tests { } #[test] - fn message_stream_include_those_sent_before_asking_for_stream() { - let mut consensus = ConsensusGossip::::new(); - consensus.register_validator_internal([0, 0, 0, 0], Arc::new(AllowAll)); + fn message_stream_include_those_sent_before_asking() { + let mut consensus = ConsensusGossip::::new(Arc::new(AllowAll), [0, 0, 0, 0]); - let engine_id = [0, 0, 0, 0]; + // Register message. let message = vec![4, 5, 6]; let topic = HashFor::::hash(&[1,2,3]); + consensus.register_message(topic, message.clone()); - consensus.register_message(topic, engine_id, message.clone()); - let mut stream = block_on_stream(consensus.messages_for([0, 0, 0, 0], topic)); - - assert_eq!(stream.next(), Some(TopicNotification { message: message, sender: None })); + assert_eq!( + consensus.messages_for(topic).next(), + Some(TopicNotification { message: message, sender: None }), + ); } #[test] fn can_keep_multiple_messages_per_topic() { - let mut consensus = ConsensusGossip::::new(); + let mut consensus = ConsensusGossip::::new(Arc::new(AllowAll), [0, 0, 0, 0]); let topic = [1; 32].into(); let msg_a = vec![1, 2, 3]; let msg_b = vec![4, 5, 6]; - consensus.register_message(topic, [0, 0, 0, 0], msg_a); - consensus.register_message(topic, [0, 0, 0, 0], msg_b); + consensus.register_message(topic, msg_a); + consensus.register_message(topic, msg_b); assert_eq!(consensus.messages.len(), 2); } #[test] - fn can_keep_multiple_subscribers_per_topic() { - let mut consensus = ConsensusGossip::::new(); - consensus.register_validator_internal([0, 0, 0, 0], Arc::new(AllowAll)); - - let message = vec![4, 5, 6]; - let topic = HashFor::::hash(&[1, 2, 3]); + fn peer_is_removed_on_disconnect() { + let mut consensus = ConsensusGossip::::new(Arc::new(AllowAll), [0, 0, 0, 0]); - consensus.register_message(topic, [0, 0, 0, 0], message.clone()); + let mut network = NoOpNetwork::default(); - let mut stream1 = block_on_stream(consensus.messages_for([0, 0, 0, 0], topic)); - let mut stream2 = block_on_stream(consensus.messages_for([0, 0, 0, 0], topic)); + let peer_id = PeerId::random(); + consensus.new_peer(&mut network, peer_id.clone(), ObservedRole::Full); + assert!(consensus.peers.contains_key(&peer_id)); - assert_eq!(stream1.next(), Some(TopicNotification { message: message.clone(), sender: None })); - assert_eq!(stream2.next(), Some(TopicNotification { message, sender: None })); + consensus.peer_disconnected(&mut network, peer_id.clone()); + assert!(!consensus.peers.contains_key(&peer_id)); } #[test] - fn topics_are_localized_to_engine_id() { - let mut consensus = ConsensusGossip::::new(); - consensus.register_validator_internal([0, 0, 0, 0], Arc::new(AllowAll)); - - let topic = [1; 32].into(); - let msg_a = vec![1, 2, 3]; - let msg_b = vec![4, 5, 6]; - - consensus.register_message(topic, [0, 0, 0, 0], msg_a); - consensus.register_message(topic, [0, 0, 0, 1], msg_b); - - let mut stream = block_on_stream(consensus.messages_for([0, 0, 0, 0], topic)); - - assert_eq!(stream.next(), Some(TopicNotification { message: vec![1, 2, 3], sender: None })); + fn on_incoming_ignores_discarded_messages() { + let to_forward = ConsensusGossip::::new( + Arc::new(DiscardAll), + [0, 0, 0, 0], + ).on_incoming( + &mut NoOpNetwork::default(), + PeerId::random(), + vec![vec![1, 2, 3]], + ); - let _ = consensus.live_message_sinks.remove(&([0, 0, 0, 0], topic)); - assert_eq!(stream.next(), None); + assert!( + to_forward.is_empty(), + "Expected `on_incoming` to ignore discarded message but got {:?}", to_forward, + ); } #[test] - fn peer_is_removed_on_disconnect() { - struct TestNetwork; - impl Network for TestNetwork { - fn event_stream( - &self, - ) -> std::pin::Pin + Send>> { - unimplemented!("Not required in tests") - } - - fn report_peer(&self, _: PeerId, _: crate::ReputationChange) { - unimplemented!("Not required in tests") - } - - fn disconnect_peer(&self, _: PeerId) { - unimplemented!("Not required in tests") - } - - fn write_notification(&self, _: PeerId, _: crate::ConsensusEngineId, _: Vec) { - unimplemented!("Not required in tests") - } - - fn register_notifications_protocol( - &self, - _: ConsensusEngineId, - _: std::borrow::Cow<'static, [u8]>, - ) { - unimplemented!("Not required in tests") - } - - fn announce(&self, _: H256, _: Vec) { - unimplemented!("Not required in tests") - } - } - - let mut consensus = ConsensusGossip::::new(); - consensus.register_validator_internal([0, 0, 0, 0], Arc::new(AllowAll)); - - let mut network = TestNetwork; - - let peer_id = PeerId::random(); - consensus.new_peer(&mut network, peer_id.clone(), ObservedRole::Full); - assert!(consensus.peers.contains_key(&peer_id)); + fn on_incoming_ignores_unregistered_peer() { + let mut network = NoOpNetwork::default(); + let remote = PeerId::random(); + + let to_forward = ConsensusGossip::::new( + Arc::new(AllowAll), + [0, 0, 0, 0], + ).on_incoming( + &mut network, + // Unregistered peer. + remote.clone(), + vec![vec![1, 2, 3]], + ); - consensus.peer_disconnected(&mut network, peer_id.clone()); - assert!(!consensus.peers.contains_key(&peer_id)); + assert!( + to_forward.is_empty(), + "Expected `on_incoming` to ignore message from unregistered peer but got {:?}", + to_forward, + ); } } diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index 8d67a15c354bdde4adfdee04a3b60293b55e86f7..56a4bda2b238839bb586b3a2082382b2f08bb918 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -1,7 +1,7 @@ [package] description = "Substrate network protocol" name = "sc-network" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" @@ -10,6 +10,10 @@ repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sc-network" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + + [build-dependencies] prost-build = "0.6.1" @@ -21,51 +25,56 @@ derive_more = "0.99.2" either = "1.5.3" erased-serde = "0.3.9" fnv = "1.0.6" -fork-tree = { version = "2.0.0-alpha.5", path = "../../utils/fork-tree" } +fork-tree = { version = "2.0.0-dev", path = "../../utils/fork-tree" } futures = "0.3.4" -futures_codec = "0.3.3" futures-timer = "3.0.1" -wasm-timer = "0.2" -libp2p = { version = "0.16.2", default-features = false, features = ["libp2p-websocket"] } +futures_codec = "0.3.3" +hex = "0.4.0" +ip_network = "0.3.4" linked-hash-map = "0.5.2" linked_hash_set = "0.1.3" log = "0.4.8" lru = "0.4.0" nohash-hasher = "0.2.0" parking_lot = "0.10.0" +pin-project = "0.4.6" +prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.8.0-dev", path = "../../utils/prometheus" } prost = "0.6.1" rand = "0.7.2" -hex = "0.4.0" -sc-block-builder = { version = "0.8.0-alpha.5", path = "../block-builder" } -sc-client = { version = "0.8.0-alpha.5", path = "../" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } -sc-peerset = { version = "2.0.0-alpha.5", path = "../peerset" } -pin-project = "0.4.6" +sc-block-builder = { version = "0.8.0-dev", path = "../block-builder" } +sc-client-api = { version = "2.0.0-dev", path = "../api" } +sc-peerset = { version = "2.0.0-dev", path = "../peerset" } serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" slog = { version = "2.5.2", features = ["nested-values"] } slog_derive = "0.2.0" smallvec = "0.6.10" -sp-arithmetic = { version = "2.0.0-alpha.5", path = "../../primitives/arithmetic" } -sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/common" } -sp-consensus-babe = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/babe" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.8.0-alpha.5", path = "../../utils/prometheus" } +sp-arithmetic = { version = "2.0.0-dev", path = "../../primitives/arithmetic" } +sp-blockchain = { version = "2.0.0-dev", path = "../../primitives/blockchain" } +sp-consensus = { version = "0.8.0-dev", path = "../../primitives/consensus/common" } +sp-consensus-babe = { version = "0.8.0-dev", path = "../../primitives/consensus/babe" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-dev", path = "../../primitives/utils" } thiserror = "1" unsigned-varint = { version = "0.3.1", features = ["futures", "futures-codec"] } void = "1.0.2" +wasm-timer = "0.2" zeroize = "1.0.0" +[dependencies.libp2p] +version = "0.18.1" +default-features = false +features = ["websocket", "kad", "mdns", "ping", "identify", "mplex", "yamux", "noise"] + [dev-dependencies] async-std = "1.5" assert_matches = "1.3" env_logger = "0.7.0" +libp2p = { version = "0.18.1", default-features = false, features = ["secio"] } quickcheck = "0.9.0" rand = "0.7.2" -sp-keyring = { version = "2.0.0-alpha.5", path = "../../primitives/keyring" } +sp-keyring = { version = "2.0.0-dev", path = "../../primitives/keyring" } sp-test-primitives = { version = "2.0.0-dev", path = "../../primitives/test-primitives" } substrate-test-runtime = { version = "2.0.0-dev", path = "../../test-utils/runtime" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } @@ -73,7 +82,3 @@ tempfile = "3.1.0" [features] default = [] - - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/network/build.rs b/client/network/build.rs index 0fd1f128660e9aaec36c4ac14e405f77e9588096..991b1cba5d6c81c6e7ade01432ee5e51252a32a4 100644 --- a/client/network/build.rs +++ b/client/network/build.rs @@ -1,5 +1,6 @@ const PROTOS: &[&str] = &[ "src/protocol/schema/api.v1.proto", + "src/protocol/schema/finality.v1.proto", "src/protocol/schema/light.v1.proto" ]; diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 203435134c6cd090bb84b9e6e8d08f384f0b79af..eae20c9031ec10130f5d9e0ae9a90fcc07f17b6c 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -15,11 +15,16 @@ // along with Substrate. If not, see . use crate::{ - config::Role, - debug_info, discovery::DiscoveryBehaviour, discovery::DiscoveryOut, + config::{ProtocolId, Role}, + debug_info, discovery::{DiscoveryBehaviour, DiscoveryConfig, DiscoveryOut}, Event, ObservedRole, DhtEvent, ExHashT, }; -use crate::protocol::{self, light_client_handler, message::Roles, CustomMessageOutcome, Protocol}; +use crate::protocol::{ + self, block_requests, light_client_handler, finality_requests, + message::{self, Roles}, CustomMessageOutcome, Protocol +}; + +use codec::Encode as _; use libp2p::NetworkBehaviour; use libp2p::core::{Multiaddr, PeerId, PublicKey}; use libp2p::kad::record; @@ -27,7 +32,7 @@ use libp2p::swarm::{NetworkBehaviourAction, NetworkBehaviourEventProcess, PollPa use log::debug; use sp_consensus::{BlockOrigin, import_queue::{IncomingBlock, Origin}}; use sp_runtime::{traits::{Block as BlockT, NumberFor}, ConsensusEngineId, Justification}; -use std::{borrow::Cow, iter, task::Context, task::Poll}; +use std::{borrow::Cow, iter, task::{Context, Poll}, time::Duration}; use void; /// General behaviour of the network. Combines all protocols together. @@ -43,6 +48,8 @@ pub struct Behaviour { discovery: DiscoveryBehaviour, /// Block request handling. block_requests: protocol::BlockRequests, + /// Finality proof request handling. + finality_proof_requests: protocol::FinalityProofRequests, /// Light client request handling. light_client_handler: protocol::LightClientHandler, @@ -60,36 +67,60 @@ pub enum BehaviourOut { BlockImport(BlockOrigin, Vec>), JustificationImport(Origin, B::Hash, NumberFor, Justification), FinalityProofImport(Origin, B::Hash, NumberFor, Vec), - /// Started a random Kademlia discovery query. - RandomKademliaStarted, + + /// Started a random iterative Kademlia discovery query. + RandomKademliaStarted(ProtocolId), + + /// We have received a request from a peer and answered it. + AnsweredRequest { + /// Peer which sent us a request. + peer: PeerId, + /// Protocol name of the request. + protocol: Vec, + /// Time it took to build the response. + build_time: Duration, + }, + /// Started a new request with the given node. + RequestStarted { + peer: PeerId, + /// Protocol name of the request. + protocol: Vec, + }, + /// Finished, successfully or not, a previously-started request. + RequestFinished { + /// Who we were requesting. + peer: PeerId, + /// Protocol name of the request. + protocol: Vec, + /// How long before the response came or the request got cancelled. + request_duration: Duration, + }, + + /// Any event represented by the [`Event`] enum. + /// + /// > **Note**: The [`Event`] enum contains the events that are available through the public + /// > API of the library. Event(Event), } impl Behaviour { /// Builds a new `Behaviour`. - pub async fn new( + pub fn new( substrate: Protocol, role: Role, user_agent: String, local_public_key: PublicKey, - known_addresses: Vec<(PeerId, Multiaddr)>, - enable_mdns: bool, - allow_private_ipv4: bool, - discovery_only_if_under_num: u64, block_requests: protocol::BlockRequests, + finality_proof_requests: protocol::FinalityProofRequests, light_client_handler: protocol::LightClientHandler, + disco_config: DiscoveryConfig, ) -> Self { Behaviour { substrate, debug_info: debug_info::DebugInfoBehaviour::new(user_agent, local_public_key.clone()), - discovery: DiscoveryBehaviour::new( - local_public_key, - known_addresses, - enable_mdns, - allow_private_ipv4, - discovery_only_if_under_num, - ).await, + discovery: disco_config.finish(), block_requests, + finality_proof_requests, light_client_handler, events: Vec::new(), role, @@ -107,10 +138,20 @@ impl Behaviour { } /// Returns the number of nodes that are in the Kademlia k-buckets. - pub fn num_kbuckets_entries(&mut self) -> usize { + pub fn num_kbuckets_entries(&mut self) -> impl ExactSizeIterator { self.discovery.num_kbuckets_entries() } + /// Returns the number of records in the Kademlia record stores. + pub fn num_kademlia_records(&mut self) -> impl ExactSizeIterator { + self.discovery.num_kademlia_records() + } + + /// Returns the total size in bytes of all the records in the Kademlia record stores. + pub fn kademlia_records_total_size(&mut self) -> impl ExactSizeIterator { + self.discovery.kademlia_records_total_size() + } + /// Borrows `self` and returns a struct giving access to the information about a node. /// /// Returns `None` if we don't know anything about this node. Always returns `Some` for nodes @@ -134,7 +175,11 @@ impl Behaviour { engine_id: ConsensusEngineId, protocol_name: impl Into>, ) { - let list = self.substrate.register_notifications_protocol(engine_id, protocol_name); + // This is the message that we will send to the remote as part of the initial handshake. + // At the moment, we force this to be an encoded `Roles`. + let handshake_message = Roles::from(&self.role).encode(); + + let list = self.substrate.register_notifications_protocol(engine_id, protocol_name, handshake_message); for (remote, roles) in list { let role = reported_roles_to_observed_role(&self.role, remote, roles); let ev = Event::NotificationStreamOpened { @@ -205,6 +250,32 @@ Behaviour { self.events.push(BehaviourOut::JustificationImport(origin, hash, nb, justification)), CustomMessageOutcome::FinalityProofImport(origin, hash, nb, proof) => self.events.push(BehaviourOut::FinalityProofImport(origin, hash, nb, proof)), + CustomMessageOutcome::BlockRequest { target, request } => { + match self.block_requests.send_request(&target, request) { + block_requests::SendRequestOutcome::Ok => { + self.events.push(BehaviourOut::RequestStarted { + peer: target, + protocol: self.block_requests.protocol_name().to_vec(), + }); + }, + block_requests::SendRequestOutcome::Replaced { request_duration, .. } => { + self.events.push(BehaviourOut::RequestFinished { + peer: target.clone(), + protocol: self.block_requests.protocol_name().to_vec(), + request_duration, + }); + self.events.push(BehaviourOut::RequestStarted { + peer: target, + protocol: self.block_requests.protocol_name().to_vec(), + }); + } + block_requests::SendRequestOutcome::NotConnected | + block_requests::SendRequestOutcome::EncodeError(_) => {}, + } + }, + CustomMessageOutcome::FinalityProofRequest { target, block_hash, request } => { + self.finality_proof_requests.send_request(&target, block_hash, request); + }, CustomMessageOutcome::NotificationStreamOpened { remote, protocols, roles } => { let role = reported_roles_to_observed_role(&self.role, &remote, roles); for engine_id in protocols { @@ -234,6 +305,69 @@ Behaviour { } } +impl NetworkBehaviourEventProcess> for Behaviour { + fn inject_event(&mut self, event: block_requests::Event) { + match event { + block_requests::Event::AnsweredRequest { peer, response_build_time } => { + self.events.push(BehaviourOut::AnsweredRequest { + peer, + protocol: self.block_requests.protocol_name().to_vec(), + build_time: response_build_time, + }); + }, + block_requests::Event::Response { peer, original_request, response, request_duration } => { + self.events.push(BehaviourOut::RequestFinished { + peer: peer.clone(), + protocol: self.block_requests.protocol_name().to_vec(), + request_duration, + }); + let ev = self.substrate.on_block_response(peer, original_request, response); + self.inject_event(ev); + } + block_requests::Event::RequestCancelled { peer, request_duration, .. } => { + // There doesn't exist any mechanism to report cancellations yet. + // We would normally disconnect the node, but this event happens as the result of + // a disconnect, so there's nothing more to do. + self.events.push(BehaviourOut::RequestFinished { + peer, + protocol: self.block_requests.protocol_name().to_vec(), + request_duration, + }); + } + block_requests::Event::RequestTimeout { peer, request_duration, .. } => { + // There doesn't exist any mechanism to report timeouts yet, so we process them by + // disconnecting the node. + self.events.push(BehaviourOut::RequestFinished { + peer: peer.clone(), + protocol: self.block_requests.protocol_name().to_vec(), + request_duration, + }); + self.substrate.disconnect_peer(&peer); + } + } + } +} + +impl NetworkBehaviourEventProcess> for Behaviour { + fn inject_event(&mut self, event: finality_requests::Event) { + match event { + finality_requests::Event::Response { peer, block_hash, proof } => { + let response = message::FinalityProofResponse { + id: 0, + block: block_hash, + proof: if !proof.is_empty() { + Some(proof) + } else { + None + }, + }; + let ev = self.substrate.on_finality_proof_response(peer, response); + self.inject_event(ev); + } + } + } +} + impl NetworkBehaviourEventProcess for Behaviour { fn inject_event(&mut self, event: debug_info::DebugInfoEvent) { @@ -277,8 +411,10 @@ impl NetworkBehaviourEventProcess DiscoveryOut::ValuePutFailed(key) => { self.events.push(BehaviourOut::Event(Event::Dht(DhtEvent::ValuePutFailed(key)))); } - DiscoveryOut::RandomKademliaStarted => { - self.events.push(BehaviourOut::RandomKademliaStarted); + DiscoveryOut::RandomKademliaStarted(protocols) => { + for protocol in protocols { + self.events.push(BehaviourOut::RandomKademliaStarted(protocol)); + } } } } diff --git a/client/network/src/config.rs b/client/network/src/config.rs index c9290927ebaff5a7814a5a3129c6381fcebc4447..66800aeeaf8d23ec01bf101179f99e3a0e67b8e1 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -21,7 +21,6 @@ pub use crate::chain::{Client, FinalityProofProvider}; pub use crate::on_demand_layer::{AlwaysBadChecker, OnDemand}; -pub use crate::service::{TransactionPool, EmptyTransactionPool}; pub use libp2p::{identity, core::PublicKey, wasm_ext::ExtTransport, build_multiaddr}; // Note: this re-export shouldn't be part of the public API of the crate and will be removed in @@ -29,18 +28,27 @@ pub use libp2p::{identity, core::PublicKey, wasm_ext::ExtTransport, build_multia #[doc(hidden)] pub use crate::protocol::ProtocolConfig; -use crate::service::ExHashT; +use crate::{ExHashT, ReportHandle}; -use sp_consensus::{block_validation::BlockAnnounceValidator, import_queue::ImportQueue}; -use sp_runtime::traits::{Block as BlockT}; -use libp2p::identity::{Keypair, ed25519}; -use libp2p::wasm_ext; -use libp2p::{PeerId, Multiaddr, multiaddr}; use core::{fmt, iter}; -use std::{convert::TryFrom, future::Future, pin::Pin, str::FromStr}; -use std::{error::Error, fs, io::{self, Write}, net::Ipv4Addr, path::{Path, PathBuf}, sync::Arc}; -use zeroize::Zeroize; +use libp2p::identity::{ed25519, Keypair}; +use libp2p::wasm_ext; +use libp2p::{multiaddr, Multiaddr, PeerId}; use prometheus_endpoint::Registry; +use sc_peerset::ReputationChange; +use sp_consensus::{block_validation::BlockAnnounceValidator, import_queue::ImportQueue}; +use sp_runtime::{traits::Block as BlockT, ConsensusEngineId}; +use std::{borrow::Cow, convert::TryFrom, future::Future, pin::Pin, str::FromStr}; +use std::{ + collections::HashMap, + error::Error, + fs, + io::{self, Write}, + net::Ipv4Addr, + path::{Path, PathBuf}, + sync::Arc, +}; +use zeroize::Zeroize; /// Network initialization parameters. pub struct Params { @@ -159,6 +167,60 @@ impl FinalityProofRequestBuilder for DummyFinalityProofRequestBuil /// Shared finality proof request builder struct used by the queue. pub type BoxFinalityProofRequestBuilder = Box + Send + Sync>; +/// Transaction pool interface +pub trait TransactionPool: Send + Sync { + /// Get transactions from the pool that are ready to be propagated. + fn transactions(&self) -> Vec<(H, B::Extrinsic)>; + /// Get hash of transaction. + fn hash_of(&self, transaction: &B::Extrinsic) -> H; + /// Import a transaction into the pool. + /// + /// Peer reputation is changed by reputation_change if transaction is accepted by the pool. + fn import( + &self, + report_handle: ReportHandle, + who: PeerId, + reputation_change_good: ReputationChange, + reputation_change_bad: ReputationChange, + transaction: B::Extrinsic, + ); + /// Notify the pool about transactions broadcast. + fn on_broadcasted(&self, propagations: HashMap>); + /// Get transaction by hash. + fn transaction(&self, hash: &H) -> Option; +} + +/// Dummy implementation of the [`TransactionPool`] trait for a transaction pool that is always +/// empty and discards all incoming transactions. +/// +/// Requires the "hash" type to implement the `Default` trait. +/// +/// Useful for testing purposes. +pub struct EmptyTransactionPool; + +impl TransactionPool for EmptyTransactionPool { + fn transactions(&self) -> Vec<(H, B::Extrinsic)> { + Vec::new() + } + + fn hash_of(&self, _transaction: &B::Extrinsic) -> H { + Default::default() + } + + fn import( + &self, + _report_handle: ReportHandle, + _who: PeerId, + _rep_change_good: ReputationChange, + _rep_change_bad: ReputationChange, + _transaction: B::Extrinsic + ) {} + + fn on_broadcasted(&self, _: HashMap>) {} + + fn transaction(&self, _h: &H) -> Option { None } +} + /// Name of a protocol, transmitted on the wire. Should be unique for each chain. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct ProtocolId(smallvec::SmallVec<[u8; 6]>); @@ -307,8 +369,6 @@ impl From for ParseErr { /// Network service configuration. #[derive(Clone, Debug)] pub struct NetworkConfiguration { - /// Directory path to store general network configuration. None means nothing will be saved. - pub config_path: Option, /// Directory path to store network-specific configuration. None means nothing will be saved. pub net_config_path: Option, /// Multiaddresses to listen for incoming connections. @@ -319,6 +379,9 @@ pub struct NetworkConfiguration { pub boot_nodes: Vec, /// The node key configuration, which determines the node's network identity keypair. pub node_key: NodeKeyConfig, + /// List of notifications protocols that the node supports. Must also include a + /// `ConsensusEngineId` for backwards-compatibility. + pub notifications_protocols: Vec<(ConsensusEngineId, Cow<'static, [u8]>)>, /// Maximum allowed number of incoming connections. pub in_peers: u32, /// Number of outgoing connections we're trying to maintain. @@ -335,23 +398,34 @@ pub struct NetworkConfiguration { pub transport: TransportConfig, /// Maximum number of peers to ask the same blocks in parallel. pub max_parallel_downloads: u32, + /// Should we insert non-global addresses into the DHT? + pub allow_non_globals_in_dht: bool, + /// If true, uses the `//block-requests/` experimental protocol rather than + /// the legacy substream. This option is meant to be hard-wired to `true` in the future. + pub use_new_block_requests_protocol: bool, } -impl Default for NetworkConfiguration { - fn default() -> Self { +impl NetworkConfiguration { + /// Create new default configuration + pub fn new, SV: Into>( + node_name: SN, + client_version: SV, + node_key: NodeKeyConfig, + net_config_path: Option, + ) -> Self { NetworkConfiguration { - config_path: None, - net_config_path: None, + net_config_path, listen_addresses: Vec::new(), public_addresses: Vec::new(), boot_nodes: Vec::new(), - node_key: NodeKeyConfig::Ed25519(Secret::New), + node_key, + notifications_protocols: Vec::new(), in_peers: 25, out_peers: 75, reserved_nodes: Vec::new(), non_reserved_mode: NonReservedPeerMode::Accept, - client_version: "unknown".into(), - node_name: "unknown".into(), + client_version: client_version.into(), + node_name: node_name.into(), transport: TransportConfig::Normal { enable_mdns: false, allow_private_ipv4: true, @@ -359,35 +433,48 @@ impl Default for NetworkConfiguration { use_yamux_flow_control: false, }, max_parallel_downloads: 5, + allow_non_globals_in_dht: false, + use_new_block_requests_protocol: true, } } } impl NetworkConfiguration { - /// Create a new instance of default settings. - pub fn new() -> Self { - Self::default() - } - /// Create new default configuration for localhost-only connection with random port (useful for testing) pub fn new_local() -> NetworkConfiguration { - let mut config = NetworkConfiguration::new(); + let mut config = NetworkConfiguration::new( + "test-node", + "test-client", + Default::default(), + None, + ); + config.listen_addresses = vec![ iter::once(multiaddr::Protocol::Ip4(Ipv4Addr::new(127, 0, 0, 1))) .chain(iter::once(multiaddr::Protocol::Tcp(0))) .collect() ]; + + config.allow_non_globals_in_dht = true; config } /// Create new default configuration for localhost-only connection with random port (useful for testing) pub fn new_memory() -> NetworkConfiguration { - let mut config = NetworkConfiguration::new(); + let mut config = NetworkConfiguration::new( + "test-node", + "test-client", + Default::default(), + None, + ); + config.listen_addresses = vec![ iter::once(multiaddr::Protocol::Ip4(Ipv4Addr::new(127, 0, 0, 1))) .chain(iter::once(multiaddr::Protocol::Tcp(0))) .collect() ]; + + config.allow_non_globals_in_dht = true; config } } @@ -452,6 +539,12 @@ pub enum NodeKeyConfig { Ed25519(Secret) } +impl Default for NodeKeyConfig { + fn default() -> NodeKeyConfig { + NodeKeyConfig::Ed25519(Secret::New) + } +} + /// The options for obtaining a Ed25519 secret key. pub type Ed25519Secret = Secret; diff --git a/client/network/src/debug_info.rs b/client/network/src/debug_info.rs index 17fb622f7cd3cab44f3b17ab508d7f1a3b258882..e2803cde35a772e5e094919ef31b364af640c487 100644 --- a/client/network/src/debug_info.rs +++ b/client/network/src/debug_info.rs @@ -17,14 +17,15 @@ use fnv::FnvHashMap; use futures::prelude::*; use libp2p::Multiaddr; -use libp2p::core::nodes::listeners::ListenerId; +use libp2p::core::connection::{ConnectionId, ListenerId}; use libp2p::core::{ConnectedPoint, either::EitherOutput, PeerId, PublicKey}; use libp2p::swarm::{IntoProtocolsHandler, IntoProtocolsHandlerSelect, ProtocolsHandler}; use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}; use libp2p::identify::{Identify, IdentifyEvent, IdentifyInfo}; use libp2p::ping::{Ping, PingConfig, PingEvent, PingSuccess}; use log::{debug, trace, error}; -use std::error; +use smallvec::SmallVec; +use std::{error, io}; use std::collections::hash_map::Entry; use std::pin::Pin; use std::task::{Context, Poll}; @@ -56,14 +57,27 @@ struct NodeInfo { /// When we will remove the entry about this node from the list, or `None` if we're connected /// to the node. info_expire: Option, - /// How we're connected to the node. - endpoint: ConnectedPoint, + /// Non-empty list of connected endpoints, one per connection. + endpoints: SmallVec<[ConnectedPoint; crate::MAX_CONNECTIONS_PER_PEER]>, /// Version reported by the remote, or `None` if unknown. client_version: Option, /// Latest ping time with this node. latest_ping: Option, } +impl NodeInfo { + fn new(endpoint: ConnectedPoint) -> Self { + let mut endpoints = SmallVec::new(); + endpoints.push(endpoint); + NodeInfo { + info_expire: None, + endpoints, + client_version: None, + latest_ping: None, + } + } +} + impl DebugInfoBehaviour { /// Builds a new `DebugInfoBehaviour`. pub fn new( @@ -121,9 +135,9 @@ impl DebugInfoBehaviour { pub struct Node<'a>(&'a NodeInfo); impl<'a> Node<'a> { - /// Returns the endpoint we are connected to or were last connected to. + /// Returns the endpoint of an established connection to the peer. pub fn endpoint(&self) -> &'a ConnectedPoint { - &self.0.endpoint + &self.0.endpoints[0] // `endpoints` are non-empty by definition } /// Returns the latest version information we know of. @@ -168,18 +182,17 @@ impl NetworkBehaviour for DebugInfoBehaviour { list } - fn inject_connected(&mut self, peer_id: PeerId, endpoint: ConnectedPoint) { - self.ping.inject_connected(peer_id.clone(), endpoint.clone()); - self.identify.inject_connected(peer_id.clone(), endpoint.clone()); + fn inject_connected(&mut self, peer_id: &PeerId) { + self.ping.inject_connected(peer_id); + self.identify.inject_connected(peer_id); + } - match self.nodes_info.entry(peer_id) { + fn inject_connection_established(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + self.ping.inject_connection_established(peer_id, conn, endpoint); + self.identify.inject_connection_established(peer_id, conn, endpoint); + match self.nodes_info.entry(peer_id.clone()) { Entry::Vacant(e) => { - e.insert(NodeInfo { - info_expire: None, - endpoint, - client_version: None, - latest_ping: None, - }); + e.insert(NodeInfo::new(endpoint.clone())); } Entry::Occupied(e) => { let e = e.into_mut(); @@ -188,14 +201,26 @@ impl NetworkBehaviour for DebugInfoBehaviour { e.latest_ping = None; } e.info_expire = None; - e.endpoint = endpoint; + e.endpoints.push(endpoint.clone()); } } } - fn inject_disconnected(&mut self, peer_id: &PeerId, endpoint: ConnectedPoint) { - self.ping.inject_disconnected(peer_id, endpoint.clone()); - self.identify.inject_disconnected(peer_id, endpoint); + fn inject_connection_closed(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + self.ping.inject_connection_closed(peer_id, conn, endpoint); + self.identify.inject_connection_closed(peer_id, conn, endpoint); + + if let Some(entry) = self.nodes_info.get_mut(peer_id) { + entry.endpoints.retain(|ep| ep != endpoint) + } else { + error!(target: "sub-libp2p", + "Unknown connection to {:?} closed: {:?}", peer_id, endpoint); + } + } + + fn inject_disconnected(&mut self, peer_id: &PeerId) { + self.ping.inject_disconnected(peer_id); + self.identify.inject_disconnected(peer_id); if let Some(entry) = self.nodes_info.get_mut(peer_id) { entry.info_expire = Some(Instant::now() + CACHE_EXPIRE); @@ -205,26 +230,15 @@ impl NetworkBehaviour for DebugInfoBehaviour { } } - fn inject_node_event( + fn inject_event( &mut self, peer_id: PeerId, + connection: ConnectionId, event: <::Handler as ProtocolsHandler>::OutEvent ) { match event { - EitherOutput::First(event) => self.ping.inject_node_event(peer_id, event), - EitherOutput::Second(event) => self.identify.inject_node_event(peer_id, event), - } - } - - fn inject_replaced(&mut self, peer_id: PeerId, closed_endpoint: ConnectedPoint, new_endpoint: ConnectedPoint) { - self.ping.inject_replaced(peer_id.clone(), closed_endpoint.clone(), new_endpoint.clone()); - self.identify.inject_replaced(peer_id.clone(), closed_endpoint, new_endpoint.clone()); - - if let Some(entry) = self.nodes_info.get_mut(&peer_id) { - entry.endpoint = new_endpoint; - } else { - error!(target: "sub-libp2p", - "Disconnected from node we were not connected to {:?}", peer_id); + EitherOutput::First(event) => self.ping.inject_event(peer_id, connection, event), + EitherOutput::Second(event) => self.identify.inject_event(peer_id, connection, event), } } @@ -258,9 +272,9 @@ impl NetworkBehaviour for DebugInfoBehaviour { self.identify.inject_listener_error(id, err); } - fn inject_listener_closed(&mut self, id: ListenerId) { - self.ping.inject_listener_closed(id); - self.identify.inject_listener_closed(id); + fn inject_listener_closed(&mut self, id: ListenerId, reason: Result<(), &io::Error>) { + self.ping.inject_listener_closed(id, reason); + self.identify.inject_listener_closed(id, reason); } fn poll( @@ -283,11 +297,12 @@ impl NetworkBehaviour for DebugInfoBehaviour { }, Poll::Ready(NetworkBehaviourAction::DialAddress { address }) => return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), - Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }) => - return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }), - Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id, event }) => - return Poll::Ready(NetworkBehaviourAction::SendEvent { + Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }) => + return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }), + Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, handler, event }) => + return Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, + handler, event: EitherOutput::First(event) }), Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address }) => @@ -312,11 +327,12 @@ impl NetworkBehaviour for DebugInfoBehaviour { }, Poll::Ready(NetworkBehaviourAction::DialAddress { address }) => return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), - Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }) => - return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }), - Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id, event }) => - return Poll::Ready(NetworkBehaviourAction::SendEvent { + Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }) => + return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }), + Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, handler, event }) => + return Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, + handler, event: EitherOutput::Second(event) }), Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address }) => diff --git a/client/network/src/discovery.rs b/client/network/src/discovery.rs index ed5016642be8b3e47e27588d2a86f3a01ea98614..56c08cc56cf1b0cd5b8a58ef2f447f7ae56f9036 100644 --- a/client/network/src/discovery.rs +++ b/client/network/src/discovery.rs @@ -45,85 +45,150 @@ //! of a node's address, you must call `add_self_reported_address`. //! +use crate::config::ProtocolId; use futures::prelude::*; use futures_timer::Delay; -use libp2p::core::{nodes::listeners::ListenerId, ConnectedPoint, Multiaddr, PeerId, PublicKey}; -use libp2p::swarm::{ProtocolsHandler, NetworkBehaviour, NetworkBehaviourAction, PollParameters}; -use libp2p::kad::{Kademlia, KademliaEvent, Quorum, Record}; +use ip_network::IpNetwork; +use libp2p::core::{connection::{ConnectionId, ListenerId}, ConnectedPoint, Multiaddr, PeerId, PublicKey}; +use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters, ProtocolsHandler}; +use libp2p::swarm::protocols_handler::multi::MultiHandler; +use libp2p::kad::{Kademlia, KademliaConfig, KademliaEvent, Quorum, Record}; use libp2p::kad::GetClosestPeersError; -use libp2p::kad::record::{self, store::MemoryStore}; +use libp2p::kad::handler::KademliaHandler; +use libp2p::kad::QueryId; +use libp2p::kad::record::{self, store::{MemoryStore, RecordStore}}; #[cfg(not(target_os = "unknown"))] -use libp2p::{swarm::toggle::Toggle}; +use libp2p::swarm::toggle::Toggle; #[cfg(not(target_os = "unknown"))] use libp2p::mdns::{Mdns, MdnsEvent}; use libp2p::multiaddr::Protocol; use log::{debug, info, trace, warn, error}; -use std::{cmp, collections::VecDeque, time::Duration}; +use std::{cmp, collections::{HashMap, HashSet, VecDeque}, io, time::Duration}; use std::task::{Context, Poll}; use sp_core::hexdisplay::HexDisplay; -/// Implementation of `NetworkBehaviour` that discovers the nodes on the network. -pub struct DiscoveryBehaviour { - /// User-defined list of nodes and their addresses. Typically includes bootstrap nodes and - /// reserved nodes. - user_defined: Vec<(PeerId, Multiaddr)>, - /// Kademlia requests and answers. - kademlia: Kademlia, - /// Discovers nodes on the local network. - #[cfg(not(target_os = "unknown"))] - mdns: Toggle, - /// Stream that fires when we need to perform the next random Kademlia query. - next_kad_random_query: Delay, - /// After `next_kad_random_query` triggers, the next one triggers after this duration. - duration_to_next_kad: Duration, - /// Discovered nodes to return. - discoveries: VecDeque, - /// Identity of our local node. +/// `DiscoveryBehaviour` configuration. +pub struct DiscoveryConfig { local_peer_id: PeerId, - /// Number of nodes we're currently connected to. - num_connections: u64, - /// If false, `addresses_of_peer` won't return any private IPv4 address, except for the ones - /// stored in `user_defined`. + user_defined: Vec<(PeerId, Multiaddr)>, allow_private_ipv4: bool, - /// Number of active connections over which we interrupt the discovery process. + allow_non_globals_in_dht: bool, discovery_only_if_under_num: u64, + enable_mdns: bool, + kademlias: HashMap> } -impl DiscoveryBehaviour { - /// Builds a new `DiscoveryBehaviour`. - /// - /// `user_defined` is a list of known address for nodes that never expire. - pub async fn new( - local_public_key: PublicKey, - user_defined: Vec<(PeerId, Multiaddr)>, - enable_mdns: bool, - allow_private_ipv4: bool, - discovery_only_if_under_num: u64, - ) -> Self { - if enable_mdns { - #[cfg(target_os = "unknown")] - warn!(target: "sub-libp2p", "mDNS is not available on this platform"); +impl DiscoveryConfig { + /// Create a default configuration with the given public key. + pub fn new(local_public_key: PublicKey) -> Self { + let mut this = DiscoveryConfig { + local_peer_id: local_public_key.into_peer_id(), + user_defined: Vec::new(), + allow_private_ipv4: true, + allow_non_globals_in_dht: false, + discovery_only_if_under_num: std::u64::MAX, + enable_mdns: false, + kademlias: HashMap::new() + }; + + // Temporary hack to retain backwards compatibility. + // We should eventually remove the special handling of DEFAULT_PROTO_NAME. + let proto_id = ProtocolId::from(libp2p::kad::protocol::DEFAULT_PROTO_NAME); + let proto_name = Vec::from(proto_id.as_bytes()); + this.add_kademlia(proto_id, proto_name); + + this + } + + /// Set the number of active connections at which we pause discovery. + pub fn discovery_limit(&mut self, limit: u64) -> &mut Self { + self.discovery_only_if_under_num = limit; + self + } + + /// Set custom nodes which never expire, e.g. bootstrap or reserved nodes. + pub fn with_user_defined(&mut self, user_defined: I) -> &mut Self + where + I: IntoIterator + { + for (peer_id, addr) in user_defined { + for kad in self.kademlias.values_mut() { + kad.add_address(&peer_id, addr.clone()) + } + self.user_defined.push((peer_id, addr)) } + self + } - let local_id = local_public_key.clone().into_peer_id(); - let store = MemoryStore::new(local_id.clone()); - let mut kademlia = Kademlia::new(local_id.clone(), store); - for (peer_id, addr) in &user_defined { - kademlia.add_address(peer_id, addr.clone()); + /// Should private IPv4 addresses be reported? + pub fn allow_private_ipv4(&mut self, value: bool) -> &mut Self { + self.allow_private_ipv4 = value; + self + } + + /// Should non-global addresses be inserted to the DHT? + pub fn allow_non_globals_in_dht(&mut self, value: bool) -> &mut Self { + self.allow_non_globals_in_dht = value; + self + } + + /// Should MDNS discovery be supported? + pub fn with_mdns(&mut self, value: bool) -> &mut Self { + if value && cfg!(target_os = "unknown") { + log::warn!(target: "sub-libp2p", "mDNS is not available on this platform") } + self.enable_mdns = value; + self + } + /// Add discovery via Kademlia for the given protocol. + pub fn add_protocol(&mut self, p: ProtocolId) -> &mut Self { + // NB: If this protocol name derivation is changed, check if + // `DiscoveryBehaviour::new_handler` is still correct. + let proto_name = { + let mut v = vec![b'/']; + v.extend_from_slice(p.as_bytes()); + v.extend_from_slice(b"/kad"); + v + }; + + self.add_kademlia(p, proto_name); + self + } + + fn add_kademlia(&mut self, id: ProtocolId, proto_name: Vec) { + if self.kademlias.contains_key(&id) { + warn!(target: "sub-libp2p", "Discovery already registered for protocol {:?}", id); + return + } + + let mut config = KademliaConfig::default(); + config.set_protocol_name(proto_name); + + let store = MemoryStore::new(self.local_peer_id.clone()); + let mut kad = Kademlia::with_config(self.local_peer_id.clone(), store, config); + + for (peer_id, addr) in &self.user_defined { + kad.add_address(peer_id, addr.clone()); + } + + self.kademlias.insert(id, kad); + } + + /// Create a `DiscoveryBehaviour` from this config. + pub fn finish(self) -> DiscoveryBehaviour { DiscoveryBehaviour { - user_defined, - kademlia, + user_defined: self.user_defined, + kademlias: self.kademlias, next_kad_random_query: Delay::new(Duration::new(0, 0)), duration_to_next_kad: Duration::from_secs(1), discoveries: VecDeque::new(), - local_peer_id: local_public_key.into_peer_id(), + local_peer_id: self.local_peer_id, num_connections: 0, - allow_private_ipv4, - discovery_only_if_under_num, + allow_private_ipv4: self.allow_private_ipv4, + discovery_only_if_under_num: self.discovery_only_if_under_num, #[cfg(not(target_os = "unknown"))] - mdns: if enable_mdns { + mdns: if self.enable_mdns { match Mdns::new() { Ok(mdns) => Some(mdns).into(), Err(err) => { @@ -134,12 +199,48 @@ impl DiscoveryBehaviour { } else { None.into() }, + allow_non_globals_in_dht: self.allow_non_globals_in_dht } } +} + +/// Implementation of `NetworkBehaviour` that discovers the nodes on the network. +pub struct DiscoveryBehaviour { + /// User-defined list of nodes and their addresses. Typically includes bootstrap nodes and + /// reserved nodes. + user_defined: Vec<(PeerId, Multiaddr)>, + /// Kademlia requests and answers. + kademlias: HashMap>, + /// Discovers nodes on the local network. + #[cfg(not(target_os = "unknown"))] + mdns: Toggle, + /// Stream that fires when we need to perform the next random Kademlia query. + next_kad_random_query: Delay, + /// After `next_kad_random_query` triggers, the next one triggers after this duration. + duration_to_next_kad: Duration, + /// Discovered nodes to return. + discoveries: VecDeque, + /// Identity of our local node. + local_peer_id: PeerId, + /// Number of nodes we're currently connected to. + num_connections: u64, + /// If false, `addresses_of_peer` won't return any private IPv4 address, except for the ones + /// stored in `user_defined`. + allow_private_ipv4: bool, + /// Number of active connections over which we interrupt the discovery process. + discovery_only_if_under_num: u64, + /// Should non-global addresses be added to the DHT? + allow_non_globals_in_dht: bool +} +impl DiscoveryBehaviour { /// Returns the list of nodes that we know exist in the network. pub fn known_peers(&mut self) -> impl Iterator { - self.kademlia.kbuckets_entries() + let mut set = HashSet::new(); + for p in self.kademlias.values_mut().map(|k| k.kbuckets_entries()).flatten() { + set.insert(p); + } + set.into_iter() } /// Adds a hard-coded address for the given peer, that never expires. @@ -149,6 +250,9 @@ impl DiscoveryBehaviour { /// If we didn't know this address before, also generates a `Discovered` event. pub fn add_known_address(&mut self, peer_id: PeerId, addr: Multiaddr) { if self.user_defined.iter().all(|(p, a)| *p != peer_id && *a != addr) { + for k in self.kademlias.values_mut() { + k.add_address(&peer_id, addr.clone()) + } self.discoveries.push_back(peer_id.clone()); self.user_defined.push((peer_id, addr)); } @@ -159,14 +263,22 @@ impl DiscoveryBehaviour { /// **Note**: It is important that you call this method, otherwise the discovery mechanism will /// not properly work. pub fn add_self_reported_address(&mut self, peer_id: &PeerId, addr: Multiaddr) { - self.kademlia.add_address(peer_id, addr); + if self.allow_non_globals_in_dht || self.can_add_to_dht(&addr) { + for k in self.kademlias.values_mut() { + k.add_address(peer_id, addr.clone()) + } + } else { + log::trace!(target: "sub-libp2p", "Ignoring self-reported address {} from {}", addr, peer_id); + } } /// Start fetching a record from the DHT. /// /// A corresponding `ValueFound` or `ValueNotFound` event will later be generated. pub fn get_value(&mut self, key: &record::Key) { - self.kademlia.get_record(key, Quorum::One) + for k in self.kademlias.values_mut() { + k.get_record(key, Quorum::One) + } } /// Start putting a record into the DHT. Other nodes can later fetch that value with @@ -174,12 +286,50 @@ impl DiscoveryBehaviour { /// /// A corresponding `ValuePut` or `ValuePutFailed` event will later be generated. pub fn put_value(&mut self, key: record::Key, value: Vec) { - self.kademlia.put_record(Record::new(key, value), Quorum::All); + for k in self.kademlias.values_mut() { + k.put_record(Record::new(key.clone(), value.clone()), Quorum::All) + } } /// Returns the number of nodes that are in the Kademlia k-buckets. - pub fn num_kbuckets_entries(&mut self) -> usize { - self.kademlia.kbuckets_entries().count() + pub fn num_kbuckets_entries(&mut self) -> impl ExactSizeIterator { + self.kademlias.iter_mut().map(|(id, kad)| (id, kad.kbuckets_entries().count())) + } + + /// Returns the number of records in the Kademlia record stores. + pub fn num_kademlia_records(&mut self) -> impl ExactSizeIterator { + // Note that this code is ok only because we use a `MemoryStore`. + self.kademlias.iter_mut().map(|(id, kad)| { + let num = kad.store_mut().records().count(); + (id, num) + }) + } + + /// Returns the total size in bytes of all the records in the Kademlia record stores. + pub fn kademlia_records_total_size(&mut self) -> impl ExactSizeIterator { + // Note that this code is ok only because we use a `MemoryStore`. If the records were + // for example stored on disk, this would load every single one of them every single time. + self.kademlias.iter_mut().map(|(id, kad)| { + let size = kad.store_mut().records().fold(0, |tot, rec| tot + rec.value.len()); + (id, size) + }) + } + + /// Can the given `Multiaddr` be put into the DHT? + /// + /// This test is successful only for global IP addresses and DNS names. + // + // NB: Currently all DNS names are allowed and no check for TLD suffixes is done + // because the set of valid domains is highly dynamic and would require frequent + // updates, for example by utilising publicsuffix.org or IANA. + pub fn can_add_to_dht(&self, addr: &Multiaddr) -> bool { + let ip = match addr.iter().next() { + Some(Protocol::Ip4(ip)) => IpNetwork::from(ip), + Some(Protocol::Ip6(ip)) => IpNetwork::from(ip), + Some(Protocol::Dns4(_)) | Some(Protocol::Dns6(_)) => return true, + _ => return false + }; + ip.is_global() } } @@ -209,16 +359,24 @@ pub enum DiscoveryOut { /// Inserting a value into the DHT failed. ValuePutFailed(record::Key), - /// Started a random Kademlia query. - RandomKademliaStarted, + /// Started a random Kademlia query for each DHT identified by the given `ProtocolId`s. + RandomKademliaStarted(Vec), } impl NetworkBehaviour for DiscoveryBehaviour { - type ProtocolsHandler = as NetworkBehaviour>::ProtocolsHandler; + type ProtocolsHandler = MultiHandler>; type OutEvent = DiscoveryOut; fn new_handler(&mut self) -> Self::ProtocolsHandler { - NetworkBehaviour::new_handler(&mut self.kademlia) + let iter = self.kademlias.iter_mut() + .map(|(p, k)| (p.clone(), NetworkBehaviour::new_handler(k))); + + MultiHandler::try_from_iter(iter) + .expect("There can be at most one handler per `ProtocolId` and \ + protocol names contain the `ProtocolId` so no two protocol \ + names in `self.kademlias` can be equal which is the only error \ + `try_from_iter` can return, therefore this call is guaranteed \ + to succeed; qed") } fn addresses_of_peer(&mut self, peer_id: &PeerId) -> Vec { @@ -227,7 +385,11 @@ impl NetworkBehaviour for DiscoveryBehaviour { .collect::>(); { - let mut list_to_filter = self.kademlia.addresses_of_peer(peer_id); + let mut list_to_filter = Vec::new(); + for k in self.kademlias.values_mut() { + list_to_filter.extend(k.addresses_of_peer(peer_id)) + } + #[cfg(not(target_os = "unknown"))] list_to_filter.extend(self.mdns.addresses_of_peer(peer_id)); @@ -246,31 +408,51 @@ impl NetworkBehaviour for DiscoveryBehaviour { list.extend(list_to_filter); } - trace!(target: "sub-libp2p", "Addresses of {:?} are {:?}", peer_id, list); - if list.is_empty() { - if self.kademlia.kbuckets_entries().any(|p| p == peer_id) { - debug!(target: "sub-libp2p", "Requested dialing to {:?} (peer in k-buckets), \ - and no address was found", peer_id); + if !list.is_empty() { + trace!(target: "sub-libp2p", "Addresses of {:?}: {:?}", peer_id, list); + + } else { + let mut has_entry = false; + for k in self.kademlias.values_mut() { + if k.kbuckets_entries().any(|p| p == peer_id) { + has_entry = true; + break + } + } + if has_entry { + trace!(target: "sub-libp2p", "Addresses of {:?}: none (peer in k-buckets)", peer_id); } else { - debug!(target: "sub-libp2p", "Requested dialing to {:?} (peer not in k-buckets), \ - and no address was found", peer_id); + trace!(target: "sub-libp2p", "Addresses of {:?}: none (peer not in k-buckets)", peer_id); } } + list } - fn inject_connected(&mut self, peer_id: PeerId, endpoint: ConnectedPoint) { + fn inject_connection_established(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { self.num_connections += 1; - NetworkBehaviour::inject_connected(&mut self.kademlia, peer_id, endpoint) + for k in self.kademlias.values_mut() { + NetworkBehaviour::inject_connection_established(k, peer_id, conn, endpoint) + } + } + + fn inject_connected(&mut self, peer_id: &PeerId) { + for k in self.kademlias.values_mut() { + NetworkBehaviour::inject_connected(k, peer_id) + } } - fn inject_disconnected(&mut self, peer_id: &PeerId, endpoint: ConnectedPoint) { + fn inject_connection_closed(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { self.num_connections -= 1; - NetworkBehaviour::inject_disconnected(&mut self.kademlia, peer_id, endpoint) + for k in self.kademlias.values_mut() { + NetworkBehaviour::inject_connection_closed(k, peer_id, conn, endpoint) + } } - fn inject_replaced(&mut self, peer_id: PeerId, closed: ConnectedPoint, opened: ConnectedPoint) { - NetworkBehaviour::inject_replaced(&mut self.kademlia, peer_id, closed, opened) + fn inject_disconnected(&mut self, peer_id: &PeerId) { + for k in self.kademlias.values_mut() { + NetworkBehaviour::inject_disconnected(k, peer_id) + } } fn inject_addr_reach_failure( @@ -279,45 +461,65 @@ impl NetworkBehaviour for DiscoveryBehaviour { addr: &Multiaddr, error: &dyn std::error::Error ) { - NetworkBehaviour::inject_addr_reach_failure(&mut self.kademlia, peer_id, addr, error) + for k in self.kademlias.values_mut() { + NetworkBehaviour::inject_addr_reach_failure(k, peer_id, addr, error) + } } - fn inject_node_event( + fn inject_event( &mut self, peer_id: PeerId, - event: ::OutEvent, + connection: ConnectionId, + (pid, event): ::OutEvent, ) { - NetworkBehaviour::inject_node_event(&mut self.kademlia, peer_id, event) + if let Some(kad) = self.kademlias.get_mut(&pid) { + return kad.inject_event(peer_id, connection, event) + } + log::error!(target: "sub-libp2p", + "inject_node_event: no kademlia instance registered for protocol {:?}", + pid) } fn inject_new_external_addr(&mut self, addr: &Multiaddr) { let new_addr = addr.clone() .with(Protocol::P2p(self.local_peer_id.clone().into())); info!(target: "sub-libp2p", "🔍 Discovered new external address for our node: {}", new_addr); - NetworkBehaviour::inject_new_external_addr(&mut self.kademlia, addr) + for k in self.kademlias.values_mut() { + NetworkBehaviour::inject_new_external_addr(k, addr) + } } fn inject_expired_listen_addr(&mut self, addr: &Multiaddr) { info!(target: "sub-libp2p", "No longer listening on {}", addr); - NetworkBehaviour::inject_expired_listen_addr(&mut self.kademlia, addr) + for k in self.kademlias.values_mut() { + NetworkBehaviour::inject_expired_listen_addr(k, addr) + } } fn inject_dial_failure(&mut self, peer_id: &PeerId) { - NetworkBehaviour::inject_dial_failure(&mut self.kademlia, peer_id) + for k in self.kademlias.values_mut() { + NetworkBehaviour::inject_dial_failure(k, peer_id) + } } fn inject_new_listen_addr(&mut self, addr: &Multiaddr) { - NetworkBehaviour::inject_new_listen_addr(&mut self.kademlia, addr) + for k in self.kademlias.values_mut() { + NetworkBehaviour::inject_new_listen_addr(k, addr) + } } fn inject_listener_error(&mut self, id: ListenerId, err: &(dyn std::error::Error + 'static)) { error!(target: "sub-libp2p", "Error on libp2p listener {:?}: {}", id, err); - NetworkBehaviour::inject_listener_error(&mut self.kademlia, id, err); + for k in self.kademlias.values_mut() { + NetworkBehaviour::inject_listener_error(k, id, err) + } } - fn inject_listener_closed(&mut self, id: ListenerId) { + fn inject_listener_closed(&mut self, id: ListenerId, reason: Result<(), &io::Error>) { error!(target: "sub-libp2p", "Libp2p listener {:?} closed", id); - NetworkBehaviour::inject_listener_closed(&mut self.kademlia, id); + for k in self.kademlias.values_mut() { + NetworkBehaviour::inject_listener_closed(k, id, reason) + } } fn poll( @@ -340,12 +542,13 @@ impl NetworkBehaviour for DiscoveryBehaviour { while let Poll::Ready(_) = self.next_kad_random_query.poll_unpin(cx) { let actually_started = if self.num_connections < self.discovery_only_if_under_num { let random_peer_id = PeerId::random(); - debug!(target: "sub-libp2p", "Libp2p <= Starting random Kademlia request for \ - {:?}", random_peer_id); - - self.kademlia.get_closest_peers(random_peer_id); + debug!(target: "sub-libp2p", + "Libp2p <= Starting random Kademlia request for {:?}", + random_peer_id); + for k in self.kademlias.values_mut() { + k.get_closest_peers(random_peer_id.clone()) + } true - } else { debug!( target: "sub-libp2p", @@ -362,101 +565,107 @@ impl NetworkBehaviour for DiscoveryBehaviour { Duration::from_secs(60)); if actually_started { - let ev = DiscoveryOut::RandomKademliaStarted; + let ev = DiscoveryOut::RandomKademliaStarted(self.kademlias.keys().cloned().collect()); return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); } } - // Poll Kademlia. - while let Poll::Ready(ev) = self.kademlia.poll(cx, params) { - match ev { - NetworkBehaviourAction::GenerateEvent(ev) => match ev { - KademliaEvent::UnroutablePeer { peer, .. } => { - let ev = DiscoveryOut::UnroutablePeer(peer); - return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); - } - KademliaEvent::RoutingUpdated { peer, .. } => { - let ev = DiscoveryOut::Discovered(peer); - return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); - } - KademliaEvent::GetClosestPeersResult(res) => { - match res { - Err(GetClosestPeersError::Timeout { key, peers }) => { - debug!(target: "sub-libp2p", - "Libp2p => Query for {:?} timed out with {} results", - HexDisplay::from(&key), peers.len()); - }, - Ok(ok) => { - trace!(target: "sub-libp2p", - "Libp2p => Query for {:?} yielded {:?} results", - HexDisplay::from(&ok.key), ok.peers.len()); - if ok.peers.is_empty() && self.num_connections != 0 { - debug!(target: "sub-libp2p", "Libp2p => Random Kademlia query has yielded empty \ - results"); + // Poll Kademlias. + for (pid, kademlia) in &mut self.kademlias { + while let Poll::Ready(ev) = kademlia.poll(cx, params) { + match ev { + NetworkBehaviourAction::GenerateEvent(ev) => match ev { + KademliaEvent::UnroutablePeer { peer, .. } => { + let ev = DiscoveryOut::UnroutablePeer(peer); + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); + } + KademliaEvent::RoutingUpdated { peer, .. } => { + let ev = DiscoveryOut::Discovered(peer); + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); + } + KademliaEvent::GetClosestPeersResult(res) => { + match res { + Err(GetClosestPeersError::Timeout { key, peers }) => { + debug!(target: "sub-libp2p", + "Libp2p => Query for {:?} timed out with {} results", + HexDisplay::from(&key), peers.len()); + }, + Ok(ok) => { + trace!(target: "sub-libp2p", + "Libp2p => Query for {:?} yielded {:?} results", + HexDisplay::from(&ok.key), ok.peers.len()); + if ok.peers.is_empty() && self.num_connections != 0 { + debug!(target: "sub-libp2p", "Libp2p => Random Kademlia query has yielded empty \ + results"); + } } } } - } - KademliaEvent::GetRecordResult(res) => { - let ev = match res { - Ok(ok) => { - let results = ok.records - .into_iter() - .map(|r| (r.key, r.value)) - .collect(); - - DiscoveryOut::ValueFound(results) - } - Err(e @ libp2p::kad::GetRecordError::NotFound { .. }) => { - trace!(target: "sub-libp2p", - "Libp2p => Failed to get record: {:?}", e); - DiscoveryOut::ValueNotFound(e.into_key()) - } - Err(e) => { - warn!(target: "sub-libp2p", - "Libp2p => Failed to get record: {:?}", e); - DiscoveryOut::ValueNotFound(e.into_key()) - } - }; - return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); - } - KademliaEvent::PutRecordResult(res) => { - let ev = match res { - Ok(ok) => DiscoveryOut::ValuePut(ok.key), - Err(e) => { - warn!(target: "sub-libp2p", - "Libp2p => Failed to put record: {:?}", e); - DiscoveryOut::ValuePutFailed(e.into_key()) + KademliaEvent::GetRecordResult(res) => { + let ev = match res { + Ok(ok) => { + let results = ok.records + .into_iter() + .map(|r| (r.key, r.value)) + .collect(); + + DiscoveryOut::ValueFound(results) + } + Err(e @ libp2p::kad::GetRecordError::NotFound { .. }) => { + trace!(target: "sub-libp2p", + "Libp2p => Failed to get record: {:?}", e); + DiscoveryOut::ValueNotFound(e.into_key()) + } + Err(e) => { + warn!(target: "sub-libp2p", + "Libp2p => Failed to get record: {:?}", e); + DiscoveryOut::ValueNotFound(e.into_key()) + } + }; + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); + } + KademliaEvent::PutRecordResult(res) => { + let ev = match res { + Ok(ok) => DiscoveryOut::ValuePut(ok.key), + Err(e) => { + warn!(target: "sub-libp2p", + "Libp2p => Failed to put record: {:?}", e); + DiscoveryOut::ValuePutFailed(e.into_key()) + } + }; + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); + } + KademliaEvent::RepublishRecordResult(res) => { + match res { + Ok(ok) => debug!(target: "sub-libp2p", + "Libp2p => Record republished: {:?}", + ok.key), + Err(e) => warn!(target: "sub-libp2p", + "Libp2p => Republishing of record {:?} failed with: {:?}", + e.key(), e) } - }; - return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); - } - KademliaEvent::RepublishRecordResult(res) => { - match res { - Ok(ok) => debug!(target: "sub-libp2p", - "Libp2p => Record republished: {:?}", - ok.key), - Err(e) => warn!(target: "sub-libp2p", - "Libp2p => Republishing of record {:?} failed with: {:?}", - e.key(), e) + } + KademliaEvent::Discovered { .. } => { + // We are not interested in these events at the moment. + } + // We never start any other type of query. + e => { + warn!(target: "sub-libp2p", "Libp2p => Unhandled Kademlia event: {:?}", e) } } - KademliaEvent::Discovered { .. } => { - // We are not interested in these events at the moment. - } - // We never start any other type of query. - e => { - warn!(target: "sub-libp2p", "Libp2p => Unhandled Kademlia event: {:?}", e) - } - }, - NetworkBehaviourAction::DialAddress { address } => - return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), - NetworkBehaviourAction::DialPeer { peer_id } => - return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }), - NetworkBehaviourAction::SendEvent { peer_id, event } => - return Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id, event }), - NetworkBehaviourAction::ReportObservedAddr { address } => - return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address }), + NetworkBehaviourAction::DialAddress { address } => + return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), + NetworkBehaviourAction::DialPeer { peer_id, condition } => + return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }), + NetworkBehaviourAction::NotifyHandler { peer_id, handler, event } => + return Poll::Ready(NetworkBehaviourAction::NotifyHandler { + peer_id, + handler, + event: (pid.clone(), event) + }), + NetworkBehaviourAction::ReportObservedAddr { address } => + return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address }), + } } } @@ -482,9 +691,9 @@ impl NetworkBehaviour for DiscoveryBehaviour { }, NetworkBehaviourAction::DialAddress { address } => return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), - NetworkBehaviourAction::DialPeer { peer_id } => - return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }), - NetworkBehaviourAction::SendEvent { event, .. } => + NetworkBehaviourAction::DialPeer { peer_id, condition } => + return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }), + NetworkBehaviourAction::NotifyHandler { event, .. } => match event {}, // `event` is an enum with no variant NetworkBehaviourAction::ReportObservedAddr { address } => return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address }), @@ -505,7 +714,7 @@ mod tests { use libp2p::core::upgrade::{InboundUpgradeExt, OutboundUpgradeExt}; use libp2p::swarm::Swarm; use std::{collections::HashSet, task::Poll}; - use super::{DiscoveryBehaviour, DiscoveryOut}; + use super::{DiscoveryConfig, DiscoveryOut}; #[test] fn discovery_working() { @@ -534,13 +743,15 @@ mod tests { upgrade::apply(stream, upgrade, endpoint, upgrade::Version::V1) }); - let behaviour = futures::executor::block_on({ - let user_defined = user_defined.clone(); - let keypair_public = keypair.public(); - async move { - DiscoveryBehaviour::new(keypair_public, user_defined, false, true, 50).await - } - }); + let behaviour = { + let mut config = DiscoveryConfig::new(keypair.public()); + config.with_user_defined(user_defined.clone()) + .allow_private_ipv4(true) + .allow_non_globals_in_dht(true) + .discovery_limit(50); + config.finish() + }; + let mut swarm = Swarm::new(transport, behaviour, keypair.public().into_peer_id()); let listen_addr: Multiaddr = format!("/memory/{}", rand::random::()).parse().unwrap(); diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index b425a7763b648fe87e234e56a992f954e2c8f369..44bb1516bd48818368c44f73130c573fac37082b 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -210,7 +210,14 @@ //! notifications protocol. //! //! At the moment, for backwards-compatibility, notification protocols are tied to the legacy -//! Substrate substream. In the future, though, it will no longer be the case. +//! Substrate substream. Additionally, the handshake message is hardcoded to be a single 8-bits +//! integer representing the role of the node: +//! +//! - 1 for a full node. +//! - 2 for a light node. +//! - 4 for an authority. +//! +//! In the future, though, these restrictions will be removed. //! //! # Usage //! @@ -246,7 +253,7 @@ pub mod config; pub mod error; pub mod network_state; -pub use service::{NetworkService, NetworkStateInfo, NetworkWorker, ExHashT, ReportHandle}; +pub use service::{NetworkService, NetworkWorker}; pub use protocol::PeerInfo; pub use protocol::event::{Event, DhtEvent, ObservedRole}; pub use protocol::sync::SyncState; @@ -255,3 +262,47 @@ pub use libp2p::{Multiaddr, PeerId}; pub use libp2p::multiaddr; pub use sc_peerset::ReputationChange; + +/// The maximum allowed number of established connections per peer. +/// +/// Typically, and by design of the network behaviours in this crate, +/// there is a single established connection per peer. However, to +/// avoid unnecessary and nondeterministic connection closure in +/// case of (possibly repeated) simultaneous dialing attempts between +/// two peers, the per-peer connection limit is not set to 1 but 2. +const MAX_CONNECTIONS_PER_PEER: usize = 2; + +/// Minimum Requirements for a Hash within Networking +pub trait ExHashT: std::hash::Hash + Eq + std::fmt::Debug + Clone + Send + Sync + 'static {} + +impl ExHashT for T where T: std::hash::Hash + Eq + std::fmt::Debug + Clone + Send + Sync + 'static +{} + +/// A cloneable handle for reporting cost/benefits of peers. +#[derive(Clone)] +pub struct ReportHandle { + inner: sc_peerset::PeersetHandle, // wraps it so we don't have to worry about breaking API. +} + +impl From for ReportHandle { + fn from(peerset_handle: sc_peerset::PeersetHandle) -> Self { + ReportHandle { inner: peerset_handle } + } +} + +impl ReportHandle { + /// Report a given peer as either beneficial (+) or costly (-) according to the + /// given scalar. + pub fn report_peer(&self, who: PeerId, cost_benefit: ReputationChange) { + self.inner.report_peer(who, cost_benefit); + } +} + +/// Trait for providing information about the local network state +pub trait NetworkStateInfo { + /// Returns the local external addresses. + fn external_addresses(&self) -> Vec; + + /// Returns the local Peer ID. + fn local_peer_id(&self) -> PeerId; +} diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index bfe8226c8d61529d3817699731c3b4cd7fdbd1c3..8222767e1a1e6973d0a95caee06002934264f1f6 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -14,17 +14,23 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use crate::config::ProtocolId; -use crate::utils::interval; +use crate::{ + ExHashT, + chain::{Client, FinalityProofProvider}, + config::{BoxFinalityProofRequestBuilder, ProtocolId, TransactionPool}, + error, + utils::interval +}; + use bytes::{Bytes, BytesMut}; use futures::prelude::*; use generic_proto::{GenericProto, GenericProtoOut}; use libp2p::{Multiaddr, PeerId}; -use libp2p::core::{ConnectedPoint, nodes::listeners::ListenerId}; +use libp2p::core::{ConnectedPoint, connection::{ConnectionId, ListenerId}}; use libp2p::swarm::{ProtocolsHandler, IntoProtocolsHandler}; use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}; use sp_core::{ - storage::{StorageKey, ChildInfo}, + storage::{StorageKey, PrefixedStorageKey, ChildInfo, ChildType}, hexdisplay::HexDisplay }; use sp_consensus::{ @@ -42,17 +48,13 @@ use message::{BlockAnnounce, Message}; use message::generic::{Message as GenericMessage, ConsensusMessage, Roles}; use prometheus_endpoint::{Registry, Gauge, GaugeVec, HistogramVec, PrometheusError, Opts, register, U64}; use sync::{ChainSync, SyncState}; -use crate::service::{TransactionPool, ExHashT}; -use crate::config::BoxFinalityProofRequestBuilder; use std::borrow::Cow; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::sync::Arc; use std::fmt::Write; -use std::{cmp, num::NonZeroUsize, pin::Pin, task::Poll, time}; +use std::{cmp, io, num::NonZeroUsize, pin::Pin, task::Poll, time}; use log::{log, Level, trace, debug, warn, error}; -use crate::chain::{Client, FinalityProofProvider}; use sc_client_api::{ChangesProof, StorageProof}; -use crate::error; use util::LruHashSet; use wasm_timer::Instant; @@ -60,6 +62,9 @@ use wasm_timer::Instant; pub mod api { pub mod v1 { include!(concat!(env!("OUT_DIR"), "/api.v1.rs")); + pub mod finality { + include!(concat!(env!("OUT_DIR"), "/api.v1.finality.rs")); + } pub mod light { include!(concat!(env!("OUT_DIR"), "/api.v1.light.rs")); } @@ -70,13 +75,16 @@ mod generic_proto; mod util; pub mod block_requests; +pub mod finality_requests; pub mod message; pub mod event; pub mod light_client_handler; pub mod sync; pub use block_requests::BlockRequests; +pub use finality_requests::FinalityProofRequests; pub use light_client_handler::LightClientHandler; +pub use generic_proto::LegacyConnectionKillError; const REQUEST_TIMEOUT_SEC: u64 = 40; /// Interval at which we perform time based maintenance @@ -229,6 +237,9 @@ pub struct Protocol { metrics: Option, /// The `PeerId`'s of all boot nodes. boot_node_ids: Arc>, + /// If true, we send back requests as `CustomMessageOutcome` events. If false, we directly + /// dispatch requests using the legacy substream. + use_new_block_requests_protocol: bool, } #[derive(Default)] @@ -300,6 +311,31 @@ impl Default for ProtocolConfig { } } +/// Handshake sent when we open a block announces substream. +#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] +struct BlockAnnouncesHandshake { + /// Roles of the node. + roles: Roles, + /// Best block number. + best_number: NumberFor, + /// Best block hash. + best_hash: B::Hash, + /// Genesis block hash. + genesis_hash: B::Hash, +} + +impl BlockAnnouncesHandshake { + fn build(protocol_config: &ProtocolConfig, chain: &Arc>) -> Self { + let info = chain.info(); + BlockAnnouncesHandshake { + genesis_hash: info.genesis_hash, + roles: protocol_config.roles.into(), + best_number: info.best_number, + best_hash: info.best_hash, + } + } +} + /// Fallback mechanism to use to send a notification if no substream is open. #[derive(Debug, Clone, PartialEq, Eq)] enum Fallback { @@ -324,6 +360,7 @@ impl Protocol { block_announce_validator: Box + Send>, metrics_registry: Option<&Registry>, boot_node_ids: Arc>, + use_new_block_requests_protocol: bool, queue_size_report: Option, ) -> error::Result<(Protocol, sc_peerset::PeersetHandle)> { let info = chain.info(); @@ -366,7 +403,10 @@ impl Protocol { proto.extend(b"/block-announces/1"); proto }); - behaviour.register_notif_protocol(block_announces_protocol.clone(), Vec::new()); + behaviour.register_notif_protocol( + block_announces_protocol.clone(), + BlockAnnouncesHandshake::build(&config, &chain).encode() + ); legacy_equiv_by_name.insert(block_announces_protocol.clone(), Fallback::BlockAnnounce); let protocol = Protocol { @@ -397,6 +437,7 @@ impl Protocol { None }, boot_node_ids, + use_new_block_requests_protocol, }; Ok((protocol, peerset_handle)) @@ -481,6 +522,8 @@ impl Protocol { self.sync.num_sync_requests() } + /// Accepts a response from the legacy substream and determines what the corresponding + /// request was. fn handle_response( &mut self, who: PeerId, @@ -770,7 +813,9 @@ impl Protocol { self.peerset_handle.report_peer(who, reputation) } - fn on_block_response( + /// Must be called in response to a [`CustomMessageOutcome::BlockRequest`] being emitted. + /// Must contain the same `PeerId` and request that have been emitted. + pub fn on_block_response( &mut self, peer: PeerId, request: message::BlockRequest, @@ -821,8 +866,15 @@ impl Protocol { Ok(sync::OnBlockData::Import(origin, blocks)) => CustomMessageOutcome::BlockImport(origin, blocks), Ok(sync::OnBlockData::Request(peer, req)) => { - self.send_request(&peer, GenericMessage::BlockRequest(req)); - CustomMessageOutcome::None + if self.use_new_block_requests_protocol { + CustomMessageOutcome::BlockRequest { + target: peer, + request: req, + } + } else { + self.send_request(&peer, GenericMessage::BlockRequest(req)); + CustomMessageOutcome::None + } } Err(sync::BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id); @@ -989,7 +1041,16 @@ impl Protocol { if info.roles.is_full() { match self.sync.new_peer(who.clone(), info.best_hash, info.best_number) { Ok(None) => (), - Ok(Some(req)) => self.send_request(&who, GenericMessage::BlockRequest(req)), + Ok(Some(req)) => { + if self.use_new_block_requests_protocol { + self.pending_messages.push_back(CustomMessageOutcome::BlockRequest { + target: who.clone(), + request: req, + }); + } else { + self.send_request(&who, GenericMessage::BlockRequest(req)) + } + }, Err(sync::BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id); self.peerset_handle.report_peer(id, repu) @@ -1040,12 +1101,13 @@ impl Protocol { &'a mut self, engine_id: ConsensusEngineId, protocol_name: impl Into>, + handshake_message: Vec, ) -> impl ExactSizeIterator + 'a { let protocol_name = protocol_name.into(); if self.protocol_name_by_engine.insert(engine_id, protocol_name.clone()).is_some() { error!(target: "sub-libp2p", "Notifications protocol already registered: {:?}", protocol_name); } else { - self.behaviour.register_notif_protocol(protocol_name.clone(), Vec::new()); + self.behaviour.register_notif_protocol(protocol_name.clone(), handshake_message); self.legacy_equiv_by_name.insert(protocol_name, Fallback::Consensus(engine_id)); } @@ -1290,29 +1352,30 @@ impl Protocol { ], }, ); + + if is_their_best { + self.pending_messages.push_back(CustomMessageOutcome::PeerNewBest(who, number)); + } + match blocks_to_import { Ok(sync::OnBlockData::Import(origin, blocks)) => { - if is_their_best { - self.pending_messages.push_back(CustomMessageOutcome::PeerNewBest(who, number)); - } CustomMessageOutcome::BlockImport(origin, blocks) }, Ok(sync::OnBlockData::Request(peer, req)) => { - self.send_request(&peer, GenericMessage::BlockRequest(req)); - if is_their_best { - CustomMessageOutcome::PeerNewBest(who, number) + if self.use_new_block_requests_protocol { + CustomMessageOutcome::BlockRequest { + target: peer, + request: req, + } } else { + self.send_request(&peer, GenericMessage::BlockRequest(req)); CustomMessageOutcome::None } } Err(sync::BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id); self.peerset_handle.report_peer(id, repu); - if is_their_best { - CustomMessageOutcome::PeerNewBest(who, number) - } else { - CustomMessageOutcome::None - } + CustomMessageOutcome::None } } } @@ -1321,6 +1384,10 @@ impl Protocol { pub fn on_block_imported(&mut self, header: &B::Header, is_best: bool) { if is_best { self.sync.update_chain_info(header); + self.behaviour.set_notif_protocol_handshake( + &self.block_announces_protocol, + BlockAnnouncesHandshake::build(&self.config, &self.context_data.chain).encode() + ); } } @@ -1402,14 +1469,21 @@ impl Protocol { for result in results { match result { Ok((id, req)) => { - let msg = GenericMessage::BlockRequest(req); - send_request( - &mut self.behaviour, - &mut self.context_data.stats, - &mut self.context_data.peers, - &id, - msg - ) + if self.use_new_block_requests_protocol { + self.pending_messages.push_back(CustomMessageOutcome::BlockRequest { + target: id, + request: req, + }); + } else { + let msg = GenericMessage::BlockRequest(req); + send_request( + &mut self.behaviour, + &mut self.context_data.stats, + &mut self.context_data.peers, + &id, + msg + ) + } } Err(sync::BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id); @@ -1519,37 +1593,28 @@ impl Protocol { trace!(target: "sync", "Remote read child request {} from {} ({} {} at {})", request.id, who, HexDisplay::from(&request.storage_key), keys_str(), request.block); - let proof = if let Some(child_info) = ChildInfo::resolve_child_info(request.child_type, &request.child_info[..]) { - match self.context_data.chain.read_child_proof( - &BlockId::Hash(request.block), - &request.storage_key, - child_info, - &mut request.keys.iter().map(AsRef::as_ref), - ) { - Ok(proof) => proof, - Err(error) => { - trace!(target: "sync", "Remote read child request {} from {} ({} {} at {}) failed with: {}", - request.id, - who, - HexDisplay::from(&request.storage_key), - keys_str(), - request.block, - error - ); - StorageProof::empty() - } + let prefixed_key = PrefixedStorageKey::new_ref(&request.storage_key); + let child_info = match ChildType::from_prefixed_key(prefixed_key) { + Some((ChildType::ParentKeyId, storage_key)) => Ok(ChildInfo::new_default(storage_key)), + None => Err("Invalid child storage key".into()), + }; + let proof = match child_info.and_then(|child_info| self.context_data.chain.read_child_proof( + &BlockId::Hash(request.block), + &child_info, + &mut request.keys.iter().map(AsRef::as_ref), + )) { + Ok(proof) => proof, + Err(error) => { + trace!(target: "sync", "Remote read child request {} from {} ({} {} at {}) failed with: {}", + request.id, + who, + HexDisplay::from(&request.storage_key), + keys_str(), + request.block, + error + ); + StorageProof::empty() } - } else { - trace!(target: "sync", "Remote read child request {} from {} ({} {} at {}) failed with: {}", - request.id, - who, - HexDisplay::from(&request.storage_key), - keys_str(), - request.block, - "invalid child info and type", - ); - - StorageProof::empty() }; self.send_message( &who, @@ -1607,14 +1672,16 @@ impl Protocol { request.first, request.last ); - let storage_key = request.storage_key.map(|sk| StorageKey(sk)); let key = StorageKey(request.key); + let prefixed_key = request.storage_key.as_ref() + .map(|storage_key| PrefixedStorageKey::new_ref(storage_key)); + let (first, last, min, max) = (request.first, request.last, request.min, request.max); let proof = match self.context_data.chain.key_changes_proof( - request.first, - request.last, - request.min, - request.max, - storage_key.as_ref(), + first, + last, + min, + max, + prefixed_key, &key, ) { Ok(proof) => proof, @@ -1622,8 +1689,8 @@ impl Protocol { trace!(target: "sync", "Remote changes proof request {} from {} for key {} ({}..{}) failed with: {}", request.id, who, - if let Some(sk) = storage_key { - format!("{} : {}", HexDisplay::from(&sk.0), HexDisplay::from(&key.0)) + if let Some(sk) = request.storage_key.as_ref() { + format!("{} : {}", HexDisplay::from(sk), HexDisplay::from(&key.0)) } else { HexDisplay::from(&key.0).to_string() }, @@ -1685,7 +1752,9 @@ impl Protocol { ); } - fn on_finality_proof_response( + /// Must be called after a [`CustomMessageOutcome::FinalityProofRequest`] has been emitted, + /// to notify of the response having arrived. + pub fn on_finality_proof_response( &mut self, who: PeerId, response: message::FinalityProofResponse, @@ -1764,6 +1833,7 @@ impl Protocol { /// Outcome of an incoming custom message. #[derive(Debug)] +#[must_use] pub enum CustomMessageOutcome { BlockImport(BlockOrigin, Vec>), JustificationImport(Origin, B::Hash, NumberFor, Justification), @@ -1774,6 +1844,18 @@ pub enum CustomMessageOutcome { NotificationStreamClosed { remote: PeerId, protocols: Vec }, /// Messages have been received on one or more notifications protocols. NotificationsReceived { remote: PeerId, messages: Vec<(ConsensusEngineId, Bytes)> }, + /// A new block request must be emitted. + /// Once you have the response, you must call `Protocol::on_block_response`. + /// It is the responsibility of the handler to ensure that a timeout exists. + /// If the request times out, or the peer responds in an invalid way, the peer has to be + /// disconnect. This will inform the state machine that the request it has emitted is stale. + BlockRequest { target: PeerId, request: message::BlockRequest }, + /// A new finality proof request must be emitted. + /// Once you have the response, you must call `Protocol::on_finality_proof_response`. + /// It is the responsibility of the handler to ensure that a timeout exists. + /// If the request times out, or the peer responds in an invalid way, the peer has to be + /// disconnect. This will inform the state machine that the request it has emitted is stale. + FinalityProofRequest { target: PeerId, block_hash: B::Hash, request: Vec }, /// Peer has a reported a new head of chain. PeerNewBest(PeerId, NumberFor), None, @@ -1830,20 +1912,29 @@ impl NetworkBehaviour for Protocol { self.behaviour.addresses_of_peer(peer_id) } - fn inject_connected(&mut self, peer_id: PeerId, endpoint: ConnectedPoint) { - self.behaviour.inject_connected(peer_id, endpoint) + fn inject_connection_established(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + self.behaviour.inject_connection_established(peer_id, conn, endpoint) } - fn inject_disconnected(&mut self, peer_id: &PeerId, endpoint: ConnectedPoint) { - self.behaviour.inject_disconnected(peer_id, endpoint) + fn inject_connection_closed(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + self.behaviour.inject_connection_closed(peer_id, conn, endpoint) } - fn inject_node_event( + fn inject_connected(&mut self, peer_id: &PeerId) { + self.behaviour.inject_connected(peer_id) + } + + fn inject_disconnected(&mut self, peer_id: &PeerId) { + self.behaviour.inject_disconnected(peer_id) + } + + fn inject_event( &mut self, peer_id: PeerId, + connection: ConnectionId, event: <::Handler as ProtocolsHandler>::OutEvent, ) { - self.behaviour.inject_node_event(peer_id, event) + self.behaviour.inject_event(peer_id, connection, event) } fn poll( @@ -1869,30 +1960,55 @@ impl NetworkBehaviour for Protocol { } for (id, r) in self.sync.block_requests() { - send_request( - &mut self.behaviour, - &mut self.context_data.stats, - &mut self.context_data.peers, - &id, - GenericMessage::BlockRequest(r) - ) + if self.use_new_block_requests_protocol { + let event = CustomMessageOutcome::BlockRequest { + target: id, + request: r, + }; + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(event)); + } else { + send_request( + &mut self.behaviour, + &mut self.context_data.stats, + &mut self.context_data.peers, + &id, + GenericMessage::BlockRequest(r) + ) + } } for (id, r) in self.sync.justification_requests() { - send_request( - &mut self.behaviour, - &mut self.context_data.stats, - &mut self.context_data.peers, - &id, - GenericMessage::BlockRequest(r) - ) + if self.use_new_block_requests_protocol { + let event = CustomMessageOutcome::BlockRequest { + target: id, + request: r, + }; + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(event)); + } else { + send_request( + &mut self.behaviour, + &mut self.context_data.stats, + &mut self.context_data.peers, + &id, + GenericMessage::BlockRequest(r) + ) + } } for (id, r) in self.sync.finality_proof_requests() { - send_request( - &mut self.behaviour, - &mut self.context_data.stats, - &mut self.context_data.peers, - &id, - GenericMessage::FinalityProofRequest(r)) + if self.use_new_block_requests_protocol { + let event = CustomMessageOutcome::FinalityProofRequest { + target: id, + block_hash: r.block, + request: r.request, + }; + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(event)); + } else { + send_request( + &mut self.behaviour, + &mut self.context_data.stats, + &mut self.context_data.peers, + &id, + GenericMessage::FinalityProofRequest(r)) + } } let event = match self.behaviour.poll(cx, params) { @@ -1900,10 +2016,10 @@ impl NetworkBehaviour for Protocol { Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)) => ev, Poll::Ready(NetworkBehaviourAction::DialAddress { address }) => return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), - Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }) => - return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id }), - Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id, event }) => - return Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id, event }), + Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }) => + return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }), + Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, handler, event }) => + return Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, handler, event }), Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address }) => return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { address }), }; @@ -1967,10 +2083,6 @@ impl NetworkBehaviour for Protocol { } } - fn inject_replaced(&mut self, peer_id: PeerId, closed_endpoint: ConnectedPoint, new_endpoint: ConnectedPoint) { - self.behaviour.inject_replaced(peer_id, closed_endpoint, new_endpoint) - } - fn inject_addr_reach_failure( &mut self, peer_id: Option<&PeerId>, @@ -2000,8 +2112,8 @@ impl NetworkBehaviour for Protocol { self.behaviour.inject_listener_error(id, err); } - fn inject_listener_closed(&mut self, id: ListenerId) { - self.behaviour.inject_listener_closed(id); + fn inject_listener_closed(&mut self, id: ListenerId, reason: Result<(), &io::Error>) { + self.behaviour.inject_listener_closed(id, reason); } } @@ -2043,6 +2155,7 @@ mod tests { Box::new(DefaultBlockAnnounceValidator::new(client.clone())), None, Default::default(), + true, None, ).unwrap(); diff --git a/client/network/src/protocol/block_requests.rs b/client/network/src/protocol/block_requests.rs index 5a947c0b6b5854ce05e393ea563976bb85656592..920d3f0e23f411a652e647fb3ed4c0e937b148b0 100644 --- a/client/network/src/protocol/block_requests.rs +++ b/client/network/src/protocol/block_requests.rs @@ -27,22 +27,26 @@ use codec::{Encode, Decode}; use crate::{ chain::Client, config::ProtocolId, - protocol::{api, message::BlockAttributes} + protocol::{api, message::{self, BlockAttributes}} }; use futures::{future::BoxFuture, prelude::*, stream::FuturesUnordered}; +use futures_timer::Delay; use libp2p::{ core::{ ConnectedPoint, Multiaddr, PeerId, - upgrade::{InboundUpgrade, ReadOneError, UpgradeInfo, Negotiated}, + connection::ConnectionId, + upgrade::{InboundUpgrade, OutboundUpgrade, ReadOneError, UpgradeInfo, Negotiated}, upgrade::{DeniedUpgrade, read_one, write_one} }, swarm::{ NegotiatedSubstream, NetworkBehaviour, NetworkBehaviourAction, + NotifyHandler, OneShotHandler, + OneShotHandlerConfig, PollParameters, SubstreamProtocol } @@ -51,23 +55,73 @@ use prost::Message; use sp_runtime::{generic::BlockId, traits::{Block, Header, One, Zero}}; use std::{ cmp::min, + collections::{HashMap, VecDeque}, io, iter, + marker::PhantomData, + pin::Pin, sync::Arc, time::Duration, task::{Context, Poll} }; use void::{Void, unreachable}; +use wasm_timer::Instant; // Type alias for convenience. pub type Error = Box; +/// Event generated by the block requests behaviour. +#[derive(Debug)] +pub enum Event { + /// A request came and we answered it. + AnsweredRequest { + /// Peer which has emitted the request. + peer: PeerId, + /// Time it took to compute the response. + response_build_time: Duration, + }, + + /// A response to a block request has arrived. + Response { + peer: PeerId, + /// The original request passed to `send_request`. + original_request: message::BlockRequest, + response: message::BlockResponse, + /// Time elapsed between the start of the request and the response. + request_duration: Duration, + }, + + /// A request has been cancelled because the peer has disconnected. + /// Disconnects can also happen as a result of violating the network protocol. + /// + /// > **Note**: This event is NOT emitted if a request is overridden by calling `send_request`. + /// > For that, you must check the value returned by `send_request`. + RequestCancelled { + peer: PeerId, + /// The original request passed to `send_request`. + original_request: message::BlockRequest, + /// Time elapsed between the start of the request and the cancellation. + request_duration: Duration, + }, + + /// A request has timed out. + RequestTimeout { + peer: PeerId, + /// The original request passed to `send_request`. + original_request: message::BlockRequest, + /// Time elapsed between the start of the request and the timeout. + request_duration: Duration, + } +} + /// Configuration options for `BlockRequests`. #[derive(Debug, Clone)] pub struct Config { max_block_data_response: u32, max_request_len: usize, + max_response_len: usize, inactivity_timeout: Duration, + request_timeout: Duration, protocol: Bytes, } @@ -76,12 +130,16 @@ impl Config { /// /// - max. block data in response = 128 /// - max. request size = 1 MiB + /// - max. response size = 16 MiB /// - inactivity timeout = 15s + /// - request timeout = 40s pub fn new(id: &ProtocolId) -> Self { let mut c = Config { max_block_data_response: 128, max_request_len: 1024 * 1024, + max_response_len: 16 * 1024 * 1024, inactivity_timeout: Duration::from_secs(15), + request_timeout: Duration::from_secs(40), protocol: Bytes::new(), }; c.set_protocol(id); @@ -100,6 +158,12 @@ impl Config { self } + /// Limit the max. size of responses to our block requests. + pub fn set_max_response_len(&mut self, v: usize) -> &mut Self { + self.max_response_len = v; + self + } + /// Limit the max. duration the substream may remain inactive before closing it. pub fn set_inactivity_timeout(&mut self, v: Duration) -> &mut Self { self.inactivity_timeout = v; @@ -123,8 +187,47 @@ pub struct BlockRequests { config: Config, /// Blockchain client. chain: Arc>, + /// List of all active connections and the requests we've sent. + peers: HashMap>>, /// Futures sending back the block request response. outgoing: FuturesUnordered>, + /// Events to return as soon as possible from `poll`. + pending_events: VecDeque, Event>>, +} + +/// Local tracking of a libp2p connection. +#[derive(Debug)] +struct Connection { + id: ConnectionId, + ongoing_request: Option>, +} + +#[derive(Debug)] +struct OngoingRequest { + /// `Instant` when the request has been emitted. Used for diagnostic purposes. + emitted: Instant, + request: message::BlockRequest, + timeout: Delay, +} + +/// Outcome of calling `send_request`. +#[derive(Debug)] +#[must_use] +pub enum SendRequestOutcome { + /// Request has been emitted. + Ok, + /// The request has been emitted and has replaced an existing request. + Replaced { + /// The previously-emitted request. + previous: message::BlockRequest, + /// Time that had elapsed since `previous` has been emitted. + request_duration: Duration, + }, + /// Didn't start a request because we have no connection to this node. + /// If `send_request` returns that, it is as if the function had never been called. + NotConnected, + /// Error while serializing the request. + EncodeError(prost::EncodeError), } impl BlockRequests @@ -135,7 +238,101 @@ where BlockRequests { config: cfg, chain, + peers: HashMap::new(), outgoing: FuturesUnordered::new(), + pending_events: VecDeque::new(), + } + } + + /// Returns the libp2p protocol name used on the wire (e.g. `/foo/sync/2`). + pub fn protocol_name(&self) -> &[u8] { + &self.config.protocol + } + + /// Issue a new block request. + /// + /// Cancels any existing request targeting the same `PeerId`. + /// + /// If the response doesn't arrive in time, or if the remote answers improperly, the target + /// will be disconnected. + pub fn send_request(&mut self, target: &PeerId, req: message::BlockRequest) -> SendRequestOutcome { + // Determine which connection to send the request to. + let connection = if let Some(peer) = self.peers.get_mut(target) { + // We don't want to have multiple requests for any given node, so in priority try to + // find a connection with an existing request, to override it. + if let Some(entry) = peer.iter_mut().find(|c| c.ongoing_request.is_some()) { + entry + } else if let Some(entry) = peer.get_mut(0) { + entry + } else { + log::error!( + target: "sync", + "State inconsistency: empty list of peer connections" + ); + return SendRequestOutcome::NotConnected; + } + } else { + return SendRequestOutcome::NotConnected; + }; + + let protobuf_rq = api::v1::BlockRequest { + fields: u32::from_be_bytes([req.fields.bits(), 0, 0, 0]), + from_block: match req.from { + message::FromBlock::Hash(h) => + Some(api::v1::block_request::FromBlock::Hash(h.encode())), + message::FromBlock::Number(n) => + Some(api::v1::block_request::FromBlock::Number(n.encode())), + }, + to_block: req.to.map(|h| h.encode()).unwrap_or_default(), + direction: match req.direction { + message::Direction::Ascending => api::v1::Direction::Ascending as i32, + message::Direction::Descending => api::v1::Direction::Descending as i32, + }, + max_blocks: req.max.unwrap_or(0), + }; + + let mut buf = Vec::with_capacity(protobuf_rq.encoded_len()); + if let Err(err) = protobuf_rq.encode(&mut buf) { + log::warn!( + target: "sync", + "Failed to encode block request {:?}: {:?}", + protobuf_rq, + err + ); + return SendRequestOutcome::EncodeError(err); + } + + let previous_request = connection.ongoing_request.take(); + connection.ongoing_request = Some(OngoingRequest { + emitted: Instant::now(), + request: req.clone(), + timeout: Delay::new(self.config.request_timeout), + }); + + log::trace!(target: "sync", "Enqueueing block request to {:?}: {:?}", target, protobuf_rq); + self.pending_events.push_back(NetworkBehaviourAction::NotifyHandler { + peer_id: target.clone(), + handler: NotifyHandler::One(connection.id), + event: OutboundProtocol { + request: buf, + original_request: req, + max_response_size: self.config.max_response_len, + protocol: self.config.protocol.clone(), + }, + }); + + if let Some(previous_request) = previous_request { + log::debug!( + target: "sync", + "Replacing existing block request on connection {:?}", + connection.id + ); + SendRequestOutcome::Replaced { + previous: previous_request.request, + request_duration: previous_request.emitted.elapsed(), + } + } else { + SendRequestOutcome::Ok } } @@ -146,7 +343,9 @@ where , request: &api::v1::BlockRequest ) -> Result { - log::trace!("block request from peer {}: from block {:?} to block {:?}, max blocks {:?}", + log::trace!( + target: "sync", + "Block request from peer {}: from block {:?} to block {:?}, max blocks {:?}", peer, request.from_block, request.to_block, @@ -200,6 +399,12 @@ where let number = header.number().clone(); let hash = header.hash(); let parent_hash = header.parent_hash().clone(); + let justification = if get_justification { + self.chain.justification(&BlockId::Hash(hash))? + } else { + None + }; + let is_empty_justification = justification.as_ref().map(|j| j.is_empty()).unwrap_or(false); let block_data = api::v1::BlockData { hash: hash.encode(), @@ -219,11 +424,8 @@ where }, receipt: Vec::new(), message_queue: Vec::new(), - justification: if get_justification { - self.chain.justification(&BlockId::Hash(hash))?.unwrap_or(Vec::new()) - } else { - Vec::new() - } + justification: justification.unwrap_or(Vec::new()), + is_empty_justification, }; blocks.push(block_data); @@ -249,64 +451,277 @@ impl NetworkBehaviour for BlockRequests where B: Block { - type ProtocolsHandler = OneShotHandler>; - type OutEvent = Void; + type ProtocolsHandler = OneShotHandler, OutboundProtocol, NodeEvent>; + type OutEvent = Event; fn new_handler(&mut self) -> Self::ProtocolsHandler { - let p = Protocol { + let p = InboundProtocol { max_request_len: self.config.max_request_len, protocol: self.config.protocol.clone(), + marker: PhantomData, }; - OneShotHandler::new(SubstreamProtocol::new(p), self.config.inactivity_timeout) + let mut cfg = OneShotHandlerConfig::default(); + cfg.inactive_timeout = self.config.inactivity_timeout; + cfg.substream_timeout = self.config.request_timeout; + OneShotHandler::new(SubstreamProtocol::new(p), cfg) } fn addresses_of_peer(&mut self, _: &PeerId) -> Vec { Vec::new() } - fn inject_connected(&mut self, _peer: PeerId, _info: ConnectedPoint) { + fn inject_connected(&mut self, _peer: &PeerId) { } - fn inject_disconnected(&mut self, _peer: &PeerId, _info: ConnectedPoint) { + fn inject_disconnected(&mut self, _peer: &PeerId) { } - fn inject_node_event(&mut self, peer: PeerId, Request(request, mut stream): Request) { - match self.on_block_request(&peer, &request) { - Ok(res) => { - log::trace!("enqueueing block response for peer {} with {} blocks", peer, res.blocks.len()); - let mut data = Vec::with_capacity(res.encoded_len()); - if let Err(e) = res.encode(&mut data) { - log::debug!("error encoding block response for peer {}: {}", peer, e) - } else { - let future = async move { - if let Err(e) = write_one(&mut stream, data).await { - log::debug!("error writing block response: {}", e) - } + fn inject_connection_established(&mut self, peer_id: &PeerId, id: &ConnectionId, _: &ConnectedPoint) { + self.peers.entry(peer_id.clone()) + .or_default() + .push(Connection { + id: *id, + ongoing_request: None, + }); + } + + fn inject_connection_closed(&mut self, peer_id: &PeerId, id: &ConnectionId, _: &ConnectedPoint) { + let mut needs_remove = false; + if let Some(entry) = self.peers.get_mut(peer_id) { + if let Some(pos) = entry.iter().position(|i| i.id == *id) { + let ongoing_request = entry.remove(pos).ongoing_request; + if let Some(ongoing_request) = ongoing_request { + log::debug!( + target: "sync", + "Connection {:?} with {} closed with ongoing sync request: {:?}", + id, + peer_id, + ongoing_request + ); + let ev = Event::RequestCancelled { + peer: peer_id.clone(), + original_request: ongoing_request.request.clone(), + request_duration: ongoing_request.emitted.elapsed(), }; - self.outgoing.push(future.boxed()) + self.pending_events.push_back(NetworkBehaviourAction::GenerateEvent(ev)); + } + if entry.is_empty() { + needs_remove = true; + } + } else { + log::error!( + target: "sync", + "State inconsistency: connection id not found in list" + ); + } + } else { + log::error!( + target: "sync", + "State inconsistency: peer_id not found in list of connections" + ); + } + if needs_remove { + self.peers.remove(peer_id); + } + } + + fn inject_event( + &mut self, + peer: PeerId, + connection_id: ConnectionId, + node_event: NodeEvent + ) { + match node_event { + NodeEvent::Request(request, mut stream) => { + let before_answer_build = Instant::now(); + + match self.on_block_request(&peer, &request) { + Ok(res) => { + log::trace!( + target: "sync", + "Enqueueing block response for peer {} with {} blocks", + peer, res.blocks.len() + ); + let mut data = Vec::with_capacity(res.encoded_len()); + if let Err(e) = res.encode(&mut data) { + log::debug!( + target: "sync", + "Error encoding block response for peer {}: {}", + peer, e + ) + } else { + let future = async move { + if let Err(e) = write_one(&mut stream, data).await { + log::debug!( + target: "sync", + "Error writing block response: {}", + e + ); + } + }; + self.outgoing.push(future.boxed()) + } + } + Err(e) => log::debug!( + target: "sync", + "Error handling block request from peer {}: {}", peer, e + ) + } + + let ev = Event::AnsweredRequest { + peer: peer.clone(), + response_build_time: before_answer_build.elapsed(), + }; + self.pending_events.push_back(NetworkBehaviourAction::GenerateEvent(ev)); + } + NodeEvent::Response(original_request, response) => { + log::trace!( + target: "sync", + "Received block response from peer {} with {} blocks", + peer, response.blocks.len() + ); + let request_duration = if let Some(connections) = self.peers.get_mut(&peer) { + if let Some(connection) = connections.iter_mut().find(|c| c.id == connection_id) { + if let Some(ongoing_request) = &mut connection.ongoing_request { + if ongoing_request.request == original_request { + let request_duration = ongoing_request.emitted.elapsed(); + connection.ongoing_request = None; + request_duration + } else { + // We're no longer interested in that request. + log::debug!( + target: "sync", + "Received response from {} to obsolete block request {:?}", + peer, + original_request + ); + return; + } + } else { + // We remove from `self.peers` requests we're no longer interested in, + // so this can legitimately happen. + return; + } + } else { + log::error!( + target: "sync", + "State inconsistency: response on non-existing connection {:?}", + connection_id + ); + return; + } + } else { + log::error!( + target: "sync", + "State inconsistency: response on non-connected peer {}", + peer + ); + return; + }; + + let blocks = response.blocks.into_iter().map(|block_data| { + Ok(message::BlockData:: { + hash: Decode::decode(&mut block_data.hash.as_ref())?, + header: if !block_data.header.is_empty() { + Some(Decode::decode(&mut block_data.header.as_ref())?) + } else { + None + }, + body: if original_request.fields.contains(message::BlockAttributes::BODY) { + Some(block_data.body.iter().map(|body| { + Decode::decode(&mut body.as_ref()) + }).collect::, _>>()?) + } else { + None + }, + receipt: if !block_data.message_queue.is_empty() { + Some(block_data.receipt) + } else { + None + }, + message_queue: if !block_data.message_queue.is_empty() { + Some(block_data.message_queue) + } else { + None + }, + justification: if !block_data.justification.is_empty() { + Some(block_data.justification) + } else if block_data.is_empty_justification { + Some(Vec::new()) + } else { + None + }, + }) + }).collect::, codec::Error>>(); + + match blocks { + Ok(blocks) => { + let id = original_request.id; + let ev = Event::Response { + peer, + original_request, + response: message::BlockResponse:: { id, blocks }, + request_duration, + }; + self.pending_events.push_back(NetworkBehaviourAction::GenerateEvent(ev)); + } + Err(err) => { + log::debug!( + target: "sync", + "Failed to decode block response from peer {}: {}", peer, err + ); + } } } - Err(e) => log::debug!("error handling block request from peer {}: {}", peer, e) } } - fn poll(&mut self, cx: &mut Context, _: &mut impl PollParameters) -> Poll> { + fn poll(&mut self, cx: &mut Context, _: &mut impl PollParameters) + -> Poll, Event>> + { + if let Some(ev) = self.pending_events.pop_front() { + return Poll::Ready(ev); + } + + // Check the request timeouts. + for (peer, connections) in &mut self.peers { + for connection in connections { + let ongoing_request = match &mut connection.ongoing_request { + Some(rq) => rq, + None => continue, + }; + + if let Poll::Ready(_) = Pin::new(&mut ongoing_request.timeout).poll(cx) { + let original_request = ongoing_request.request.clone(); + let request_duration = ongoing_request.emitted.elapsed(); + connection.ongoing_request = None; + log::debug!( + target: "sync", + "Request timeout for {}: {:?}", + peer, original_request + ); + let ev = Event::RequestTimeout { + peer: peer.clone(), + original_request, + request_duration, + }; + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); + } + } + } + while let Poll::Ready(Some(_)) = self.outgoing.poll_next_unpin(cx) {} Poll::Pending } } -/// The incoming block request. -/// -/// Holds the protobuf value and the connection substream which made the -/// request and over which to send the response. +/// Output type of inbound and outbound substream upgrades. #[derive(Debug)] -pub struct Request(api::v1::BlockRequest, T); - -impl From for Request { - fn from(v: Void) -> Self { - unreachable(v) - } +pub enum NodeEvent { + /// Incoming request from remote and substream to use for the response. + Request(api::v1::BlockRequest, T), + /// Incoming response from remote. + Response(message::BlockRequest, api::v1::BlockResponse), } /// Substream upgrade protocol. @@ -316,36 +731,39 @@ impl From for Request { /// will become visible via `inject_node_event` which then dispatches to the /// relevant callback to process the message and prepare a response. #[derive(Debug, Clone)] -pub struct Protocol { +pub struct InboundProtocol { /// The max. request length in bytes. max_request_len: usize, /// The protocol to use during upgrade negotiation. protocol: Bytes, + /// Type of the block. + marker: PhantomData, } -impl UpgradeInfo for Protocol { - type Info = Bytes; - type InfoIter = iter::Once; +impl UpgradeInfo for InboundProtocol { + type Info = Bytes; + type InfoIter = iter::Once; - fn protocol_info(&self) -> Self::InfoIter { - iter::once(self.protocol.clone()) - } + fn protocol_info(&self) -> Self::InfoIter { + iter::once(self.protocol.clone()) + } } -impl InboundUpgrade for Protocol +impl InboundUpgrade for InboundProtocol where + B: Block, T: AsyncRead + AsyncWrite + Unpin + Send + 'static { - type Output = Request; - type Error = ReadOneError; - type Future = BoxFuture<'static, Result>; + type Output = NodeEvent; + type Error = ReadOneError; + type Future = BoxFuture<'static, Result>; - fn upgrade_inbound(self, mut s: T, _: Self::Info) -> Self::Future { + fn upgrade_inbound(self, mut s: T, _: Self::Info) -> Self::Future { let future = async move { let len = self.max_request_len; let vec = read_one(&mut s, len).await?; match api::v1::BlockRequest::decode(&vec[..]) { - Ok(r) => Ok(Request(r, s)), + Ok(r) => Ok(NodeEvent::Request(r, s)), Err(e) => Err(ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e))) } }; @@ -353,3 +771,49 @@ where } } +/// Substream upgrade protocol. +/// +/// Sends a request to remote and awaits the response. +#[derive(Debug, Clone)] +pub struct OutboundProtocol { + /// The serialized protobuf request. + request: Vec, + /// The original request. Passed back through the API when the response comes back. + original_request: message::BlockRequest, + /// The max. response length in bytes. + max_response_size: usize, + /// The protocol to use for upgrade negotiation. + protocol: Bytes, +} + +impl UpgradeInfo for OutboundProtocol { + type Info = Bytes; + type InfoIter = iter::Once; + + fn protocol_info(&self) -> Self::InfoIter { + iter::once(self.protocol.clone()) + } +} + +impl OutboundUpgrade for OutboundProtocol +where + B: Block, + T: AsyncRead + AsyncWrite + Unpin + Send + 'static +{ + type Output = NodeEvent; + type Error = ReadOneError; + type Future = BoxFuture<'static, Result>; + + fn upgrade_outbound(self, mut s: T, _: Self::Info) -> Self::Future { + async move { + write_one(&mut s, &self.request).await?; + let vec = read_one(&mut s, self.max_response_size).await?; + + api::v1::BlockResponse::decode(&vec[..]) + .map(|r| NodeEvent::Response(self.original_request, r)) + .map_err(|e| { + ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e)) + }) + }.boxed() + } +} diff --git a/client/network/src/protocol/finality_requests.rs b/client/network/src/protocol/finality_requests.rs new file mode 100644 index 0000000000000000000000000000000000000000..061675961795422f26120be6b513f9d7b7d66b37 --- /dev/null +++ b/client/network/src/protocol/finality_requests.rs @@ -0,0 +1,402 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. +// +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! `NetworkBehaviour` implementation which handles incoming finality proof requests. +//! +//! Every request is coming in on a separate connection substream which gets +//! closed after we have sent the response back. Incoming requests are encoded +//! as protocol buffers (cf. `finality.v1.proto`). + +#![allow(unused)] + +use bytes::Bytes; +use codec::{Encode, Decode}; +use crate::{ + chain::FinalityProofProvider, + config::ProtocolId, + protocol::{api, message} +}; +use futures::{future::BoxFuture, prelude::*, stream::FuturesUnordered}; +use libp2p::{ + core::{ + ConnectedPoint, + Multiaddr, + PeerId, + connection::ConnectionId, + upgrade::{InboundUpgrade, OutboundUpgrade, ReadOneError, UpgradeInfo, Negotiated}, + upgrade::{DeniedUpgrade, read_one, write_one} + }, + swarm::{ + NegotiatedSubstream, + NetworkBehaviour, + NetworkBehaviourAction, + NotifyHandler, + OneShotHandler, + OneShotHandlerConfig, + PollParameters, + SubstreamProtocol + } +}; +use prost::Message; +use sp_runtime::{generic::BlockId, traits::{Block, Header, One, Zero}}; +use std::{ + cmp::min, + collections::VecDeque, + io, + iter, + marker::PhantomData, + sync::Arc, + time::Duration, + task::{Context, Poll} +}; +use void::{Void, unreachable}; + +// Type alias for convenience. +pub type Error = Box; + +/// Event generated by the finality proof requests behaviour. +#[derive(Debug)] +pub enum Event { + /// A response to a finality proof request has arrived. + Response { + peer: PeerId, + /// Block hash originally passed to `send_request`. + block_hash: B::Hash, + /// Finality proof returned by the remote. + proof: Vec, + }, +} + +/// Configuration options for `FinalityProofRequests`. +#[derive(Debug, Clone)] +pub struct Config { + max_request_len: usize, + max_response_len: usize, + inactivity_timeout: Duration, + protocol: Bytes, +} + +impl Config { + /// Create a fresh configuration with the following options: + /// + /// - max. request size = 1 MiB + /// - max. response size = 1 MiB + /// - inactivity timeout = 15s + pub fn new(id: &ProtocolId) -> Self { + let mut c = Config { + max_request_len: 1024 * 1024, + max_response_len: 1024 * 1024, + inactivity_timeout: Duration::from_secs(15), + protocol: Bytes::new(), + }; + c.set_protocol(id); + c + } + + /// Limit the max. length of incoming finality proof request bytes. + pub fn set_max_request_len(&mut self, v: usize) -> &mut Self { + self.max_request_len = v; + self + } + + /// Limit the max. length of incoming finality proof response bytes. + pub fn set_max_response_len(&mut self, v: usize) -> &mut Self { + self.max_response_len = v; + self + } + + /// Limit the max. duration the substream may remain inactive before closing it. + pub fn set_inactivity_timeout(&mut self, v: Duration) -> &mut Self { + self.inactivity_timeout = v; + self + } + + /// Set protocol to use for upgrade negotiation. + pub fn set_protocol(&mut self, id: &ProtocolId) -> &mut Self { + let mut v = Vec::new(); + v.extend_from_slice(b"/"); + v.extend_from_slice(id.as_bytes()); + v.extend_from_slice(b"/finality-proof/1"); + self.protocol = v.into(); + self + } +} + +/// The finality proof request handling behaviour. +pub struct FinalityProofRequests { + /// This behaviour's configuration. + config: Config, + /// How to construct finality proofs. + finality_proof_provider: Option>>, + /// Futures sending back the finality proof request responses. + outgoing: FuturesUnordered>, + /// Events to return as soon as possible from `poll`. + pending_events: VecDeque, Event>>, +} + +impl FinalityProofRequests +where + B: Block, +{ + /// Initializes the behaviour. + /// + /// If the proof provider is `None`, then the behaviour will not support the finality proof + /// requests protocol. + pub fn new(cfg: Config, finality_proof_provider: Option>>) -> Self { + FinalityProofRequests { + config: cfg, + finality_proof_provider, + outgoing: FuturesUnordered::new(), + pending_events: VecDeque::new(), + } + } + + /// Issue a new finality proof request. + /// + /// If the response doesn't arrive in time, or if the remote answers improperly, the target + /// will be disconnected. + pub fn send_request(&mut self, target: &PeerId, block_hash: B::Hash, request: Vec) { + let protobuf_rq = api::v1::finality::FinalityProofRequest { + block_hash: block_hash.encode(), + request, + }; + + let mut buf = Vec::with_capacity(protobuf_rq.encoded_len()); + if let Err(err) = protobuf_rq.encode(&mut buf) { + log::warn!("failed to encode finality proof request {:?}: {:?}", protobuf_rq, err); + return; + } + + log::trace!("enqueueing finality proof request to {:?}: {:?}", target, protobuf_rq); + self.pending_events.push_back(NetworkBehaviourAction::NotifyHandler { + peer_id: target.clone(), + handler: NotifyHandler::Any, + event: OutboundProtocol { + request: buf, + block_hash, + max_response_size: self.config.max_response_len, + protocol: self.config.protocol.clone(), + }, + }); + } + + /// Callback, invoked when a new finality request has been received from remote. + fn on_finality_request(&mut self, peer: &PeerId, request: &api::v1::finality::FinalityProofRequest) + -> Result + { + let block_hash = Decode::decode(&mut request.block_hash.as_ref())?; + + log::trace!(target: "sync", "Finality proof request from {} for {}", peer, block_hash); + + // Note that an empty Vec is sent if no proof is available. + let finality_proof = if let Some(provider) = &self.finality_proof_provider { + provider + .prove_finality(block_hash, &request.request)? + .unwrap_or(Vec::new()) + } else { + log::error!("Answering a finality proof request while finality provider is empty"); + return Err(From::from("Empty finality proof provider".to_string())) + }; + + Ok(api::v1::finality::FinalityProofResponse { proof: finality_proof }) + } +} + +impl NetworkBehaviour for FinalityProofRequests +where + B: Block +{ + type ProtocolsHandler = OneShotHandler, OutboundProtocol, NodeEvent>; + type OutEvent = Event; + + fn new_handler(&mut self) -> Self::ProtocolsHandler { + let p = InboundProtocol { + max_request_len: self.config.max_request_len, + protocol: if self.finality_proof_provider.is_some() { + Some(self.config.protocol.clone()) + } else { + None + }, + marker: PhantomData, + }; + let mut cfg = OneShotHandlerConfig::default(); + cfg.inactive_timeout = self.config.inactivity_timeout; + OneShotHandler::new(SubstreamProtocol::new(p), cfg) + } + + fn addresses_of_peer(&mut self, _: &PeerId) -> Vec { + Vec::new() + } + + fn inject_connected(&mut self, _peer: &PeerId) { + } + + fn inject_disconnected(&mut self, _peer: &PeerId) { + } + + fn inject_event( + &mut self, + peer: PeerId, + connection: ConnectionId, + event: NodeEvent + ) { + match event { + NodeEvent::Request(request, mut stream) => { + match self.on_finality_request(&peer, &request) { + Ok(res) => { + log::trace!("enqueueing finality response for peer {}", peer); + let mut data = Vec::with_capacity(res.encoded_len()); + if let Err(e) = res.encode(&mut data) { + log::debug!("error encoding finality response for peer {}: {}", peer, e) + } else { + let future = async move { + if let Err(e) = write_one(&mut stream, data).await { + log::debug!("error writing finality response: {}", e) + } + }; + self.outgoing.push(future.boxed()) + } + } + Err(e) => log::debug!("error handling finality request from peer {}: {}", peer, e) + } + } + NodeEvent::Response(response, block_hash) => { + let ev = Event::Response { + peer, + block_hash, + proof: response.proof, + }; + self.pending_events.push_back(NetworkBehaviourAction::GenerateEvent(ev)); + } + } + } + + fn poll(&mut self, cx: &mut Context, _: &mut impl PollParameters) + -> Poll, Event>> + { + if let Some(ev) = self.pending_events.pop_front() { + return Poll::Ready(ev); + } + + while let Poll::Ready(Some(_)) = self.outgoing.poll_next_unpin(cx) {} + Poll::Pending + } +} + +/// Output type of inbound and outbound substream upgrades. +#[derive(Debug)] +pub enum NodeEvent { + /// Incoming request from remote and substream to use for the response. + Request(api::v1::finality::FinalityProofRequest, T), + /// Incoming response from remote. + Response(api::v1::finality::FinalityProofResponse, B::Hash), +} + +/// Substream upgrade protocol. +/// +/// We attempt to parse an incoming protobuf encoded request (cf. `Request`) +/// which will be handled by the `FinalityProofRequests` behaviour, i.e. the request +/// will become visible via `inject_node_event` which then dispatches to the +/// relevant callback to process the message and prepare a response. +#[derive(Debug, Clone)] +pub struct InboundProtocol { + /// The max. request length in bytes. + max_request_len: usize, + /// The protocol to use during upgrade negotiation. If `None`, then the incoming protocol + /// is simply disabled. + protocol: Option, + /// Marker to pin the block type. + marker: PhantomData, +} + +impl UpgradeInfo for InboundProtocol { + type Info = Bytes; + // This iterator will return either 0 elements if `self.protocol` is `None`, or 1 element if + // it is `Some`. + type InfoIter = std::option::IntoIter; + + fn protocol_info(&self) -> Self::InfoIter { + self.protocol.clone().into_iter() + } +} + +impl InboundUpgrade for InboundProtocol +where + B: Block, + T: AsyncRead + AsyncWrite + Unpin + Send + 'static +{ + type Output = NodeEvent; + type Error = ReadOneError; + type Future = BoxFuture<'static, Result>; + + fn upgrade_inbound(self, mut s: T, _: Self::Info) -> Self::Future { + async move { + let len = self.max_request_len; + let vec = read_one(&mut s, len).await?; + match api::v1::finality::FinalityProofRequest::decode(&vec[..]) { + Ok(r) => Ok(NodeEvent::Request(r, s)), + Err(e) => Err(ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e))) + } + }.boxed() + } +} + +/// Substream upgrade protocol. +/// +/// Sends a request to remote and awaits the response. +#[derive(Debug, Clone)] +pub struct OutboundProtocol { + /// The serialized protobuf request. + request: Vec, + /// Block hash that has been requested. + block_hash: B::Hash, + /// The max. response length in bytes. + max_response_size: usize, + /// The protocol to use for upgrade negotiation. + protocol: Bytes, +} + +impl UpgradeInfo for OutboundProtocol { + type Info = Bytes; + type InfoIter = iter::Once; + + fn protocol_info(&self) -> Self::InfoIter { + iter::once(self.protocol.clone()) + } +} + +impl OutboundUpgrade for OutboundProtocol +where + B: Block, + T: AsyncRead + AsyncWrite + Unpin + Send + 'static +{ + type Output = NodeEvent; + type Error = ReadOneError; + type Future = BoxFuture<'static, Result>; + + fn upgrade_outbound(self, mut s: T, _: Self::Info) -> Self::Future { + async move { + write_one(&mut s, &self.request).await?; + let vec = read_one(&mut s, self.max_response_size).await?; + + api::v1::finality::FinalityProofResponse::decode(&vec[..]) + .map(|r| NodeEvent::Response(r, self.block_hash)) + .map_err(|e| { + ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e)) + }) + }.boxed() + } +} diff --git a/client/network/src/protocol/generic_proto.rs b/client/network/src/protocol/generic_proto.rs index f703287f386fdcf9aebcfc8d3fa3d0c5971c7eff..cf8434d8bceffc5140a55d23d4aae1fc4fc4fa43 100644 --- a/client/network/src/protocol/generic_proto.rs +++ b/client/network/src/protocol/generic_proto.rs @@ -21,6 +21,7 @@ //! network, then performs the Substrate protocol handling on top. pub use self::behaviour::{GenericProto, GenericProtoOut}; +pub use self::handler::LegacyConnectionKillError; mod behaviour; mod handler; diff --git a/client/network/src/protocol/generic_proto/behaviour.rs b/client/network/src/protocol/generic_proto/behaviour.rs index c398f6df2d409e93cd774e8b2b2de6dcd048ae18..3ce98dc11ed3c5c675f697ed520bf8e01667647b 100644 --- a/client/network/src/protocol/generic_proto/behaviour.rs +++ b/client/network/src/protocol/generic_proto/behaviour.rs @@ -21,8 +21,14 @@ use crate::protocol::generic_proto::upgrade::RegisteredProtocol; use bytes::BytesMut; use fnv::FnvHashMap; use futures::prelude::*; -use libp2p::core::{ConnectedPoint, Multiaddr, PeerId}; -use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}; +use libp2p::core::{ConnectedPoint, Multiaddr, PeerId, connection::ConnectionId}; +use libp2p::swarm::{ + DialPeerCondition, + NetworkBehaviour, + NetworkBehaviourAction, + NotifyHandler, + PollParameters +}; use log::{debug, error, trace, warn}; use prometheus_endpoint::HistogramVec; use rand::distributions::{Distribution as _, Uniform}; @@ -32,15 +38,16 @@ use std::{borrow::Cow, cmp, collections::hash_map::Entry}; use std::{error, mem, pin::Pin, str, time::Duration}; use wasm_timer::Instant; -/// Network behaviour that handles opening substreams for custom protocols with other nodes. +/// Network behaviour that handles opening substreams for custom protocols with other peers. /// /// ## Legacy vs new protocol /// /// The `GenericProto` behaves as following: /// -/// - Whenever a connection is established, we open a single substream (called "legay protocol" in -/// the source code). This substream name depends on the `protocol_id` and `versions` passed at -/// initialization. If the remote refuses this substream, we close the connection. +/// - Whenever a connection is established, we open a single substream (called "legacy protocol" in +/// the source code) on that connection. This substream name depends on the `protocol_id` and +/// `versions` passed at initialization. If the remote refuses this substream, we close the +/// connection. /// /// - For each registered protocol, we also open an additional substream for this protocol. If the /// remote refuses this substream, then it's fine. @@ -55,33 +62,59 @@ use wasm_timer::Instant; /// /// - The libp2p swarm that opens new connections and reports disconnects. /// - The connection handler (see `handler.rs`) that handles individual connections. -/// - The peerset manager (PSM) that requests links to nodes to be established or broken. +/// - The peerset manager (PSM) that requests links to peers to be established or broken. /// - The external API, that requires knowledge of the links that have been established. /// /// Each connection handler can be in four different states: Enabled+Open, Enabled+Closed, /// Disabled+Open, or Disabled+Closed. The Enabled/Disabled component must be in sync with the /// peerset manager. For example, if the peerset manager requires a disconnection, we disable the -/// existing handler. The Open/Closed component must be in sync with the external API. +/// connection handlers of that peer. The Open/Closed component must be in sync with the external +/// API. /// -/// However a connection handler only exists if we are actually connected to a node. What this -/// means is that there are six possible states for each node: Disconnected, Dialing (trying to -/// reach it), Enabled+Open, Enabled+Closed, Disabled+open, Disabled+Closed. Most notably, the -/// Dialing state must correspond to a "link established" state in the peerset manager. In other -/// words, the peerset manager doesn't differentiate whether we are dialing a node or connected -/// to it. +/// However, a connection handler for a peer only exists if we are actually connected to that peer. +/// What this means is that there are six possible states for each peer: Disconnected, Dialing +/// (trying to connect), Enabled+Open, Enabled+Closed, Disabled+Open, Disabled+Closed. +/// Most notably, the Dialing state must correspond to a "link established" state in the peerset +/// manager. In other words, the peerset manager doesn't differentiate whether we are dialing a +/// peer or connected to it. /// -/// Additionally, there also exists a "banning" system. If we fail to dial a node, we "ban" it for -/// a few seconds. If the PSM requests a node that is in the "banned" state, then we delay the -/// actual dialing attempt until after the ban expires, but the PSM will still consider the link -/// to be established. -/// Note that this "banning" system is not an actual ban. If a "banned" node tries to connect to -/// us, we accept the connection. The "banning" system is only about delaying dialing attempts. +/// There may be multiple connections to a peer. However, the status of a peer on +/// the API of this behaviour and towards the peerset manager is aggregated in +/// the following way: +/// +/// 1. The enabled/disabled status is the same across all connections, as +/// decided by the peerset manager. +/// 2. `send_packet` and `write_notification` always send all data over +/// the same connection to preserve the ordering provided by the transport, +/// as long as that connection is open. If it closes, a second open +/// connection may take over, if one exists, but that case should be no +/// different than a single connection failing and being re-established +/// in terms of potential reordering and dropped messages. Messages can +/// be received on any connection. +/// 3. The behaviour reports `GenericProtoOut::CustomProtocolOpen` when the +/// first connection reports `NotifsHandlerOut::Open`. +/// 4. The behaviour reports `GenericProtoOut::CustomProtocolClosed` when the +/// last connection reports `NotifsHandlerOut::Closed`. +/// +/// In this way, the number of actual established connections to the peer is +/// an implementation detail of this behaviour. Note that, in practice and at +/// the time of this writing, there may be at most two connections to a peer +/// and only as a result of simultaneous dialing. However, the implementation +/// accommodates for any number of connections. +/// +/// Additionally, there also exists a "banning" system. If we fail to dial a peer, we "ban" it for +/// a few seconds. If the PSM requests connecting to a peer that is currently "banned", the next +/// dialing attempt is delayed until after the ban expires. However, the PSM will still consider +/// the peer to be connected. This "ban" is thus not a ban in a strict sense: If a "banned" peer +/// tries to connect, the connection is accepted. A ban only delays dialing attempts. /// pub struct GenericProto { /// Legacy protocol to open with peers. Never modified. legacy_protocol: RegisteredProtocol, /// Notification protocols. Entries are only ever added and not removed. + /// Contains, for each protocol, the protocol name and the message to send as part of the + /// initial handshake. notif_protocols: Vec<(Cow<'static, [u8]>, Vec)>, /// Receiver for instructions about who to connect to or disconnect from. @@ -113,14 +146,14 @@ enum PeerState { /// the state machine code. Poisoned, - /// The peer misbehaved. If the PSM wants us to connect to this node, we will add an artificial + /// The peer misbehaved. If the PSM wants us to connect to this peer, we will add an artificial /// delay to the connection. Banned { - /// Until when the node is banned. + /// Until when the peer is banned. until: Instant, }, - /// The peerset requested that we connect to this peer. We are not connected to this node. + /// The peerset requested that we connect to this peer. We are currently not connected. PendingRequest { /// When to actually start dialing. timer: futures_timer::Delay, @@ -131,16 +164,13 @@ enum PeerState { /// The peerset requested that we connect to this peer. We are currently dialing this peer. Requested, - /// We are connected to this peer but the peerset refused it. This peer can still perform - /// Kademlia queries and such, but should get disconnected in a few seconds. + /// We are connected to this peer but the peerset refused it. + /// + /// We may still have ongoing traffic with that peer, but it should cease shortly. Disabled { - /// How we are connected to this peer. - connected_point: ConnectedPoint, - /// If true, we still have a custom protocol open with it. It will likely get closed in - /// a short amount of time, but we need to keep the information in order to not have a - /// state mismatch. - open: bool, - /// If `Some`, the node is banned until the given `Instant`. + /// The connections that are currently open for custom protocol traffic. + open: SmallVec<[ConnectionId; crate::MAX_CONNECTIONS_PER_PEER]>, + /// If `Some`, any dial attempts to this peer are delayed until the given `Instant`. banned_until: Option, }, @@ -148,12 +178,8 @@ enum PeerState { /// will be enabled when `timer` fires. This peer can still perform Kademlia queries and such, /// but should get disconnected in a few seconds. DisabledPendingEnable { - /// How we are connected to this peer. - connected_point: ConnectedPoint, - /// If true, we still have a custom protocol open with it. It will likely get closed in - /// a short amount of time, but we need to keep the information in order to not have a - /// state mismatch. - open: bool, + /// The connections that are currently open for custom protocol traffic. + open: SmallVec<[ConnectionId; crate::MAX_CONNECTIONS_PER_PEER]>, /// When to enable this remote. timer: futures_timer::Delay, /// When the `timer` will trigger. @@ -163,33 +189,55 @@ enum PeerState { /// We are connected to this peer and the peerset has accepted it. The handler is in the /// enabled state. Enabled { - /// How we are connected to this peer. - connected_point: ConnectedPoint, - /// If true, we have a custom protocol open with this peer. - open: bool, + /// The connections that are currently open for custom protocol traffic. + open: SmallVec<[ConnectionId; crate::MAX_CONNECTIONS_PER_PEER]>, }, - /// We are connected to this peer, and we sent an incoming message to the peerset. The handler - /// is in initialization mode. We are waiting for the Accept or Reject from the peerset. There - /// is a corresponding entry in `incoming`. - Incoming { - /// How we are connected to this peer. - connected_point: ConnectedPoint, - }, + /// We received an incoming connection from this peer and forwarded that + /// connection request to the peerset. The connection handlers are waiting + /// for initialisation, i.e. to be enabled or disabled based on whether + /// the peerset accepts or rejects the peer. + Incoming, } impl PeerState { - /// True if we have an open channel with that node. - fn is_open(&self) -> bool { + /// True if there exists any established connection to the peer. + fn is_connected(&self) -> bool { match self { - PeerState::Poisoned => false, + PeerState::Disabled { .. } | + PeerState::DisabledPendingEnable { .. } | + PeerState::Enabled { .. } | + PeerState::PendingRequest { .. } | + PeerState::Requested | + PeerState::Incoming { .. } => true, + PeerState::Poisoned | PeerState::Banned { .. } => false, - PeerState::PendingRequest { .. } => false, - PeerState::Requested => false, - PeerState::Disabled { open, .. } => *open, - PeerState::DisabledPendingEnable { open, .. } => *open, - PeerState::Enabled { open, .. } => *open, - PeerState::Incoming { .. } => false, + } + } + + /// True if there exists an established connection to the peer + /// that is open for custom protocol traffic. + fn is_open(&self) -> bool { + self.get_open().is_some() + } + + /// Returns the connection ID of the first established connection + /// that is open for custom protocol traffic. + fn get_open(&self) -> Option { + match self { + PeerState::Disabled { open, .. } | + PeerState::DisabledPendingEnable { open, .. } | + PeerState::Enabled { open, .. } => + if !open.is_empty() { + Some(open[0]) + } else { + None + } + PeerState::Poisoned => None, + PeerState::Banned { .. } => None, + PeerState::PendingRequest { .. } => None, + PeerState::Requested => None, + PeerState::Incoming { .. } => None, } } @@ -211,7 +259,7 @@ impl PeerState { /// State of an "incoming" message sent to the peer set manager. #[derive(Debug)] struct IncomingPeer { - /// Id of the node that is concerned. + /// Id of the remote peer of the incoming connection. peer_id: PeerId, /// If true, this "incoming" still corresponds to an actual connection. If false, then the /// connection corresponding to it has been closed or replaced already. @@ -225,10 +273,8 @@ struct IncomingPeer { pub enum GenericProtoOut { /// Opened a custom protocol with the remote. CustomProtocolOpen { - /// Id of the node we have opened a connection with. + /// Id of the peer we are connected to. peer_id: PeerId, - /// Endpoint used for this custom protocol. - endpoint: ConnectedPoint, }, /// Closed a custom protocol with the remote. @@ -306,6 +352,34 @@ impl GenericProto { self.notif_protocols.push((protocol_name.into(), handshake_msg.into())); } + /// Modifies the handshake of the given notifications protocol. + /// + /// Has no effect if the protocol is unknown. + pub fn set_notif_protocol_handshake( + &mut self, + protocol_name: &[u8], + handshake_message: impl Into> + ) { + let handshake_message = handshake_message.into(); + if let Some(protocol) = self.notif_protocols.iter_mut().find(|(name, _)| name == &protocol_name) { + protocol.1 = handshake_message.clone(); + } else { + return; + } + + // Send an event to all the peers we're connected to, updating the handshake message. + for (peer_id, _) in self.peers.iter().filter(|(_, state)| state.is_connected()) { + self.events.push(NetworkBehaviourAction::NotifyHandler { + peer_id: peer_id.clone(), + handler: NotifyHandler::All, + event: NotifsHandlerIn::UpdateHandshake { + protocol_name: Cow::Owned(protocol_name.to_owned()), + handshake_message: handshake_message.clone(), + }, + }); + } + } + /// Returns the number of discovered nodes that we keep in memory. pub fn num_discovered_peers(&self) -> usize { self.peerset.num_discovered_peers() @@ -316,7 +390,7 @@ impl GenericProto { self.peers.iter().filter(|(_, state)| state.is_open()).map(|(id, _)| id) } - /// Returns true if we have a channel open with this node. + /// Returns true if we have an open connection to the given peer. pub fn is_open(&self, peer_id: &PeerId) -> bool { self.peers.get(peer_id).map(|p| p.is_open()).unwrap_or(false) } @@ -327,8 +401,8 @@ impl GenericProto { self.disconnect_peer_inner(peer_id, None); } - /// Inner implementation of `disconnect_peer`. If `ban` is `Some`, we ban the node for the - /// specific duration. + /// Inner implementation of `disconnect_peer`. If `ban` is `Some`, we ban the peer + /// for the specific duration. fn disconnect_peer_inner(&mut self, peer_id: &PeerId, ban: Option) { let mut entry = if let Entry::Occupied(entry) = self.peers.entry(peer_id.clone()) { entry @@ -344,7 +418,11 @@ impl GenericProto { st @ PeerState::Banned { .. } => *entry.into_mut() = st, // DisabledPendingEnable => Disabled. - PeerState::DisabledPendingEnable { open, connected_point, timer_deadline, .. } => { + PeerState::DisabledPendingEnable { + open, + timer_deadline, + timer: _ + } => { debug!(target: "sub-libp2p", "PSM <= Dropped({:?})", peer_id); self.peerset.dropped(peer_id.clone()); let banned_until = Some(if let Some(ban) = ban { @@ -352,24 +430,31 @@ impl GenericProto { } else { timer_deadline }); - *entry.into_mut() = PeerState::Disabled { open, connected_point, banned_until } + *entry.into_mut() = PeerState::Disabled { + open, + banned_until + } }, // Enabled => Disabled. - PeerState::Enabled { open, connected_point } => { + PeerState::Enabled { open } => { debug!(target: "sub-libp2p", "PSM <= Dropped({:?})", peer_id); self.peerset.dropped(peer_id.clone()); debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", peer_id); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), + handler: NotifyHandler::All, event: NotifsHandlerIn::Disable, }); let banned_until = ban.map(|dur| Instant::now() + dur); - *entry.into_mut() = PeerState::Disabled { open, connected_point, banned_until } + *entry.into_mut() = PeerState::Disabled { + open, + banned_until + } }, // Incoming => Disabled. - PeerState::Incoming { connected_point, .. } => { + PeerState::Incoming => { let inc = if let Some(inc) = self.incoming.iter_mut() .find(|i| i.peer_id == *entry.key() && i.alive) { inc @@ -381,12 +466,16 @@ impl GenericProto { inc.alive = false; debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", peer_id); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), + handler: NotifyHandler::All, event: NotifsHandlerIn::Disable, }); let banned_until = ban.map(|dur| Instant::now() + dur); - *entry.into_mut() = PeerState::Disabled { open: false, connected_point, banned_until } + *entry.into_mut() = PeerState::Disabled { + open: SmallVec::new(), + banned_until + } }, PeerState::Poisoned => @@ -441,9 +530,15 @@ impl GenericProto { message: impl Into>, encoded_fallback_message: Vec, ) { - if !self.is_open(target) { - return; - } + let conn = match self.peers.get(target).and_then(|p| p.get_open()) { + None => { + debug!(target: "sub-libp2p", + "Tried to sent notification to {:?} without an open channel.", + target); + return + }, + Some(conn) => conn + }; trace!( target: "sub-libp2p", @@ -453,8 +548,9 @@ impl GenericProto { ); trace!(target: "sub-libp2p", "Handler({:?}) <= Packet", target); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: target.clone(), + handler: NotifyHandler::One(conn), event: NotifsHandlerIn::SendNotification { message: message.into(), encoded_fallback_message, @@ -470,14 +566,21 @@ impl GenericProto { /// Also note that even we have a valid open substream, it may in fact be already closed /// without us knowing, in which case the packet will not be received. pub fn send_packet(&mut self, target: &PeerId, message: Vec) { - if !self.is_open(target) { - return; - } + let conn = match self.peers.get(target).and_then(|p| p.get_open()) { + None => { + debug!(target: "sub-libp2p", + "Tried to sent packet to {:?} without an open channel.", + target); + return + } + Some(conn) => conn + }; trace!(target: "sub-libp2p", "External API => Packet for {:?}", target); trace!(target: "sub-libp2p", "Handler({:?}) <= Packet", target); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: target.clone(), + handler: NotifyHandler::One(conn), event: NotifsHandlerIn::SendLegacy { message, } @@ -489,7 +592,7 @@ impl GenericProto { self.peerset.debug_info() } - /// Function that is called when the peerset wants us to connect to a node. + /// Function that is called when the peerset wants us to connect to a peer. fn peerset_report_connect(&mut self, peer_id: PeerId) { let mut occ_entry = match self.peers.entry(peer_id) { Entry::Occupied(entry) => entry, @@ -497,7 +600,10 @@ impl GenericProto { // If there's no entry in `self.peers`, start dialing. debug!(target: "sub-libp2p", "PSM => Connect({:?}): Starting to connect", entry.key()); debug!(target: "sub-libp2p", "Libp2p <= Dial {:?}", entry.key()); - self.events.push(NetworkBehaviourAction::DialPeer { peer_id: entry.key().clone() }); + self.events.push(NetworkBehaviourAction::DialPeer { + peer_id: entry.key().clone(), + condition: DialPeerCondition::Disconnected + }); entry.insert(PeerState::Requested); return; } @@ -518,36 +624,41 @@ impl GenericProto { PeerState::Banned { .. } => { debug!(target: "sub-libp2p", "PSM => Connect({:?}): Starting to connect", occ_entry.key()); debug!(target: "sub-libp2p", "Libp2p <= Dial {:?}", occ_entry.key()); - self.events.push(NetworkBehaviourAction::DialPeer { peer_id: occ_entry.key().clone() }); + self.events.push(NetworkBehaviourAction::DialPeer { + peer_id: occ_entry.key().clone(), + condition: DialPeerCondition::Disconnected + }); *occ_entry.into_mut() = PeerState::Requested; }, - PeerState::Disabled { open, ref connected_point, banned_until: Some(ref banned) } - if *banned > now => { - debug!(target: "sub-libp2p", "PSM => Connect({:?}): Has idle connection through \ - {:?} but node is banned until {:?}", occ_entry.key(), connected_point, banned); + PeerState::Disabled { + open, + banned_until: Some(ref banned) + } if *banned > now => { + debug!(target: "sub-libp2p", "PSM => Connect({:?}): But peer is banned until {:?}", + occ_entry.key(), banned); *occ_entry.into_mut() = PeerState::DisabledPendingEnable { - connected_point: connected_point.clone(), open, timer: futures_timer::Delay::new(banned.clone() - now), timer_deadline: banned.clone(), }; }, - PeerState::Disabled { open, connected_point, banned_until: _ } => { - debug!(target: "sub-libp2p", "PSM => Connect({:?}): Enabling previously-idle \ - connection through {:?}", occ_entry.key(), connected_point); + PeerState::Disabled { open, banned_until: _ } => { + debug!(target: "sub-libp2p", "PSM => Connect({:?}): Enabling connections.", + occ_entry.key()); debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", occ_entry.key()); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: occ_entry.key().clone(), + handler: NotifyHandler::All, event: NotifsHandlerIn::Enable, }); - *occ_entry.into_mut() = PeerState::Enabled { connected_point, open }; + *occ_entry.into_mut() = PeerState::Enabled { open }; }, - PeerState::Incoming { connected_point, .. } => { - debug!(target: "sub-libp2p", "PSM => Connect({:?}): Enabling incoming \ - connection through {:?}", occ_entry.key(), connected_point); + PeerState::Incoming => { + debug!(target: "sub-libp2p", "PSM => Connect({:?}): Enabling connections.", + occ_entry.key()); if let Some(inc) = self.incoming.iter_mut() .find(|i| i.peer_id == *occ_entry.key() && i.alive) { inc.alive = false; @@ -556,26 +667,30 @@ impl GenericProto { incoming for incoming peer") } debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", occ_entry.key()); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: occ_entry.key().clone(), + handler: NotifyHandler::All, event: NotifsHandlerIn::Enable, }); - *occ_entry.into_mut() = PeerState::Enabled { connected_point, open: false }; + *occ_entry.into_mut() = PeerState::Enabled { open: SmallVec::new() }; }, st @ PeerState::Enabled { .. } => { - warn!(target: "sub-libp2p", "PSM => Connect({:?}): Already connected to this \ - peer", occ_entry.key()); + warn!(target: "sub-libp2p", + "PSM => Connect({:?}): Already connected.", + occ_entry.key()); *occ_entry.into_mut() = st; }, st @ PeerState::DisabledPendingEnable { .. } => { - warn!(target: "sub-libp2p", "PSM => Connect({:?}): Already have an idle \ - connection to this peer and waiting to enable it", occ_entry.key()); + warn!(target: "sub-libp2p", + "PSM => Connect({:?}): Already pending enabling.", + occ_entry.key()); *occ_entry.into_mut() = st; }, st @ PeerState::Requested { .. } | st @ PeerState::PendingRequest { .. } => { - warn!(target: "sub-libp2p", "PSM => Connect({:?}): Received a previous \ - request for that peer", occ_entry.key()); + warn!(target: "sub-libp2p", + "PSM => Connect({:?}): Duplicate request.", + occ_entry.key()); *occ_entry.into_mut() = st; }, @@ -584,55 +699,63 @@ impl GenericProto { } } - /// Function that is called when the peerset wants us to disconnect from a node. + /// Function that is called when the peerset wants us to disconnect from a peer. fn peerset_report_disconnect(&mut self, peer_id: PeerId) { let mut entry = match self.peers.entry(peer_id) { Entry::Occupied(entry) => entry, Entry::Vacant(entry) => { - debug!(target: "sub-libp2p", "PSM => Drop({:?}): Node already disabled", entry.key()); + debug!(target: "sub-libp2p", "PSM => Drop({:?}): Already disabled.", entry.key()); return } }; match mem::replace(entry.get_mut(), PeerState::Poisoned) { st @ PeerState::Disabled { .. } | st @ PeerState::Banned { .. } => { - debug!(target: "sub-libp2p", "PSM => Drop({:?}): Node already disabled", entry.key()); + debug!(target: "sub-libp2p", "PSM => Drop({:?}): Already disabled.", entry.key()); *entry.into_mut() = st; }, - PeerState::DisabledPendingEnable { open, connected_point, timer_deadline, .. } => { - debug!(target: "sub-libp2p", "PSM => Drop({:?}): Interrupting pending \ - enable", entry.key()); + PeerState::DisabledPendingEnable { + open, + timer_deadline, + timer: _ + } => { + debug!(target: "sub-libp2p", + "PSM => Drop({:?}): Interrupting pending enabling.", + entry.key()); *entry.into_mut() = PeerState::Disabled { open, - connected_point, banned_until: Some(timer_deadline), }; }, - PeerState::Enabled { open, connected_point } => { - debug!(target: "sub-libp2p", "PSM => Drop({:?}): Disabling connection", entry.key()); + PeerState::Enabled { open } => { + debug!(target: "sub-libp2p", "PSM => Drop({:?}): Disabling connections.", entry.key()); debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", entry.key()); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: entry.key().clone(), + handler: NotifyHandler::All, event: NotifsHandlerIn::Disable, }); - *entry.into_mut() = PeerState::Disabled { open, connected_point, banned_until: None } + *entry.into_mut() = PeerState::Disabled { + open, + banned_until: None + } }, - st @ PeerState::Incoming { .. } => { - error!(target: "sub-libp2p", "PSM => Drop({:?}): Was in incoming mode", + st @ PeerState::Incoming => { + error!(target: "sub-libp2p", "PSM => Drop({:?}): Not enabled (Incoming).", entry.key()); *entry.into_mut() = st; }, PeerState::Requested => { // We don't cancel dialing. Libp2p doesn't expose that on purpose, as other - // sub-systems (such as the discovery mechanism) may require dialing this node as + // sub-systems (such as the discovery mechanism) may require dialing this peer as // well at the same time. - debug!(target: "sub-libp2p", "PSM => Drop({:?}): Was not yet connected", entry.key()); + debug!(target: "sub-libp2p", "PSM => Drop({:?}): Not yet connected.", entry.key()); entry.remove(); }, PeerState::PendingRequest { timer_deadline, .. } => { - debug!(target: "sub-libp2p", "PSM => Drop({:?}): Was not yet connected", entry.key()); + debug!(target: "sub-libp2p", "PSM => Drop({:?}): Not yet connected", entry.key()); *entry.into_mut() = PeerState::Banned { until: timer_deadline } }, @@ -641,7 +764,8 @@ impl GenericProto { } } - /// Function that is called when the peerset wants us to accept an incoming node. + /// Function that is called when the peerset wants us to accept a connection + /// request from a peer. fn peerset_report_accept(&mut self, index: sc_peerset::IncomingIndex) { let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { self.incoming.remove(pos) @@ -658,34 +782,25 @@ impl GenericProto { return } - let state = if let Some(state) = self.peers.get_mut(&incoming.peer_id) { - state - } else { - error!(target: "sub-libp2p", "State mismatch in libp2p: no entry in peers \ - corresponding to an alive incoming"); - return - }; - - let connected_point = if let PeerState::Incoming { connected_point } = state { - connected_point.clone() - } else { - error!(target: "sub-libp2p", "State mismatch in libp2p: entry in peers corresponding \ - to an alive incoming is not in incoming state"); - return - }; - - debug!(target: "sub-libp2p", "PSM => Accept({:?}, {:?}): Enabling connection \ - through {:?}", index, incoming.peer_id, connected_point); - debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", incoming.peer_id); - self.events.push(NetworkBehaviourAction::SendEvent { - peer_id: incoming.peer_id, - event: NotifsHandlerIn::Enable, - }); - - *state = PeerState::Enabled { open: false, connected_point }; + match self.peers.get_mut(&incoming.peer_id) { + Some(state @ PeerState::Incoming) => { + debug!(target: "sub-libp2p", "PSM => Accept({:?}, {:?}): Enabling connections.", + index, incoming.peer_id); + debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", incoming.peer_id); + self.events.push(NetworkBehaviourAction::NotifyHandler { + peer_id: incoming.peer_id, + handler: NotifyHandler::All, + event: NotifsHandlerIn::Enable, + }); + *state = PeerState::Enabled { open: SmallVec::new() }; + } + peer => error!(target: "sub-libp2p", + "State mismatch in libp2p: Expected alive incoming. Got {:?}.", + peer) + } } - /// Function that is called when the peerset wants us to reject an incoming node. + /// Function that is called when the peerset wants us to reject an incoming peer. fn peerset_report_reject(&mut self, index: sc_peerset::IncomingIndex) { let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { self.incoming.remove(pos) @@ -700,30 +815,25 @@ impl GenericProto { return } - let state = if let Some(state) = self.peers.get_mut(&incoming.peer_id) { - state - } else { - error!(target: "sub-libp2p", "State mismatch in libp2p: no entry in peers \ - corresponding to an alive incoming"); - return - }; - - let connected_point = if let PeerState::Incoming { connected_point } = state { - connected_point.clone() - } else { - error!(target: "sub-libp2p", "State mismatch in libp2p: entry in peers corresponding \ - to an alive incoming is not in incoming state"); - return - }; - - debug!(target: "sub-libp2p", "PSM => Reject({:?}, {:?}): Rejecting connection through \ - {:?}", index, incoming.peer_id, connected_point); - debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", incoming.peer_id); - self.events.push(NetworkBehaviourAction::SendEvent { - peer_id: incoming.peer_id, - event: NotifsHandlerIn::Disable, - }); - *state = PeerState::Disabled { open: false, connected_point, banned_until: None }; + match self.peers.get_mut(&incoming.peer_id) { + Some(state @ PeerState::Incoming) => { + debug!(target: "sub-libp2p", "PSM => Reject({:?}, {:?}): Rejecting connections.", + index, incoming.peer_id); + debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", incoming.peer_id); + self.events.push(NetworkBehaviourAction::NotifyHandler { + peer_id: incoming.peer_id, + handler: NotifyHandler::All, + event: NotifsHandlerIn::Disable, + }); + *state = PeerState::Disabled { + open: SmallVec::new(), + banned_until: None + }; + } + peer => error!(target: "sub-libp2p", + "State mismatch in libp2p: Expected alive incoming. Got {:?}.", + peer) + } } } @@ -743,26 +853,32 @@ impl NetworkBehaviour for GenericProto { Vec::new() } - fn inject_connected(&mut self, peer_id: PeerId, connected_point: ConnectedPoint) { - match (self.peers.entry(peer_id.clone()).or_insert(PeerState::Poisoned), connected_point) { - (st @ &mut PeerState::Requested, connected_point) | - (st @ &mut PeerState::PendingRequest { .. }, connected_point) => { - debug!(target: "sub-libp2p", "Libp2p => Connected({:?}): Connection \ - requested by PSM (through {:?})", peer_id, connected_point + fn inject_connected(&mut self, _: &PeerId) { + } + + fn inject_connection_established(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + debug!(target: "sub-libp2p", "Libp2p => Connection ({:?},{:?}) to {} established.", + conn, endpoint, peer_id); + match (self.peers.entry(peer_id.clone()).or_insert(PeerState::Poisoned), endpoint) { + (st @ &mut PeerState::Requested, endpoint) | + (st @ &mut PeerState::PendingRequest { .. }, endpoint) => { + debug!(target: "sub-libp2p", + "Libp2p => Connected({}, {:?}): Connection was requested by PSM.", + peer_id, endpoint ); - debug!(target: "sub-libp2p", "Handler({:?}) <= Enable", peer_id); - self.events.push(NetworkBehaviourAction::SendEvent { + *st = PeerState::Enabled { open: SmallVec::new() }; + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), - event: NotifsHandlerIn::Enable, + handler: NotifyHandler::One(*conn), + event: NotifsHandlerIn::Enable }); - *st = PeerState::Enabled { open: false, connected_point }; } - // Note: it may seem weird that "Banned" nodes get treated as if there were absent. + // Note: it may seem weird that "Banned" peers get treated as if they were absent. // This is because the word "Banned" means "temporarily prevent outgoing connections to - // this node", and not "banned" in the sense that we would refuse the node altogether. - (st @ &mut PeerState::Poisoned, connected_point @ ConnectedPoint::Listener { .. }) | - (st @ &mut PeerState::Banned { .. }, connected_point @ ConnectedPoint::Listener { .. }) => { + // this peer", and not "banned" in the sense that we would refuse the peer altogether. + (st @ &mut PeerState::Poisoned, endpoint @ ConnectedPoint::Listener { .. }) | + (st @ &mut PeerState::Banned { .. }, endpoint @ ConnectedPoint::Listener { .. }) => { let incoming_id = self.next_incoming_index.clone(); self.next_incoming_index.0 = match self.next_incoming_index.0.checked_add(1) { Some(v) => v, @@ -771,61 +887,79 @@ impl NetworkBehaviour for GenericProto { return } }; - debug!(target: "sub-libp2p", "Libp2p => Connected({:?}): Incoming connection", - peer_id); - debug!(target: "sub-libp2p", "PSM <= Incoming({:?}, {:?}): Through {:?}", - incoming_id, peer_id, connected_point); + debug!(target: "sub-libp2p", "Libp2p => Connected({}, {:?}): Incoming connection", + peer_id, endpoint); + debug!(target: "sub-libp2p", "PSM <= Incoming({}, {:?}).", + peer_id, incoming_id); self.peerset.incoming(peer_id.clone(), incoming_id); self.incoming.push(IncomingPeer { peer_id: peer_id.clone(), alive: true, incoming_id, }); - *st = PeerState::Incoming { connected_point }; + *st = PeerState::Incoming { }; } - (st @ &mut PeerState::Poisoned, connected_point) | - (st @ &mut PeerState::Banned { .. }, connected_point) => { + (st @ &mut PeerState::Poisoned, endpoint) | + (st @ &mut PeerState::Banned { .. }, endpoint) => { let banned_until = if let PeerState::Banned { until } = st { Some(*until) } else { None }; - debug!(target: "sub-libp2p", "Libp2p => Connected({:?}): Requested by something \ - else than PSM, disabling", peer_id); - debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", peer_id); - self.events.push(NetworkBehaviourAction::SendEvent { + debug!(target: "sub-libp2p", + "Libp2p => Connected({},{:?}): Not requested by PSM, disabling.", + peer_id, endpoint); + *st = PeerState::Disabled { open: SmallVec::new(), banned_until }; + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), - event: NotifsHandlerIn::Disable, + handler: NotifyHandler::One(*conn), + event: NotifsHandlerIn::Disable }); - *st = PeerState::Disabled { open: false, connected_point, banned_until }; } - st => { - // This is a serious bug either in this state machine or in libp2p. - error!(target: "sub-libp2p", "Received inject_connected for \ - already-connected node; state is {:?}", st - ); + (PeerState::Incoming { .. }, _) => { + debug!(target: "sub-libp2p", + "Secondary connection {:?} to {} waiting for PSM decision.", + conn, peer_id); + }, + + (PeerState::Enabled { .. }, _) => { + debug!(target: "sub-libp2p", "Handler({},{:?}) <= Enable secondary connection", + peer_id, conn); + self.events.push(NetworkBehaviourAction::NotifyHandler { + peer_id: peer_id.clone(), + handler: NotifyHandler::One(*conn), + event: NotifsHandlerIn::Enable + }); + } + + (PeerState::Disabled { .. }, _) | (PeerState::DisabledPendingEnable { .. }, _) => { + debug!(target: "sub-libp2p", "Handler({},{:?}) <= Disable secondary connection", + peer_id, conn); + self.events.push(NetworkBehaviourAction::NotifyHandler { + peer_id: peer_id.clone(), + handler: NotifyHandler::One(*conn), + event: NotifsHandlerIn::Disable + }); } } } - fn inject_disconnected(&mut self, peer_id: &PeerId, endpoint: ConnectedPoint) { - match self.peers.remove(peer_id) { - None | Some(PeerState::Requested) | Some(PeerState::PendingRequest { .. }) | - Some(PeerState::Banned { .. }) => - // This is a serious bug either in this state machine or in libp2p. - error!(target: "sub-libp2p", "Received inject_disconnected for non-connected \ - node {:?}", peer_id), - - Some(PeerState::Disabled { open, banned_until, .. }) => { - debug!(target: "sub-libp2p", "Libp2p => Disconnected({:?}): Was disabled \ - (through {:?})", peer_id, endpoint); - if let Some(until) = banned_until { - self.peers.insert(peer_id.clone(), PeerState::Banned { until }); - } - if open { - debug!(target: "sub-libp2p", "External API <= Closed({:?})", peer_id); + fn inject_connection_closed(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + debug!(target: "sub-libp2p", "Libp2p => Connection ({:?},{:?}) to {} closed.", + conn, endpoint, peer_id); + match self.peers.get_mut(peer_id) { + Some(PeerState::Disabled { open, .. }) | + Some(PeerState::DisabledPendingEnable { open, .. }) | + Some(PeerState::Enabled { open, .. }) => { + // Check if the "link" to the peer is already considered closed, + // i.e. there is no connection that is open for custom protocols, + // in which case `CustomProtocolClosed` was already emitted. + let closed = open.is_empty(); + open.retain(|c| c != conn); + if open.is_empty() && !closed { + debug!(target: "sub-libp2p", "External API <= Closed({})", peer_id); let event = GenericProtoOut::CustomProtocolClosed { peer_id: peer_id.clone(), reason: "Disconnected by libp2p".into(), @@ -834,52 +968,52 @@ impl NetworkBehaviour for GenericProto { self.events.push(NetworkBehaviourAction::GenerateEvent(event)); } } + _ => {} + } + } - Some(PeerState::DisabledPendingEnable { open, timer_deadline, .. }) => { - debug!(target: "sub-libp2p", "Libp2p => Disconnected({:?}): Was disabled \ - (through {:?}) but pending enable", peer_id, endpoint); - debug!(target: "sub-libp2p", "PSM <= Dropped({:?})", peer_id); - self.peerset.dropped(peer_id.clone()); - self.peers.insert(peer_id.clone(), PeerState::Banned { until: timer_deadline }); - if open { - debug!(target: "sub-libp2p", "External API <= Closed({:?})", peer_id); - let event = GenericProtoOut::CustomProtocolClosed { - peer_id: peer_id.clone(), - reason: "Disconnected by libp2p".into(), - }; + fn inject_disconnected(&mut self, peer_id: &PeerId) { + match self.peers.remove(peer_id) { + None | Some(PeerState::Requested) | Some(PeerState::PendingRequest { .. }) | + Some(PeerState::Banned { .. }) => + // This is a serious bug either in this state machine or in libp2p. + error!(target: "sub-libp2p", + "`inject_disconnected` called for unknown peer {}", + peer_id), - self.events.push(NetworkBehaviourAction::GenerateEvent(event)); + Some(PeerState::Disabled { banned_until, .. }) => { + debug!(target: "sub-libp2p", "Libp2p => Disconnected({}): Was disabled.", peer_id); + if let Some(until) = banned_until { + self.peers.insert(peer_id.clone(), PeerState::Banned { until }); } } - Some(PeerState::Enabled { open, .. }) => { - debug!(target: "sub-libp2p", "Libp2p => Disconnected({:?}): Was enabled \ - (through {:?})", peer_id, endpoint); - debug!(target: "sub-libp2p", "PSM <= Dropped({:?})", peer_id); + Some(PeerState::DisabledPendingEnable { timer_deadline, .. }) => { + debug!(target: "sub-libp2p", + "Libp2p => Disconnected({}): Was disabled but pending enable.", + peer_id); + debug!(target: "sub-libp2p", "PSM <= Dropped({})", peer_id); self.peerset.dropped(peer_id.clone()); + self.peers.insert(peer_id.clone(), PeerState::Banned { until: timer_deadline }); + } + Some(PeerState::Enabled { .. }) => { + debug!(target: "sub-libp2p", "Libp2p => Disconnected({}): Was enabled.", peer_id); + debug!(target: "sub-libp2p", "PSM <= Dropped({})", peer_id); + self.peerset.dropped(peer_id.clone()); let ban_dur = Uniform::new(5, 10).sample(&mut rand::thread_rng()); self.peers.insert(peer_id.clone(), PeerState::Banned { until: Instant::now() + Duration::from_secs(ban_dur) }); - - if open { - debug!(target: "sub-libp2p", "External API <= Closed({:?})", peer_id); - let event = GenericProtoOut::CustomProtocolClosed { - peer_id: peer_id.clone(), - reason: "Disconnected by libp2p".into(), - }; - - self.events.push(NetworkBehaviourAction::GenerateEvent(event)); - } } // In the incoming state, we don't report "Dropped". Instead we will just ignore the // corresponding Accept/Reject. - Some(PeerState::Incoming { .. }) => { + Some(PeerState::Incoming { }) => { if let Some(state) = self.incoming.iter_mut().find(|i| i.peer_id == *peer_id) { - debug!(target: "sub-libp2p", "Libp2p => Disconnected({:?}): Was in incoming \ - mode (id {:?}, through {:?})", peer_id, state.incoming_id, endpoint); + debug!(target: "sub-libp2p", + "Libp2p => Disconnected({}): Was in incoming mode with id {:?}.", + peer_id, state.incoming_id); state.alive = false; } else { error!(target: "sub-libp2p", "State mismatch in libp2p: no entry in incoming \ @@ -888,7 +1022,7 @@ impl NetworkBehaviour for GenericProto { } Some(PeerState::Poisoned) => - error!(target: "sub-libp2p", "State of {:?} is poisoned", peer_id), + error!(target: "sub-libp2p", "State of peer {} is poisoned", peer_id), } } @@ -899,13 +1033,13 @@ impl NetworkBehaviour for GenericProto { fn inject_dial_failure(&mut self, peer_id: &PeerId) { if let Entry::Occupied(mut entry) = self.peers.entry(peer_id.clone()) { match mem::replace(entry.get_mut(), PeerState::Poisoned) { - // The node is not in our list. + // The peer is not in our list. st @ PeerState::Banned { .. } => { trace!(target: "sub-libp2p", "Libp2p => Dial failure for {:?}", peer_id); *entry.into_mut() = st; }, - // "Basic" situation: we failed to reach a node that the peerset requested. + // "Basic" situation: we failed to reach a peer that the peerset requested. PeerState::Requested | PeerState::PendingRequest { .. } => { debug!(target: "sub-libp2p", "Libp2p => Dial failure for {:?}", peer_id); *entry.into_mut() = PeerState::Banned { @@ -915,7 +1049,7 @@ impl NetworkBehaviour for GenericProto { self.peerset.dropped(peer_id.clone()) }, - // We can still get dial failures even if we are already connected to the node, + // We can still get dial failures even if we are already connected to the peer, // as an extra diagnostic for an earlier attempt. st @ PeerState::Disabled { .. } | st @ PeerState::Enabled { .. } | st @ PeerState::DisabledPendingEnable { .. } | st @ PeerState::Incoming { .. } => { @@ -928,92 +1062,130 @@ impl NetworkBehaviour for GenericProto { } } else { - // The node is not in our list. + // The peer is not in our list. trace!(target: "sub-libp2p", "Libp2p => Dial failure for {:?}", peer_id); } } - fn inject_node_event( + fn inject_event( &mut self, source: PeerId, + connection: ConnectionId, event: NotifsHandlerOut, ) { match event { - NotifsHandlerOut::Closed { reason } => { - debug!(target: "sub-libp2p", "Handler({:?}) => Closed: {}", source, reason); + NotifsHandlerOut::Closed { endpoint, reason } => { + debug!(target: "sub-libp2p", + "Handler({:?}) => Endpoint {:?} closed for custom protocols: {}", + source, endpoint, reason); let mut entry = if let Entry::Occupied(entry) = self.peers.entry(source.clone()) { entry } else { - error!(target: "sub-libp2p", "State mismatch in the custom protos handler"); + error!(target: "sub-libp2p", "Closed: State mismatch in the custom protos handler"); return }; - debug!(target: "sub-libp2p", "External API <= Closed({:?})", source); - let event = GenericProtoOut::CustomProtocolClosed { - reason, - peer_id: source.clone(), - }; - self.events.push(NetworkBehaviourAction::GenerateEvent(event)); - - match mem::replace(entry.get_mut(), PeerState::Poisoned) { - PeerState::Enabled { open, connected_point } => { - debug_assert!(open); - - debug!(target: "sub-libp2p", "PSM <= Dropped({:?})", source); - self.peerset.dropped(source.clone()); + let last = match mem::replace(entry.get_mut(), PeerState::Poisoned) { + PeerState::Enabled { mut open } => { + debug_assert!(open.iter().any(|c| c == &connection)); + open.retain(|c| c != &connection); debug!(target: "sub-libp2p", "Handler({:?}) <= Disable", source); - self.events.push(NetworkBehaviourAction::SendEvent { + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: source.clone(), + handler: NotifyHandler::One(connection), event: NotifsHandlerIn::Disable, }); + let last = open.is_empty(); + + if last { + debug!(target: "sub-libp2p", "PSM <= Dropped({:?})", source); + self.peerset.dropped(source.clone()); + *entry.into_mut() = PeerState::Disabled { + open, + banned_until: None + }; + } else { + *entry.into_mut() = PeerState::Enabled { open }; + } + + last + }, + PeerState::Disabled { mut open, banned_until } => { + debug_assert!(open.iter().any(|c| c == &connection)); + open.retain(|c| c != &connection); + let last = open.is_empty(); *entry.into_mut() = PeerState::Disabled { - open: false, - connected_point, - banned_until: None + open, + banned_until }; + last }, - PeerState::Disabled { open, connected_point, banned_until } => { - debug_assert!(open); - *entry.into_mut() = PeerState::Disabled { open: false, connected_point, banned_until }; - }, - PeerState::DisabledPendingEnable { open, connected_point, timer, timer_deadline } => { - debug_assert!(open); + PeerState::DisabledPendingEnable { + mut open, + timer, + timer_deadline + } => { + debug_assert!(open.iter().any(|c| c == &connection)); + open.retain(|c| c != &connection); + let last = open.is_empty(); *entry.into_mut() = PeerState::DisabledPendingEnable { - open: false, - connected_point, + open, timer, timer_deadline }; + last }, - _ => error!(target: "sub-libp2p", "State mismatch in the custom protos handler"), + state => { + error!(target: "sub-libp2p", + "Unexpected state in the custom protos handler: {:?}", + state); + return + } + }; + + if last { + debug!(target: "sub-libp2p", "External API <= Closed({:?})", source); + let event = GenericProtoOut::CustomProtocolClosed { + reason, + peer_id: source.clone(), + }; + self.events.push(NetworkBehaviourAction::GenerateEvent(event)); + } else { + debug!(target: "sub-libp2p", "Secondary connection closed custom protocol."); } } - NotifsHandlerOut::Open => { - debug!(target: "sub-libp2p", "Handler({:?}) => Open", source); - let endpoint = match self.peers.get_mut(&source) { - Some(PeerState::Enabled { ref mut open, ref connected_point }) | - Some(PeerState::DisabledPendingEnable { ref mut open, ref connected_point, .. }) | - Some(PeerState::Disabled { ref mut open, ref connected_point, .. }) if !*open => { - *open = true; - connected_point.clone() + NotifsHandlerOut::Open { endpoint } => { + debug!(target: "sub-libp2p", + "Handler({:?}) => Endpoint {:?} open for custom protocols.", + source, endpoint); + + let first = match self.peers.get_mut(&source) { + Some(PeerState::Enabled { ref mut open, .. }) | + Some(PeerState::DisabledPendingEnable { ref mut open, .. }) | + Some(PeerState::Disabled { ref mut open, .. }) => { + let first = open.is_empty(); + open.push(connection); + first } - _ => { - error!(target: "sub-libp2p", "State mismatch in the custom protos handler"); + state => { + error!(target: "sub-libp2p", + "Open: Unexpected state in the custom protos handler: {:?}", + state); return } }; - debug!(target: "sub-libp2p", "External API <= Open({:?})", source); - let event = GenericProtoOut::CustomProtocolOpen { - peer_id: source, - endpoint, - }; - - self.events.push(NetworkBehaviourAction::GenerateEvent(event)); + if first { + debug!(target: "sub-libp2p", "External API <= Open({:?})", source); + let event = GenericProtoOut::CustomProtocolOpen { peer_id: source }; + self.events.push(NetworkBehaviourAction::GenerateEvent(event)); + } else { + debug!(target: "sub-libp2p", "Secondary connection opened custom protocol."); + } } NotifsHandlerOut::CustomMessage { message } => { @@ -1065,11 +1237,12 @@ impl NetworkBehaviour for GenericProto { } NotifsHandlerOut::ProtocolError { error, .. } => { - debug!(target: "sub-libp2p", "Handler({:?}) => Severe protocol error: {:?}", + debug!(target: "sub-libp2p", + "Handler({:?}) => Severe protocol error: {:?}", source, error); - // A severe protocol error happens when we detect a "bad" node, such as a node on - // a different chain, or a node that doesn't speak the same protocol(s). We - // decrease the node's reputation, hence lowering the chances we try this node + // A severe protocol error happens when we detect a "bad" peer, such as a peer on + // a different chain, or a peer that doesn't speak the same protocol(s). We + // decrease the peer's reputation, hence lowering the chances we try this peer // again in the short term. self.peerset.report_peer( source.clone(), @@ -1123,27 +1296,34 @@ impl NetworkBehaviour for GenericProto { } debug!(target: "sub-libp2p", "Libp2p <= Dial {:?} now that ban has expired", peer_id); - self.events.push(NetworkBehaviourAction::DialPeer { peer_id: peer_id.clone() }); + self.events.push(NetworkBehaviourAction::DialPeer { + peer_id: peer_id.clone(), + condition: DialPeerCondition::Disconnected + }); *peer_state = PeerState::Requested; } - PeerState::DisabledPendingEnable { mut timer, connected_point, open, timer_deadline } => { + PeerState::DisabledPendingEnable { + mut timer, + open, + timer_deadline + } => { if let Poll::Pending = Pin::new(&mut timer).poll(cx) { *peer_state = PeerState::DisabledPendingEnable { timer, - connected_point, open, timer_deadline }; continue; } - debug!(target: "sub-libp2p", "Handler({:?}) <= Enable now that ban has expired", peer_id); - self.events.push(NetworkBehaviourAction::SendEvent { + debug!(target: "sub-libp2p", "Handler({:?}) <= Enable (ban expired)", peer_id); + self.events.push(NetworkBehaviourAction::NotifyHandler { peer_id: peer_id.clone(), + handler: NotifyHandler::All, event: NotifsHandlerIn::Enable, }); - *peer_state = PeerState::Enabled { connected_point, open }; + *peer_state = PeerState::Enabled { open }; } st @ _ => *peer_state = st, diff --git a/client/network/src/protocol/generic_proto/handler.rs b/client/network/src/protocol/generic_proto/handler.rs index e97176cfbbfbb98cfa6f1f1d2f25dc9f5f777990..f0e2fc4bb8a8d70af5ef58298fa6d45be07228d8 100644 --- a/client/network/src/protocol/generic_proto/handler.rs +++ b/client/network/src/protocol/generic_proto/handler.rs @@ -15,6 +15,7 @@ // along with Substrate. If not, see . pub use self::group::{NotifsHandlerProto, NotifsHandler, NotifsHandlerIn, NotifsHandlerOut}; +pub use self::legacy::ConnectionKillError as LegacyConnectionKillError; mod group; mod legacy; diff --git a/client/network/src/protocol/generic_proto/handler/group.rs b/client/network/src/protocol/generic_proto/handler/group.rs index 21dc4091c0c4e70beec71c31f8263362ade80d3d..0e453a368c222c38795439d8510240aa3fd93268 100644 --- a/client/network/src/protocol/generic_proto/handler/group.rs +++ b/client/network/src/protocol/generic_proto/handler/group.rs @@ -75,11 +75,12 @@ use std::{borrow::Cow, error, io, str, task::{Context, Poll}}; /// /// See the documentation at the module level for more information. pub struct NotifsHandlerProto { - /// Prototypes for handlers for inbound substreams. - in_handlers: Vec, + /// Prototypes for handlers for inbound substreams, and the message we respond with in the + /// handshake. + in_handlers: Vec<(NotifsInHandlerProto, Vec)>, - /// Prototypes for handlers for outbound substreams. - out_handlers: Vec, + /// Prototypes for handlers for outbound substreams, and the initial handshake message we send. + out_handlers: Vec<(NotifsOutHandlerProto, Vec)>, /// Prototype for handler for backwards-compatibility. legacy: LegacyProtoHandlerProto, @@ -89,11 +90,11 @@ pub struct NotifsHandlerProto { /// /// See the documentation at the module level for more information. pub struct NotifsHandler { - /// Handlers for inbound substreams. - in_handlers: Vec, + /// Handlers for inbound substreams, and the message we respond with in the handshake. + in_handlers: Vec<(NotifsInHandler, Vec)>, - /// Handlers for outbound substreams. - out_handlers: Vec, + /// Handlers for outbound substreams, and the initial handshake message we send. + out_handlers: Vec<(NotifsOutHandler, Vec)>, /// Handler for backwards-compatibility. legacy: LegacyProtoHandler, @@ -119,7 +120,7 @@ impl IntoProtocolsHandler for NotifsHandlerProto { fn inbound_protocol(&self) -> SelectUpgrade, RegisteredProtocol> { let in_handlers = self.in_handlers.iter() - .map(|h| h.inbound_protocol()) + .map(|(h, _)| h.inbound_protocol()) .collect::>(); SelectUpgrade::new(in_handlers, self.legacy.inbound_protocol()) @@ -129,11 +130,11 @@ impl IntoProtocolsHandler for NotifsHandlerProto { NotifsHandler { in_handlers: self.in_handlers .into_iter() - .map(|p| p.into_handler(remote_peer_id, connected_point)) + .map(|(proto, msg)| (proto.into_handler(remote_peer_id, connected_point), msg)) .collect(), out_handlers: self.out_handlers .into_iter() - .map(|p| p.into_handler(remote_peer_id, connected_point)) + .map(|(proto, msg)| (proto.into_handler(remote_peer_id, connected_point), msg)) .collect(), legacy: self.legacy.into_handler(remote_peer_id, connected_point), enabled: EnabledState::Initial, @@ -143,7 +144,7 @@ impl IntoProtocolsHandler for NotifsHandlerProto { } /// Event that can be received by a `NotifsHandler`. -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum NotifsHandlerIn { /// The node should start using custom protocols. Enable, @@ -160,6 +161,18 @@ pub enum NotifsHandlerIn { message: Vec, }, + /// Modifies the handshake message of a notifications protocol. + UpdateHandshake { + /// Name of the protocol for the message. + /// + /// Must match one of the registered protocols. + protocol_name: Cow<'static, [u8]>, + + /// The new handshake message to send if we open a substream or if the remote opens a + /// substream towards us. + handshake_message: Vec, + }, + /// Sends a notifications message. SendNotification { /// Name of the protocol for the message. @@ -181,13 +194,18 @@ pub enum NotifsHandlerIn { /// Event that can be emitted by a `NotifsHandler`. #[derive(Debug)] pub enum NotifsHandlerOut { - /// Opened the substreams with the remote. - Open, + /// The connection is open for custom protocols. + Open { + /// The endpoint of the connection that is open for custom protocols. + endpoint: ConnectedPoint, + }, - /// Closed the substreams with the remote. + /// The connection is closed for custom protocols. Closed { - /// Reason why the substream closed, for diagnostic purposes. + /// The reason for closing, for diagnostic purposes. reason: Cow<'static, str>, + /// The endpoint of the connection that closed for custom protocols. + endpoint: ConnectedPoint, }, /// Received a non-gossiping message on the legacy substream. @@ -227,28 +245,42 @@ pub enum NotifsHandlerOut { impl NotifsHandlerProto { /// Builds a new handler. /// + /// `list` is a list of notification protocols names, and the message to send as part of the + /// handshake. At the moment, the message is always the same whether we open a substream + /// ourselves or respond to handshake from the remote. + /// /// The `queue_size_report` is an optional Prometheus metric that can report the size of the /// messages queue. If passed, it must have one label for the protocol name. - pub fn new(legacy: RegisteredProtocol, list: impl Into, Vec)>>, queue_size_report: Option) -> Self { + pub fn new( + legacy: RegisteredProtocol, + list: impl Into, Vec)>>, + queue_size_report: Option + ) -> Self { let list = list.into(); let out_handlers = list .clone() .into_iter() - .map(|(p, _)| { + .map(|(proto_name, initial_message)| { let queue_size_report = queue_size_report.as_ref().and_then(|qs| { - if let Ok(utf8) = str::from_utf8(&p) { + if let Ok(utf8) = str::from_utf8(&proto_name) { Some(qs.with_label_values(&[utf8])) } else { - log::warn!("Ignoring Prometheus metric because {:?} isn't UTF-8", p); + log::warn!("Ignoring Prometheus metric because {:?} isn't UTF-8", proto_name); None } }); - NotifsOutHandlerProto::new(p, queue_size_report) + + (NotifsOutHandlerProto::new(proto_name, queue_size_report), initial_message) }).collect(); + let in_handlers = list.clone() + .into_iter() + .map(|(proto_name, msg)| (NotifsInHandlerProto::new(proto_name), msg)) + .collect(); + NotifsHandlerProto { - in_handlers: list.clone().into_iter().map(|(p, _)| NotifsInHandlerProto::new(p)).collect(), + in_handlers, out_handlers, legacy: LegacyProtoHandlerProto::new(legacy), } @@ -272,7 +304,7 @@ impl ProtocolsHandler for NotifsHandler { fn listen_protocol(&self) -> SubstreamProtocol { let in_handlers = self.in_handlers.iter() - .map(|h| h.listen_protocol().into_upgrade().1) + .map(|(h, _)| h.listen_protocol().into_upgrade().1) .collect::>(); let proto = SelectUpgrade::new(in_handlers, self.legacy.listen_protocol().into_upgrade().1); @@ -285,7 +317,7 @@ impl ProtocolsHandler for NotifsHandler { ) { match out { EitherOutput::First((out, num)) => - self.in_handlers[num].inject_fully_negotiated_inbound(out), + self.in_handlers[num].0.inject_fully_negotiated_inbound(out), EitherOutput::Second(out) => self.legacy.inject_fully_negotiated_inbound(out), } @@ -298,7 +330,7 @@ impl ProtocolsHandler for NotifsHandler { ) { match (out, num) { (EitherOutput::First(out), Some(num)) => - self.out_handlers[num].inject_fully_negotiated_outbound(out, ()), + self.out_handlers[num].0.inject_fully_negotiated_outbound(out, ()), (EitherOutput::Second(out), None) => self.legacy.inject_fully_negotiated_outbound(out, ()), _ => error!("inject_fully_negotiated_outbound called with wrong parameters"), @@ -313,13 +345,15 @@ impl ProtocolsHandler for NotifsHandler { } self.enabled = EnabledState::Enabled; self.legacy.inject_event(LegacyProtoHandlerIn::Enable); - for handler in &mut self.out_handlers { + for (handler, initial_message) in &mut self.out_handlers { handler.inject_event(NotifsOutHandlerIn::Enable { - initial_message: vec![] + initial_message: initial_message.clone(), }); } for num in self.pending_in.drain(..) { - self.in_handlers[num].inject_event(NotifsInHandlerIn::Accept(vec![])); + let handshake_message = self.in_handlers[num].1.clone(); + self.in_handlers[num].0 + .inject_event(NotifsInHandlerIn::Accept(handshake_message)); } }, NotifsHandlerIn::Disable => { @@ -330,19 +364,31 @@ impl ProtocolsHandler for NotifsHandler { // The notifications protocols start in the disabled state. If we were in the // "Initial" state, then we shouldn't disable the notifications protocols again. if self.enabled != EnabledState::Initial { - for handler in &mut self.out_handlers { + for (handler, _) in &mut self.out_handlers { handler.inject_event(NotifsOutHandlerIn::Disable); } } self.enabled = EnabledState::Disabled; for num in self.pending_in.drain(..) { - self.in_handlers[num].inject_event(NotifsInHandlerIn::Refuse); + self.in_handlers[num].0.inject_event(NotifsInHandlerIn::Refuse); } }, NotifsHandlerIn::SendLegacy { message } => self.legacy.inject_event(LegacyProtoHandlerIn::SendCustomMessage { message }), + NotifsHandlerIn::UpdateHandshake { protocol_name, handshake_message } => { + for (handler, current_handshake) in &mut self.in_handlers { + if handler.protocol_name() == &*protocol_name { + *current_handshake = handshake_message.clone(); + } + } + for (handler, current_handshake) in &mut self.out_handlers { + if handler.protocol_name() == &*protocol_name { + *current_handshake = handshake_message.clone(); + } + } + } NotifsHandlerIn::SendNotification { message, encoded_fallback_message, protocol_name } => { - for handler in &mut self.out_handlers { + for (handler, _) in &mut self.out_handlers { if handler.protocol_name() != &protocol_name[..] { continue; } @@ -367,21 +413,21 @@ impl ProtocolsHandler for NotifsHandler { ) { match (err, num) { (ProtocolsHandlerUpgrErr::Timeout, Some(num)) => - self.out_handlers[num].inject_dial_upgrade_error( + self.out_handlers[num].0.inject_dial_upgrade_error( (), ProtocolsHandlerUpgrErr::Timeout ), (ProtocolsHandlerUpgrErr::Timeout, None) => self.legacy.inject_dial_upgrade_error((), ProtocolsHandlerUpgrErr::Timeout), (ProtocolsHandlerUpgrErr::Timer, Some(num)) => - self.out_handlers[num].inject_dial_upgrade_error( + self.out_handlers[num].0.inject_dial_upgrade_error( (), ProtocolsHandlerUpgrErr::Timer ), (ProtocolsHandlerUpgrErr::Timer, None) => self.legacy.inject_dial_upgrade_error((), ProtocolsHandlerUpgrErr::Timer), (ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)), Some(num)) => - self.out_handlers[num].inject_dial_upgrade_error( + self.out_handlers[num].0.inject_dial_upgrade_error( (), ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)) ), @@ -391,7 +437,7 @@ impl ProtocolsHandler for NotifsHandler { ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Select(err)) ), (ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(EitherError::A(err))), Some(num)) => - self.out_handlers[num].inject_dial_upgrade_error( + self.out_handlers[num].0.inject_dial_upgrade_error( (), ProtocolsHandlerUpgrErr::Upgrade(UpgradeError::Apply(err)) ), @@ -412,7 +458,7 @@ impl ProtocolsHandler for NotifsHandler { return KeepAlive::Yes; } - for handler in &self.in_handlers { + for (handler, _) in &self.in_handlers { let val = handler.connection_keep_alive(); if val.is_yes() { return KeepAlive::Yes; @@ -420,7 +466,7 @@ impl ProtocolsHandler for NotifsHandler { if ret < val { ret = val; } } - for handler in &self.out_handlers { + for (handler, _) in &self.out_handlers { let val = handler.connection_keep_alive(); if val.is_yes() { return KeepAlive::Yes; @@ -437,7 +483,39 @@ impl ProtocolsHandler for NotifsHandler { ) -> Poll< ProtocolsHandlerEvent > { - for (handler_num, handler) in self.in_handlers.iter_mut().enumerate() { + while let Poll::Ready(ev) = self.legacy.poll(cx) { + match ev { + ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () } => + return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { + protocol: protocol.map_upgrade(EitherUpgrade::B), + info: None, + }), + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolOpen { endpoint, .. }) => + return Poll::Ready(ProtocolsHandlerEvent::Custom( + NotifsHandlerOut::Open { endpoint } + )), + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolClosed { endpoint, reason }) => + return Poll::Ready(ProtocolsHandlerEvent::Custom( + NotifsHandlerOut::Closed { endpoint, reason } + )), + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomMessage { message }) => + return Poll::Ready(ProtocolsHandlerEvent::Custom( + NotifsHandlerOut::CustomMessage { message } + )), + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::Clogged { messages }) => + return Poll::Ready(ProtocolsHandlerEvent::Custom( + NotifsHandlerOut::Clogged { messages } + )), + ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::ProtocolError { is_severe, error }) => + return Poll::Ready(ProtocolsHandlerEvent::Custom( + NotifsHandlerOut::ProtocolError { is_severe, error } + )), + ProtocolsHandlerEvent::Close(err) => + return Poll::Ready(ProtocolsHandlerEvent::Close(EitherError::B(err))), + } + } + + for (handler_num, (handler, handshake_message)) in self.in_handlers.iter_mut().enumerate() { while let Poll::Ready(ev) = handler.poll(cx) { match ev { ProtocolsHandlerEvent::OutboundSubstreamRequest { .. } => @@ -447,7 +525,7 @@ impl ProtocolsHandler for NotifsHandler { match self.enabled { EnabledState::Initial => self.pending_in.push(handler_num), EnabledState::Enabled => - handler.inject_event(NotifsInHandlerIn::Accept(vec![])), + handler.inject_event(NotifsInHandlerIn::Accept(handshake_message.clone())), EnabledState::Disabled => handler.inject_event(NotifsInHandlerIn::Refuse), }, @@ -467,7 +545,7 @@ impl ProtocolsHandler for NotifsHandler { } } - for (handler_num, handler) in self.out_handlers.iter_mut().enumerate() { + for (handler_num, (handler, _)) in self.out_handlers.iter_mut().enumerate() { while let Poll::Ready(ev) = handler.poll(cx) { match ev { ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () } => @@ -490,38 +568,6 @@ impl ProtocolsHandler for NotifsHandler { } } - while let Poll::Ready(ev) = self.legacy.poll(cx) { - match ev { - ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol, info: () } => - return Poll::Ready(ProtocolsHandlerEvent::OutboundSubstreamRequest { - protocol: protocol.map_upgrade(EitherUpgrade::B), - info: None, - }), - ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolOpen { .. }) => - return Poll::Ready(ProtocolsHandlerEvent::Custom( - NotifsHandlerOut::Open - )), - ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomProtocolClosed { reason }) => - return Poll::Ready(ProtocolsHandlerEvent::Custom( - NotifsHandlerOut::Closed { reason } - )), - ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::CustomMessage { message }) => - return Poll::Ready(ProtocolsHandlerEvent::Custom( - NotifsHandlerOut::CustomMessage { message } - )), - ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::Clogged { messages }) => - return Poll::Ready(ProtocolsHandlerEvent::Custom( - NotifsHandlerOut::Clogged { messages } - )), - ProtocolsHandlerEvent::Custom(LegacyProtoHandlerOut::ProtocolError { is_severe, error }) => - return Poll::Ready(ProtocolsHandlerEvent::Custom( - NotifsHandlerOut::ProtocolError { is_severe, error } - )), - ProtocolsHandlerEvent::Close(err) => - return Poll::Ready(ProtocolsHandlerEvent::Close(EitherError::B(err))), - } - } - Poll::Pending } } diff --git a/client/network/src/protocol/generic_proto/handler/legacy.rs b/client/network/src/protocol/generic_proto/handler/legacy.rs index a2d2fc9246d1c79b761d5e5eb1ce375fba33b49a..bc84fd847c9c4ba44dbffcf5a7735c18c39f7776 100644 --- a/client/network/src/protocol/generic_proto/handler/legacy.rs +++ b/client/network/src/protocol/generic_proto/handler/legacy.rs @@ -40,9 +40,8 @@ use std::{pin::Pin, task::{Context, Poll}}; /// it is turned into a `LegacyProtoHandler`. It then handles all communications that are specific /// to Substrate on that single connection. /// -/// Note that there can be multiple instance of this struct simultaneously for same peer. However -/// if that happens, only one main instance can communicate with the outer layers of the code. In -/// other words, the outer layers of the code only ever see one handler. +/// Note that there can be multiple instance of this struct simultaneously for same peer, +/// if there are multiple established connections to the peer. /// /// ## State of the handler /// @@ -61,6 +60,7 @@ use std::{pin::Pin, task::{Context, Poll}}; /// these states. For example, if the handler reports a network misbehaviour, it will close the /// substreams but it is the role of the user to send a `Disabled` event if it wants the connection /// to close. Otherwise, the handler will try to reopen substreams. +/// /// The handler starts in the "Initializing" state and must be transitionned to Enabled or Disabled /// as soon as possible. /// @@ -111,7 +111,7 @@ impl IntoProtocolsHandler for LegacyProtoHandlerProto { fn into_handler(self, remote_peer_id: &PeerId, connected_point: &ConnectedPoint) -> Self::Handler { LegacyProtoHandler { protocol: self.protocol, - endpoint: connected_point.to_endpoint(), + endpoint: connected_point.clone(), remote_peer_id: remote_peer_id.clone(), state: ProtocolState::Init { substreams: SmallVec::new(), @@ -136,7 +136,7 @@ pub struct LegacyProtoHandler { /// Whether we are the connection dialer or listener. Used to determine who, between the local /// node and the remote node, has priority. - endpoint: Endpoint, + endpoint: ConnectedPoint, /// Queue of events to send to the outside. /// @@ -218,12 +218,16 @@ pub enum LegacyProtoHandlerOut { CustomProtocolOpen { /// Version of the protocol that has been opened. version: u8, + /// The connected endpoint. + endpoint: ConnectedPoint, }, /// Closed a custom protocol with the remote. CustomProtocolClosed { /// Reason why the substream closed, for diagnostic purposes. reason: Cow<'static, str>, + /// The connected endpoint. + endpoint: ConnectedPoint, }, /// Receives a message on a custom protocol substream. @@ -272,7 +276,7 @@ impl LegacyProtoHandler { ProtocolState::Init { substreams: incoming, .. } => { if incoming.is_empty() { - if let Endpoint::Dialer = self.endpoint { + if let ConnectedPoint::Dialer { .. } = self.endpoint { self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest { protocol: SubstreamProtocol::new(self.protocol.clone()), info: (), @@ -281,10 +285,10 @@ impl LegacyProtoHandler { ProtocolState::Opening { deadline: Delay::new(Duration::from_secs(60)) } - } else { let event = LegacyProtoHandlerOut::CustomProtocolOpen { - version: incoming[0].protocol_version() + version: incoming[0].protocol_version(), + endpoint: self.endpoint.clone() }; self.events_queue.push(ProtocolsHandlerEvent::Custom(event)); ProtocolState::Normal { @@ -404,6 +408,7 @@ impl LegacyProtoHandler { if substreams.is_empty() { let event = LegacyProtoHandlerOut::CustomProtocolClosed { reason: "All substreams have been closed by the remote".into(), + endpoint: self.endpoint.clone() }; self.state = ProtocolState::Disabled { shutdown: shutdown.into_iter().collect(), @@ -416,6 +421,7 @@ impl LegacyProtoHandler { if substreams.is_empty() { let event = LegacyProtoHandlerOut::CustomProtocolClosed { reason: format!("Error on the last substream: {:?}", err).into(), + endpoint: self.endpoint.clone() }; self.state = ProtocolState::Disabled { shutdown: shutdown.into_iter().collect(), @@ -479,7 +485,8 @@ impl LegacyProtoHandler { ProtocolState::Opening { .. } => { let event = LegacyProtoHandlerOut::CustomProtocolOpen { - version: substream.protocol_version() + version: substream.protocol_version(), + endpoint: self.endpoint.clone() }; self.events_queue.push(ProtocolsHandlerEvent::Custom(event)); ProtocolState::Normal { diff --git a/client/network/src/protocol/generic_proto/handler/notif_in.rs b/client/network/src/protocol/generic_proto/handler/notif_in.rs index 7558d1d361fd7cf445271b4351cd0c7c5fdd45d7..83923154bd6256de3bb4e983796807093f7019ad 100644 --- a/client/network/src/protocol/generic_proto/handler/notif_in.rs +++ b/client/network/src/protocol/generic_proto/handler/notif_in.rs @@ -72,7 +72,7 @@ pub struct NotifsInHandler { } /// Event that can be received by a `NotifsInHandler`. -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum NotifsInHandlerIn { /// Can be sent back as a response to an `OpenRequest`. Contains the status message to send /// to the remote. diff --git a/client/network/src/protocol/generic_proto/handler/notif_out.rs b/client/network/src/protocol/generic_proto/handler/notif_out.rs index dd38826496e7a3f8289131183d27f7f2c235c230..b5d6cd61ada2a9e54bfa5873e0f7992ab1dfcfcb 100644 --- a/client/network/src/protocol/generic_proto/handler/notif_out.rs +++ b/client/network/src/protocol/generic_proto/handler/notif_out.rs @@ -33,7 +33,7 @@ use libp2p::swarm::{ SubstreamProtocol, NegotiatedSubstream, }; -use log::error; +use log::{debug, warn, error}; use prometheus_endpoint::Histogram; use smallvec::SmallVec; use std::{borrow::Cow, fmt, mem, pin::Pin, task::{Context, Poll}, time::Duration}; @@ -280,7 +280,7 @@ impl ProtocolsHandler for NotifsOutHandler { // be recovered. When in doubt, let's drop the existing substream and // open a new one. if sub.close().now_or_never().is_none() { - log::warn!( + warn!( target: "sub-libp2p", "📞 Improperly closed outbound notifications substream" ); @@ -293,16 +293,22 @@ impl ProtocolsHandler for NotifsOutHandler { }); self.state = State::Opening { initial_message }; }, - State::Opening { .. } | State::Refused | State::Open { .. } => - error!("☎️ Tried to enable notifications handler that was already enabled"), - State::Poisoned => error!("☎️ Notifications handler in a poisoned state"), + st @ State::Opening { .. } | st @ State::Refused | st @ State::Open { .. } => { + debug!(target: "sub-libp2p", + "Tried to enable notifications handler that was already enabled"); + self.state = st; + } + State::Poisoned => error!("Notifications handler in a poisoned state"), } } NotifsOutHandlerIn::Disable => { match mem::replace(&mut self.state, State::Poisoned) { - State::Disabled | State::DisabledOpen(_) | State::DisabledOpening => - error!("☎️ Tried to disable notifications handler that was already disabled"), + st @ State::Disabled | st @ State::DisabledOpen(_) | st @ State::DisabledOpening => { + debug!(target: "sub-libp2p", + "Tried to disable notifications handler that was already disabled"); + self.state = st; + } State::Opening { .. } => self.state = State::DisabledOpening, State::Refused => self.state = State::Disabled, State::Open { substream, .. } => self.state = State::DisabledOpen(substream), @@ -313,7 +319,7 @@ impl ProtocolsHandler for NotifsOutHandler { NotifsOutHandlerIn::Send(msg) => if let State::Open { substream, .. } = &mut self.state { if substream.push_message(msg).is_err() { - log::warn!( + warn!( target: "sub-libp2p", "📞 Notifications queue with peer {} is full, dropped message (protocol: {:?})", self.peer_id, @@ -325,7 +331,7 @@ impl ProtocolsHandler for NotifsOutHandler { } } else { // This is an API misuse. - log::warn!( + warn!( target: "sub-libp2p", "📞 Tried to send a notification on a disabled handler" ); diff --git a/client/network/src/protocol/generic_proto/tests.rs b/client/network/src/protocol/generic_proto/tests.rs index 4548859ac420cd0ab52ed9c9c5cffd9167aef92b..1bc6e745f887685af41a6c9cfdfc05da84af7918 100644 --- a/client/network/src/protocol/generic_proto/tests.rs +++ b/client/network/src/protocol/generic_proto/tests.rs @@ -18,7 +18,7 @@ use futures::{prelude::*, ready}; use codec::{Encode, Decode}; -use libp2p::core::nodes::listeners::ListenerId; +use libp2p::core::connection::{ConnectionId, ListenerId}; use libp2p::core::ConnectedPoint; use libp2p::swarm::{Swarm, ProtocolsHandler, IntoProtocolsHandler}; use libp2p::swarm::{PollParameters, NetworkBehaviour, NetworkBehaviourAction}; @@ -148,20 +148,29 @@ impl NetworkBehaviour for CustomProtoWithAddr { list } - fn inject_connected(&mut self, peer_id: PeerId, endpoint: ConnectedPoint) { - self.inner.inject_connected(peer_id, endpoint) + fn inject_connected(&mut self, peer_id: &PeerId) { + self.inner.inject_connected(peer_id) } - fn inject_disconnected(&mut self, peer_id: &PeerId, endpoint: ConnectedPoint) { - self.inner.inject_disconnected(peer_id, endpoint) + fn inject_disconnected(&mut self, peer_id: &PeerId) { + self.inner.inject_disconnected(peer_id) } - fn inject_node_event( + fn inject_connection_established(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + self.inner.inject_connection_established(peer_id, conn, endpoint) + } + + fn inject_connection_closed(&mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint) { + self.inner.inject_connection_closed(peer_id, conn, endpoint) + } + + fn inject_event( &mut self, peer_id: PeerId, + connection: ConnectionId, event: <::Handler as ProtocolsHandler>::OutEvent ) { - self.inner.inject_node_event(peer_id, event) + self.inner.inject_event(peer_id, connection, event) } fn poll( @@ -177,10 +186,6 @@ impl NetworkBehaviour for CustomProtoWithAddr { self.inner.poll(cx, params) } - fn inject_replaced(&mut self, peer_id: PeerId, closed_endpoint: ConnectedPoint, new_endpoint: ConnectedPoint) { - self.inner.inject_replaced(peer_id, closed_endpoint, new_endpoint) - } - fn inject_addr_reach_failure(&mut self, peer_id: Option<&PeerId>, addr: &Multiaddr, error: &dyn std::error::Error) { self.inner.inject_addr_reach_failure(peer_id, addr, error) } @@ -205,8 +210,8 @@ impl NetworkBehaviour for CustomProtoWithAddr { self.inner.inject_listener_error(id, err); } - fn inject_listener_closed(&mut self, id: ListenerId) { - self.inner.inject_listener_closed(id); + fn inject_listener_closed(&mut self, id: ListenerId, reason: Result<(), &io::Error>) { + self.inner.inject_listener_closed(id, reason); } } diff --git a/client/network/src/protocol/generic_proto/upgrade/notifications.rs b/client/network/src/protocol/generic_proto/upgrade/notifications.rs index f626110a3346cd414bab6a936b6f9fcd6290bd76..cf271016e728adec53b7e56ffc86fc71281b7f59 100644 --- a/client/network/src/protocol/generic_proto/upgrade/notifications.rs +++ b/client/network/src/protocol/generic_proto/upgrade/notifications.rs @@ -44,7 +44,7 @@ use unsigned_varint::codec::UviBytes; /// Maximum allowed size of the two handshake messages, in bytes. const MAX_HANDSHAKE_SIZE: usize = 1024; /// Maximum number of buffered messages before we refuse to accept more. -const MAX_PENDING_MESSAGES: usize = 256; +const MAX_PENDING_MESSAGES: usize = 512; /// Upgrade that accepts a substream, sends back a status message, then becomes a unidirectional /// stream of messages. diff --git a/client/network/src/protocol/light_client_handler.rs b/client/network/src/protocol/light_client_handler.rs index 88a951914944ec04aaa7b2986836a6f2ae242506..2de6f56e2bb90fe9189765d5973022825c36a703 100644 --- a/client/network/src/protocol/light_client_handler.rs +++ b/client/network/src/protocol/light_client_handler.rs @@ -37,6 +37,7 @@ use libp2p::{ ConnectedPoint, Multiaddr, PeerId, + connection::ConnectionId, upgrade::{InboundUpgrade, ReadOneError, UpgradeInfo, Negotiated}, upgrade::{OutboundUpgrade, read_one, write_one} }, @@ -44,20 +45,28 @@ use libp2p::{ NegotiatedSubstream, NetworkBehaviour, NetworkBehaviourAction, + NotifyHandler, OneShotHandler, + OneShotHandlerConfig, PollParameters, - SubstreamProtocol + SubstreamProtocol, } }; use nohash_hasher::IntMap; use prost::Message; -use sc_client::light::fetcher; -use sc_client_api::StorageProof; +use sc_client_api::{ + StorageProof, + light::{ + self, RemoteReadRequest, RemoteBodyRequest, ChangesProof, + RemoteCallRequest, RemoteChangesRequest, RemoteHeaderRequest, + } +}; use sc_peerset::ReputationChange; use sp_core::{ - storage::{ChildInfo, StorageKey}, + storage::{ChildInfo, ChildType,StorageKey, PrefixedStorageKey}, hexdisplay::HexDisplay, }; +use smallvec::SmallVec; use sp_blockchain::{Error as ClientError}; use sp_runtime::{ traits::{Block, Header, NumberFor, Zero}, @@ -189,27 +198,27 @@ pub enum Error { #[derive(Debug)] pub enum Request { Body { - request: fetcher::RemoteBodyRequest, + request: RemoteBodyRequest, sender: oneshot::Sender, ClientError>> }, Header { - request: fetcher::RemoteHeaderRequest, + request: light::RemoteHeaderRequest, sender: oneshot::Sender> }, Read { - request: fetcher::RemoteReadRequest, + request: light::RemoteReadRequest, sender: oneshot::Sender, Option>>, ClientError>> }, ReadChild { - request: fetcher::RemoteReadChildRequest, + request: light::RemoteReadChildRequest, sender: oneshot::Sender, Option>>, ClientError>> }, Call { - request: fetcher::RemoteCallRequest, + request: light::RemoteCallRequest, sender: oneshot::Sender, ClientError>> }, Changes { - request: fetcher::RemoteChangesRequest, + request: light::RemoteChangesRequest, sender: oneshot::Sender, u32)>, ClientError>> } } @@ -237,25 +246,39 @@ struct RequestWrapper { retries: usize, /// The actual request. request: Request, - /// Peer information, e.g. `PeerId`. - peer: P + /// The peer to send the request to, e.g. `PeerId`. + peer: P, + /// The connection to use for sending the request. + connection: Option, } /// Information we have about some peer. #[derive(Debug)] struct PeerInfo { - address: Multiaddr, + connections: SmallVec<[(ConnectionId, Multiaddr); crate::MAX_CONNECTIONS_PER_PEER]>, best_block: Option>, status: PeerStatus, } +impl Default for PeerInfo { + fn default() -> Self { + PeerInfo { + connections: SmallVec::new(), + best_block: None, + status: PeerStatus::Idle, + } + } +} + +type RequestId = u64; + /// A peer is either idle or busy processing a request from us. #[derive(Debug, Clone, PartialEq, Eq)] enum PeerStatus { /// The peer is available. Idle, /// We wait for the peer to return us a response for the given request ID. - BusyWith(u64), + BusyWith(RequestId), } /// The light client handler behaviour. @@ -265,7 +288,7 @@ pub struct LightClientHandler { /// Blockchain client. chain: Arc>, /// Verifies that received responses are correct. - checker: Arc>, + checker: Arc>, /// Peer information (addresses, their best block, etc.) peers: HashMap>, /// Futures sending back response to remote clients. @@ -273,9 +296,9 @@ pub struct LightClientHandler { /// Pending (local) requests. pending_requests: VecDeque>, /// Requests on their way to remote peers. - outstanding: IntMap>, + outstanding: IntMap>, /// (Local) Request ID counter - next_request_id: u64, + next_request_id: RequestId, /// Handle to use for reporting misbehaviour of peers. peerset: sc_peerset::PeersetHandle, } @@ -288,7 +311,7 @@ where pub fn new( cfg: Config, chain: Arc>, - checker: Arc>, + checker: Arc>, peerset: sc_peerset::PeersetHandle, ) -> Self { LightClientHandler { @@ -323,35 +346,18 @@ where retries: retries(&req), request: req, peer: (), // we do not know the peer yet + connection: None, }; self.pending_requests.push_back(rw); Ok(()) } - fn next_request_id(&mut self) -> u64 { + fn next_request_id(&mut self) -> RequestId { let id = self.next_request_id; self.next_request_id += 1; id } - // Iterate over peers known to possess a certain block. - fn idle_peers_with_block(&mut self, num: NumberFor) -> impl Iterator + '_ { - self.peers.iter() - .filter(move |(_, info)| { - info.status == PeerStatus::Idle && info.best_block >= Some(num) - }) - .map(|(peer, _)| peer.clone()) - } - - // Iterate over peers without a known block. - fn idle_peers_with_unknown_block(&mut self) -> impl Iterator + '_ { - self.peers.iter() - .filter(|(_, info)| { - info.status == PeerStatus::Idle && info.best_block.is_none() - }) - .map(|(peer, _)| peer.clone()) - } - /// Remove the given peer. /// /// If we have a request to this peer in flight, we move it back to @@ -364,12 +370,50 @@ where retries: rw.retries, request: rw.request, peer: (), // need to find another peer + connection: None, }; self.pending_requests.push_back(rw); } self.peers.remove(peer); } + /// Prepares a request by selecting a suitable peer and connection to send it to. + /// + /// If there is currently no suitable peer for the request, the given request + /// is returned as `Err`. + fn prepare_request(&self, req: RequestWrapper) + -> Result<(PeerId, RequestWrapper), RequestWrapper> + { + let number = required_block(&req.request); + + let mut peer = None; + for (peer_id, peer_info) in self.peers.iter() { + if peer_info.status == PeerStatus::Idle { + match peer_info.best_block { + Some(n) => if n >= number { + peer = Some((peer_id, peer_info)); + break + }, + None => peer = Some((peer_id, peer_info)) + } + } + } + + if let Some((peer_id, peer_info)) = peer { + let connection = peer_info.connections.iter().next().map(|(id, _)| *id); + let rw = RequestWrapper { + timestamp: req.timestamp, + retries: req.retries, + request: req.request, + peer: peer_id.clone(), + connection, + }; + Ok((peer_id.clone(), rw)) + } else { + Err(req) + } + } + /// Process a local request's response from remote. /// /// If successful, this will give us the actual, checked data we should be @@ -432,7 +476,7 @@ where } r }; - let reply = self.checker.check_changes_proof(&request, fetcher::ChangesProof { + let reply = self.checker.check_changes_proof(&request, light::ChangesProof { max_block, proof: response.proof, roots, @@ -578,35 +622,27 @@ where let block = Decode::decode(&mut request.block.as_ref())?; - let proof = - if let Some(info) = ChildInfo::resolve_child_info(request.child_type, &request.child_info[..]) { - match self.chain.read_child_proof( - &BlockId::Hash(block), - &request.storage_key, - info, - &mut request.keys.iter().map(AsRef::as_ref) - ) { - Ok(proof) => proof, - Err(error) => { - log::trace!("remote read child request from {} ({} {} at {:?}) failed with: {}", - peer, - HexDisplay::from(&request.storage_key), - fmt_keys(request.keys.first(), request.keys.last()), - request.block, - error); - StorageProof::empty() - } - } - } else { + let prefixed_key = PrefixedStorageKey::new_ref(&request.storage_key); + let child_info = match ChildType::from_prefixed_key(prefixed_key) { + Some((ChildType::ParentKeyId, storage_key)) => Ok(ChildInfo::new_default(storage_key)), + None => Err("Invalid child storage key".into()), + }; + let proof = match child_info.and_then(|child_info| self.chain.read_child_proof( + &BlockId::Hash(block), + &child_info, + &mut request.keys.iter().map(AsRef::as_ref) + )) { + Ok(proof) => proof, + Err(error) => { log::trace!("remote read child request from {} ({} {} at {:?}) failed with: {}", peer, HexDisplay::from(&request.storage_key), fmt_keys(request.keys.first(), request.keys.last()), request.block, - "invalid child info and type" - ); + error); StorageProof::empty() - }; + } + }; let response = { let r = api::v1::light::RemoteReadResponse { proof: proof.encode() }; @@ -665,28 +701,23 @@ where let min = Decode::decode(&mut request.min.as_ref())?; let max = Decode::decode(&mut request.max.as_ref())?; let key = StorageKey(request.key.clone()); - let storage_key = - if request.storage_key.is_empty() { - None - } else { - Some(StorageKey(request.storage_key.clone())) - }; + let storage_key = if request.storage_key.is_empty() { + None + } else { + Some(PrefixedStorageKey::new_ref(&request.storage_key)) + }; - let proof = match self.chain.key_changes_proof(first, last, min, max, storage_key.as_ref(), &key) { + let proof = match self.chain.key_changes_proof(first, last, min, max, storage_key, &key) { Ok(proof) => proof, Err(error) => { log::trace!("remote changes proof request from {} for key {} ({:?}..{:?}) failed with: {}", peer, - if let Some(sk) = storage_key { - format!("{} : {}", HexDisplay::from(&sk.0), HexDisplay::from(&key.0)) - } else { - HexDisplay::from(&key.0).to_string() - }, + format!("{} : {}", HexDisplay::from(&request.storage_key), HexDisplay::from(&key.0)), request.first, request.last, error); - fetcher::ChangesProof:: { + light::ChangesProof:: { max_block: Zero::zero(), proof: Vec::new(), roots: BTreeMap::new(), @@ -723,38 +754,68 @@ where max_request_size: self.config.max_request_size, protocol: self.config.light_protocol.clone(), }; - OneShotHandler::new(SubstreamProtocol::new(p), self.config.inactivity_timeout) + let mut cfg = OneShotHandlerConfig::default(); + cfg.inactive_timeout = self.config.inactivity_timeout; + OneShotHandler::new(SubstreamProtocol::new(p), cfg) } fn addresses_of_peer(&mut self, peer: &PeerId) -> Vec { self.peers.get(peer) - .map(|info| vec![info.address.clone()]) + .map(|info| info.connections.iter().map(|(_, a)| a.clone()).collect()) .unwrap_or_default() } - fn inject_connected(&mut self, peer: PeerId, info: ConnectedPoint) { + fn inject_connected(&mut self, peer: &PeerId) { + } + + fn inject_connection_established(&mut self, peer: &PeerId, conn: &ConnectionId, info: &ConnectedPoint) { let peer_address = match info { - ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr, - ConnectedPoint::Dialer { address } => address + ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr.clone(), + ConnectedPoint::Dialer { address } => address.clone() }; log::trace!("peer {} connected with address {}", peer, peer_address); - let info = PeerInfo { - address: peer_address, - best_block: None, - status: PeerStatus::Idle, - }; - - self.peers.insert(peer, info); + let entry = self.peers.entry(peer.clone()).or_default(); + entry.connections.push((*conn, peer_address)); } - fn inject_disconnected(&mut self, peer: &PeerId, _: ConnectedPoint) { + fn inject_disconnected(&mut self, peer: &PeerId) { log::trace!("peer {} disconnected", peer); self.remove_peer(peer) } - fn inject_node_event(&mut self, peer: PeerId, event: Event) { + fn inject_connection_closed(&mut self, peer: &PeerId, conn: &ConnectionId, info: &ConnectedPoint) { + let peer_address = match info { + ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr, + ConnectedPoint::Dialer { address } => address + }; + + log::trace!("connection to peer {} closed: {}", peer, peer_address); + + if let Some(info) = self.peers.get_mut(peer) { + info.connections.retain(|(c, _)| c != conn) + } + + // Add any outstanding requests on the closed connection back to the + // pending requests. + if let Some(id) = self.outstanding.iter() + .find(|(_, rw)| &rw.peer == peer && rw.connection == Some(*conn)) // (*) + .map(|(id, _)| *id) + { + let rw = self.outstanding.remove(&id).expect("by (*)"); + let rw = RequestWrapper { + timestamp: rw.timestamp, + retries: rw.retries, + request: rw.request, + peer: (), // need to find another peer + connection: None, + }; + self.pending_requests.push_back(rw); + } + } + + fn inject_event(&mut self, peer: PeerId, conn: ConnectionId, event: Event) { match event { // An incoming request from remote has been received. Event::Request(request, mut stream) => { @@ -800,9 +861,10 @@ where // A response to one of our own requests has been received. Event::Response(id, response) => { if let Some(request) = self.outstanding.remove(&id) { - // We first just check if the response originates from the expected peer. + // We first just check if the response originates from the expected peer + // and connection. if request.peer != peer { - log::debug!("was expecting response from {} instead of {}", request.peer, peer); + log::debug!("Expected response from {} instead of {}.", request.peer, peer); self.outstanding.insert(id, request); self.remove_peer(&peer); self.peerset.report_peer(peer, ReputationChange::new_fatal("response from unexpected peer")); @@ -834,6 +896,7 @@ where retries: request.retries, request: request.request, peer: (), + connection: None, }; self.pending_requests.push_back(rw); } @@ -847,6 +910,7 @@ where retries: request.retries - 1, request: request.request, peer: (), + connection: None, }; self.pending_requests.push_back(rw) } else { @@ -886,54 +950,54 @@ where request.timestamp = Instant::now(); request.retries -= 1 } - let number = required_block(&request.request); - let available_peer = { - let p = self.idle_peers_with_block(number).next(); - if p.is_none() { - self.idle_peers_with_unknown_block().next() - } else { - p + + + match self.prepare_request(request) { + Err(request) => { + self.pending_requests.push_front(request); + log::debug!("no peer available to send request to"); + break } - }; - if let Some(peer) = available_peer { - let buf = match serialize_request(&request.request) { - Ok(b) => b, - Err(e) => { - log::debug!("failed to serialize request: {}", e); - send_reply(Err(ClientError::RemoteFetchFailed), request.request); - continue; - } - }; + Ok((peer, request)) => { + let request_bytes = match serialize_request(&request.request) { + Ok(bytes) => bytes, + Err(error) => { + log::debug!("failed to serialize request: {}", error); + send_reply(Err(ClientError::RemoteFetchFailed), request.request); + continue + } + }; - let id = self.next_request_id(); - log::trace!("sending request {} to peer {}", id, peer); - let protocol = OutboundProtocol { - request: buf, - request_id: id, - expected: match request.request { - Request::Body { .. } => ExpectedResponseTy::Block, - _ => ExpectedResponseTy::Light, - }, - max_response_size: self.config.max_response_size, - protocol: match request.request { - Request::Body { .. } => self.config.block_protocol.clone(), - _ => self.config.light_protocol.clone(), - }, - }; - self.peers.get_mut(&peer).map(|info| info.status = PeerStatus::BusyWith(id)); - let rw = RequestWrapper { - timestamp: request.timestamp, - retries: request.retries, - request: request.request, - peer: peer.clone(), - }; - self.outstanding.insert(id, rw); - return Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id: peer, event: protocol }) + let (expected, protocol) = match request.request { + Request::Body { .. } => + (ExpectedResponseTy::Block, self.config.block_protocol.clone()), + _ => + (ExpectedResponseTy::Light, self.config.light_protocol.clone()), + }; - } else { - self.pending_requests.push_front(request); - log::debug!("no peer available to send request to"); - break + let peer_id = peer.clone(); + let handler = request.connection.map_or(NotifyHandler::Any, NotifyHandler::One); + + let request_id = self.next_request_id(); + self.peers.get_mut(&peer).map(|p| p.status = PeerStatus::BusyWith(request_id)); + self.outstanding.insert(request_id, request); + + let event = OutboundProtocol { + request_id, + request: request_bytes, + expected, + max_response_size: self.config.max_response_size, + protocol, + }; + + log::trace!("sending request {} to peer {}", request_id, peer_id); + + return Poll::Ready(NetworkBehaviourAction::NotifyHandler { + peer_id, + handler, + event, + }) + } } } @@ -959,6 +1023,7 @@ where retries: rw.retries - 1, request: rw.request, peer: (), + connection: None, }; self.pending_requests.push_back(rw) } @@ -1019,9 +1084,7 @@ fn serialize_request(request: &Request) -> Result, prost::E Request::ReadChild { request, .. } => { let r = api::v1::light::RemoteReadChildRequest { block: request.block.encode(), - storage_key: request.storage_key.clone(), - child_type: request.child_type.clone(), - child_info: request.child_info.clone(), + storage_key: request.storage_key.clone().into_inner(), keys: request.keys.clone(), }; api::v1::light::request::Request::RemoteReadChildRequest(r) @@ -1040,7 +1103,8 @@ fn serialize_request(request: &Request) -> Result, prost::E last: request.last_block.1.encode(), min: request.tries_roots.1.encode(), max: request.max_block.1.encode(), - storage_key: request.storage_key.clone().unwrap_or_default(), + storage_key: request.storage_key.clone().map(|s| s.into_inner()) + .unwrap_or_default(), key: request.key.clone(), }; api::v1::light::request::Request::RemoteChangesRequest(r) @@ -1097,7 +1161,7 @@ pub enum Event { /// Incoming request from remote and substream to use for the response. Request(api::v1::light::Request, T), /// Incoming response from remote. - Response(u64, Response), + Response(RequestId, Response), } /// Incoming response from remote. @@ -1157,7 +1221,7 @@ pub struct OutboundProtocol { /// The serialized protobuf request. request: Vec, /// Local identifier for the request. Used to associate it with a response. - request_id: u64, + request_id: RequestId, /// Kind of response expected for this request. expected: ExpectedResponseTy, /// The max. response length in bytes. @@ -1230,6 +1294,7 @@ fn fmt_keys(first: Option<&Vec>, last: Option<&Vec>) -> String { #[cfg(test)] mod tests { + use super::*; use async_std::task; use assert_matches::assert_matches; use codec::Encode; @@ -1244,6 +1309,7 @@ mod tests { Multiaddr, core::{ ConnectedPoint, + connection::ConnectionId, identity, muxing::{StreamMuxerBox, SubstreamRef}, transport::{Transport, boxed::Boxed, memory::MemoryTransport}, @@ -1253,8 +1319,7 @@ mod tests { swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}, yamux }; - use sc_client_api::StorageProof; - use sc_client::light::fetcher; + use sc_client_api::{StorageProof, RemoteReadChildRequest, FetchChecker}; use sp_blockchain::{Error as ClientError}; use sp_core::storage::ChildInfo; use std::{ @@ -1269,8 +1334,6 @@ mod tests { use super::{Event, LightClientHandler, Request, Response, OutboundProtocol, PeerStatus}; use void::Void; - const CHILD_INFO: ChildInfo<'static> = ChildInfo::new_default(b"foobarbaz"); - type Block = sp_runtime::generic::Block, substrate_test_runtime::Extrinsic>; type Handler = LightClientHandler; type Swarm = libp2p::swarm::Swarm; @@ -1300,12 +1363,12 @@ mod tests { _mark: std::marker::PhantomData } - impl fetcher::FetchChecker for DummyFetchChecker { + impl light::FetchChecker for DummyFetchChecker { fn check_header_proof( &self, - _request: &fetcher::RemoteHeaderRequest, + _request: &RemoteHeaderRequest, header: Option, - _remote_proof: fetcher::StorageProof, + _remote_proof: StorageProof, ) -> Result { match self.ok { true if header.is_some() => Ok(header.unwrap()), @@ -1315,8 +1378,8 @@ mod tests { fn check_read_proof( &self, - request: &fetcher::RemoteReadRequest, - _: fetcher::StorageProof, + request: &RemoteReadRequest, + _: StorageProof, ) -> Result, Option>>, ClientError> { match self.ok { true => Ok(request.keys @@ -1331,8 +1394,8 @@ mod tests { fn check_read_child_proof( &self, - request: &fetcher::RemoteReadChildRequest, - _: fetcher::StorageProof, + request: &RemoteReadChildRequest, + _: StorageProof, ) -> Result, Option>>, ClientError> { match self.ok { true => Ok(request.keys @@ -1347,8 +1410,8 @@ mod tests { fn check_execution_proof( &self, - _: &fetcher::RemoteCallRequest, - _: fetcher::StorageProof, + _: &RemoteCallRequest, + _: StorageProof, ) -> Result, ClientError> { match self.ok { true => Ok(vec![42]), @@ -1358,8 +1421,8 @@ mod tests { fn check_changes_proof( &self, - _: &fetcher::RemoteChangesRequest, - _: fetcher::ChangesProof + _: &RemoteChangesRequest, + _: ChangesProof ) -> Result, u32)>, ClientError> { match self.ok { true => Ok(vec![(100.into(), 2)]), @@ -1369,7 +1432,7 @@ mod tests { fn check_body_proof( &self, - _: &fetcher::RemoteBodyRequest, + _: &RemoteBodyRequest, body: Vec ) -> Result, ClientError> { match self.ok { @@ -1457,10 +1520,12 @@ mod tests { let pset = peerset(); let mut behaviour = make_behaviour(true, pset.1, make_config()); - behaviour.inject_connected(peer.clone(), empty_dialer()); + behaviour.inject_connection_established(&peer, &ConnectionId::new(1), &empty_dialer()); + behaviour.inject_connected(&peer); assert_eq!(1, behaviour.peers.len()); - behaviour.inject_disconnected(&peer, empty_dialer()); + behaviour.inject_connection_closed(&peer, &ConnectionId::new(1), &empty_dialer()); + behaviour.inject_disconnected(&peer); assert_eq!(0, behaviour.peers.len()) } @@ -1471,8 +1536,10 @@ mod tests { let pset = peerset(); let mut behaviour = make_behaviour(true, pset.1, make_config()); - behaviour.inject_connected(peer0.clone(), empty_dialer()); - behaviour.inject_connected(peer1.clone(), empty_dialer()); + behaviour.inject_connection_established(&peer0, &ConnectionId::new(1), &empty_dialer()); + behaviour.inject_connected(&peer0); + behaviour.inject_connection_established(&peer1, &ConnectionId::new(2), &empty_dialer()); + behaviour.inject_connected(&peer1); // We now know about two peers. assert_eq!(HashSet::from_iter(&[peer0.clone(), peer1.clone()]), behaviour.peers.keys().collect::>()); @@ -1483,7 +1550,7 @@ mod tests { // Issue our first request! let chan = oneshot::channel(); - let request = fetcher::RemoteCallRequest { + let request = light::RemoteCallRequest { block: Default::default(), header: dummy_header(), method: "test".into(), @@ -1494,7 +1561,7 @@ mod tests { assert_eq!(1, behaviour.pending_requests.len()); // The behaviour should now attempt to send the request. - assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id, .. }) => { + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, .. }) => { assert!(peer_id == peer0 || peer_id == peer1) }); @@ -1534,11 +1601,13 @@ mod tests { let mut behaviour = make_behaviour(false, pset.1, make_config()); // ^--- Making sure the response data check fails. - behaviour.inject_connected(peer.clone(), empty_dialer()); + let conn = ConnectionId::new(1); + behaviour.inject_connection_established(&peer, &conn, &empty_dialer()); + behaviour.inject_connected(&peer); assert_eq!(1, behaviour.peers.len()); let chan = oneshot::channel(); - let request = fetcher::RemoteCallRequest { + let request = light::RemoteCallRequest { block: Default::default(), header: dummy_header(), method: "test".into(), @@ -1562,7 +1631,7 @@ mod tests { } }; - behaviour.inject_node_event(peer.clone(), Event::Response(request_id, Response::Light(response))); + behaviour.inject_event(peer.clone(), conn, Event::Response(request_id, Response::Light(response))); assert!(behaviour.peers.is_empty()); poll(&mut behaviour); // More progress @@ -1578,7 +1647,9 @@ mod tests { let pset = peerset(); let mut behaviour = make_behaviour(true, pset.1, make_config()); - behaviour.inject_connected(peer.clone(), empty_dialer()); + let conn = ConnectionId::new(1); + behaviour.inject_connection_established(&peer, &conn, &empty_dialer()); + behaviour.inject_connected(&peer); assert_eq!(1, behaviour.peers.len()); assert_eq!(0, behaviour.pending_requests.len()); assert_eq!(0, behaviour.outstanding.len()); @@ -1591,7 +1662,7 @@ mod tests { } }; - behaviour.inject_node_event(peer.clone(), Event::Response(2347895932, Response::Light(response))); + behaviour.inject_event(peer.clone(), conn, Event::Response(2347895932, Response::Light(response))); assert!(behaviour.peers.is_empty()); poll(&mut behaviour); @@ -1605,11 +1676,13 @@ mod tests { let pset = peerset(); let mut behaviour = make_behaviour(true, pset.1, make_config()); - behaviour.inject_connected(peer.clone(), empty_dialer()); + let conn = ConnectionId::new(1); + behaviour.inject_connection_established(&peer, &conn, &empty_dialer()); + behaviour.inject_connected(&peer); assert_eq!(1, behaviour.peers.len()); let chan = oneshot::channel(); - let request = fetcher::RemoteCallRequest { + let request = light::RemoteCallRequest { block: Default::default(), header: dummy_header(), method: "test".into(), @@ -1633,7 +1706,7 @@ mod tests { } }; - behaviour.inject_node_event(peer.clone(), Event::Response(request_id, Response::Light(response))); + behaviour.inject_event(peer.clone(), conn, Event::Response(request_id, Response::Light(response))); assert!(behaviour.peers.is_empty()); poll(&mut behaviour); // More progress @@ -1653,14 +1726,22 @@ mod tests { let mut behaviour = make_behaviour(false, pset.1, make_config()); // ^--- Making sure the response data check fails. - behaviour.inject_connected(peer1.clone(), empty_dialer()); - behaviour.inject_connected(peer2.clone(), empty_dialer()); - behaviour.inject_connected(peer3.clone(), empty_dialer()); - behaviour.inject_connected(peer4.clone(), empty_dialer()); + let conn1 = ConnectionId::new(1); + behaviour.inject_connection_established(&peer1, &conn1, &empty_dialer()); + behaviour.inject_connected(&peer1); + let conn2 = ConnectionId::new(2); + behaviour.inject_connection_established(&peer2, &conn2, &empty_dialer()); + behaviour.inject_connected(&peer2); + let conn3 = ConnectionId::new(3); + behaviour.inject_connection_established(&peer3, &conn3, &empty_dialer()); + behaviour.inject_connected(&peer3); + let conn4 = ConnectionId::new(3); + behaviour.inject_connection_established(&peer4, &conn4, &empty_dialer()); + behaviour.inject_connected(&peer4); assert_eq!(4, behaviour.peers.len()); let mut chan = oneshot::channel(); - let request = fetcher::RemoteCallRequest { + let request = light::RemoteCallRequest { block: Default::default(), header: dummy_header(), method: "test".into(), @@ -1671,11 +1752,11 @@ mod tests { assert_eq!(1, behaviour.pending_requests.len()); assert_eq!(0, behaviour.outstanding.len()); - assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { .. })); + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::NotifyHandler { .. })); assert_eq!(0, behaviour.pending_requests.len()); assert_eq!(1, behaviour.outstanding.len()); - for _ in 0 .. 3 { + for i in 1 ..= 3 { // Construct an invalid response let request_id = *behaviour.outstanding.keys().next().unwrap(); let responding_peer = behaviour.outstanding.values().next().unwrap().peer.clone(); @@ -1685,8 +1766,9 @@ mod tests { response: Some(api::v1::light::response::Response::RemoteCallResponse(r)) } }; - behaviour.inject_node_event(responding_peer, Event::Response(request_id, Response::Light(response.clone()))); - assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { .. })); + let conn = ConnectionId::new(i); + behaviour.inject_event(responding_peer, conn, Event::Response(request_id, Response::Light(response.clone()))); + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::NotifyHandler { .. })); assert_matches!(chan.1.try_recv(), Ok(None)) } // Final invalid response @@ -1698,7 +1780,7 @@ mod tests { response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), } }; - behaviour.inject_node_event(responding_peer, Event::Response(request_id, Response::Light(response))); + behaviour.inject_event(responding_peer, conn4, Event::Response(request_id, Response::Light(response))); assert_matches!(poll(&mut behaviour), Poll::Pending); assert_matches!(chan.1.try_recv(), Ok(Some(Err(ClientError::RemoteFetchFailed)))) } @@ -1708,7 +1790,9 @@ mod tests { let pset = peerset(); let mut behaviour = make_behaviour(true, pset.1, make_config()); - behaviour.inject_connected(peer.clone(), empty_dialer()); + let conn = ConnectionId::new(1); + behaviour.inject_connection_established(&peer, &conn, &empty_dialer()); + behaviour.inject_connected(&peer); assert_eq!(1, behaviour.peers.len()); let response = match request { @@ -1757,12 +1841,12 @@ mod tests { assert_eq!(1, behaviour.pending_requests.len()); assert_eq!(0, behaviour.outstanding.len()); - assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { .. })); + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::NotifyHandler { .. })); assert_eq!(0, behaviour.pending_requests.len()); assert_eq!(1, behaviour.outstanding.len()); assert_eq!(1, *behaviour.outstanding.keys().next().unwrap()); - behaviour.inject_node_event(peer.clone(), Event::Response(1, Response::Light(response))); + behaviour.inject_event(peer.clone(), conn, Event::Response(1, Response::Light(response))); poll(&mut behaviour); @@ -1773,7 +1857,7 @@ mod tests { #[test] fn receives_remote_call_response() { let mut chan = oneshot::channel(); - let request = fetcher::RemoteCallRequest { + let request = light::RemoteCallRequest { block: Default::default(), header: dummy_header(), method: "test".into(), @@ -1787,7 +1871,7 @@ mod tests { #[test] fn receives_remote_read_response() { let mut chan = oneshot::channel(); - let request = fetcher::RemoteReadRequest { + let request = light::RemoteReadRequest { header: dummy_header(), block: Default::default(), keys: vec![b":key".to_vec()], @@ -1799,15 +1883,13 @@ mod tests { #[test] fn receives_remote_read_child_response() { - let info = CHILD_INFO.info(); let mut chan = oneshot::channel(); - let request = fetcher::RemoteReadChildRequest { + let child_info = ChildInfo::new_default(&b":child_storage:default:sub"[..]); + let request = light::RemoteReadChildRequest { header: dummy_header(), block: Default::default(), - storage_key: b":child_storage:sub".to_vec(), + storage_key: child_info.prefixed_storage_key(), keys: vec![b":key".to_vec()], - child_info: info.0.to_vec(), - child_type: info.1, retry_count: None, }; issue_request(Request::ReadChild { request, sender: chan.0 }); @@ -1817,7 +1899,7 @@ mod tests { #[test] fn receives_remote_header_response() { let mut chan = oneshot::channel(); - let request = fetcher::RemoteHeaderRequest { + let request = light::RemoteHeaderRequest { cht_root: Default::default(), block: 1, retry_count: None, @@ -1829,7 +1911,7 @@ mod tests { #[test] fn receives_remote_changes_response() { let mut chan = oneshot::channel(); - let request = fetcher::RemoteChangesRequest { + let request = light::RemoteChangesRequest { changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { zero: (0, Default::default()), end: None, @@ -1874,7 +1956,7 @@ mod tests { #[test] fn send_receive_call() { let chan = oneshot::channel(); - let request = fetcher::RemoteCallRequest { + let request = light::RemoteCallRequest { block: Default::default(), header: dummy_header(), method: "test".into(), @@ -1889,7 +1971,7 @@ mod tests { #[test] fn send_receive_read() { let chan = oneshot::channel(); - let request = fetcher::RemoteReadRequest { + let request = light::RemoteReadRequest { header: dummy_header(), block: Default::default(), keys: vec![b":key".to_vec()], @@ -1902,15 +1984,13 @@ mod tests { #[test] fn send_receive_read_child() { - let info = CHILD_INFO.info(); let chan = oneshot::channel(); - let request = fetcher::RemoteReadChildRequest { + let child_info = ChildInfo::new_default(&b":child_storage:default:sub"[..]); + let request = light::RemoteReadChildRequest { header: dummy_header(), block: Default::default(), - storage_key: b":child_storage:sub".to_vec(), + storage_key: child_info.prefixed_storage_key(), keys: vec![b":key".to_vec()], - child_info: info.0.to_vec(), - child_type: info.1, retry_count: None, }; send_receive(Request::ReadChild { request, sender: chan.0 }); @@ -1922,7 +2002,7 @@ mod tests { fn send_receive_header() { let _ = env_logger::try_init(); let chan = oneshot::channel(); - let request = fetcher::RemoteHeaderRequest { + let request = light::RemoteHeaderRequest { cht_root: Default::default(), block: 1, retry_count: None, @@ -1935,7 +2015,7 @@ mod tests { #[test] fn send_receive_changes() { let chan = oneshot::channel(); - let request = fetcher::RemoteChangesRequest { + let request = light::RemoteChangesRequest { changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { zero: (0, Default::default()), end: None, diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index ae83b49e60f57e475bb3399ec6af3e9e672e6c8f..8638e9afc59b93fc134b53d71e01770fdecd01d3 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -477,11 +477,6 @@ pub mod generic { pub block: H, /// Child Storage key. pub storage_key: Vec, - /// Child trie source information. - pub child_info: Vec, - /// Child type, its required to resolve `child_info` - /// content and choose child implementation. - pub child_type: u32, /// Storage key. pub keys: Vec>, } diff --git a/client/network/src/protocol/schema/api.v1.proto b/client/network/src/protocol/schema/api.v1.proto index ccbf49d666115ea50fbe08885c9242c891d805a1..a933c5811c109616c7adec903b17293ddebaeab7 100644 --- a/client/network/src/protocol/schema/api.v1.proto +++ b/client/network/src/protocol/schema/api.v1.proto @@ -51,5 +51,10 @@ message BlockData { bytes message_queue = 5; // optional // Justification if requested. bytes justification = 6; // optional + // True if justification should be treated as present but empty. + // This hack is unfortunately necessary because shortcomings in the protobuf format otherwise + // doesn't make in possible to differentiate between a lack of justification and an empty + // justification. + bool is_empty_justification = 7; // optional, false if absent } diff --git a/client/network/src/protocol/schema/finality.v1.proto b/client/network/src/protocol/schema/finality.v1.proto new file mode 100644 index 0000000000000000000000000000000000000000..843bc4eca0990cc01b1479e19d68a721395266c4 --- /dev/null +++ b/client/network/src/protocol/schema/finality.v1.proto @@ -0,0 +1,19 @@ +// Schema definition for finality proof request/responses. + +syntax = "proto3"; + +package api.v1.finality; + +// Request a finality proof from a peer. +message FinalityProofRequest { + // SCALE-encoded hash of the block to request. + bytes block_hash = 1; + // Opaque chain-specific additional request data. + bytes request = 2; +} + +// Response to a finality proof request. +message FinalityProofResponse { + // Opaque chain-specific finality proof. Empty if no such proof exists. + bytes proof = 1; // optional +} diff --git a/client/network/src/protocol/schema/light.v1.proto b/client/network/src/protocol/schema/light.v1.proto index 1c98d49730cf98dcafb9999056682f6c40efb101..9b5d47719dc28e7930e449ef2fa05a6333a4a18d 100644 --- a/client/network/src/protocol/schema/light.v1.proto +++ b/client/network/src/protocol/schema/light.v1.proto @@ -67,13 +67,9 @@ message RemoteReadResponse { message RemoteReadChildRequest { // Block at which to perform call. bytes block = 2; - // Child Storage key. + // Child Storage key, this is relative + // to the child type storage location. bytes storage_key = 3; - // Child trie source information. - bytes child_info = 4; - /// Child type, its required to resolve `child_info` - /// content and choose child implementation. - uint32 child_type = 5; // Storage keys. repeated bytes keys = 6; } diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index 9feded784fe78871646879ba49e3b19a866a6d7c..98fbd4ae4f7b87b393e95388c153f71e2b023b94 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -90,6 +90,12 @@ mod rep { /// Reputation change for peers which send us a known bad block. pub const BAD_BLOCK: Rep = Rep::new(-(1 << 29), "Bad block"); + /// Peer did not provide us with advertised block data. + pub const NO_BLOCK: Rep = Rep::new(-(1 << 29), "No requested block data"); + + /// Reputation change for peers which send us a known block. + pub const KNOWN_BLOCK: Rep = Rep::new(-(1 << 29), "Duplicate block"); + /// Reputation change for peers which send us a block with bad justifications. pub const BAD_JUSTIFICATION: Rep = Rep::new(-(1 << 16), "Bad justification"); @@ -184,7 +190,11 @@ pub enum PeerSyncState { /// Available for sync requests. Available, /// Searching for ancestors the Peer has in common with us. - AncestorSearch(NumberFor, AncestorSearchState), + AncestorSearch { + start: NumberFor, + current: NumberFor, + state: AncestorSearchState, + }, /// Actively downloading new blocks, starting from the given Number. DownloadingNew(NumberFor), /// Downloading a stale block with given Hash. Stale means that it is a @@ -432,10 +442,11 @@ impl ChainSync { common_number: Zero::zero(), best_hash, best_number, - state: PeerSyncState::AncestorSearch( - common_best, - AncestorSearchState::ExponentialBackoff(One::one()) - ), + state: PeerSyncState::AncestorSearch { + current: common_best, + start: self.best_queued_number, + state: AncestorSearchState::ExponentialBackoff(One::one()), + }, recently_announced: Default::default() }); self.is_idle = false; @@ -508,7 +519,7 @@ impl ChainSync { self.is_idle = false; for peer_id in &peers { if let Some(peer) = self.peers.get_mut(peer_id) { - if let PeerSyncState::AncestorSearch(_, _) = peer.state { + if let PeerSyncState::AncestorSearch {..} = peer.state { continue; } @@ -669,7 +680,7 @@ impl ChainSync { match &mut peer.state { PeerSyncState::DownloadingNew(start_block) => { self.blocks.clear_peer_download(&who); - self.blocks.insert(*start_block, blocks, who); + self.blocks.insert(*start_block, blocks, who.clone()); peer.state = PeerSyncState::Available; self.blocks .drain(self.best_queued_number + One::one()) @@ -688,6 +699,10 @@ impl ChainSync { } PeerSyncState::DownloadingStale(_) => { peer.state = PeerSyncState::Available; + if blocks.is_empty() { + debug!(target: "sync", "Empty block response from {}", who); + return Err(BadPeer(who, rep::NO_BLOCK)); + } blocks.into_iter().map(|b| { IncomingBlock { hash: b.hash, @@ -700,10 +715,10 @@ impl ChainSync { } }).collect() } - PeerSyncState::AncestorSearch(num, state) => { - let matching_hash = match (blocks.get(0), self.client.hash(*num)) { + PeerSyncState::AncestorSearch { current, start, state } => { + let matching_hash = match (blocks.get(0), self.client.hash(*current)) { (Some(block), Ok(maybe_our_block_hash)) => { - trace!(target: "sync", "Got ancestry block #{} ({}) from peer {}", num, block.hash, who); + trace!(target: "sync", "Got ancestry block #{} ({}) from peer {}", current, block.hash, who); maybe_our_block_hash.filter(|x| x == &block.hash) }, (None, _) => { @@ -715,15 +730,27 @@ impl ChainSync { return Err(BadPeer(who, rep::BLOCKCHAIN_READ_ERROR)) } }; - if matching_hash.is_some() && peer.common_number < *num { - peer.common_number = *num; + if matching_hash.is_some() { + if *start < self.best_queued_number && self.best_queued_number <= peer.best_number { + // We've made progress on this chain since the search was started. + // Opportunistically set common number to updated number + // instead of the one that started the search. + peer.common_number = self.best_queued_number; + } + else if peer.common_number < *current { + peer.common_number = *current; + } } - if matching_hash.is_none() && num.is_zero() { + if matching_hash.is_none() && current.is_zero() { trace!(target:"sync", "Ancestry search: genesis mismatch for peer {}", who); return Err(BadPeer(who, rep::GENESIS_MISMATCH)) } - if let Some((next_state, next_num)) = handle_ancestor_search_state(state, *num, matching_hash.is_some()) { - peer.state = PeerSyncState::AncestorSearch(next_num, next_state); + if let Some((next_state, next_num)) = handle_ancestor_search_state(state, *current, matching_hash.is_some()) { + peer.state = PeerSyncState::AncestorSearch { + current: next_num, + start: *start, + state: next_state, + }; return Ok(OnBlockData::Request(who, ancestry_request::(next_num))) } else { // Ancestry search is complete. Check if peer is on a stale fork unknown to us and @@ -747,7 +774,7 @@ impl ChainSync { parent_hash: None, peers: Default::default(), }) - .peers.insert(who); + .peers.insert(who.clone()); } peer.state = PeerSyncState::Available; Vec::new() @@ -776,18 +803,28 @@ impl ChainSync { Vec::new() }; - let orig_len = new_blocks.len(); - new_blocks.retain(|b| !self.queue_blocks.contains(&b.hash)); - if new_blocks.len() != orig_len { - debug!(target: "sync", "Ignoring {} blocks that are already queued", orig_len - new_blocks.len()); - } - + // When doing initial sync we don't request blocks in parallel. + // So the only way this can happen is when peers lie about the + // common block. let is_recent = new_blocks.first() .map(|block| { self.peers.iter().any(|(_, peer)| peer.recently_announced.contains(&block.hash)) }) .unwrap_or(false); + if !is_recent && new_blocks.last().map_or(false, |b| self.is_known(&b.hash)) { + // When doing initial sync we don't request blocks in parallel. + // So the only way this can happen is when peers lie about the + // common block. + debug!(target: "sync", "Ignoring known blocks from {}", who); + return Err(BadPeer(who, rep::KNOWN_BLOCK)); + } + let orig_len = new_blocks.len(); + new_blocks.retain(|b| !self.queue_blocks.contains(&b.hash)); + if new_blocks.len() != orig_len { + debug!(target: "sync", "Ignoring {} blocks that are already queued", orig_len - new_blocks.len()); + } + let origin = if is_recent { BlockOrigin::NetworkBroadcast @@ -1043,7 +1080,7 @@ impl ChainSync { self.best_queued_hash = *hash; // Update common blocks for (n, peer) in self.peers.iter_mut() { - if let PeerSyncState::AncestorSearch(_, _) = peer.state { + if let PeerSyncState::AncestorSearch {..} = peer.state { // Wait for ancestry search to complete first. continue; } @@ -1103,7 +1140,7 @@ impl ChainSync { peer.best_number = number; peer.best_hash = hash; } - if let PeerSyncState::AncestorSearch(_, _) = peer.state { + if let PeerSyncState::AncestorSearch {..} = peer.state { return OnBlockAnnounce::Nothing } // If the announced block is the best they have and is not ahead of us, our common number diff --git a/client/network/src/service.rs b/client/network/src/service.rs index abe9231abf22e51a19c52c987ac650edbfc9663c..3fbfd2dd14252517108eb8f0995643c41714861b 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -26,20 +26,24 @@ //! which is then processed by [`NetworkWorker::poll`]. use crate::{ + ExHashT, NetworkStateInfo, behaviour::{Behaviour, BehaviourOut}, config::{parse_addr, parse_str_addr, NonReservedPeerMode, Params, Role, TransportConfig}, + discovery::DiscoveryConfig, error::Error, network_state::{ NetworkState, NotConnectedPeer as NetworkStateNotConnectedPeer, Peer as NetworkStatePeer, }, on_demand_layer::AlwaysBadChecker, - protocol::{self, event::Event, light_client_handler, sync::SyncState, PeerInfo, Protocol}, + protocol::{self, event::Event, light_client_handler, LegacyConnectionKillError, sync::SyncState, PeerInfo, Protocol}, transport, ReputationChange, }; use futures::prelude::*; -use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver}; -use libp2p::swarm::{NetworkBehaviour, SwarmBuilder, SwarmEvent}; -use libp2p::{kad::record, Multiaddr, PeerId}; +use libp2p::{PeerId, Multiaddr}; +use libp2p::core::{ConnectedPoint, Executor, connection::{ConnectionError, PendingConnectionError}, either::EitherError}; +use libp2p::kad::record; +use libp2p::ping::handler::PingFailure; +use libp2p::swarm::{NetworkBehaviour, SwarmBuilder, SwarmEvent, protocols_handler::NodeHandlerWrapperError}; use log::{error, info, trace, warn}; use parking_lot::Mutex; use prometheus_endpoint::{ @@ -51,98 +55,24 @@ use sp_runtime::{ traits::{Block as BlockT, NumberFor}, ConsensusEngineId, }; +use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use std::{ borrow::Cow, - collections::{HashMap, HashSet}, + collections::HashSet, fs, io, marker::PhantomData, - path::Path, pin::Pin, str, - sync::{atomic::{AtomicBool, AtomicUsize, Ordering}, Arc}, + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, + }, task::Poll, }; -/// Minimum Requirements for a Hash within Networking -pub trait ExHashT: std::hash::Hash + Eq + std::fmt::Debug + Clone + Send + Sync + 'static {} - -impl ExHashT for T where - T: std::hash::Hash + Eq + std::fmt::Debug + Clone + Send + Sync + 'static -{} - -/// Transaction pool interface -pub trait TransactionPool: Send + Sync { - /// Get transactions from the pool that are ready to be propagated. - fn transactions(&self) -> Vec<(H, B::Extrinsic)>; - /// Get hash of transaction. - fn hash_of(&self, transaction: &B::Extrinsic) -> H; - /// Import a transaction into the pool. - /// - /// Peer reputation is changed by reputation_change if transaction is accepted by the pool. - fn import( - &self, - report_handle: ReportHandle, - who: PeerId, - reputation_change_good: ReputationChange, - reputation_change_bad: ReputationChange, - transaction: B::Extrinsic, - ); - /// Notify the pool about transactions broadcast. - fn on_broadcasted(&self, propagations: HashMap>); - /// Get transaction by hash. - fn transaction(&self, hash: &H) -> Option; -} - -/// Dummy implementation of the [`TransactionPool`] trait for a transaction pool that is always -/// empty and discards all incoming transactions. -/// -/// Requires the "hash" type to implement the `Default` trait. -/// -/// Useful for testing purposes. -pub struct EmptyTransactionPool; - -impl TransactionPool for EmptyTransactionPool { - fn transactions(&self) -> Vec<(H, B::Extrinsic)> { - Vec::new() - } - - fn hash_of(&self, _transaction: &B::Extrinsic) -> H { - Default::default() - } - - fn import( - &self, - _report_handle: ReportHandle, - _who: PeerId, - _rep_change_good: ReputationChange, - _rep_change_bad: ReputationChange, - _transaction: B::Extrinsic - ) {} - - fn on_broadcasted(&self, _: HashMap>) {} - - fn transaction(&self, _h: &H) -> Option { None } -} - -/// A cloneable handle for reporting cost/benefits of peers. -#[derive(Clone)] -pub struct ReportHandle { - inner: PeersetHandle, // wraps it so we don't have to worry about breaking API. -} - -impl From for ReportHandle { - fn from(peerset_handle: PeersetHandle) -> Self { - ReportHandle { inner: peerset_handle } - } -} - -impl ReportHandle { - /// Report a given peer as either beneficial (+) or costly (-) according to the - /// given scalar. - pub fn report_peer(&self, who: PeerId, cost_benefit: ReputationChange) { - self.inner.report_peer(who, cost_benefit); - } -} +mod out_events; +#[cfg(test)] +mod tests; /// Substrate network service. Handles network IO and manages connectivity. pub struct NetworkService { @@ -175,8 +105,8 @@ impl NetworkWorker { pub fn new(params: Params) -> Result, Error> { let (to_worker, from_worker) = tracing_unbounded("mpsc_network_worker"); - if let Some(ref path) = params.network_config.net_config_path { - fs::create_dir_all(Path::new(path))?; + if let Some(path) = params.network_config.net_config_path { + fs::create_dir_all(&path)?; } // List of multiaddresses that we know in the network. @@ -281,11 +211,12 @@ impl NetworkWorker { params.block_announce_validator, params.metrics_registry.as_ref(), boot_node_ids.clone(), + params.network_config.use_new_block_requests_protocol, metrics.as_ref().map(|m| m.notifications_queues_size.clone()), )?; // Build the swarm. - let (mut swarm, bandwidth): (Swarm::, _) = { + let (mut swarm, bandwidth): (Swarm, _) = { let user_agent = format!( "{} ({})", params.network_config.client_version, @@ -295,28 +226,55 @@ impl NetworkWorker { let config = protocol::block_requests::Config::new(¶ms.protocol_id); protocol::BlockRequests::new(config, params.chain.clone()) }; + let finality_proof_requests = { + let config = protocol::finality_requests::Config::new(¶ms.protocol_id); + protocol::FinalityProofRequests::new(config, params.finality_proof_provider.clone()) + }; let light_client_handler = { let config = protocol::light_client_handler::Config::new(¶ms.protocol_id); - protocol::LightClientHandler::new(config, params.chain, checker, peerset_handle.clone()) + protocol::LightClientHandler::new( + config, + params.chain, + checker, + peerset_handle.clone(), + ) + }; + + let discovery_config = { + let mut config = DiscoveryConfig::new(local_public.clone()); + config.with_user_defined(known_addresses); + config.discovery_limit(u64::from(params.network_config.out_peers) + 15); + config.add_protocol(params.protocol_id.clone()); + config.allow_non_globals_in_dht(params.network_config.allow_non_globals_in_dht); + + match params.network_config.transport { + TransportConfig::MemoryOnly => { + config.with_mdns(false); + config.allow_private_ipv4(false); + } + TransportConfig::Normal { enable_mdns, allow_private_ipv4, .. } => { + config.with_mdns(enable_mdns); + config.allow_private_ipv4(allow_private_ipv4); + } + } + + config }; - let behaviour = futures::executor::block_on(Behaviour::new( + + let mut behaviour = Behaviour::new( protocol, params.role, user_agent, local_public, - known_addresses, - match params.network_config.transport { - TransportConfig::MemoryOnly => false, - TransportConfig::Normal { enable_mdns, .. } => enable_mdns, - }, - match params.network_config.transport { - TransportConfig::MemoryOnly => false, - TransportConfig::Normal { allow_private_ipv4, .. } => allow_private_ipv4, - }, - u64::from(params.network_config.out_peers) + 15, block_requests, - light_client_handler - )); + finality_proof_requests, + light_client_handler, + discovery_config + ); + + for (engine_id, protocol_name) in ¶ms.network_config.notifications_protocols { + behaviour.register_notifications_protocol(*engine_id, protocol_name.clone()); + } let (transport, bandwidth) = { let (config_mem, config_wasm, flowctrl) = match params.network_config.transport { TransportConfig::MemoryOnly => (true, None, false), @@ -325,9 +283,16 @@ impl NetworkWorker { }; transport::build_transport(local_identity, config_mem, config_wasm, flowctrl) }; - let mut builder = SwarmBuilder::new(transport, behaviour, local_peer_id.clone()); + let mut builder = SwarmBuilder::new(transport, behaviour, local_peer_id.clone()) + .peer_connection_limit(crate::MAX_CONNECTIONS_PER_PEER); if let Some(spawner) = params.executor { - builder = builder.executor_fn(spawner); + struct SpawnImpl(F); + impl + Send>>)> Executor for SpawnImpl { + fn exec(&self, f: Pin + Send>>) { + (self.0)(f) + } + } + builder = builder.executor(Box::new(SpawnImpl(spawner))); } (builder.build(), bandwidth) }; @@ -366,7 +331,7 @@ impl NetworkWorker { import_queue: params.import_queue, from_worker, light_client_rqs: params.on_demand.and_then(|od| od.extract_receiver()), - event_streams: Vec::new(), + event_streams: out_events::OutChannels::new(params.metrics_registry.as_ref())?, metrics, boot_node_ids, }) @@ -443,6 +408,18 @@ impl NetworkWorker { self.network_service.user_protocol_mut().on_block_finalized(hash, &header); } + /// Returns the local `PeerId`. + pub fn local_peer_id(&self) -> &PeerId { + Swarm::::local_peer_id(&self.network_service) + } + + /// Returns the list of addresses we are listening on. + /// + /// Does **NOT** include a trailing `/p2p/` with our `PeerId`. + pub fn listen_addresses(&self) -> impl Iterator { + Swarm::::listeners(&self.network_service) + } + /// Get network state. /// /// **Note**: Use this only for debugging. This API is unstable. There are warnings literally @@ -525,6 +502,11 @@ impl NetworkWorker { } impl NetworkService { + /// Returns the local `PeerId`. + pub fn local_peer_id(&self) -> &PeerId { + &self.local_peer_id + } + /// Writes a message on an open notifications channel. Has no effect if the notifications /// channel with this protocol name is closed. /// @@ -549,9 +531,13 @@ impl NetworkService { /// If this method is called multiple times, the events are duplicated. /// /// The stream never ends (unless the `NetworkWorker` gets shut down). - pub fn event_stream(&self) -> impl Stream { + /// + /// The name passed is used to identify the channel in the Prometheus metrics. Note that the + /// parameter is a `&'static str`, and not a `String`, in order to avoid accidentally having + /// an unbounded set of Prometheus metrics, which would be quite bad in terms of memory + pub fn event_stream(&self, name: &'static str) -> impl Stream { // Note: when transitioning to stable futures, remove the `Error` entirely - let (tx, rx) = tracing_unbounded("mpsc_network_event_stream"); + let (tx, rx) = out_events::channel(name); let _ = self.to_worker.unbounded_send(ServiceToWorkerMsg::EventStream(tx)); rx } @@ -734,15 +720,6 @@ impl<'a, B: BlockT + 'static, H: ExHashT> sp_consensus::SyncOracle } } -/// Trait for providing information about the local network state -pub trait NetworkStateInfo { - /// Returns the local external addresses. - fn external_addresses(&self) -> Vec; - - /// Returns the local Peer ID. - fn local_peer_id(&self) -> PeerId; -} - impl NetworkStateInfo for NetworkService where B: sp_runtime::traits::Block, @@ -771,7 +748,7 @@ enum ServiceToWorkerMsg { PutValue(record::Key, Vec), AddKnownAddress(PeerId, Multiaddr), SyncFork(Vec, B::Hash, NumberFor), - EventStream(TracingUnboundedSender), + EventStream(out_events::Sender), WriteNotification { message: Vec, engine_id: ConsensusEngineId, @@ -806,7 +783,7 @@ pub struct NetworkWorker { /// Receiver for queries from the light client that must be processed. light_client_rqs: Option>>, /// Senders for events that happen on the network. - event_streams: Vec>, + event_streams: out_events::OutChannels, /// Prometheus network metrics. metrics: Option, /// The `PeerId`'s of all boot nodes. @@ -815,30 +792,53 @@ pub struct NetworkWorker { struct Metrics { // This list is ordered alphabetically - connections: Gauge, + connections_closed_total: CounterVec, + connections_opened_total: CounterVec, import_queue_blocks_submitted: Counter, import_queue_finality_proofs_submitted: Counter, import_queue_justifications_submitted: Counter, + incoming_connections_errors_total: CounterVec, + incoming_connections_total: Counter, is_major_syncing: Gauge, issued_light_requests: Counter, - kbuckets_num_nodes: Gauge, + kademlia_random_queries_total: CounterVec, + kademlia_records_count: GaugeVec, + kademlia_records_sizes_total: GaugeVec, + kbuckets_num_nodes: GaugeVec, + listeners_local_addresses: Gauge, + listeners_errors_total: Counter, network_per_sec_bytes: GaugeVec, notifications_queues_size: HistogramVec, - notifications_total: CounterVec, - num_event_stream_channels: Gauge, - opened_notification_streams: GaugeVec, + notifications_sizes: HistogramVec, + notifications_streams_closed_total: CounterVec, + notifications_streams_opened_total: CounterVec, peers_count: Gauge, peerset_num_discovered: Gauge, peerset_num_requested: Gauge, - random_kademalia_queries_total: Counter, + pending_connections: Gauge, + pending_connections_errors_total: CounterVec, + requests_in_total: HistogramVec, + requests_out_finished: HistogramVec, + requests_out_started_total: CounterVec, } impl Metrics { fn register(registry: &Registry) -> Result { Ok(Self { // This list is ordered alphabetically - connections: register(Gauge::new( - "sub_libp2p_connections", "Number of libp2p connections" + connections_closed_total: register(CounterVec::new( + Opts::new( + "sub_libp2p_connections_closed_total", + "Total number of connections closed, by reason and direction" + ), + &["direction", "reason"] + )?, registry)?, + connections_opened_total: register(CounterVec::new( + Opts::new( + "sub_libp2p_connections_opened_total", + "Total number of connections opened" + ), + &["direction"] )?, registry)?, import_queue_blocks_submitted: register(Counter::new( "import_queue_blocks_submitted", @@ -852,6 +852,18 @@ impl Metrics { "import_queue_justifications_submitted", "Number of justifications submitted to the import queue.", )?, registry)?, + incoming_connections_errors_total: register(CounterVec::new( + Opts::new( + "sub_libp2p_incoming_connections_handshake_errors_total", + "Total number of incoming connections that have failed during the \ + initial handshake" + ), + &["reason"] + )?, registry)?, + incoming_connections_total: register(Counter::new( + "sub_libp2p_incoming_connections_total", + "Total number of incoming connections on the listening sockets" + )?, registry)?, is_major_syncing: register(Gauge::new( "sub_libp2p_is_major_syncing", "Whether the node is performing a major sync or not.", )?, registry)?, @@ -859,8 +871,40 @@ impl Metrics { "issued_light_requests", "Number of light client requests that our node has issued.", )?, registry)?, - kbuckets_num_nodes: register(Gauge::new( - "sub_libp2p_kbuckets_num_nodes", "Number of nodes in the Kademlia k-buckets" + kademlia_random_queries_total: register(CounterVec::new( + Opts::new( + "sub_libp2p_kademlia_random_queries_total", + "Number of random Kademlia queries started" + ), + &["protocol"] + )?, registry)?, + kademlia_records_count: register(GaugeVec::new( + Opts::new( + "sub_libp2p_kademlia_records_count", + "Number of records in the Kademlia records store" + ), + &["protocol"] + )?, registry)?, + kademlia_records_sizes_total: register(GaugeVec::new( + Opts::new( + "sub_libp2p_kademlia_records_sizes_total", + "Total size of all the records in the Kademlia records store" + ), + &["protocol"] + )?, registry)?, + kbuckets_num_nodes: register(GaugeVec::new( + Opts::new( + "sub_libp2p_kbuckets_num_nodes", + "Number of nodes in the Kademlia k-buckets" + ), + &["protocol"] + )?, registry)?, + listeners_local_addresses: register(Gauge::new( + "sub_libp2p_listeners_local_addresses", "Number of local addresses we're listening on" + )?, registry)?, + listeners_errors_total: register(Counter::new( + "sub_libp2p_listeners_errors_total", + "Total number of non-fatal errors reported by a listener" )?, registry)?, network_per_sec_bytes: register(GaugeVec::new( Opts::new( @@ -875,25 +919,32 @@ impl Metrics { "sub_libp2p_notifications_queues_size", "Total size of all the notification queues" ), - buckets: vec![0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0], + buckets: vec![0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 511.0, 512.0], }, &["protocol"] )?, registry)?, - notifications_total: register(CounterVec::new( - Opts::new( - "sub_libp2p_notifications_total", - "Number of notification received from all nodes" - ), + notifications_sizes: register(HistogramVec::new( + HistogramOpts { + common_opts: Opts::new( + "sub_libp2p_notifications_sizes", + "Sizes of the notifications send to and received from all nodes" + ), + buckets: prometheus_endpoint::exponential_buckets(64.0, 4.0, 8) + .expect("parameters are always valid values; qed"), + }, &["direction", "protocol"] )?, registry)?, - num_event_stream_channels: register(Gauge::new( - "sub_libp2p_num_event_stream_channels", - "Number of internal active channels that broadcast network events", + notifications_streams_closed_total: register(CounterVec::new( + Opts::new( + "sub_libp2p_notifications_streams_closed_total", + "Total number of notification substreams that have been closed" + ), + &["protocol"] )?, registry)?, - opened_notification_streams: register(GaugeVec::new( + notifications_streams_opened_total: register(CounterVec::new( Opts::new( - "sub_libp2p_opened_notification_streams", - "Number of open notification substreams" + "sub_libp2p_notifications_streams_opened_total", + "Total number of notification substreams that have been opened" ), &["protocol"] )?, registry)?, @@ -906,8 +957,45 @@ impl Metrics { peerset_num_requested: register(Gauge::new( "sub_libp2p_peerset_num_requested", "Number of nodes that the peerset manager wants us to be connected to", )?, registry)?, - random_kademalia_queries_total: register(Counter::new( - "sub_libp2p_random_kademalia_queries_total", "Number of random Kademlia queries started", + pending_connections: register(Gauge::new( + "sub_libp2p_pending_connections", + "Number of connections in the process of being established", + )?, registry)?, + pending_connections_errors_total: register(CounterVec::new( + Opts::new( + "sub_libp2p_pending_connections_errors_total", + "Total number of pending connection errors" + ), + &["reason"] + )?, registry)?, + requests_in_total: register(HistogramVec::new( + HistogramOpts { + common_opts: Opts::new( + "sub_libp2p_requests_in_total", + "Total number of requests received and answered" + ), + buckets: prometheus_endpoint::exponential_buckets(0.001, 2.0, 16) + .expect("parameters are always valid values; qed"), + }, + &["protocol"] + )?, registry)?, + requests_out_finished: register(HistogramVec::new( + HistogramOpts { + common_opts: Opts::new( + "sub_libp2p_requests_out_finished", + "Time between a request's start and finish (successful or not)" + ), + buckets: prometheus_endpoint::exponential_buckets(0.001, 2.0, 16) + .expect("parameters are always valid values; qed"), + }, + &["protocol"] + )?, registry)?, + requests_out_started_total: register(CounterVec::new( + Opts::new( + "sub_libp2p_requests_out_started_total", + "Total number of requests emitted" + ), + &["protocol"] )?, registry)?, }) } @@ -915,14 +1003,18 @@ impl Metrics { fn update_with_network_event(&self, event: &Event) { match event { Event::NotificationStreamOpened { engine_id, .. } => { - self.opened_notification_streams.with_label_values(&[&engine_id_to_string(&engine_id)]).inc(); + self.notifications_streams_opened_total + .with_label_values(&[&maybe_utf8_bytes_to_string(engine_id)]).inc(); }, Event::NotificationStreamClosed { engine_id, .. } => { - self.opened_notification_streams.with_label_values(&[&engine_id_to_string(&engine_id)]).dec(); + self.notifications_streams_closed_total + .with_label_values(&[&maybe_utf8_bytes_to_string(engine_id)]).inc(); }, Event::NotificationsReceived { messages, .. } => { - for (engine_id, _) in messages { - self.notifications_total.with_label_values(&["in", &engine_id_to_string(&engine_id)]).inc(); + for (engine_id, message) in messages { + self.notifications_sizes + .with_label_values(&["in", &maybe_utf8_bytes_to_string(engine_id)]) + .observe(message.len() as f64); } }, _ => {} @@ -983,7 +1075,9 @@ impl Future for NetworkWorker { this.event_streams.push(sender), ServiceToWorkerMsg::WriteNotification { message, engine_id, target } => { if let Some(metrics) = this.metrics.as_ref() { - metrics.notifications_total.with_label_values(&["out", &engine_id_to_string(&engine_id)]).inc(); + metrics.notifications_sizes + .with_label_values(&["out", &maybe_utf8_bytes_to_string(&engine_id)]) + .observe(message.len() as f64); } this.network_service.user_protocol_mut().write_notification(target, engine_id, message) }, @@ -1022,36 +1116,90 @@ impl Future for NetworkWorker { } this.import_queue.import_finality_proof(origin, hash, nb, proof); }, - Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RandomKademliaStarted)) => { + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::AnsweredRequest { protocol, build_time, .. })) => { + if let Some(metrics) = this.metrics.as_ref() { + metrics.requests_in_total + .with_label_values(&[&maybe_utf8_bytes_to_string(&protocol)]) + .observe(build_time.as_secs_f64()); + } + }, + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestStarted { protocol, .. })) => { if let Some(metrics) = this.metrics.as_ref() { - metrics.random_kademalia_queries_total.inc(); + metrics.requests_out_started_total + .with_label_values(&[&maybe_utf8_bytes_to_string(&protocol)]) + .inc(); + } + }, + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RequestFinished { protocol, request_duration, .. })) => { + if let Some(metrics) = this.metrics.as_ref() { + metrics.requests_out_finished + .with_label_values(&[&maybe_utf8_bytes_to_string(&protocol)]) + .observe(request_duration.as_secs_f64()); + } + }, + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::RandomKademliaStarted(protocol))) => { + if let Some(metrics) = this.metrics.as_ref() { + metrics.kademlia_random_queries_total + .with_label_values(&[&maybe_utf8_bytes_to_string(protocol.as_bytes())]) + .inc(); } }, Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::Event(ev))) => { - this.event_streams.retain(|sender| sender.unbounded_send(ev.clone()).is_ok()); if let Some(metrics) = this.metrics.as_ref() { metrics.update_with_network_event(&ev); } + this.event_streams.send(ev); }, - Poll::Ready(SwarmEvent::Connected(peer_id)) => { + Poll::Ready(SwarmEvent::ConnectionEstablished { peer_id, endpoint, .. }) => { trace!(target: "sub-libp2p", "Libp2p => Connected({:?})", peer_id); if let Some(metrics) = this.metrics.as_ref() { - metrics.connections.inc(); + match endpoint { + ConnectedPoint::Dialer { .. } => + metrics.connections_opened_total.with_label_values(&["out"]).inc(), + ConnectedPoint::Listener { .. } => + metrics.connections_opened_total.with_label_values(&["in"]).inc(), + } } }, - Poll::Ready(SwarmEvent::Disconnected(peer_id)) => { - trace!(target: "sub-libp2p", "Libp2p => Disconnected({:?})", peer_id); + Poll::Ready(SwarmEvent::ConnectionClosed { peer_id, cause, endpoint, .. }) => { + trace!(target: "sub-libp2p", "Libp2p => Disconnected({:?}, {:?})", peer_id, cause); if let Some(metrics) = this.metrics.as_ref() { - metrics.connections.dec(); + let dir = match endpoint { + ConnectedPoint::Dialer { .. } => "out", + ConnectedPoint::Listener { .. } => "in", + }; + + match cause { + ConnectionError::IO(_) => + metrics.connections_closed_total.with_label_values(&[dir, "transport-error"]).inc(), + ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A( + EitherError::A(EitherError::A(EitherError::B( + EitherError::A(PingFailure::Timeout)))))))) => + metrics.connections_closed_total.with_label_values(&[dir, "ping-timeout"]).inc(), + ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A( + EitherError::A(EitherError::A(EitherError::A( + EitherError::B(LegacyConnectionKillError)))))))) => + metrics.connections_closed_total.with_label_values(&[dir, "force-closed"]).inc(), + ConnectionError::Handler(NodeHandlerWrapperError::Handler(_)) => + metrics.connections_closed_total.with_label_values(&[dir, "protocol-error"]).inc(), + ConnectionError::Handler(NodeHandlerWrapperError::KeepAliveTimeout) => + metrics.connections_closed_total.with_label_values(&[dir, "keep-alive-timeout"]).inc(), + } } }, - Poll::Ready(SwarmEvent::NewListenAddr(addr)) => - trace!(target: "sub-libp2p", "Libp2p => NewListenAddr({})", addr), - Poll::Ready(SwarmEvent::ExpiredListenAddr(addr)) => - trace!(target: "sub-libp2p", "Libp2p => ExpiredListenAddr({})", addr), - Poll::Ready(SwarmEvent::UnreachableAddr { peer_id, address, error }) => { - let error = error.to_string(); - + Poll::Ready(SwarmEvent::NewListenAddr(addr)) => { + trace!(target: "sub-libp2p", "Libp2p => NewListenAddr({})", addr); + if let Some(metrics) = this.metrics.as_ref() { + metrics.listeners_local_addresses.inc(); + } + }, + Poll::Ready(SwarmEvent::ExpiredListenAddr(addr)) => { + trace!(target: "sub-libp2p", "Libp2p => ExpiredListenAddr({})", addr); + if let Some(metrics) = this.metrics.as_ref() { + metrics.listeners_local_addresses.dec(); + } + }, + Poll::Ready(SwarmEvent::UnreachableAddr { peer_id, address, error, .. }) => { trace!( target: "sub-libp2p", "Libp2p => Failed to reach {:?} through {:?}: {}", peer_id, @@ -1059,21 +1207,72 @@ impl Future for NetworkWorker { error, ); - if let Some(peer_id) = peer_id { - if this.boot_node_ids.contains(&peer_id) - && error.contains("Peer ID mismatch") - { + if this.boot_node_ids.contains(&peer_id) { + if let PendingConnectionError::InvalidPeerId = error { error!( - "Connecting to bootnode with peer id `{}` and address `{}` failed \ - because it returned a different peer id!", - peer_id, + "💔 The bootnode you want to connect to at `{}` provided a different peer ID than the one you expect: `{}`.", address, + peer_id, ); } } + + if let Some(metrics) = this.metrics.as_ref() { + match error { + PendingConnectionError::ConnectionLimit(_) => + metrics.pending_connections_errors_total.with_label_values(&["limit-reached"]).inc(), + PendingConnectionError::InvalidPeerId => + metrics.pending_connections_errors_total.with_label_values(&["invalid-peer-id"]).inc(), + PendingConnectionError::Transport(_) | PendingConnectionError::IO(_) => + metrics.pending_connections_errors_total.with_label_values(&["transport-error"]).inc(), + } + } + } + Poll::Ready(SwarmEvent::Dialing(peer_id)) => + trace!(target: "sub-libp2p", "Libp2p => Dialing({:?})", peer_id), + Poll::Ready(SwarmEvent::IncomingConnection { local_addr, send_back_addr }) => { + trace!(target: "sub-libp2p", "Libp2p => IncomingConnection({},{}))", + local_addr, send_back_addr); + if let Some(metrics) = this.metrics.as_ref() { + metrics.incoming_connections_total.inc(); + } + }, + Poll::Ready(SwarmEvent::IncomingConnectionError { local_addr, send_back_addr, error }) => { + trace!(target: "sub-libp2p", "Libp2p => IncomingConnectionError({},{}): {}", + local_addr, send_back_addr, error); + if let Some(metrics) = this.metrics.as_ref() { + let reason = match error { + PendingConnectionError::ConnectionLimit(_) => "limit-reached", + PendingConnectionError::InvalidPeerId => "invalid-peer-id", + PendingConnectionError::Transport(_) | + PendingConnectionError::IO(_) => "transport-error", + }; + + metrics.incoming_connections_errors_total.with_label_values(&[reason]).inc(); + } + }, + Poll::Ready(SwarmEvent::BannedPeer { peer_id, endpoint }) => { + trace!(target: "sub-libp2p", "Libp2p => BannedPeer({}). Connected via {:?}.", + peer_id, endpoint); + if let Some(metrics) = this.metrics.as_ref() { + metrics.incoming_connections_errors_total.with_label_values(&["banned"]).inc(); + } + }, + Poll::Ready(SwarmEvent::UnknownPeerUnreachableAddr { address, error }) => + trace!(target: "sub-libp2p", "Libp2p => UnknownPeerUnreachableAddr({}): {}", + address, error), + Poll::Ready(SwarmEvent::ListenerClosed { reason, addresses }) => { + warn!(target: "sub-libp2p", "Libp2p => ListenerClosed: {:?}", reason); + if let Some(metrics) = this.metrics.as_ref() { + metrics.listeners_local_addresses.sub(addresses.len() as u64); + } + }, + Poll::Ready(SwarmEvent::ListenerError { error }) => { + trace!(target: "sub-libp2p", "Libp2p => ListenerError: {}", error); + if let Some(metrics) = this.metrics.as_ref() { + metrics.listeners_errors_total.inc(); + } }, - Poll::Ready(SwarmEvent::StartConnect(peer_id)) => - trace!(target: "sub-libp2p", "Libp2p => StartConnect({:?})", peer_id), }; } @@ -1097,11 +1296,22 @@ impl Future for NetworkWorker { metrics.network_per_sec_bytes.with_label_values(&["in"]).set(this.service.bandwidth.average_download_per_sec()); metrics.network_per_sec_bytes.with_label_values(&["out"]).set(this.service.bandwidth.average_upload_per_sec()); metrics.is_major_syncing.set(is_major_syncing as u64); - metrics.kbuckets_num_nodes.set(this.network_service.num_kbuckets_entries() as u64); - metrics.num_event_stream_channels.set(this.event_streams.len() as u64); + for (proto, num_entries) in this.network_service.num_kbuckets_entries() { + let proto = maybe_utf8_bytes_to_string(proto.as_bytes()); + metrics.kbuckets_num_nodes.with_label_values(&[&proto]).set(num_entries as u64); + } + for (proto, num_entries) in this.network_service.num_kademlia_records() { + let proto = maybe_utf8_bytes_to_string(proto.as_bytes()); + metrics.kademlia_records_count.with_label_values(&[&proto]).set(num_entries as u64); + } + for (proto, num_entries) in this.network_service.kademlia_records_total_size() { + let proto = maybe_utf8_bytes_to_string(proto.as_bytes()); + metrics.kademlia_records_sizes_total.with_label_values(&[&proto]).set(num_entries as u64); + } metrics.peers_count.set(num_connected_peers as u64); metrics.peerset_num_discovered.set(this.network_service.user_protocol().num_discovered_peers() as u64); metrics.peerset_num_requested.set(this.network_service.user_protocol().requested_peers().count() as u64); + metrics.pending_connections.set(Swarm::network_info(&this.network_service).num_connections_pending as u64); } Poll::Pending @@ -1111,8 +1321,10 @@ impl Future for NetworkWorker { impl Unpin for NetworkWorker { } -/// Turns a `ConsensusEngineId` into a representable string. -fn engine_id_to_string(id: &ConsensusEngineId) -> Cow { +/// Turns bytes that are potentially UTF-8 into a reasonable representable string. +/// +/// Meant to be used only for debugging or metrics-reporting purposes. +fn maybe_utf8_bytes_to_string(id: &[u8]) -> Cow { if let Ok(s) = std::str::from_utf8(&id[..]) { Cow::Borrowed(s) } else { diff --git a/client/network/src/service/out_events.rs b/client/network/src/service/out_events.rs new file mode 100644 index 0000000000000000000000000000000000000000..2d4d7ded213e57578daf2433ec48c9e009470bb5 --- /dev/null +++ b/client/network/src/service/out_events.rs @@ -0,0 +1,279 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Registering events streams. +//! +//! This code holds the logic that is used for the network service to inform other parts of +//! Substrate about what is happening. +//! +//! # Usage +//! +//! - Create an instance of [`OutChannels`]. +//! - Create channels using the [`channel`] function. The receiving side implements the `Stream` +//! trait. +//! - You cannot directly send an event on a sender. Instead, you have to call +//! [`OutChannels::push`] to put the sender within a [`OutChannels`]. +//! - Send events by calling [`OutChannels::send`]. Events are cloned for each sender in the +//! collection. +//! + +use crate::Event; +use super::maybe_utf8_bytes_to_string; + +use futures::{prelude::*, channel::mpsc, ready}; +use parking_lot::Mutex; +use prometheus_endpoint::{register, CounterVec, GaugeVec, Opts, PrometheusError, Registry, U64}; +use std::{ + convert::TryFrom as _, + fmt, pin::Pin, sync::Arc, + task::{Context, Poll} +}; + +/// Creates a new channel that can be associated to a [`OutChannels`]. +/// +/// The name is used in Prometheus reports. +pub fn channel(name: &'static str) -> (Sender, Receiver) { + let (tx, rx) = mpsc::unbounded(); + let metrics = Arc::new(Mutex::new(None)); + let tx = Sender { inner: tx, name, metrics: metrics.clone() }; + let rx = Receiver { inner: rx, name, metrics }; + (tx, rx) +} + +/// Sending side of a channel. +/// +/// Must be associated with an [`OutChannels`] before anything can be sent on it +/// +/// > **Note**: Contrary to regular channels, this `Sender` is purposefully designed to not +/// implement the `Clone` trait e.g. in Order to not complicate the logic keeping the metrics in +/// sync on drop. If someone adds a `#[derive(Clone)]` below, it is **wrong**. +pub struct Sender { + inner: mpsc::UnboundedSender, + name: &'static str, + /// Clone of [`Receiver::metrics`]. + metrics: Arc>>>>, +} + +impl fmt::Debug for Sender { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_tuple("Sender").finish() + } +} + +impl Drop for Sender { + fn drop(&mut self) { + let metrics = self.metrics.lock(); + if let Some(Some(metrics)) = metrics.as_ref().map(|m| &**m) { + metrics.num_channels.with_label_values(&[self.name]).dec(); + } + } +} + +/// Receiving side of a channel. +pub struct Receiver { + inner: mpsc::UnboundedReceiver, + name: &'static str, + /// Initially contains `None`, and will be set to a value once the corresponding [`Sender`] + /// is assigned to an instance of [`OutChannels`]. + metrics: Arc>>>>, +} + +impl Stream for Receiver { + type Item = Event; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + if let Some(ev) = ready!(Pin::new(&mut self.inner).poll_next(cx)) { + let metrics = self.metrics.lock().clone(); + match metrics.as_ref().map(|m| m.as_ref()) { + Some(Some(metrics)) => metrics.event_out(&ev, self.name), + Some(None) => (), // no registry + None => log::warn!("Inconsistency in out_events: event happened before sender associated"), + } + Poll::Ready(Some(ev)) + } else { + Poll::Ready(None) + } + } +} + +impl fmt::Debug for Receiver { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_tuple("Receiver").finish() + } +} + +impl Drop for Receiver { + fn drop(&mut self) { + // Empty the list to properly decrease the metrics. + while let Some(Some(_)) = self.next().now_or_never() {} + } +} + +/// Collection of senders. +pub struct OutChannels { + event_streams: Vec, + /// The metrics we collect. A clone of this is sent to each [`Receiver`] associated with this + /// object. + metrics: Arc>, +} + +impl OutChannels { + /// Creates a new empty collection of senders. + pub fn new(registry: Option<&Registry>) -> Result { + let metrics = if let Some(registry) = registry { + Some(Metrics::register(registry)?) + } else { + None + }; + + Ok(OutChannels { + event_streams: Vec::new(), + metrics: Arc::new(metrics), + }) + } + + /// Adds a new [`Sender`] to the collection. + pub fn push(&mut self, sender: Sender) { + let mut metrics = sender.metrics.lock(); + debug_assert!(metrics.is_none()); + *metrics = Some(self.metrics.clone()); + drop(metrics); + + if let Some(metrics) = &*self.metrics { + metrics.num_channels.with_label_values(&[sender.name]).inc(); + } + + self.event_streams.push(sender); + } + + /// Sends an event. + pub fn send(&mut self, event: Event) { + self.event_streams.retain(|sender| { + sender.inner.unbounded_send(event.clone()).is_ok() + }); + + if let Some(metrics) = &*self.metrics { + for ev in &self.event_streams { + metrics.event_in(&event, 1, ev.name); + } + } + } +} + +impl fmt::Debug for OutChannels { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("OutChannels") + .field("num_channels", &self.event_streams.len()) + .finish() + } +} + +struct Metrics { + // This list is ordered alphabetically + events_total: CounterVec, + notifications_sizes: CounterVec, + num_channels: GaugeVec, +} + +impl Metrics { + fn register(registry: &Registry) -> Result { + Ok(Self { + events_total: register(CounterVec::new( + Opts::new( + "sub_libp2p_out_events_events_total", + "Number of broadcast network events that have been sent or received across all \ + channels" + ), + &["event_name", "action", "name"] + )?, registry)?, + notifications_sizes: register(CounterVec::new( + Opts::new( + "sub_libp2p_out_events_notifications_sizes", + "Size of notification events that have been sent or received across all \ + channels" + ), + &["protocol", "action", "name"] + )?, registry)?, + num_channels: register(GaugeVec::new( + Opts::new( + "sub_libp2p_out_events_num_channels", + "Number of internal active channels that broadcast network events", + ), + &["name"] + )?, registry)?, + }) + } + + fn event_in(&self, event: &Event, num: u64, name: &str) { + match event { + Event::Dht(_) => { + self.events_total + .with_label_values(&["dht", "sent", name]) + .inc_by(num); + } + Event::NotificationStreamOpened { engine_id, .. } => { + self.events_total + .with_label_values(&[&format!("notif-open-{:?}", engine_id), "sent", name]) + .inc_by(num); + }, + Event::NotificationStreamClosed { engine_id, .. } => { + self.events_total + .with_label_values(&[&format!("notif-closed-{:?}", engine_id), "sent", name]) + .inc_by(num); + }, + Event::NotificationsReceived { messages, .. } => { + for (engine_id, message) in messages { + self.events_total + .with_label_values(&[&format!("notif-{:?}", engine_id), "sent", name]) + .inc_by(num); + self.notifications_sizes + .with_label_values(&[&maybe_utf8_bytes_to_string(engine_id), "sent", name]) + .inc_by(num.saturating_mul(u64::try_from(message.len()).unwrap_or(u64::max_value()))); + } + }, + } + } + + fn event_out(&self, event: &Event, name: &str) { + match event { + Event::Dht(_) => { + self.events_total + .with_label_values(&["dht", "received", name]) + .inc(); + } + Event::NotificationStreamOpened { engine_id, .. } => { + self.events_total + .with_label_values(&[&format!("notif-open-{:?}", engine_id), "received", name]) + .inc(); + }, + Event::NotificationStreamClosed { engine_id, .. } => { + self.events_total + .with_label_values(&[&format!("notif-closed-{:?}", engine_id), "received", name]) + .inc(); + }, + Event::NotificationsReceived { messages, .. } => { + for (engine_id, message) in messages { + self.events_total + .with_label_values(&[&format!("notif-{:?}", engine_id), "received", name]) + .inc(); + self.notifications_sizes + .with_label_values(&[&maybe_utf8_bytes_to_string(engine_id), "received", name]) + .inc_by(u64::try_from(message.len()).unwrap_or(u64::max_value())); + } + }, + } + } +} diff --git a/client/network/src/service/tests.rs b/client/network/src/service/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..84c393fbd2f425ba432e13e3dad863147ae3e415 --- /dev/null +++ b/client/network/src/service/tests.rs @@ -0,0 +1,270 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::{config, Event, NetworkService, NetworkWorker}; + +use futures::prelude::*; +use sp_runtime::traits::{Block as BlockT, Header as _}; +use std::{sync::Arc, time::Duration}; +use substrate_test_runtime_client::{TestClientBuilder, TestClientBuilderExt as _}; + +type TestNetworkService = NetworkService< + substrate_test_runtime_client::runtime::Block, + substrate_test_runtime_client::runtime::Hash, +>; + +/// Builds a full node to be used for testing. Returns the node service and its associated events +/// stream. +/// +/// > **Note**: We return the events stream in order to not possibly lose events between the +/// > construction of the service and the moment the events stream is grabbed. +fn build_test_full_node(config: config::NetworkConfiguration) + -> (Arc, impl Stream) +{ + let client = Arc::new( + TestClientBuilder::with_default_backend() + .build_with_longest_chain() + .0, + ); + + #[derive(Clone)] + struct PassThroughVerifier(bool); + impl sp_consensus::import_queue::Verifier for PassThroughVerifier { + fn verify( + &mut self, + origin: sp_consensus::BlockOrigin, + header: B::Header, + justification: Option, + body: Option>, + ) -> Result< + ( + sp_consensus::BlockImportParams, + Option)>>, + ), + String, + > { + let maybe_keys = header + .digest() + .log(|l| { + l.try_as_raw(sp_runtime::generic::OpaqueDigestItemId::Consensus(b"aura")) + .or_else(|| { + l.try_as_raw(sp_runtime::generic::OpaqueDigestItemId::Consensus(b"babe")) + }) + }) + .map(|blob| { + vec![( + sp_blockchain::well_known_cache_keys::AUTHORITIES, + blob.to_vec(), + )] + }); + + let mut import = sp_consensus::BlockImportParams::new(origin, header); + import.body = body; + import.finalized = self.0; + import.justification = justification; + import.fork_choice = Some(sp_consensus::ForkChoiceStrategy::LongestChain); + Ok((import, maybe_keys)) + } + } + + let import_queue = Box::new(sp_consensus::import_queue::BasicQueue::new( + PassThroughVerifier(false), + Box::new(client.clone()), + None, + None, + )); + + let worker = NetworkWorker::new(config::Params { + role: config::Role::Full, + executor: None, + network_config: config, + chain: client.clone(), + finality_proof_provider: None, + finality_proof_request_builder: None, + on_demand: None, + transaction_pool: Arc::new(crate::config::EmptyTransactionPool), + protocol_id: config::ProtocolId::from(&b"/test-protocol-name"[..]), + import_queue, + block_announce_validator: Box::new( + sp_consensus::block_validation::DefaultBlockAnnounceValidator::new(client.clone()), + ), + metrics_registry: None, + }) + .unwrap(); + + let service = worker.service().clone(); + let event_stream = service.event_stream("test"); + + async_std::task::spawn(async move { + futures::pin_mut!(worker); + let _ = worker.await; + }); + + (service, event_stream) +} + +const ENGINE_ID: sp_runtime::ConsensusEngineId = *b"foo\0"; + +/// Builds two nodes and their associated events stream. +/// The nodes are connected together and have the `ENGINE_ID` protocol registered. +fn build_nodes_one_proto() + -> (Arc, impl Stream, Arc, impl Stream) +{ + let listen_addr = config::build_multiaddr![Memory(rand::random::())]; + + let (node1, events_stream1) = build_test_full_node(config::NetworkConfiguration { + notifications_protocols: vec![(ENGINE_ID, From::from(&b"/foo"[..]))], + listen_addresses: vec![listen_addr.clone()], + transport: config::TransportConfig::MemoryOnly, + .. config::NetworkConfiguration::new_local() + }); + + let (node2, events_stream2) = build_test_full_node(config::NetworkConfiguration { + notifications_protocols: vec![(ENGINE_ID, From::from(&b"/foo"[..]))], + reserved_nodes: vec![config::MultiaddrWithPeerId { + multiaddr: listen_addr, + peer_id: node1.local_peer_id().clone(), + }], + transport: config::TransportConfig::MemoryOnly, + .. config::NetworkConfiguration::new_local() + }); + + (node1, events_stream1, node2, events_stream2) +} + +#[test] +fn notifications_state_consistent() { + // Runs two nodes and ensures that events are propagated out of the API in a consistent + // correct order, which means no notification received on a closed substream. + + let (node1, mut events_stream1, node2, mut events_stream2) = build_nodes_one_proto(); + + // Write some initial notifications that shouldn't get through. + for _ in 0..(rand::random::() % 5) { + node1.write_notification(node2.local_peer_id().clone(), ENGINE_ID, b"hello world".to_vec()); + } + for _ in 0..(rand::random::() % 5) { + node2.write_notification(node1.local_peer_id().clone(), ENGINE_ID, b"hello world".to_vec()); + } + + async_std::task::block_on(async move { + // True if we have an active substream from node1 to node2. + let mut node1_to_node2_open = false; + // True if we have an active substream from node2 to node1. + let mut node2_to_node1_open = false; + // We stop the test after a certain number of iterations. + let mut iterations = 0; + // Safe guard because we don't want the test to pass if no substream has been open. + let mut something_happened = false; + + loop { + iterations += 1; + if iterations >= 1_000 { + assert!(something_happened); + break; + } + + // Start by sending a notification from node1 to node2 and vice-versa. Part of the + // test consists in ensuring that notifications get ignored if the stream isn't open. + if rand::random::() % 5 >= 3 { + node1.write_notification(node2.local_peer_id().clone(), ENGINE_ID, b"hello world".to_vec()); + } + if rand::random::() % 5 >= 3 { + node2.write_notification(node1.local_peer_id().clone(), ENGINE_ID, b"hello world".to_vec()); + } + + // Also randomly disconnect the two nodes from time to time. + if rand::random::() % 20 == 0 { + node1.disconnect_peer(node2.local_peer_id().clone()); + } + if rand::random::() % 20 == 0 { + node2.disconnect_peer(node1.local_peer_id().clone()); + } + + // Grab next event from either `events_stream1` or `events_stream2`. + let next_event = { + let next1 = events_stream1.next(); + let next2 = events_stream2.next(); + // We also await on a small timer, otherwise it is possible for the test to wait + // forever while nothing at all happens on the network. + let continue_test = futures_timer::Delay::new(Duration::from_millis(20)); + match future::select(future::select(next1, next2), continue_test).await { + future::Either::Left((future::Either::Left((Some(ev), _)), _)) => + future::Either::Left(ev), + future::Either::Left((future::Either::Right((Some(ev), _)), _)) => + future::Either::Right(ev), + future::Either::Right(_) => continue, + _ => break, + } + }; + + match next_event { + future::Either::Left(Event::NotificationStreamOpened { remote, engine_id, .. }) => { + something_happened = true; + assert!(!node1_to_node2_open); + node1_to_node2_open = true; + assert_eq!(remote, *node2.local_peer_id()); + assert_eq!(engine_id, ENGINE_ID); + } + future::Either::Right(Event::NotificationStreamOpened { remote, engine_id, .. }) => { + something_happened = true; + assert!(!node2_to_node1_open); + node2_to_node1_open = true; + assert_eq!(remote, *node1.local_peer_id()); + assert_eq!(engine_id, ENGINE_ID); + } + future::Either::Left(Event::NotificationStreamClosed { remote, engine_id, .. }) => { + assert!(node1_to_node2_open); + node1_to_node2_open = false; + assert_eq!(remote, *node2.local_peer_id()); + assert_eq!(engine_id, ENGINE_ID); + } + future::Either::Right(Event::NotificationStreamClosed { remote, engine_id, .. }) => { + assert!(node2_to_node1_open); + node2_to_node1_open = false; + assert_eq!(remote, *node1.local_peer_id()); + assert_eq!(engine_id, ENGINE_ID); + } + future::Either::Left(Event::NotificationsReceived { remote, .. }) => { + assert!(node1_to_node2_open); + assert_eq!(remote, *node2.local_peer_id()); + if rand::random::() % 5 >= 4 { + node1.write_notification( + node2.local_peer_id().clone(), + ENGINE_ID, + b"hello world".to_vec() + ); + } + } + future::Either::Right(Event::NotificationsReceived { remote, .. }) => { + assert!(node2_to_node1_open); + assert_eq!(remote, *node1.local_peer_id()); + if rand::random::() % 5 >= 4 { + node2.write_notification( + node1.local_peer_id().clone(), + ENGINE_ID, + b"hello world".to_vec() + ); + } + } + + // Add new events here. + future::Either::Left(Event::Dht(_)) => {} + future::Either::Right(Event::Dht(_)) => {} + }; + } + }); +} diff --git a/client/network/test/Cargo.toml b/client/network/test/Cargo.toml index 769e0faca9e5cd8a1055515f5c9abd66df22b915..154694c692adbb786c397529bf397509cee9bc08 100644 --- a/client/network/test/Cargo.toml +++ b/client/network/test/Cargo.toml @@ -9,26 +9,27 @@ publish = false homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sc-network = { version = "0.8.0-alpha.5", path = "../" } +sc-network = { version = "0.8.0-dev", path = "../" } log = "0.4.8" parking_lot = "0.10.0" futures = "0.3.4" futures-timer = "3.0.1" rand = "0.7.2" -libp2p = { version = "0.16.2", default-features = false, features = ["libp2p-websocket"] } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } -sc-client = { version = "0.8.0-alpha.5", path = "../../" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../../api" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sc-block-builder = { version = "0.8.0-alpha.5", path = "../../block-builder" } -sp-consensus-babe = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/babe" } +libp2p = { version = "0.18.1", default-features = false, features = ["libp2p-websocket"] } +sp-consensus = { version = "0.8.0-dev", path = "../../../primitives/consensus/common" } +sc-consensus = { version = "0.8.0-dev", path = "../../../client/consensus/common" } +sc-client-api = { version = "2.0.0-dev", path = "../../api" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sc-block-builder = { version = "0.8.0-dev", path = "../../block-builder" } +sp-consensus-babe = { version = "0.8.0-dev", path = "../../../primitives/consensus/babe" } env_logger = "0.7.0" substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } substrate-test-runtime = { version = "2.0.0-dev", path = "../../../test-utils/runtime" } tempfile = "3.1.0" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sc-service = { version = "0.8.0-dev", default-features = false, features = ["test-helpers"], path = "../../service" } diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index aa887bf5ca6765b48e9507f977c01b67522d6557..5be5de9078ef117934938e16358ce1b2ca5405e7 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -27,11 +27,16 @@ use libp2p::build_multiaddr; use log::trace; use sc_network::config::FinalityProofProvider; use sp_blockchain::{ - Result as ClientResult, well_known_cache_keys::{self, Id as CacheKeyId}, Info as BlockchainInfo, + HeaderBackend, Result as ClientResult, + well_known_cache_keys::{self, Id as CacheKeyId}, + Info as BlockchainInfo, }; -use sc_client_api::{BlockchainEvents, BlockImportNotification, FinalityNotifications, ImportNotifications, FinalityNotification, backend::{TransactionFor, AuxStore, Backend, Finalizer}, BlockBackend}; +use sc_client_api::{ + BlockchainEvents, BlockImportNotification, FinalityNotifications, ImportNotifications, FinalityNotification, + backend::{TransactionFor, AuxStore, Backend, Finalizer}, BlockBackend, +}; +use sc_consensus::LongestChain; use sc_block_builder::{BlockBuilder, BlockBuilderProvider}; -use sc_client::LongestChain; use sc_network::config::Role; use sp_consensus::block_validation::DefaultBlockAnnounceValidator; use sp_consensus::import_queue::{ @@ -41,7 +46,7 @@ use sp_consensus::block_import::{BlockImport, ImportResult}; use sp_consensus::Error as ConsensusError; use sp_consensus::{BlockOrigin, ForkChoiceStrategy, BlockImportParams, BlockCheckParams, JustificationImport}; use futures::prelude::*; -use sc_network::{NetworkWorker, NetworkStateInfo, NetworkService, config::ProtocolId}; +use sc_network::{NetworkWorker, NetworkService, config::ProtocolId}; use sc_network::config::{NetworkConfiguration, TransportConfig, BoxFinalityProofRequestBuilder}; use libp2p::PeerId; use parking_lot::Mutex; @@ -51,7 +56,7 @@ use sp_runtime::generic::{BlockId, OpaqueDigestItemId}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; use sp_runtime::Justification; use substrate_test_runtime_client::{self, AccountKeyring}; - +use sc_service::client::Client; pub use sc_network::config::EmptyTransactionPool; pub use substrate_test_runtime_client::runtime::{Block, Extrinsic, Hash, Transfer}; pub use substrate_test_runtime_client::{TestClient, TestClientBuilder, TestClientBuilderExt}; @@ -87,10 +92,18 @@ impl Verifier for PassThroughVerifier { } } -pub type PeersFullClient = - sc_client::Client; -pub type PeersLightClient = - sc_client::Client; +pub type PeersFullClient = Client< + substrate_test_runtime_client::Backend, + substrate_test_runtime_client::Executor, + Block, + substrate_test_runtime_client::runtime::RuntimeApi +>; +pub type PeersLightClient = Client< + substrate_test_runtime_client::LightBackend, + substrate_test_runtime_client::LightExecutor, + Block, + substrate_test_runtime_client::runtime::RuntimeApi +>; #[derive(Clone)] pub enum PeersClient { @@ -189,7 +202,7 @@ pub struct Peer { impl Peer { /// Get this peer ID. pub fn id(&self) -> PeerId { - self.network.service().local_peer_id() + self.network.service().local_peer_id().clone() } /// Returns true if we're major syncing. @@ -359,13 +372,9 @@ impl Peer { /// Test helper to compare the blockchain state of multiple (networked) /// clients. - /// Potentially costly, as it creates in-memory copies of both blockchains in order - /// to compare them. If you have easier/softer checks that are sufficient, e.g. - /// by using .info(), you should probably use it instead of this. pub fn blockchain_canon_equals(&self, other: &Self) -> bool { if let (Some(mine), Some(others)) = (self.backend.clone(), other.backend.clone()) { - mine.as_in_memory().blockchain() - .canon_equals_to(others.as_in_memory().blockchain()) + mine.blockchain().info().best_hash == others.blockchain().info().best_hash } else { false } @@ -374,7 +383,7 @@ impl Peer { /// Count the total number of imported blocks. pub fn blocks_count(&self) -> u64 { self.backend.as_ref().map( - |backend| backend.blocks_count() + |backend| backend.blockchain().info().best_number ).unwrap_or(0) } @@ -382,6 +391,12 @@ impl Peer { pub fn failed_verifications(&self) -> HashMap<::Hash, String> { self.verifier.failed_verifications.lock().clone() } + + pub fn has_block(&self, hash: &H256) -> bool { + self.backend.as_ref().map( + |backend| backend.blockchain().header(BlockId::hash(*hash)).unwrap().is_some() + ).unwrap_or(false) + } } /// Implements `BlockImport` for any `Transaction`. Internally the transaction is @@ -600,14 +615,20 @@ pub trait TestNetFactory: Sized { let listen_addr = build_multiaddr![Memory(rand::random::())]; + let mut network_config = NetworkConfiguration::new( + "test-node", + "test-client", + Default::default(), + None, + ); + network_config.transport = TransportConfig::MemoryOnly; + network_config.listen_addresses = vec![listen_addr.clone()]; + network_config.allow_non_globals_in_dht = true; + let network = NetworkWorker::new(sc_network::config::Params { role: Role::Full, executor: None, - network_config: NetworkConfiguration { - listen_addresses: vec![listen_addr.clone()], - transport: TransportConfig::MemoryOnly, - ..NetworkConfiguration::default() - }, + network_config, chain: client.clone(), finality_proof_provider: self.make_finality_proof_provider( PeersClient::Full(client.clone(), backend.clone()), @@ -623,7 +644,7 @@ pub trait TestNetFactory: Sized { self.mut_peers(|peers| { for peer in peers.iter_mut() { - peer.network.add_known_address(network.service().local_peer_id(), listen_addr.clone()); + peer.network.add_known_address(network.service().local_peer_id().clone(), listen_addr.clone()); } let imported_blocks_stream = Box::pin(client.import_notification_stream().fuse()); @@ -671,14 +692,20 @@ pub trait TestNetFactory: Sized { let listen_addr = build_multiaddr![Memory(rand::random::())]; + let mut network_config = NetworkConfiguration::new( + "test-node", + "test-client", + Default::default(), + None, + ); + network_config.transport = TransportConfig::MemoryOnly; + network_config.listen_addresses = vec![listen_addr.clone()]; + network_config.allow_non_globals_in_dht = true; + let network = NetworkWorker::new(sc_network::config::Params { role: Role::Light, executor: None, - network_config: NetworkConfiguration { - listen_addresses: vec![listen_addr.clone()], - transport: TransportConfig::MemoryOnly, - ..NetworkConfiguration::default() - }, + network_config, chain: client.clone(), finality_proof_provider: self.make_finality_proof_provider( PeersClient::Light(client.clone(), backend.clone()) @@ -694,7 +721,7 @@ pub trait TestNetFactory: Sized { self.mut_peers(|peers| { for peer in peers.iter_mut() { - peer.network.add_known_address(network.service().local_peer_id(), listen_addr.clone()); + peer.network.add_known_address(network.service().local_peer_id().clone(), listen_addr.clone()); } let imported_blocks_stream = Box::pin(client.import_notification_stream().fuse()); diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 8acf265e9189288758471f69db6873b0418a3edd..60e9e558c5fffafd0863ce47202b6129d8e2dd47 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -339,13 +339,15 @@ fn syncs_all_forks() { net.peer(0).push_blocks(2, false); net.peer(1).push_blocks(2, false); - net.peer(0).push_blocks(2, true); - net.peer(1).push_blocks(4, false); + let b1 = net.peer(0).push_blocks(2, true); + let b2 = net.peer(1).push_blocks(4, false); net.block_until_sync(); - // Check that all peers have all of the blocks. - assert_eq!(9, net.peer(0).blocks_count()); - assert_eq!(9, net.peer(1).blocks_count()); + // Check that all peers have all of the branches. + assert!(net.peer(0).has_block(&b1)); + assert!(net.peer(0).has_block(&b2)); + assert!(net.peer(1).has_block(&b1)); + assert!(net.peer(1).has_block(&b2)); } #[test] @@ -587,24 +589,11 @@ fn syncs_header_only_forks() { net.peer(0).push_blocks(2, true); let small_hash = net.peer(0).client().info().best_hash; - let small_number = net.peer(0).client().info().best_number; net.peer(1).push_blocks(4, false); net.block_until_sync(); // Peer 1 will sync the small fork even though common block state is missing - assert_eq!(9, net.peer(0).blocks_count()); - assert_eq!(9, net.peer(1).blocks_count()); - - // Request explicit header-only sync request for the ancient fork. - let first_peer_id = net.peer(0).id(); - net.peer(1).set_sync_fork_request(vec![first_peer_id], small_hash, small_number); - block_on(futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - if net.peer(1).client().header(&BlockId::Hash(small_hash)).unwrap().is_none() { - return Poll::Pending - } - Poll::Ready(()) - })); + assert!(net.peer(1).has_block(&small_hash)); } #[test] diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml index e7292439e82eb0d4ad27e9f64c897c764b2f18f4..ac3a71ad0d9c8f89745817d926481b71cecba196 100644 --- a/client/offchain/Cargo.toml +++ b/client/offchain/Cargo.toml @@ -1,32 +1,35 @@ [package] description = "Substrate offchain workers" name = "sc-offchain" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] bytes = "0.5" -sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } -sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } +sc-client-api = { version = "2.0.0-dev", path = "../api" } +sp-api = { version = "2.0.0-dev", path = "../../primitives/api" } fnv = "1.0.6" futures = "0.3.4" futures-timer = "3.0.1" log = "0.4.8" threadpool = "1.7" num_cpus = "1.10" -sp-offchain = { version = "2.0.0-alpha.5", path = "../../primitives/offchain" } +sp-offchain = { version = "2.0.0-dev", path = "../../primitives/offchain" } codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } parking_lot = "0.10.0" -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } rand = "0.7.2" -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } -sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } -sc-network = { version = "0.8.0-alpha.5", path = "../network" } -sc-keystore = { version = "2.0.0-alpha.5", path = "../keystore" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-dev", path = "../../primitives/utils" } +sc-network = { version = "0.8.0-dev", path = "../network" } +sc-keystore = { version = "2.0.0-dev", path = "../keystore" } [target.'cfg(not(target_os = "unknown"))'.dependencies] hyper = "0.13.2" @@ -35,14 +38,11 @@ hyper-rustls = "0.20" [dev-dependencies] env_logger = "0.7.0" fdlimit = "0.1.4" -sc-client-db = { version = "0.8.0-alpha.5", default-features = true, path = "../db/" } -sc-transaction-pool = { version = "2.0.0-alpha.5", path = "../../client/transaction-pool" } -sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../primitives/transaction-pool" } +sc-client-db = { version = "0.8.0-dev", default-features = true, path = "../db/" } +sc-transaction-pool = { version = "2.0.0-dev", path = "../../client/transaction-pool" } +sp-transaction-pool = { version = "2.0.0-dev", path = "../../primitives/transaction-pool" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } tokio = "0.2" [features] default = [] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 94850e3fd3461c3cf79ecd48e905051991324941..332e9f779a8e27f3e655238ed5a70510db21605a 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -206,6 +206,7 @@ mod tests { let pool = Arc::new(TestPool(BasicPool::new( Default::default(), Arc::new(FullChainApi::new(client.clone())), + None, ).0)); client.execution_extensions() .register_transaction_pool(Arc::downgrade(&pool.clone()) as _); diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index 78d488a9899fc75ec68aad74e48c9017e8ae5de4..d5aa08f2f4981203b4c94b8299b174c771b25459 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -3,23 +3,23 @@ description = "Connectivity manager based on reputation" homepage = "http://parity.io" license = "GPL-3.0" name = "sc-peerset" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sc-peerset" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] futures = "0.3.4" -libp2p = { version = "0.16.2", default-features = false } -sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils"} +libp2p = { version = "0.18.1", default-features = false } +sp-utils = { version = "2.0.0-dev", path = "../../primitives/utils"} log = "0.4.8" serde_json = "1.0.41" wasm-timer = "0.2" [dev-dependencies] rand = "0.7.2" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/rpc-api/Cargo.toml b/client/rpc-api/Cargo.toml index c5043b5b0782b72d00e0f9940d0f6e87f9963c9c..e0dac773bf0991a9b685c2c2ebb9621f9319d4d6 100644 --- a/client/rpc-api/Cargo.toml +++ b/client/rpc-api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-rpc-api" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,6 +8,9 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Substrate RPC interfaces." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0" } derive_more = "0.99.2" @@ -18,13 +21,11 @@ jsonrpc-derive = "14.0.3" jsonrpc-pubsub = "14.0.3" log = "0.4.8" parking_lot = "0.10.0" -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-version = { version = "2.0.0-alpha.5", path = "../../primitives/version" } -sp-runtime = { path = "../../primitives/runtime" , version = "2.0.0-alpha.5"} +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-version = { version = "2.0.0-dev", path = "../../primitives/version" } +sp-runtime = { path = "../../primitives/runtime" , version = "2.0.0-dev"} +sp-chain-spec = { path = "../../primitives/chain-spec" , version = "2.0.0-dev"} serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" -sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../primitives/transaction-pool" } -sp-rpc = { version = "2.0.0-alpha.5", path = "../../primitives/rpc" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sp-transaction-pool = { version = "2.0.0-dev", path = "../../primitives/transaction-pool" } +sp-rpc = { version = "2.0.0-dev", path = "../../primitives/rpc" } diff --git a/client/rpc-api/src/author/error.rs b/client/rpc-api/src/author/error.rs index f1b56910086b5cb671689d612d9d2062d4722141..dfd488e5da37fd86d5a94b40fb893ee3cfcd2ac6 100644 --- a/client/rpc-api/src/author/error.rs +++ b/client/rpc-api/src/author/error.rs @@ -57,6 +57,8 @@ pub enum Error { /// Invalid session keys encoding. #[display(fmt="Session keys are not encoded correctly")] InvalidSessionKeys, + /// Call to an unsafe RPC was denied. + UnsafeRpcCalled(crate::policy::UnsafeRpcError), } impl std::error::Error for Error { @@ -65,6 +67,7 @@ impl std::error::Error for Error { Error::Client(ref err) => Some(&**err), Error::Pool(ref err) => Some(err), Error::Verification(ref err) => Some(&**err), + Error::UnsafeRpcCalled(ref err) => Some(err), _ => None, } } @@ -152,6 +155,7 @@ impl From for rpc::Error { request to insert the key successfully.".into() ), }, + Error::UnsafeRpcCalled(e) => e.into(), e => errors::internal(e), } } diff --git a/client/rpc-api/src/child_state/mod.rs b/client/rpc-api/src/child_state/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..a46269cad6c0c72c7d6478632a7d9971032d5ab9 --- /dev/null +++ b/client/rpc-api/src/child_state/mod.rs @@ -0,0 +1,69 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Substrate state API. + +use jsonrpc_derive::rpc; +use sp_core::storage::{StorageKey, PrefixedStorageKey, StorageData}; +use crate::state::error::FutureResult; + +pub use self::gen_client::Client as ChildStateClient; + +/// Substrate child state API +/// +/// Note that all `PrefixedStorageKey` are desierialized +/// from json and not guaranted valid. +#[rpc] +pub trait ChildStateApi { + /// RPC Metadata + type Metadata; + + /// Returns the keys with prefix from a child storage, leave empty to get all the keys + #[rpc(name = "childstate_getKeys")] + fn storage_keys( + &self, + child_storage_key: PrefixedStorageKey, + prefix: StorageKey, + hash: Option + ) -> FutureResult>; + + /// Returns a child storage entry at a specific block's state. + #[rpc(name = "childstate_getStorage")] + fn storage( + &self, + child_storage_key: PrefixedStorageKey, + key: StorageKey, + hash: Option + ) -> FutureResult>; + + /// Returns the hash of a child storage entry at a block's state. + #[rpc(name = "childstate_getStorageHash")] + fn storage_hash( + &self, + child_storage_key: PrefixedStorageKey, + key: StorageKey, + hash: Option + ) -> FutureResult>; + + /// Returns the size of a child storage entry at a block's state. + #[rpc(name = "childstate_getStorageSize")] + fn storage_size( + &self, + child_storage_key: PrefixedStorageKey, + key: StorageKey, + hash: Option + ) -> FutureResult>; +} diff --git a/client/rpc-api/src/lib.rs b/client/rpc-api/src/lib.rs index 8ad2d94bfd271ea0732ba811f2285e3f585b1554..f742d73b692c00de347e8a8e6f7994b529889267 100644 --- a/client/rpc-api/src/lib.rs +++ b/client/rpc-api/src/lib.rs @@ -22,14 +22,17 @@ mod errors; mod helpers; +mod policy; mod subscriptions; pub use jsonrpc_core::IoHandlerExtension as RpcExtension; pub use subscriptions::{Subscriptions, TaskExecutor}; pub use helpers::Receiver; +pub use policy::DenyUnsafe; pub mod author; pub mod chain; pub mod offchain; pub mod state; +pub mod child_state; pub mod system; diff --git a/client/rpc-api/src/offchain/error.rs b/client/rpc-api/src/offchain/error.rs index c28a2a2f3911d7e5cd9bedc9e0e84cb60905d6af..695c0cf41fd6da092c970b55a6638dee8ff4ea06 100644 --- a/client/rpc-api/src/offchain/error.rs +++ b/client/rpc-api/src/offchain/error.rs @@ -27,11 +27,16 @@ pub enum Error { /// Unavailable storage kind error. #[display(fmt="This storage kind is not available yet.")] UnavailableStorageKind, + /// Call to an unsafe RPC was denied. + UnsafeRpcCalled(crate::policy::UnsafeRpcError), } impl std::error::Error for Error { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - None + match self { + Self::UnsafeRpcCalled(err) => Some(err), + _ => None, + } } } @@ -46,6 +51,7 @@ impl From for rpc::Error { message: "This storage kind is not available yet" .into(), data: None, }, + Error::UnsafeRpcCalled(e) => e.into(), } } } diff --git a/client/rpc-api/src/policy.rs b/client/rpc-api/src/policy.rs new file mode 100644 index 0000000000000000000000000000000000000000..c01b5232f359c2224708f86b43467b4cdf4ca8d2 --- /dev/null +++ b/client/rpc-api/src/policy.rs @@ -0,0 +1,60 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Policy-related types. +//! +//! Contains a `DenyUnsafe` type that can be used to deny potentially unsafe +//! RPC when accessed externally. + +use jsonrpc_core as rpc; + +/// Signifies whether a potentially unsafe RPC should be denied. +#[derive(Clone, Copy, Debug)] +pub enum DenyUnsafe { + /// Denies only potentially unsafe RPCs. + Yes, + /// Allows calling every RPCs. + No +} + +impl DenyUnsafe { + /// Returns `Ok(())` if the RPCs considered unsafe are safe to call, + /// otherwise returns `Err(UnsafeRpcError)`. + pub fn check_if_safe(self) -> Result<(), UnsafeRpcError> { + match self { + DenyUnsafe::Yes => Err(UnsafeRpcError), + DenyUnsafe::No => Ok(()) + } + } +} + +/// Signifies whether an RPC considered unsafe is denied to be called externally. +#[derive(Debug)] +pub struct UnsafeRpcError; + +impl std::fmt::Display for UnsafeRpcError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "RPC call is unsafe to be called externally") + } +} + +impl std::error::Error for UnsafeRpcError {} + +impl From for rpc::Error { + fn from(_: UnsafeRpcError) -> rpc::Error { + rpc::Error::method_not_found() + } +} diff --git a/client/rpc-api/src/state/helpers.rs b/client/rpc-api/src/state/helpers.rs new file mode 100644 index 0000000000000000000000000000000000000000..516b6c80c49a7bfcb931f10bbdc771a81d685b19 --- /dev/null +++ b/client/rpc-api/src/state/helpers.rs @@ -0,0 +1,30 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Substrate state API helpers. + +use sp_core::Bytes; +use serde::{Serialize, Deserialize}; + +/// ReadProof struct returned by the RPC +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReadProof { + /// Block hash used to generate the proof + pub at: Hash, + /// A proof used to prove that storage entries are included in the storage trie + pub proof: Vec, +} diff --git a/client/rpc-api/src/state/mod.rs b/client/rpc-api/src/state/mod.rs index d29e46a4b5637cf85d0745d59a73b0ad69afde7c..3d38a16eb4a988137cd79905daa24b0c98153ad1 100644 --- a/client/rpc-api/src/state/mod.rs +++ b/client/rpc-api/src/state/mod.rs @@ -17,6 +17,7 @@ //! Substrate state API. pub mod error; +pub mod helpers; use jsonrpc_core::Result as RpcResult; use jsonrpc_core::futures::Future; @@ -28,6 +29,7 @@ use sp_version::RuntimeVersion; use self::error::FutureResult; pub use self::gen_client::Client as StateClient; +pub use self::helpers::ReadProof; /// Substrate state API #[rpc] @@ -72,50 +74,6 @@ pub trait StateApi { #[rpc(name = "state_getStorageSize", alias("state_getStorageSizeAt"))] fn storage_size(&self, key: StorageKey, hash: Option) -> FutureResult>; - /// Returns the keys with prefix from a child storage, leave empty to get all the keys - #[rpc(name = "state_getChildKeys")] - fn child_storage_keys( - &self, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - prefix: StorageKey, - hash: Option - ) -> FutureResult>; - - /// Returns a child storage entry at a specific block's state. - #[rpc(name = "state_getChildStorage")] - fn child_storage( - &self, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - key: StorageKey, - hash: Option - ) -> FutureResult>; - - /// Returns the hash of a child storage entry at a block's state. - #[rpc(name = "state_getChildStorageHash")] - fn child_storage_hash( - &self, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - key: StorageKey, - hash: Option - ) -> FutureResult>; - - /// Returns the size of a child storage entry at a block's state. - #[rpc(name = "state_getChildStorageSize")] - fn child_storage_size( - &self, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - key: StorageKey, - hash: Option - ) -> FutureResult>; - /// Returns the runtime metadata as an opaque blob. #[rpc(name = "state_getMetadata")] fn metadata(&self, hash: Option) -> FutureResult; @@ -144,6 +102,10 @@ pub trait StateApi { at: Option, ) -> FutureResult>>; + /// Returns proof of storage entries at a specific block's state. + #[rpc(name = "state_getReadProof")] + fn read_proof(&self, keys: Vec, hash: Option) -> FutureResult>; + /// New runtime version subscription #[pubsub( subscription = "state_runtimeVersion", diff --git a/client/rpc-api/src/system/helpers.rs b/client/rpc-api/src/system/helpers.rs index 80718cf487c8999df8f14c26a40673d41c57723a..46461d69888c18b86b4de44eb9c1f4941547a103 100644 --- a/client/rpc-api/src/system/helpers.rs +++ b/client/rpc-api/src/system/helpers.rs @@ -18,10 +18,7 @@ use std::fmt; use serde::{Serialize, Deserialize}; -use serde_json::{Value, map::Map}; - -/// Node properties -pub type Properties = Map; +use sp_chain_spec::{Properties, ChainType}; /// Running node's static details. #[derive(Clone, Debug)] @@ -34,6 +31,8 @@ pub struct SystemInfo { pub chain_name: String, /// A custom set of properties defined in the chain spec. pub properties: Properties, + /// The type of this chain. + pub chain_type: ChainType, } /// Health struct returned by the RPC diff --git a/client/rpc-api/src/system/mod.rs b/client/rpc-api/src/system/mod.rs index f12a11e0b36dd7edf14c80e5a8d99e3f995f8b0e..486623477ecf1fc87c950ded18052e9b0b240a56 100644 --- a/client/rpc-api/src/system/mod.rs +++ b/client/rpc-api/src/system/mod.rs @@ -25,7 +25,7 @@ use futures::{future::BoxFuture, compat::Compat}; use self::error::Result as SystemResult; -pub use self::helpers::{Properties, SystemInfo, Health, PeerInfo, NodeRole}; +pub use self::helpers::{SystemInfo, Health, PeerInfo, NodeRole}; pub use self::gen_client::Client as SystemClient; /// Substrate system RPC API @@ -39,13 +39,17 @@ pub trait SystemApi { #[rpc(name = "system_version")] fn system_version(&self) -> SystemResult; - /// Get the chain's type. Given as a string identifier. + /// Get the chain's name. Given as a string identifier. #[rpc(name = "system_chain")] fn system_chain(&self) -> SystemResult; + /// Get the chain's type. + #[rpc(name = "system_chainType")] + fn system_type(&self) -> SystemResult; + /// Get a custom set of properties as a JSON object, defined in the chain spec. #[rpc(name = "system_properties")] - fn system_properties(&self) -> SystemResult; + fn system_properties(&self) -> SystemResult; /// Return health status of the node. /// @@ -55,16 +59,29 @@ pub trait SystemApi { #[rpc(name = "system_health", returns = "Health")] fn system_health(&self) -> Receiver; + /// Returns the base58-encoded PeerId of the node. + #[rpc(name = "system_localPeerId", returns = "String")] + fn system_local_peer_id(&self) -> Receiver; + + /// Returns the multiaddresses that the local node is listening on + /// + /// The addresses include a trailing `/p2p/` with the local PeerId, and are thus suitable to + /// be passed to `system_addReservedPeer` or as a bootnode address for example. + #[rpc(name = "system_localListenAddresses", returns = "Vec")] + fn system_local_listen_addresses(&self) -> Receiver>; + /// Returns currently connected peers #[rpc(name = "system_peers", returns = "Vec>")] - fn system_peers(&self) -> Receiver>>; + fn system_peers(&self) + -> Compat>>>>; /// Returns current state of the network. /// /// **Warning**: This API is not stable. // TODO: make this stable and move structs https://github.com/paritytech/substrate/issues/1890 #[rpc(name = "system_networkState", returns = "jsonrpc_core::Value")] - fn system_network_state(&self) -> Receiver; + fn system_network_state(&self) + -> Compat>>; /// Adds a reserved peer. Returns the empty string or an error. The string /// parameter should encode a `p2p` multiaddr. diff --git a/client/rpc-servers/Cargo.toml b/client/rpc-servers/Cargo.toml index c834d7dbf7396b89a6a72ccaaf7feca3059e7507..345aff13d8da75553bf7d80cdf5e9e214bf0bf1e 100644 --- a/client/rpc-servers/Cargo.toml +++ b/client/rpc-servers/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-rpc-server" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,17 +8,17 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Substrate RPC servers." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] jsonrpc-core = "14.0.3" pubsub = { package = "jsonrpc-pubsub", version = "14.0.3" } log = "0.4.8" serde = "1.0.101" serde_json = "1.0.41" -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } [target.'cfg(not(target_os = "unknown"))'.dependencies] http = { package = "jsonrpc-http-server", version = "14.0.3" } ws = { package = "jsonrpc-ws-server", version = "14.0.3" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/rpc/Cargo.toml b/client/rpc/Cargo.toml index 7092959b83e9c450984b14b2394dd738ea440a2f..66f7cb50e6d1272f8aecfe4975187fd16153674b 100644 --- a/client/rpc/Cargo.toml +++ b/client/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-rpc" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,41 +8,41 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Substrate Client RPC" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sc-rpc-api = { version = "0.8.0-alpha.5", path = "../rpc-api" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } -sc-client = { version = "0.8.0-alpha.5", path = "../" } -sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } +sc-rpc-api = { version = "0.8.0-dev", path = "../rpc-api" } +sc-client-api = { version = "2.0.0-dev", path = "../api" } +sp-api = { version = "2.0.0-dev", path = "../../primitives/api" } codec = { package = "parity-scale-codec", version = "1.3.0" } futures = { version = "0.3.1", features = ["compat"] } jsonrpc-pubsub = "14.0.3" log = "0.4.8" -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } rpc = { package = "jsonrpc-core", version = "14.0.3" } -sp-version = { version = "2.0.0-alpha.5", path = "../../primitives/version" } +sp-version = { version = "2.0.0-dev", path = "../../primitives/version" } serde_json = "1.0.41" -sp-session = { version = "2.0.0-alpha.5", path = "../../primitives/session" } -sp-offchain = { version = "2.0.0-alpha.5", path = "../../primitives/offchain" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } -sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } -sp-rpc = { version = "2.0.0-alpha.5", path = "../../primitives/rpc" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../primitives/state-machine" } -sc-executor = { version = "0.8.0-alpha.5", path = "../executor" } -sc-block-builder = { version = "0.8.0-alpha.5", path = "../../client/block-builder" } -sc-keystore = { version = "2.0.0-alpha.5", path = "../keystore" } -sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../primitives/transaction-pool" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } +sp-session = { version = "2.0.0-dev", path = "../../primitives/session" } +sp-offchain = { version = "2.0.0-dev", path = "../../primitives/offchain" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } +sp-utils = { version = "2.0.0-dev", path = "../../primitives/utils" } +sp-rpc = { version = "2.0.0-dev", path = "../../primitives/rpc" } +sp-state-machine = { version = "0.8.0-dev", path = "../../primitives/state-machine" } +sp-chain-spec = { version = "2.0.0-dev", path = "../../primitives/chain-spec" } +sc-executor = { version = "0.8.0-dev", path = "../executor" } +sc-block-builder = { version = "0.8.0-dev", path = "../../client/block-builder" } +sc-keystore = { version = "2.0.0-dev", path = "../keystore" } +sp-transaction-pool = { version = "2.0.0-dev", path = "../../primitives/transaction-pool" } +sp-blockchain = { version = "2.0.0-dev", path = "../../primitives/blockchain" } hash-db = { version = "0.15.2", default-features = false } parking_lot = "0.10.0" [dev-dependencies] assert_matches = "1.3.0" futures01 = { package = "futures", version = "0.1.29" } -sc-network = { version = "0.8.0-alpha.5", path = "../network" } -sp-io = { version = "2.0.0-alpha.5", path = "../../primitives/io" } +sc-network = { version = "0.8.0-dev", path = "../network" } +sp-io = { version = "2.0.0-dev", path = "../../primitives/io" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } tokio = "0.1.22" -sc-transaction-pool = { version = "2.0.0-alpha.5", path = "../transaction-pool" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sc-transaction-pool = { version = "2.0.0-dev", path = "../transaction-pool" } diff --git a/client/rpc/src/author/mod.rs b/client/rpc/src/author/mod.rs index a3f23e8e1437a146932410d08ce6835b40907a6d..23aed953d01df3151f926e55e7b310cad4accf53 100644 --- a/client/rpc/src/author/mod.rs +++ b/client/rpc/src/author/mod.rs @@ -30,7 +30,7 @@ use rpc::futures::{ }; use futures::{StreamExt as _, compat::Compat}; use futures::future::{ready, FutureExt, TryFutureExt}; -use sc_rpc_api::Subscriptions; +use sc_rpc_api::{DenyUnsafe, Subscriptions}; use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId}; use codec::{Encode, Decode}; use sp_core::{Bytes, traits::BareCryptoStorePtr}; @@ -56,6 +56,8 @@ pub struct Author { subscriptions: Subscriptions, /// The key store. keystore: BareCryptoStorePtr, + /// Whether to deny unsafe calls + deny_unsafe: DenyUnsafe, } impl Author { @@ -65,12 +67,14 @@ impl Author { pool: Arc

, subscriptions: Subscriptions, keystore: BareCryptoStorePtr, + deny_unsafe: DenyUnsafe, ) -> Self { Author { client, pool, subscriptions, keystore, + deny_unsafe, } } } @@ -97,6 +101,8 @@ impl AuthorApi, BlockHash

> for Author suri: String, public: Bytes, ) -> Result<()> { + self.deny_unsafe.check_if_safe()?; + let key_type = key_type.as_str().try_into().map_err(|_| Error::BadKeyType)?; let mut keystore = self.keystore.write(); keystore.insert_unknown(key_type, &suri, &public[..]) @@ -105,6 +111,8 @@ impl AuthorApi, BlockHash

> for Author } fn rotate_keys(&self) -> Result { + self.deny_unsafe.check_if_safe()?; + let best_block_hash = self.client.info().best_hash; self.client.runtime_api().generate_session_keys( &generic::BlockId::Hash(best_block_hash), @@ -113,6 +121,8 @@ impl AuthorApi, BlockHash

> for Author } fn has_session_keys(&self, session_keys: Bytes) -> Result { + self.deny_unsafe.check_if_safe()?; + let best_block_hash = self.client.info().best_hash; let keys = self.client.runtime_api().decode_session_keys( &generic::BlockId::Hash(best_block_hash), @@ -124,6 +134,8 @@ impl AuthorApi, BlockHash

> for Author } fn has_key(&self, public_key: Bytes, key_type: String) -> Result { + self.deny_unsafe.check_if_safe()?; + let key_type = key_type.as_str().try_into().map_err(|_| Error::BadKeyType)?; Ok(self.keystore.read().has_keys(&[(public_key.to_vec(), key_type)])) } @@ -151,6 +163,8 @@ impl AuthorApi, BlockHash

> for Author &self, bytes_or_hash: Vec>>, ) -> Result>> { + self.deny_unsafe.check_if_safe()?; + let hashes = bytes_or_hash.into_iter() .map(|x| match x { hash::ExtrinsicOrHash::Hash(h) => Ok(h), diff --git a/client/rpc/src/author/tests.rs b/client/rpc/src/author/tests.rs index 8b956c23a5e660cd685c9d6381395068decb4ebc..de2ee92fe3674c869bb4d954b973c81d28160f01 100644 --- a/client/rpc/src/author/tests.rs +++ b/client/rpc/src/author/tests.rs @@ -65,6 +65,7 @@ impl Default for TestSetup { let pool = Arc::new(BasicPool::new( Default::default(), Arc::new(FullChainApi::new(client.clone())), + None, ).0); TestSetup { runtime: runtime::Runtime::new().expect("Failed to create runtime in test setup"), @@ -82,6 +83,7 @@ impl TestSetup { pool: self.pool.clone(), subscriptions: Subscriptions::new(Arc::new(self.runtime.executor())), keystore: self.keystore.clone(), + deny_unsafe: DenyUnsafe::No, } } } diff --git a/client/rpc/src/chain/chain_light.rs b/client/rpc/src/chain/chain_light.rs index b258c8dd3bc2561f65e10c6045cb7d2c1762840e..059233089d05d1559880c2ace86975d2d562fffd 100644 --- a/client/rpc/src/chain/chain_light.rs +++ b/client/rpc/src/chain/chain_light.rs @@ -21,9 +21,7 @@ use futures::{future::ready, FutureExt, TryFutureExt}; use rpc::futures::future::{result, Future, Either}; use sc_rpc_api::Subscriptions; -use sc_client::{ - light::{fetcher::{Fetcher, RemoteBodyRequest}, blockchain::RemoteBlockchain}, -}; +use sc_client_api::light::{Fetcher, RemoteBodyRequest, RemoteBlockchain}; use sp_runtime::{ generic::{BlockId, SignedBlock}, traits::{Block as BlockT}, @@ -80,7 +78,7 @@ impl ChainBackend for LightChain { /// Offchain storage storage: Arc>, + deny_unsafe: DenyUnsafe, } impl Offchain { /// Create new instance of Offchain API. - pub fn new(storage: T) -> Self { + pub fn new(storage: T, deny_unsafe: DenyUnsafe) -> Self { Offchain { storage: Arc::new(RwLock::new(storage)), + deny_unsafe, } } } @@ -48,6 +51,8 @@ impl Offchain { impl OffchainApi for Offchain { /// Set offchain local storage under given key and prefix. fn set_local_storage(&self, kind: StorageKind, key: Bytes, value: Bytes) -> Result<()> { + self.deny_unsafe.check_if_safe()?; + let prefix = match kind { StorageKind::PERSISTENT => sp_offchain::STORAGE_PREFIX, StorageKind::LOCAL => return Err(Error::UnavailableStorageKind), @@ -58,6 +63,8 @@ impl OffchainApi for Offchain { /// Get offchain local storage under given key and prefix. fn get_local_storage(&self, kind: StorageKind, key: Bytes) -> Result> { + self.deny_unsafe.check_if_safe()?; + let prefix = match kind { StorageKind::PERSISTENT => sp_offchain::STORAGE_PREFIX, StorageKind::LOCAL => return Err(Error::UnavailableStorageKind), diff --git a/client/rpc/src/offchain/tests.rs b/client/rpc/src/offchain/tests.rs index ac1a6a4de31bcc95bb1f385c3f7d78a75a83fbe1..cb05f3d4dbb890499ff140ac6e913e62b29c46d0 100644 --- a/client/rpc/src/offchain/tests.rs +++ b/client/rpc/src/offchain/tests.rs @@ -21,7 +21,7 @@ use sp_core::{Bytes, offchain::storage::InMemOffchainStorage}; #[test] fn local_storage_should_work() { let storage = InMemOffchainStorage::default(); - let offchain = Offchain::new(storage); + let offchain = Offchain::new(storage, DenyUnsafe::No); let key = Bytes(b"offchain_storage".to_vec()); let value = Bytes(b"offchain_value".to_vec()); @@ -34,3 +34,20 @@ fn local_storage_should_work() { Ok(Some(ref v)) if *v == value ); } + +#[test] +fn offchain_calls_considered_unsafe() { + let storage = InMemOffchainStorage::default(); + let offchain = Offchain::new(storage, DenyUnsafe::Yes); + let key = Bytes(b"offchain_storage".to_vec()); + let value = Bytes(b"offchain_value".to_vec()); + + assert_matches!( + offchain.set_local_storage(StorageKind::PERSISTENT, key.clone(), value.clone()), + Err(Error::UnsafeRpcCalled(_)) + ); + assert_matches!( + offchain.get_local_storage(StorageKind::PERSISTENT, key), + Err(Error::UnsafeRpcCalled(_)) + ); +} diff --git a/client/rpc/src/state/mod.rs b/client/rpc/src/state/mod.rs index 2747405c04fcac0c7534ea802bb5e1f08a7e1911..d1f4e2c5f9c164ac800c0c56f45f41d49a6089dc 100644 --- a/client/rpc/src/state/mod.rs +++ b/client/rpc/src/state/mod.rs @@ -26,9 +26,9 @@ use std::sync::Arc; use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId}; use rpc::{Result as RpcResult, futures::{Future, future::result}}; -use sc_rpc_api::Subscriptions; -use sc_client::{light::{blockchain::RemoteBlockchain, fetcher::Fetcher}}; -use sp_core::{Bytes, storage::{StorageKey, StorageData, StorageChangeSet}}; +use sc_rpc_api::{Subscriptions, state::ReadProof}; +use sc_client_api::light::{RemoteBlockchain, Fetcher}; +use sp_core::{Bytes, storage::{StorageKey, PrefixedStorageKey, StorageData, StorageChangeSet}}; use sp_version::RuntimeVersion; use sp_runtime::traits::Block as BlockT; @@ -37,7 +37,8 @@ use sp_api::{Metadata, ProvideRuntimeApi, CallApiAt}; use self::error::{Error, FutureResult}; pub use sc_rpc_api::state::*; -use sc_client_api::{ExecutorProvider, StorageProvider, BlockchainEvents, Backend}; +pub use sc_rpc_api::child_state::*; +use sc_client_api::{ExecutorProvider, StorageProvider, BlockchainEvents, Backend, ProofProvider}; use sp_blockchain::{HeaderMetadata, HeaderBackend}; const STORAGE_KEYS_PAGED_MAX_COUNT: u32 = 1000; @@ -103,49 +104,6 @@ pub trait StateBackend: Send + Sync + 'static .map(|x| x.map(|x| x.0.len() as u64))) } - /// Returns the keys with prefix from a child storage, leave empty to get all the keys - fn child_storage_keys( - &self, - block: Option, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - prefix: StorageKey, - ) -> FutureResult>; - - /// Returns a child storage entry at a specific block's state. - fn child_storage( - &self, - block: Option, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - key: StorageKey, - ) -> FutureResult>; - - /// Returns the hash of a child storage entry at a block's state. - fn child_storage_hash( - &self, - block: Option, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - key: StorageKey, - ) -> FutureResult>; - - /// Returns the size of a child storage entry at a block's state. - fn child_storage_size( - &self, - block: Option, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - key: StorageKey, - ) -> FutureResult> { - Box::new(self.child_storage(block, child_storage_key, child_info, child_type, key) - .map(|x| x.map(|x| x.0.len() as u64))) - } - /// Returns the runtime metadata as an opaque blob. fn metadata(&self, block: Option) -> FutureResult; @@ -170,6 +128,13 @@ pub trait StateBackend: Send + Sync + 'static at: Option ) -> FutureResult>>; + /// Returns proof of storage entries at a specific block's state. + fn read_proof( + &self, + block: Option, + keys: Vec, + ) -> FutureResult>; + /// New runtime version subscription fn subscribe_runtime_version( &self, @@ -204,19 +169,21 @@ pub trait StateBackend: Send + Sync + 'static pub fn new_full( client: Arc, subscriptions: Subscriptions, -) -> State +) -> (State, ChildState) where Block: BlockT + 'static, BE: Backend + 'static, - Client: ExecutorProvider + StorageProvider + HeaderBackend + Client: ExecutorProvider + StorageProvider + ProofProvider + HeaderBackend + HeaderMetadata + BlockchainEvents + CallApiAt + ProvideRuntimeApi + Send + Sync + 'static, Client::Api: Metadata, { - State { - backend: Box::new(self::state_full::FullState::new(client, subscriptions)), - } + let child_backend = Box::new( + self::state_full::FullState::new(client.clone(), subscriptions.clone()) + ); + let backend = Box::new(self::state_full::FullState::new(client, subscriptions)); + (State { backend }, ChildState { backend: child_backend }) } /// Create new state API that works on light node. @@ -225,7 +192,7 @@ pub fn new_light>( subscriptions: Subscriptions, remote_blockchain: Arc>, fetcher: Arc, -) -> State +) -> (State, ChildState) where Block: BlockT + 'static, BE: Backend + 'static, @@ -235,14 +202,20 @@ pub fn new_light>( + Send + Sync + 'static, F: Send + Sync + 'static, { - State { - backend: Box::new(self::state_light::LightState::new( + let child_backend = Box::new(self::state_light::LightState::new( + client.clone(), + subscriptions.clone(), + remote_blockchain.clone(), + fetcher.clone(), + )); + + let backend = Box::new(self::state_light::LightState::new( client, subscriptions, remote_blockchain, fetcher, - )), - } + )); + (State { backend }, ChildState { backend: child_backend }) } /// State API with subscriptions support. @@ -307,50 +280,6 @@ impl StateApi for State self.backend.storage_size(block, key) } - fn child_storage( - &self, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - key: StorageKey, - block: Option - ) -> FutureResult> { - self.backend.child_storage(block, child_storage_key, child_info, child_type, key) - } - - fn child_storage_keys( - &self, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - key_prefix: StorageKey, - block: Option - ) -> FutureResult> { - self.backend.child_storage_keys(block, child_storage_key, child_info, child_type, key_prefix) - } - - fn child_storage_hash( - &self, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - key: StorageKey, - block: Option - ) -> FutureResult> { - self.backend.child_storage_hash(block, child_storage_key, child_info, child_type, key) - } - - fn child_storage_size( - &self, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - key: StorageKey, - block: Option - ) -> FutureResult> { - self.backend.child_storage_size(block, child_storage_key, child_info, child_type, key) - } - fn metadata(&self, block: Option) -> FutureResult { self.backend.metadata(block) } @@ -372,6 +301,10 @@ impl StateApi for State self.backend.query_storage_at(keys, at) } + fn read_proof(&self, keys: Vec, block: Option) -> FutureResult> { + self.backend.read_proof(block, keys) + } + fn subscribe_storage( &self, meta: Self::Metadata, @@ -402,12 +335,98 @@ impl StateApi for State } } -fn client_err(err: sp_blockchain::Error) -> Error { - Error::Client(Box::new(err)) +/// Child state backend API. +pub trait ChildStateBackend: Send + Sync + 'static + where + Block: BlockT + 'static, + Client: Send + Sync + 'static, +{ + /// Returns the keys with prefix from a child storage, + /// leave prefix empty to get all the keys. + fn storage_keys( + &self, + block: Option, + storage_key: PrefixedStorageKey, + prefix: StorageKey, + ) -> FutureResult>; + + /// Returns a child storage entry at a specific block's state. + fn storage( + &self, + block: Option, + storage_key: PrefixedStorageKey, + key: StorageKey, + ) -> FutureResult>; + + /// Returns the hash of a child storage entry at a block's state. + fn storage_hash( + &self, + block: Option, + storage_key: PrefixedStorageKey, + key: StorageKey, + ) -> FutureResult>; + + /// Returns the size of a child storage entry at a block's state. + fn storage_size( + &self, + block: Option, + storage_key: PrefixedStorageKey, + key: StorageKey, + ) -> FutureResult> { + Box::new(self.storage(block, storage_key, key) + .map(|x| x.map(|x| x.0.len() as u64))) + } } -const CHILD_RESOLUTION_ERROR: &str = "Unexpected child info and type"; +/// Child state API with subscriptions support. +pub struct ChildState { + backend: Box>, +} -fn child_resolution_error() -> sp_blockchain::Error { - sp_blockchain::Error::Msg(CHILD_RESOLUTION_ERROR.to_string()) +impl ChildStateApi for ChildState + where + Block: BlockT + 'static, + Client: Send + Sync + 'static, +{ + type Metadata = crate::metadata::Metadata; + + fn storage( + &self, + storage_key: PrefixedStorageKey, + key: StorageKey, + block: Option + ) -> FutureResult> { + self.backend.storage(block, storage_key, key) + } + + fn storage_keys( + &self, + storage_key: PrefixedStorageKey, + key_prefix: StorageKey, + block: Option + ) -> FutureResult> { + self.backend.storage_keys(block, storage_key, key_prefix) + } + + fn storage_hash( + &self, + storage_key: PrefixedStorageKey, + key: StorageKey, + block: Option + ) -> FutureResult> { + self.backend.storage_hash(block, storage_key, key) + } + + fn storage_size( + &self, + storage_key: PrefixedStorageKey, + key: StorageKey, + block: Option + ) -> FutureResult> { + self.backend.storage_size(block, storage_key, key) + } +} + +fn client_err(err: sp_blockchain::Error) -> Error { + Error::Client(Box::new(err)) } diff --git a/client/rpc/src/state/state_full.rs b/client/rpc/src/state/state_full.rs index bf80820543102483ae5c41a301ad1ab0d322890d..82f87e9acf22352c122412ee9f7cba5522414786 100644 --- a/client/rpc/src/state/state_full.rs +++ b/client/rpc/src/state/state_full.rs @@ -24,12 +24,13 @@ use log::warn; use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId}; use rpc::{Result as RpcResult, futures::{stream, Future, Sink, Stream, future::result}}; -use sc_rpc_api::Subscriptions; +use sc_rpc_api::{Subscriptions, state::ReadProof}; use sc_client_api::backend::Backend; use sp_blockchain::{Result as ClientResult, Error as ClientError, HeaderMetadata, CachedHeaderMetadata, HeaderBackend}; -use sc_client::BlockchainEvents; +use sc_client_api::BlockchainEvents; use sp_core::{ - Bytes, storage::{well_known_keys, StorageKey, StorageData, StorageChangeSet, ChildInfo}, + Bytes, storage::{well_known_keys, StorageKey, StorageData, StorageChangeSet, + ChildInfo, ChildType, PrefixedStorageKey}, }; use sp_version::RuntimeVersion; use sp_runtime::{ @@ -38,9 +39,9 @@ use sp_runtime::{ use sp_api::{Metadata, ProvideRuntimeApi, CallApiAt}; -use super::{StateBackend, error::{FutureResult, Error, Result}, client_err, child_resolution_error}; +use super::{StateBackend, ChildStateBackend, error::{FutureResult, Error, Result}, client_err}; use std::marker::PhantomData; -use sc_client_api::{CallExecutor, StorageProvider, ExecutorProvider}; +use sc_client_api::{CallExecutor, StorageProvider, ExecutorProvider, ProofProvider}; /// Ranges to query in state_queryStorage. struct QueryStorageRange { @@ -218,7 +219,7 @@ impl FullState impl StateBackend for FullState where Block: BlockT + 'static, BE: Backend + 'static, - Client: ExecutorProvider + StorageProvider + HeaderBackend + Client: ExecutorProvider + StorageProvider + ProofProvider + HeaderBackend + HeaderMetadata + BlockchainEvents + CallApiAt + ProvideRuntimeApi + Send + Sync + 'static, @@ -308,66 +309,6 @@ impl StateBackend for FullState, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - prefix: StorageKey, - ) -> FutureResult> { - Box::new(result( - self.block_or_best(block) - .and_then(|block| self.client.child_storage_keys( - &BlockId::Hash(block), - &child_storage_key, - ChildInfo::resolve_child_info(child_type, &child_info.0[..]) - .ok_or_else(child_resolution_error)?, - &prefix, - )) - .map_err(client_err))) - } - - fn child_storage( - &self, - block: Option, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - key: StorageKey, - ) -> FutureResult> { - Box::new(result( - self.block_or_best(block) - .and_then(|block| self.client.child_storage( - &BlockId::Hash(block), - &child_storage_key, - ChildInfo::resolve_child_info(child_type, &child_info.0[..]) - .ok_or_else(child_resolution_error)?, - &key, - )) - .map_err(client_err))) - } - - fn child_storage_hash( - &self, - block: Option, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - key: StorageKey, - ) -> FutureResult> { - Box::new(result( - self.block_or_best(block) - .and_then(|block| self.client.child_storage_hash( - &BlockId::Hash(block), - &child_storage_key, - ChildInfo::resolve_child_info(child_type, &child_info.0[..]) - .ok_or_else(child_resolution_error)?, - &key, - )) - .map_err(client_err))) - } - fn metadata(&self, block: Option) -> FutureResult { Box::new(result( self.block_or_best(block) @@ -410,6 +351,26 @@ impl StateBackend for FullState, + keys: Vec, + ) -> FutureResult> { + Box::new(result( + self.block_or_best(block) + .and_then(|block| { + self.client + .read_proof( + &BlockId::Hash(block), + &mut keys.iter().map(|key| key.0.as_ref()), + ) + .map(|proof| proof.iter_nodes().map(|node| node.into()).collect()) + .map(|proof| ReadProof { at: block, proof }) + }) + .map_err(client_err), + )) + } + fn subscribe_runtime_version( &self, _meta: crate::metadata::Metadata, @@ -493,7 +454,7 @@ impl StateBackend for FullState StateBackend for FullState ChildStateBackend for FullState where + Block: BlockT + 'static, + BE: Backend + 'static, + Client: ExecutorProvider + StorageProvider + HeaderBackend + + HeaderMetadata + BlockchainEvents + + CallApiAt + ProvideRuntimeApi + + Send + Sync + 'static, + Client::Api: Metadata, +{ + fn storage_keys( + &self, + block: Option, + storage_key: PrefixedStorageKey, + prefix: StorageKey, + ) -> FutureResult> { + Box::new(result( + self.block_or_best(block) + .and_then(|block| { + let child_info = match ChildType::from_prefixed_key(&storage_key) { + Some((ChildType::ParentKeyId, storage_key)) => ChildInfo::new_default(storage_key), + None => return Err("Invalid child storage key".into()), + }; + self.client.child_storage_keys( + &BlockId::Hash(block), + &child_info, + &prefix, + ) + }) + .map_err(client_err))) + } + + fn storage( + &self, + block: Option, + storage_key: PrefixedStorageKey, + key: StorageKey, + ) -> FutureResult> { + Box::new(result( + self.block_or_best(block) + .and_then(|block| { + let child_info = match ChildType::from_prefixed_key(&storage_key) { + Some((ChildType::ParentKeyId, storage_key)) => ChildInfo::new_default(storage_key), + None => return Err("Invalid child storage key".into()), + }; + self.client.child_storage( + &BlockId::Hash(block), + &child_info, + &key, + ) + }) + .map_err(client_err))) + } + + fn storage_hash( + &self, + block: Option, + storage_key: PrefixedStorageKey, + key: StorageKey, + ) -> FutureResult> { + Box::new(result( + self.block_or_best(block) + .and_then(|block| { + let child_info = match ChildType::from_prefixed_key(&storage_key) { + Some((ChildType::ParentKeyId, storage_key)) => ChildInfo::new_default(storage_key), + None => return Err("Invalid child storage key".into()), + }; + self.client.child_storage_hash( + &BlockId::Hash(block), + &child_info, + &key, + ) + }) + .map_err(client_err))) + } +} + /// Splits passed range into two subranges where: /// - first range has at least one element in it; /// - second range (optionally) starts at given `middle` element. diff --git a/client/rpc/src/state/state_light.rs b/client/rpc/src/state/state_light.rs index 092419ad0129e6f4109a19d3f916dc0eb24f5738..af5d4248e3a422e2bfc759bc40c0b269ee68ccfc 100644 --- a/client/rpc/src/state/state_light.rs +++ b/client/rpc/src/state/state_light.rs @@ -38,27 +38,29 @@ use rpc::{ futures::stream::Stream, }; -use sc_rpc_api::Subscriptions; +use sc_rpc_api::{Subscriptions, state::ReadProof}; use sp_blockchain::{Error as ClientError, HeaderBackend}; -use sc_client::{ +use sc_client_api::{ BlockchainEvents, light::{ - blockchain::{future_header, RemoteBlockchain}, - fetcher::{Fetcher, RemoteCallRequest, RemoteReadRequest, RemoteReadChildRequest}, + RemoteCallRequest, RemoteReadRequest, RemoteReadChildRequest, + RemoteBlockchain, Fetcher, future_header, }, }; use sp_core::{ - Bytes, OpaqueMetadata, storage::{StorageKey, StorageData, StorageChangeSet}, + Bytes, OpaqueMetadata, + storage::{StorageKey, PrefixedStorageKey, StorageData, StorageChangeSet}, }; use sp_version::RuntimeVersion; use sp_runtime::{generic::BlockId, traits::{Block as BlockT, HashFor}}; -use super::{StateBackend, error::{FutureResult, Error}, client_err}; +use super::{StateBackend, ChildStateBackend, error::{FutureResult, Error}, client_err}; /// Storage data map of storage keys => (optional) storage value. type StorageMap = HashMap>; /// State API backend for light nodes. +#[derive(Clone)] pub struct LightState, Client> { client: Arc, subscriptions: Subscriptions, @@ -233,69 +235,7 @@ impl StateBackend for LightState, key: StorageKey, ) -> FutureResult> { - Box::new(self - .storage(block, key) - .and_then(|maybe_storage| - result(Ok(maybe_storage.map(|storage| HashFor::::hash(&storage.0)))) - ) - ) - } - - fn child_storage_keys( - &self, - _block: Option, - _child_storage_key: StorageKey, - _child_info: StorageKey, - _child_type: u32, - _prefix: StorageKey, - ) -> FutureResult> { - Box::new(result(Err(client_err(ClientError::NotAvailableOnLightClient)))) - } - - fn child_storage( - &self, - block: Option, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - key: StorageKey, - ) -> FutureResult> { - let block = self.block_or_best(block); - let fetcher = self.fetcher.clone(); - let child_storage = resolve_header(&*self.remote_blockchain, &*self.fetcher, block) - .then(move |result| match result { - Ok(header) => Either::Left(fetcher.remote_read_child(RemoteReadChildRequest { - block, - header, - storage_key: child_storage_key.0, - child_info: child_info.0, - child_type, - keys: vec![key.0.clone()], - retry_count: Default::default(), - }).then(move |result| ready(result - .map(|mut data| data - .remove(&key.0) - .expect("successful result has entry for all keys; qed") - .map(StorageData) - ) - .map_err(client_err) - ))), - Err(error) => Either::Right(ready(Err(error))), - }); - - Box::new(child_storage.boxed().compat()) - } - - fn child_storage_hash( - &self, - block: Option, - child_storage_key: StorageKey, - child_info: StorageKey, - child_type: u32, - key: StorageKey, - ) -> FutureResult> { - Box::new(self - .child_storage(block, child_storage_key, child_info, child_type, key) + Box::new(StateBackend::storage(self, block, key) .and_then(|maybe_storage| result(Ok(maybe_storage.map(|storage| HashFor::::hash(&storage.0)))) ) @@ -339,6 +279,14 @@ impl StateBackend for LightState, + _keys: Vec, + ) -> FutureResult> { + Box::new(result(Err(client_err(ClientError::NotAvailableOnLightClient)))) + } + fn subscribe_storage( &self, _meta: crate::metadata::Metadata, @@ -518,6 +466,65 @@ impl StateBackend for LightState ChildStateBackend for LightState + where + Block: BlockT, + Client: BlockchainEvents + HeaderBackend + Send + Sync + 'static, + F: Fetcher + 'static +{ + fn storage_keys( + &self, + _block: Option, + _storage_key: PrefixedStorageKey, + _prefix: StorageKey, + ) -> FutureResult> { + Box::new(result(Err(client_err(ClientError::NotAvailableOnLightClient)))) + } + + fn storage( + &self, + block: Option, + storage_key: PrefixedStorageKey, + key: StorageKey, + ) -> FutureResult> { + let block = self.block_or_best(block); + let fetcher = self.fetcher.clone(); + let child_storage = resolve_header(&*self.remote_blockchain, &*self.fetcher, block) + .then(move |result| match result { + Ok(header) => Either::Left(fetcher.remote_read_child(RemoteReadChildRequest { + block, + header, + storage_key, + keys: vec![key.0.clone()], + retry_count: Default::default(), + }).then(move |result| ready(result + .map(|mut data| data + .remove(&key.0) + .expect("successful result has entry for all keys; qed") + .map(StorageData) + ) + .map_err(client_err) + ))), + Err(error) => Either::Right(ready(Err(error))), + }); + + Box::new(child_storage.boxed().compat()) + } + + fn storage_hash( + &self, + block: Option, + storage_key: PrefixedStorageKey, + key: StorageKey, + ) -> FutureResult> { + Box::new(ChildStateBackend::storage(self, block, storage_key, key) + .and_then(|maybe_storage| + result(Ok(maybe_storage.map(|storage| HashFor::::hash(&storage.0)))) + ) + ) + } +} + /// Resolve header by hash. fn resolve_header>( remote_blockchain: &dyn RemoteBlockchain, diff --git a/client/rpc/src/state/tests.rs b/client/rpc/src/state/tests.rs index 4a9b701959c8c292a3b4dba25d89762fedfd8f62..da904f3fdc6c5f17e294e0b118d5174d3268cf91 100644 --- a/client/rpc/src/state/tests.rs +++ b/client/rpc/src/state/tests.rs @@ -21,7 +21,7 @@ use self::error::Error; use std::sync::Arc; use assert_matches::assert_matches; use futures01::stream::Stream; -use sp_core::{storage::{well_known_keys, ChildInfo}, ChangesTrieConfiguration}; +use sp_core::{storage::ChildInfo, ChangesTrieConfiguration}; use sp_core::hash::H256; use sc_block_builder::BlockBuilderProvider; use sp_io::hashing::blake2_256; @@ -32,26 +32,28 @@ use substrate_test_runtime_client::{ }; use sp_runtime::generic::BlockId; -const CHILD_INFO: ChildInfo<'static> = ChildInfo::new_default(b"unique_id"); +const STORAGE_KEY: &[u8] = b"child"; + +fn prefixed_storage_key() -> PrefixedStorageKey { + let child_info = ChildInfo::new_default(&STORAGE_KEY[..]); + child_info.prefixed_storage_key() +} #[test] fn should_return_storage() { const KEY: &[u8] = b":mock"; const VALUE: &[u8] = b"hello world"; - const STORAGE_KEY: &[u8] = b":child_storage:default:child"; const CHILD_VALUE: &[u8] = b"hello world !"; + let child_info = ChildInfo::new_default(STORAGE_KEY); let mut core = tokio::runtime::Runtime::new().unwrap(); let client = TestClientBuilder::new() .add_extra_storage(KEY.to_vec(), VALUE.to_vec()) - .add_extra_child_storage(STORAGE_KEY.to_vec(), CHILD_INFO, KEY.to_vec(), CHILD_VALUE.to_vec()) + .add_extra_child_storage(&child_info, KEY.to_vec(), CHILD_VALUE.to_vec()) .build(); let genesis_hash = client.genesis_hash(); - let client = new_full(Arc::new(client), Subscriptions::new(Arc::new(core.executor()))); + let (client, child) = new_full(Arc::new(client), Subscriptions::new(Arc::new(core.executor()))); let key = StorageKey(KEY.to_vec()); - let storage_key = StorageKey(STORAGE_KEY.to_vec()); - let (child_info, child_type) = CHILD_INFO.info(); - let child_info = StorageKey(child_info.to_vec()); assert_eq!( client.storage(key.clone(), Some(genesis_hash).into()).wait() @@ -69,7 +71,7 @@ fn should_return_storage() { ); assert_eq!( core.block_on( - client.child_storage(storage_key, child_info, child_type, key, Some(genesis_hash).into()) + child.storage(prefixed_storage_key(), key, Some(genesis_hash).into()) .map(|x| x.map(|x| x.0.len())) ).unwrap().unwrap() as usize, CHILD_VALUE.len(), @@ -79,45 +81,36 @@ fn should_return_storage() { #[test] fn should_return_child_storage() { - let (child_info, child_type) = CHILD_INFO.info(); - let child_info = StorageKey(child_info.to_vec()); + let child_info = ChildInfo::new_default(STORAGE_KEY); let core = tokio::runtime::Runtime::new().unwrap(); let client = Arc::new(substrate_test_runtime_client::TestClientBuilder::new() - .add_child_storage("test", "key", CHILD_INFO, vec![42_u8]) + .add_child_storage(&child_info, "key", vec![42_u8]) .build()); let genesis_hash = client.genesis_hash(); - let client = new_full(client, Subscriptions::new(Arc::new(core.executor()))); - let child_key = StorageKey( - well_known_keys::CHILD_STORAGE_KEY_PREFIX.iter().chain(b"test").cloned().collect() - ); + let (_client, child) = new_full(client, Subscriptions::new(Arc::new(core.executor()))); + let child_key = prefixed_storage_key(); let key = StorageKey(b"key".to_vec()); assert_matches!( - client.child_storage( + child.storage( child_key.clone(), - child_info.clone(), - child_type, key.clone(), Some(genesis_hash).into(), ).wait(), Ok(Some(StorageData(ref d))) if d[0] == 42 && d.len() == 1 ); assert_matches!( - client.child_storage_hash( + child.storage_hash( child_key.clone(), - child_info.clone(), - child_type, key.clone(), Some(genesis_hash).into(), ).wait().map(|x| x.is_some()), Ok(true) ); assert_matches!( - client.child_storage_size( + child.storage_size( child_key.clone(), - child_info.clone(), - child_type, key.clone(), None, ).wait(), @@ -130,7 +123,7 @@ fn should_call_contract() { let core = tokio::runtime::Runtime::new().unwrap(); let client = Arc::new(substrate_test_runtime_client::new()); let genesis_hash = client.genesis_hash(); - let client = new_full(client, Subscriptions::new(Arc::new(core.executor()))); + let (client, _child) = new_full(client, Subscriptions::new(Arc::new(core.executor()))); assert_matches!( client.call("balanceOf".into(), Bytes(vec![1,2,3]), Some(genesis_hash).into()).wait(), @@ -146,7 +139,7 @@ fn should_notify_about_storage_changes() { { let mut client = Arc::new(substrate_test_runtime_client::new()); - let api = new_full(client.clone(), Subscriptions::new(Arc::new(remote))); + let (api, _child) = new_full(client.clone(), Subscriptions::new(Arc::new(remote))); api.subscribe_storage(Default::default(), subscriber, None.into()); @@ -179,7 +172,7 @@ fn should_send_initial_storage_changes_and_notifications() { { let mut client = Arc::new(substrate_test_runtime_client::new()); - let api = new_full(client.clone(), Subscriptions::new(Arc::new(remote))); + let (api, _child) = new_full(client.clone(), Subscriptions::new(Arc::new(remote))); let alice_balance_key = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Alice.into())); @@ -215,7 +208,7 @@ fn should_send_initial_storage_changes_and_notifications() { fn should_query_storage() { fn run_tests(mut client: Arc, has_changes_trie_config: bool) { let core = tokio::runtime::Runtime::new().unwrap(); - let api = new_full(client.clone(), Subscriptions::new(Arc::new(core.executor()))); + let (api, _child) = new_full(client.clone(), Subscriptions::new(Arc::new(core.executor()))); let mut add_block = |nonce| { let mut builder = client.new_block(Default::default()).unwrap(); @@ -434,13 +427,14 @@ fn should_return_runtime_version() { let core = tokio::runtime::Runtime::new().unwrap(); let client = Arc::new(substrate_test_runtime_client::new()); - let api = new_full(client.clone(), Subscriptions::new(Arc::new(core.executor()))); + let (api, _child) = new_full(client.clone(), Subscriptions::new(Arc::new(core.executor()))); let result = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\ - \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",2],\ + \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",3],\ [\"0x37e397fc7c91f5e4\",1],[\"0xd2bc9897eed08f15\",2],[\"0x40fe3ad401f8959a\",4],\ - [\"0xc6e9a76309f39b09\",1],[\"0xdd718d5cc53262d4\",1],[\"0xcbca25e39f142387\",1],\ - [\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],[\"0xbc9d89904f5b923f\",1]]}"; + [\"0xc6e9a76309f39b09\",1],[\"0xdd718d5cc53262d4\",1],[\"0xcbca25e39f142387\",2],\ + [\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],[\"0xbc9d89904f5b923f\",1]],\ + \"transactionVersion\":1}"; let runtime_version = api.runtime_version(None.into()).wait().unwrap(); let serialized = serde_json::to_string(&runtime_version).unwrap(); @@ -457,7 +451,7 @@ fn should_notify_on_runtime_version_initially() { { let client = Arc::new(substrate_test_runtime_client::new()); - let api = new_full(client.clone(), Subscriptions::new(Arc::new(core.executor()))); + let (api, _child) = new_full(client.clone(), Subscriptions::new(Arc::new(core.executor()))); api.subscribe_runtime_version(Default::default(), subscriber); diff --git a/client/rpc/src/system/mod.rs b/client/rpc/src/system/mod.rs index 9565992cf44c526ecf235185be4cfd3243fe1020..2a19e5412ed4f6a01c74f76377c314de5fecc738 100644 --- a/client/rpc/src/system/mod.rs +++ b/client/rpc/src/system/mod.rs @@ -21,26 +21,40 @@ mod tests; use futures::{future::BoxFuture, FutureExt, TryFutureExt}; use futures::{channel::oneshot, compat::Compat}; -use sc_rpc_api::Receiver; +use sc_rpc_api::{DenyUnsafe, Receiver}; use sp_utils::mpsc::TracingUnboundedSender; use sp_runtime::traits::{self, Header as HeaderT}; use self::error::Result; pub use sc_rpc_api::system::*; -pub use self::helpers::{Properties, SystemInfo, Health, PeerInfo, NodeRole}; +pub use self::helpers::{SystemInfo, Health, PeerInfo, NodeRole}; pub use self::gen_client::Client as SystemClient; +macro_rules! bail_if_unsafe { + ($value: expr) => { + if let Err(err) = $value.check_if_safe() { + return async move { Err(err.into()) }.boxed().compat(); + } + }; +} + /// System API implementation pub struct System { info: SystemInfo, send_back: TracingUnboundedSender>, + deny_unsafe: DenyUnsafe, } /// Request to be processed. pub enum Request { /// Must return the health of the network. Health(oneshot::Sender), + /// Must return the base58-encoded local `PeerId`. + LocalPeerId(oneshot::Sender), + /// Must return the string representation of the addresses we listen on, including the + /// trailing `/p2p/`. + LocalListenAddresses(oneshot::Sender>), /// Must return information about the peers we are connected to. Peers(oneshot::Sender::Number>>>), /// Must return the state of the network. @@ -61,10 +75,12 @@ impl System { pub fn new( info: SystemInfo, send_back: TracingUnboundedSender>, + deny_unsafe: DenyUnsafe, ) -> Self { System { info, send_back, + deny_unsafe, } } } @@ -82,7 +98,11 @@ impl SystemApi::Number> for Sy Ok(self.info.chain_name.clone()) } - fn system_properties(&self) -> Result { + fn system_type(&self) -> Result { + Ok(self.info.chain_type.clone()) + } + + fn system_properties(&self) -> Result { Ok(self.info.properties.clone()) } @@ -92,21 +112,49 @@ impl SystemApi::Number> for Sy Receiver(Compat::new(rx)) } - fn system_peers(&self) -> Receiver::Number>>> { + fn system_local_peer_id(&self) -> Receiver { let (tx, rx) = oneshot::channel(); - let _ = self.send_back.unbounded_send(Request::Peers(tx)); + let _ = self.send_back.unbounded_send(Request::LocalPeerId(tx)); Receiver(Compat::new(rx)) } - fn system_network_state(&self) -> Receiver { + fn system_local_listen_addresses(&self) -> Receiver> { let (tx, rx) = oneshot::channel(); - let _ = self.send_back.unbounded_send(Request::NetworkState(tx)); + let _ = self.send_back.unbounded_send(Request::LocalListenAddresses(tx)); Receiver(Compat::new(rx)) } + fn system_peers(&self) + -> Compat::Number>>>>> + { + bail_if_unsafe!(self.deny_unsafe); + + let (tx, rx) = oneshot::channel(); + let _ = self.send_back.unbounded_send(Request::Peers(tx)); + + async move { + rx.await.map_err(|_| rpc::Error::internal_error()) + }.boxed().compat() + } + + fn system_network_state(&self) + -> Compat>> + { + bail_if_unsafe!(self.deny_unsafe); + + let (tx, rx) = oneshot::channel(); + let _ = self.send_back.unbounded_send(Request::NetworkState(tx)); + + async move { + rx.await.map_err(|_| rpc::Error::internal_error()) + }.boxed().compat() + } + fn system_add_reserved_peer(&self, peer: String) -> Compat>> { + bail_if_unsafe!(self.deny_unsafe); + let (tx, rx) = oneshot::channel(); let _ = self.send_back.unbounded_send(Request::NetworkAddReservedPeer(peer, tx)); async move { @@ -121,6 +169,8 @@ impl SystemApi::Number> for Sy fn system_remove_reserved_peer(&self, peer: String) -> Compat>> { + bail_if_unsafe!(self.deny_unsafe); + let (tx, rx) = oneshot::channel(); let _ = self.send_back.unbounded_send(Request::NetworkRemoveReservedPeer(peer, tx)); async move { diff --git a/client/rpc/src/system/tests.rs b/client/rpc/src/system/tests.rs index d45894743c104013d4eee67b85cebdb9b1bc3360..d0ec6951260fac346973e9c2b35f8992b8315f7a 100644 --- a/client/rpc/src/system/tests.rs +++ b/client/rpc/src/system/tests.rs @@ -55,6 +55,15 @@ fn api>>(sync: T) -> System { should_have_peers, }); }, + Request::LocalPeerId(sender) => { + let _ = sender.send("QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV".to_string()); + }, + Request::LocalListenAddresses(sender) => { + let _ = sender.send(vec![ + "/ip4/198.51.100.19/tcp/30333/p2p/QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV".to_string(), + "/ip4/127.0.0.1/tcp/30334/ws/p2p/QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV".to_string(), + ]); + }, Request::Peers(sender) => { let mut peers = vec![]; for _peer in 0..status.peers { @@ -100,12 +109,17 @@ fn api>>(sync: T) -> System { future::ready(()) })) }); - System::new(SystemInfo { - impl_name: "testclient".into(), - impl_version: "0.2.0".into(), - chain_name: "testchain".into(), - properties: Default::default(), - }, tx) + System::new( + SystemInfo { + impl_name: "testclient".into(), + impl_version: "0.2.0".into(), + chain_name: "testchain".into(), + properties: Default::default(), + chain_type: Default::default(), + }, + tx, + sc_rpc_api::DenyUnsafe::No + ) } fn wait_receiver(rx: Receiver) -> T { @@ -117,7 +131,7 @@ fn wait_receiver(rx: Receiver) -> T { fn system_name_works() { assert_eq!( api(None).system_name().unwrap(), - "testclient".to_owned() + "testclient".to_owned(), ); } @@ -125,7 +139,7 @@ fn system_name_works() { fn system_version_works() { assert_eq!( api(None).system_version().unwrap(), - "0.2.0".to_owned() + "0.2.0".to_owned(), ); } @@ -133,7 +147,7 @@ fn system_version_works() { fn system_chain_works() { assert_eq!( api(None).system_chain().unwrap(), - "testchain".to_owned() + "testchain".to_owned(), ); } @@ -141,7 +155,15 @@ fn system_chain_works() { fn system_properties_works() { assert_eq!( api(None).system_properties().unwrap(), - serde_json::map::Map::new() + serde_json::map::Map::new(), + ); +} + +#[test] +fn system_type_works() { + assert_eq!( + api(None).system_type().unwrap(), + Default::default(), ); } @@ -199,16 +221,40 @@ fn system_health() { ); } +#[test] +fn system_local_peer_id_works() { + assert_eq!( + wait_receiver(api(None).system_local_peer_id()), + "QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV".to_owned(), + ); +} + +#[test] +fn system_local_listen_addresses_works() { + assert_eq!( + wait_receiver(api(None).system_local_listen_addresses()), + vec![ + "/ip4/198.51.100.19/tcp/30333/p2p/QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV".to_string(), + "/ip4/127.0.0.1/tcp/30334/ws/p2p/QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV".to_string(), + ] + ); +} + #[test] fn system_peers() { + let mut runtime = tokio::runtime::current_thread::Runtime::new().unwrap(); + let peer_id = PeerId::random(); + let req = api(Status { + peer_id: peer_id.clone(), + peers: 1, + is_syncing: false, + is_dev: true, + }).system_peers(); + let res = runtime.block_on(req).unwrap(); + assert_eq!( - wait_receiver(api(Status { - peer_id: peer_id.clone(), - peers: 1, - is_syncing: false, - is_dev: true, - }).system_peers()), + res, vec![PeerInfo { peer_id: peer_id.to_base58(), roles: "FULL".into(), @@ -221,7 +267,10 @@ fn system_peers() { #[test] fn system_network_state() { - let res = wait_receiver(api(None).system_network_state()); + let mut runtime = tokio::runtime::current_thread::Runtime::new().unwrap(); + let req = api(None).system_network_state(); + let res = runtime.block_on(req).unwrap(); + assert_eq!( serde_json::from_value::(res).unwrap(), sc_network::network_state::NetworkState { diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml index 530142c745eeddb2501dbd46cfa81fec3bc6bb92..82fdc1ca10c22301223e85787110269c5659dc33 100644 --- a/client/service/Cargo.toml +++ b/client/service/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-service" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,20 +8,25 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Substrate service. Starts a thread that spins up the network, client, and extrinsic pool. Manages communication between them." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [features] -default = ["rocksdb"] +default = ["db"] # The RocksDB feature activates the RocksDB database backend. If it is not activated, and you pass # a path to a database, an error will be produced at runtime. -rocksdb = ["sc-client-db/kvdb-rocksdb"] +db = ["sc-client-db/kvdb-rocksdb", "sc-client-db/parity-db"] wasmtime = [ "sc-executor/wasmtime", ] +# exposes the client type +test-helpers = [] [dependencies] derive_more = "0.99.2" futures01 = { package = "futures", version = "0.1.29" } futures = "0.3.4" -futures-diagnose = "1.0" +rand = "0.7.3" parking_lot = "0.10.0" lazy_static = "1.4.0" log = "0.4.8" @@ -29,41 +34,48 @@ slog = { version = "2.5.2", features = ["nested-values"] } futures-timer = "3.0.1" wasm-timer = "0.2" exit-future = "0.2.0" +pin-project = "0.4.8" +hash-db = "0.15.2" serde = "1.0.101" serde_json = "1.0.41" -sysinfo = "0.12.0" -target_info = "0.1.0" -sc-keystore = { version = "2.0.0-alpha.5", path = "../keystore" } -sp-io = { version = "2.0.0-alpha.5", path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } -sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-session = { version = "2.0.0-alpha.5", path = "../../primitives/session" } -sp-application-crypto = { version = "2.0.0-alpha.5", path = "../../primitives/application-crypto" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/common" } -sc-network = { version = "0.8.0-alpha.5", path = "../network" } -sc-chain-spec = { version = "2.0.0-alpha.5", path = "../chain-spec" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } -sc-client = { version = "0.8.0-alpha.5", path = "../" } -sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } -sc-client-db = { version = "0.8.0-alpha.5", path = "../db" } +sysinfo = "0.13.3" +sc-keystore = { version = "2.0.0-dev", path = "../keystore" } +sp-io = { version = "2.0.0-dev", path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } +sp-trie = { version = "2.0.0-dev", path = "../../primitives/trie" } +sp-externalities = { version = "0.8.0-dev", path = "../../primitives/externalities" } +sp-utils = { version = "2.0.0-dev", path = "../../primitives/utils" } +sp-version = { version = "2.0.0-dev", path = "../../primitives/version" } +sp-blockchain = { version = "2.0.0-dev", path = "../../primitives/blockchain" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-session = { version = "2.0.0-dev", path = "../../primitives/session" } +sp-state-machine = { version = "0.8.0-dev", path = "../../primitives/state-machine" } +sp-application-crypto = { version = "2.0.0-dev", path = "../../primitives/application-crypto" } +sp-consensus = { version = "0.8.0-dev", path = "../../primitives/consensus/common" } +sc-network = { version = "0.8.0-dev", path = "../network" } +sc-chain-spec = { version = "2.0.0-dev", path = "../chain-spec" } +sc-client-api = { version = "2.0.0-dev", path = "../api" } +sp-api = { version = "2.0.0-dev", path = "../../primitives/api" } +sc-client-db = { version = "0.8.0-dev", default-features = false, path = "../db" } codec = { package = "parity-scale-codec", version = "1.3.0" } -sc-executor = { version = "0.8.0-alpha.5", path = "../executor" } -sc-transaction-pool = { version = "2.0.0-alpha.5", path = "../transaction-pool" } -sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../primitives/transaction-pool" } -sc-rpc-server = { version = "2.0.0-alpha.5", path = "../rpc-servers" } -sc-rpc = { version = "2.0.0-alpha.5", path = "../rpc" } -sc-telemetry = { version = "2.0.0-alpha.5", path = "../telemetry" } -sc-offchain = { version = "2.0.0-alpha.5", path = "../offchain" } +sc-executor = { version = "0.8.0-dev", path = "../executor" } +sc-transaction-pool = { version = "2.0.0-dev", path = "../transaction-pool" } +sp-transaction-pool = { version = "2.0.0-dev", path = "../../primitives/transaction-pool" } +sc-rpc-server = { version = "2.0.0-dev", path = "../rpc-servers" } +sc-rpc = { version = "2.0.0-dev", path = "../rpc" } +sc-block-builder = { version = "0.8.0-dev", path = "../block-builder" } +sp-block-builder = { version = "2.0.0-dev", path = "../../primitives/block-builder" } + +sc-telemetry = { version = "2.0.0-dev", path = "../telemetry" } +sc-offchain = { version = "2.0.0-dev", path = "../offchain" } parity-multiaddr = { package = "parity-multiaddr", version = "0.7.3" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" , version = "0.8.0-alpha.5"} -sc-tracing = { version = "2.0.0-alpha.5", path = "../tracing" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" , version = "0.8.0-dev"} +sc-tracing = { version = "2.0.0-dev", path = "../tracing" } tracing = "0.1.10" -parity-util-mem = { version = "0.6.0", default-features = false, features = ["primitive-types"] } +parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } -[target.'cfg(any(unix, windows))'.dependencies] +[target.'cfg(all(any(unix, windows), not(target_os = "android")))'.dependencies] netstat2 = "0.8.1" [target.'cfg(target_os = "linux")'.dependencies] @@ -72,9 +84,6 @@ procfs = '0.7.8' [dev-dependencies] substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } -sp-consensus-babe = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/babe" } -grandpa = { version = "0.8.0-alpha.5", package = "sc-finality-grandpa", path = "../finality-grandpa" } -grandpa-primitives = { version = "2.0.0-alpha.5", package = "sp-finality-grandpa", path = "../../primitives/finality-grandpa" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sp-consensus-babe = { version = "0.8.0-dev", path = "../../primitives/consensus/babe" } +grandpa = { version = "0.8.0-dev", package = "sc-finality-grandpa", path = "../finality-grandpa" } +grandpa-primitives = { version = "2.0.0-dev", package = "sp-finality-grandpa", path = "../../primitives/finality-grandpa" } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 205f877999f82665e250914e9978df45fb5e802d..a10856113442540fd237a0fe0e15c86f255063e1 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -15,21 +15,21 @@ // along with Substrate. If not, see . use crate::{Service, NetworkStatus, NetworkState, error::Error, DEFAULT_PROTOCOL_ID, MallocSizeOfWasm}; -use crate::{TaskManagerBuilder, start_rpc_servers, build_network_future, TransactionPoolAdapter}; +use crate::{start_rpc_servers, build_network_future, TransactionPoolAdapter, TaskManager}; use crate::status_sinks; -use crate::config::{Configuration, DatabaseConfig, KeystoreConfig, PrometheusConfig}; +use crate::config::{Configuration, KeystoreConfig, PrometheusConfig, OffchainWorkerConfig}; use crate::metrics::MetricsService; use sc_client_api::{ - self, - BlockchainEvents, - backend::RemoteBackend, light::RemoteBlockchain, - execution_extensions::ExtensionsFactory, - ExecutorProvider, CallExecutor + self, BlockchainEvents, backend::RemoteBackend, light::RemoteBlockchain, execution_extensions::ExtensionsFactory, + ExecutorProvider, CallExecutor, ForkBlocks, BadBlocks, CloneableSpawn, UsageProvider, }; +use crate::client::{Client, ClientConfig}; use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; -use sc_client::Client; use sc_chain_spec::get_extension; -use sp_consensus::import_queue::ImportQueue; +use sp_consensus::{ + block_validation::{BlockAnnounceValidator, DefaultBlockAnnounceValidator}, + import_queue::ImportQueue, +}; use futures::{ Future, FutureExt, StreamExt, future::ready, @@ -44,8 +44,9 @@ use sp_runtime::traits::{ Block as BlockT, NumberFor, SaturatedConversion, HashFor, }; use sp_api::ProvideRuntimeApi; -use sc_executor::{NativeExecutor, NativeExecutionDispatch}; +use sc_executor::{NativeExecutor, NativeExecutionDispatch, RuntimeInfo}; use std::{ + collections::HashMap, io::{Read, Write, Seek}, marker::PhantomData, sync::Arc, pin::Pin }; @@ -53,6 +54,11 @@ use wasm_timer::SystemTime; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; use sp_transaction_pool::{MaintainedTransactionPool, ChainEvent}; use sp_blockchain; +use prometheus_endpoint::Registry; +use sc_client_db::{Backend, DatabaseSettings}; +use sp_core::traits::CodeExecutor; +use sp_runtime::BuildStorage; +use sc_client_api::execution_extensions::ExecutionExtensions; pub type BackgroundTask = Pin + Send>>; @@ -79,7 +85,7 @@ pub struct ServiceBuilder, backend: Arc, - tasks_builder: TaskManagerBuilder, + task_manager: TaskManager, keystore: Arc>, fetcher: Option, select_chain: Option, @@ -91,6 +97,7 @@ pub struct ServiceBuilder>>, marker: PhantomData<(TBl, TRtApi)>, background_tasks: Vec<(&'static str, BackgroundTask)>, + block_announce_validator_builder: Option) -> Box + Send>>>, } /// Full client type. @@ -105,7 +112,7 @@ pub type TFullClient = Client< pub type TFullBackend = sc_client_db::Backend; /// Full client call executor type. -pub type TFullCallExecutor = sc_client::LocalCallExecutor< +pub type TFullCallExecutor = crate::client::LocalCallExecutor< sc_client_db::Backend, NativeExecutor, >; @@ -119,19 +126,19 @@ pub type TLightClient = Client< >; /// Light client backend type. -pub type TLightBackend = sc_client::light::backend::Backend< +pub type TLightBackend = crate::client::light::backend::Backend< sc_client_db::light::LightStorage, HashFor, >; /// Light call executor type. -pub type TLightCallExecutor = sc_client::light::call_executor::GenesisCallExecutor< - sc_client::light::backend::Backend< +pub type TLightCallExecutor = crate::client::light::call_executor::GenesisCallExecutor< + crate::client::light::backend::Backend< sc_client_db::light::LightStorage, HashFor >, - sc_client::LocalCallExecutor< - sc_client::light::backend::Backend< + crate::client::LocalCallExecutor< + crate::client::light::backend::Backend< sc_client_db::light::LightStorage, HashFor >, @@ -143,7 +150,7 @@ type TFullParts = ( TFullClient, Arc>, Arc>, - TaskManagerBuilder, + TaskManager, ); /// Creates a new full client for the given config. @@ -168,10 +175,12 @@ fn new_full_parts( password.clone() )?, KeystoreConfig::InMemory => Keystore::new_in_memory(), - KeystoreConfig::None => return Err("No keystore config provided!".into()), }; - let tasks_builder = TaskManagerBuilder::new(); + let task_manager = { + let registry = config.prometheus_config.as_ref().map(|cfg| &cfg.registry); + TaskManager::new(config.task_executor.clone(), registry)? + }; let executor = NativeExecutor::::new( config.wasm_method, @@ -179,12 +188,12 @@ fn new_full_parts( config.max_runtime_instances, ); - let chain_spec = config.expect_chain_spec(); - let fork_blocks = get_extension::>(chain_spec.extensions()) + let chain_spec = &config.chain_spec; + let fork_blocks = get_extension::>(chain_spec.extensions()) .cloned() .unwrap_or_default(); - let bad_blocks = get_extension::>(chain_spec.extensions()) + let bad_blocks = get_extension::>(chain_spec.extensions()) .cloned() .unwrap_or_default(); @@ -194,15 +203,7 @@ fn new_full_parts( state_cache_child_ratio: config.state_cache_child_ratio.map(|v| (v, 100)), pruning: config.pruning.clone(), - source: match config.expect_database() { - DatabaseConfig::Path { path, cache_size } => - sc_client_db::DatabaseSettingsSrc::Path { - path: path.clone(), - cache_size: cache_size.clone().map(|u| u as usize), - }, - DatabaseConfig::Custom(db) => - sc_client_db::DatabaseSettingsSrc::Custom(db.clone()), - }, + source: config.database.clone(), }; let extensions = sc_client_api::execution_extensions::ExecutionExtensions::new( @@ -210,19 +211,69 @@ fn new_full_parts( Some(keystore.clone()), ); - sc_client_db::new_client( + new_client( db_config, executor, - config.expect_chain_spec().as_storage_builder(), + chain_spec.as_storage_builder(), fork_blocks, bad_blocks, extensions, - Box::new(tasks_builder.spawn_handle()), + Box::new(task_manager.spawn_handle()), config.prometheus_config.as_ref().map(|config| config.registry.clone()), + ClientConfig { + offchain_worker_enabled : config.offchain_worker.enabled , + offchain_indexing_api: config.offchain_worker.indexing_enabled, + }, )? }; - Ok((client, backend, keystore, tasks_builder)) + Ok((client, backend, keystore, task_manager)) +} + + +/// Create an instance of db-backed client. +pub fn new_client( + settings: DatabaseSettings, + executor: E, + genesis_storage: &dyn BuildStorage, + fork_blocks: ForkBlocks, + bad_blocks: BadBlocks, + execution_extensions: ExecutionExtensions, + spawn_handle: Box, + prometheus_registry: Option, + config: ClientConfig, +) -> Result<( + crate::client::Client< + Backend, + crate::client::LocalCallExecutor, E>, + Block, + RA, + >, + Arc>, +), + sp_blockchain::Error, +> + where + Block: BlockT, + E: CodeExecutor + RuntimeInfo, +{ + const CANONICALIZATION_DELAY: u64 = 4096; + + let backend = Arc::new(Backend::new(settings, CANONICALIZATION_DELAY)?); + let executor = crate::client::LocalCallExecutor::new(backend.clone(), executor, spawn_handle, config.clone()); + Ok(( + crate::client::Client::new( + backend.clone(), + executor, + genesis_storage, + fork_blocks, + bad_blocks, + execution_extensions, + prometheus_registry, + config, + )?, + backend, + )) } impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { @@ -242,7 +293,7 @@ impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { (), TFullBackend, >, Error> { - let (client, backend, keystore, tasks_builder) = new_full_parts(&config)?; + let (client, backend, keystore, task_manager) = new_full_parts(&config)?; let client = Arc::new(client); @@ -251,7 +302,7 @@ impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { client, backend, keystore, - tasks_builder, + task_manager, fetcher: None, select_chain: None, import_queue: (), @@ -261,6 +312,7 @@ impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { rpc_extensions: Default::default(), remote_backend: None, background_tasks: Default::default(), + block_announce_validator_builder: None, marker: PhantomData, }) } @@ -281,7 +333,10 @@ impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { (), TLightBackend, >, Error> { - let tasks_builder = TaskManagerBuilder::new(); + let task_manager = { + let registry = config.prometheus_config.as_ref().map(|cfg| &cfg.registry); + TaskManager::new(config.task_executor.clone(), registry)? + }; let keystore = match &config.keystore { KeystoreConfig::Path { path, password } => Keystore::open( @@ -289,7 +344,6 @@ impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { password.clone() )?, KeystoreConfig::InMemory => Keystore::new_in_memory(), - KeystoreConfig::None => return Err("No keystore config provided!".into()), }; let executor = NativeExecutor::::new( @@ -304,34 +358,26 @@ impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { state_cache_child_ratio: config.state_cache_child_ratio.map(|v| (v, 100)), pruning: config.pruning.clone(), - source: match config.expect_database() { - DatabaseConfig::Path { path, cache_size } => - sc_client_db::DatabaseSettingsSrc::Path { - path: path.clone(), - cache_size: cache_size.clone().map(|u| u as usize), - }, - DatabaseConfig::Custom(db) => - sc_client_db::DatabaseSettingsSrc::Custom(db.clone()), - }, + source: config.database.clone(), }; sc_client_db::light::LightStorage::new(db_settings)? }; - let light_blockchain = sc_client::light::new_light_blockchain(db_storage); + let light_blockchain = crate::client::light::new_light_blockchain(db_storage); let fetch_checker = Arc::new( - sc_client::light::new_fetch_checker::<_, TBl, _>( + crate::client::light::new_fetch_checker::<_, TBl, _>( light_blockchain.clone(), executor.clone(), - Box::new(tasks_builder.spawn_handle()), + Box::new(task_manager.spawn_handle()), ), ); let fetcher = Arc::new(sc_network::config::OnDemand::new(fetch_checker)); - let backend = sc_client::light::new_light_backend(light_blockchain); + let backend = crate::client::light::new_light_backend(light_blockchain); let remote_blockchain = backend.remote_blockchain(); - let client = Arc::new(sc_client::light::new_light( + let client = Arc::new(crate::client::light::new_light( backend.clone(), - config.expect_chain_spec().as_storage_builder(), + config.chain_spec.as_storage_builder(), executor, - Box::new(tasks_builder.spawn_handle()), + Box::new(task_manager.spawn_handle()), config.prometheus_config.as_ref().map(|config| config.registry.clone()), )?); @@ -339,7 +385,7 @@ impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { config, client, backend, - tasks_builder, + task_manager, keystore, fetcher: Some(fetcher.clone()), select_chain: None, @@ -350,6 +396,7 @@ impl ServiceBuilder<(), (), (), (), (), (), (), (), (), (), ()> { rpc_extensions: Default::default(), remote_backend: Some(remote_blockchain), background_tasks: Default::default(), + block_announce_validator_builder: None, marker: PhantomData, }) } @@ -412,7 +459,7 @@ impl config: self.config, client: self.client, backend: self.backend, - tasks_builder: self.tasks_builder, + task_manager: self.task_manager, keystore: self.keystore, fetcher: self.fetcher, select_chain, @@ -423,6 +470,7 @@ impl rpc_extensions: self.rpc_extensions, remote_backend: self.remote_backend, background_tasks: self.background_tasks, + block_announce_validator_builder: self.block_announce_validator_builder, marker: self.marker, }) } @@ -455,7 +503,7 @@ impl config: self.config, client: self.client, backend: self.backend, - tasks_builder: self.tasks_builder, + task_manager: self.task_manager, keystore: self.keystore, fetcher: self.fetcher, select_chain: self.select_chain, @@ -466,6 +514,7 @@ impl rpc_extensions: self.rpc_extensions, remote_backend: self.remote_backend, background_tasks: self.background_tasks, + block_announce_validator_builder: self.block_announce_validator_builder, marker: self.marker, }) } @@ -493,7 +542,7 @@ impl config: self.config, client: self.client, backend: self.backend, - tasks_builder: self.tasks_builder, + task_manager: self.task_manager, keystore: self.keystore, fetcher: self.fetcher, select_chain: self.select_chain, @@ -504,6 +553,7 @@ impl rpc_extensions: self.rpc_extensions, remote_backend: self.remote_backend, background_tasks: self.background_tasks, + block_announce_validator_builder: self.block_announce_validator_builder, marker: self.marker, }) } @@ -555,7 +605,7 @@ impl config: self.config, client: self.client, backend: self.backend, - tasks_builder: self.tasks_builder, + task_manager: self.task_manager, keystore: self.keystore, fetcher: self.fetcher, select_chain: self.select_chain, @@ -566,6 +616,7 @@ impl rpc_extensions: self.rpc_extensions, remote_backend: self.remote_backend, background_tasks: self.background_tasks, + block_announce_validator_builder: self.block_announce_validator_builder, marker: self.marker, }) } @@ -597,6 +648,7 @@ impl sc_transaction_pool::txpool::Options, Arc, Option, + Option<&Registry>, ) -> Result<(UExPool, Option), Error> ) -> Result, Error> @@ -605,6 +657,7 @@ impl self.config.transaction_pool.clone(), self.client.clone(), self.fetcher.clone(), + self.config.prometheus_config.as_ref().map(|config| &config.registry), )?; if let Some(background_task) = background_task{ @@ -614,7 +667,7 @@ impl Ok(ServiceBuilder { config: self.config, client: self.client, - tasks_builder: self.tasks_builder, + task_manager: self.task_manager, backend: self.backend, keystore: self.keystore, fetcher: self.fetcher, @@ -626,6 +679,7 @@ impl rpc_extensions: self.rpc_extensions, remote_backend: self.remote_backend, background_tasks: self.background_tasks, + block_announce_validator_builder: self.block_announce_validator_builder, marker: self.marker, }) } @@ -643,7 +697,7 @@ impl config: self.config, client: self.client, backend: self.backend, - tasks_builder: self.tasks_builder, + task_manager: self.task_manager, keystore: self.keystore, fetcher: self.fetcher, select_chain: self.select_chain, @@ -654,6 +708,36 @@ impl rpc_extensions, remote_backend: self.remote_backend, background_tasks: self.background_tasks, + block_announce_validator_builder: self.block_announce_validator_builder, + marker: self.marker, + }) + } + + /// Defines the `BlockAnnounceValidator` to use. `DefaultBlockAnnounceValidator` will be used by + /// default. + pub fn with_block_announce_validator( + self, + block_announce_validator_builder: + impl FnOnce(Arc) -> Box + Send> + 'static, + ) -> Result, Error> + where TSc: Clone, TFchr: Clone { + Ok(ServiceBuilder { + config: self.config, + client: self.client, + backend: self.backend, + task_manager: self.task_manager, + keystore: self.keystore, + fetcher: self.fetcher, + select_chain: self.select_chain, + import_queue: self.import_queue, + finality_proof_request_builder: self.finality_proof_request_builder, + finality_proof_provider: self.finality_proof_provider, + transaction_pool: self.transaction_pool, + rpc_extensions: self.rpc_extensions, + remote_backend: self.remote_backend, + background_tasks: self.background_tasks, + block_announce_validator_builder: Some(Box::new(block_announce_validator_builder)), marker: self.marker, }) } @@ -720,7 +804,7 @@ ServiceBuilder< TBl: BlockT, TRtApi: 'static + Send + Sync, TBackend: 'static + sc_client_api::backend::Backend + Send, - TExec: 'static + sc_client::CallExecutor + Send + Sync + Clone, + TExec: 'static + CallExecutor + Send + Sync + Clone, TSc: Clone, TImpQu: 'static + ImportQueue, TExPool: MaintainedTransactionPool::Hash> + MallocSizeOfWasm + 'static, @@ -753,7 +837,7 @@ ServiceBuilder< marker: _, mut config, client, - tasks_builder, + task_manager, fetcher: on_demand, backend, keystore, @@ -765,6 +849,7 @@ ServiceBuilder< rpc_extensions, remote_backend, background_tasks, + block_announce_validator_builder, } = self; sp_session::generate_initial_session_keys( @@ -778,9 +863,9 @@ ServiceBuilder< let import_queue = Box::new(import_queue); let chain_info = client.chain_info(); - let chain_spec = config.expect_chain_spec(); + let chain_spec = &config.chain_spec; - let version = config.full_version(); + let version = config.impl_version; info!("📦 Highest known block at #{}", chain_info.best_number); telemetry!( SUBSTRATE_INFO; @@ -797,7 +882,7 @@ ServiceBuilder< imports_external_transactions: !matches!(config.role, Role::Light), pool: transaction_pool.clone(), client: client.clone(), - executor: tasks_builder.spawn_handle(), + executor: task_manager.spawn_handle(), }); let protocol_id = { @@ -813,13 +898,16 @@ ServiceBuilder< sc_network::config::ProtocolId::from(protocol_id_full) }; - let block_announce_validator = - Box::new(sp_consensus::block_validation::DefaultBlockAnnounceValidator::new(client.clone())); + let block_announce_validator = if let Some(f) = block_announce_validator_builder { + f(client.clone()) + } else { + Box::new(DefaultBlockAnnounceValidator::new(client.clone())) + }; let network_params = sc_network::config::Params { role: config.role.clone(), executor: { - let spawn_handle = tasks_builder.spawn_handle(); + let spawn_handle = task_manager.spawn_handle(); Some(Box::new(move |fut| { spawn_handle.spawn("libp2p-node", fut); })) @@ -842,18 +930,18 @@ ServiceBuilder< let network_status_sinks = Arc::new(Mutex::new(status_sinks::StatusSinks::new())); let offchain_storage = backend.offchain_storage(); - let offchain_workers = match (config.offchain_worker, offchain_storage.clone()) { - (true, Some(db)) => { + let offchain_workers = match (config.offchain_worker.clone(), offchain_storage.clone()) { + (OffchainWorkerConfig {enabled: true, .. }, Some(db)) => { Some(Arc::new(sc_offchain::OffchainWorkers::new(client.clone(), db))) }, - (true, None) => { + (OffchainWorkerConfig {enabled: true, .. }, None) => { warn!("Offchain workers disabled, due to lack of offchain storage support in backend."); None }, _ => None, }; - let spawn_handle = tasks_builder.spawn_handle(); + let spawn_handle = task_manager.spawn_handle(); // Spawn background tasks which were stacked during the // service building. @@ -865,7 +953,7 @@ ServiceBuilder< // block notifications let txpool = Arc::downgrade(&transaction_pool); let offchain = offchain_workers.as_ref().map(Arc::downgrade); - let notifications_spawn_handle = tasks_builder.spawn_handle(); + let notifications_spawn_handle = task_manager.spawn_handle(); let network_state_info: Arc = network.clone(); let is_validator = config.role.is_authority(); @@ -958,7 +1046,7 @@ ServiceBuilder< }; let metrics = MetricsService::with_prometheus( ®istry, - &config.name, + &config.network.node_name, &config.impl_version, role_bits, )?; @@ -1010,7 +1098,7 @@ ServiceBuilder< // RPC let (system_rpc_tx, system_rpc_rx) = tracing_unbounded("mpsc_system_rpc"); - let gen_handler = || { + let gen_handler = |deny_unsafe: sc_rpc::DenyUnsafe| { use sc_rpc::{chain, state, author, system, offchain}; let system_info = sc_rpc::system::SystemInfo { @@ -1018,11 +1106,12 @@ ServiceBuilder< impl_name: config.impl_name.into(), impl_version: config.impl_version.into(), properties: chain_spec.properties().clone(), + chain_type: chain_spec.chain_type().clone(), }; - let subscriptions = sc_rpc::Subscriptions::new(Arc::new(tasks_builder.spawn_handle())); + let subscriptions = sc_rpc::Subscriptions::new(Arc::new(task_manager.spawn_handle())); - let (chain, state) = if let (Some(remote_backend), Some(on_demand)) = + let (chain, state, child_state) = if let (Some(remote_backend), Some(on_demand)) = (remote_backend.as_ref(), on_demand.as_ref()) { // Light clients let chain = sc_rpc::chain::new_light( @@ -1031,19 +1120,19 @@ ServiceBuilder< remote_backend.clone(), on_demand.clone() ); - let state = sc_rpc::state::new_light( + let (state, child_state) = sc_rpc::state::new_light( client.clone(), subscriptions.clone(), remote_backend.clone(), on_demand.clone() ); - (chain, state) + (chain, state, child_state) } else { // Full nodes let chain = sc_rpc::chain::new_full(client.clone(), subscriptions.clone()); - let state = sc_rpc::state::new_full(client.clone(), subscriptions.clone()); - (chain, state) + let (state, child_state) = sc_rpc::state::new_full(client.clone(), subscriptions.clone()); + (chain, state, child_state) }; let author = sc_rpc::author::Author::new( @@ -1051,32 +1140,32 @@ ServiceBuilder< transaction_pool.clone(), subscriptions, keystore.clone(), + deny_unsafe, ); - let system = system::System::new(system_info, system_rpc_tx.clone()); - - match offchain_storage.clone() { - Some(storage) => { - let offchain = sc_rpc::offchain::Offchain::new(storage); - sc_rpc_server::rpc_handler(( - state::StateApi::to_delegate(state), - chain::ChainApi::to_delegate(chain), - offchain::OffchainApi::to_delegate(offchain), - author::AuthorApi::to_delegate(author), - system::SystemApi::to_delegate(system), - rpc_extensions.clone(), - )) - }, - None => sc_rpc_server::rpc_handler(( - state::StateApi::to_delegate(state), - chain::ChainApi::to_delegate(chain), - author::AuthorApi::to_delegate(author), - system::SystemApi::to_delegate(system), - rpc_extensions.clone(), - )) - } + let system = system::System::new(system_info, system_rpc_tx.clone(), deny_unsafe); + + let maybe_offchain_rpc = offchain_storage.clone() + .map(|storage| { + let offchain = sc_rpc::offchain::Offchain::new(storage, deny_unsafe); + // FIXME: Use plain Option (don't collect into HashMap) when we upgrade to jsonrpc 14.1 + // https://github.com/paritytech/jsonrpc/commit/20485387ed06a48f1a70bf4d609a7cde6cf0accf + let delegate = offchain::OffchainApi::to_delegate(offchain); + delegate.into_iter().collect::>() + }).unwrap_or_default(); + + sc_rpc_server::rpc_handler(( + state::StateApi::to_delegate(state), + state::ChildStateApi::to_delegate(child_state), + chain::ChainApi::to_delegate(chain), + maybe_offchain_rpc, + author::AuthorApi::to_delegate(author), + system::SystemApi::to_delegate(system), + rpc_extensions.clone(), + )) }; - let rpc_handlers = gen_handler(); let rpc = start_rpc_servers(&config, gen_handler)?; + // This is used internally, so don't restrict access to unsafe RPC + let rpc_handlers = gen_handler(sc_rpc::DenyUnsafe::No); spawn_handle.spawn( "network-worker", @@ -1097,10 +1186,10 @@ ServiceBuilder< let telemetry = config.telemetry_endpoints.clone().map(|endpoints| { let is_authority = config.role.is_authority(); let network_id = network.local_peer_id().to_base58(); - let name = config.name.clone(); + let name = config.network.node_name.clone(); let impl_name = config.impl_name.to_owned(); let version = version.clone(); - let chain_name = config.expect_chain_spec().name().to_owned(); + let chain_name = config.chain_spec.name().to_owned(); let telemetry_connection_sinks_ = telemetry_connection_sinks.clone(); let telemetry = sc_telemetry::init_telemetry(sc_telemetry::TelemetryConfig { endpoints, @@ -1152,7 +1241,7 @@ ServiceBuilder< Ok(Service { client, - task_manager: tasks_builder.into_task_manager(config.task_executor.ok_or(Error::TaskExecutorRequired)?), + task_manager, network, network_status_sinks, select_chain, diff --git a/client/service/src/chain_ops.rs b/client/service/src/chain_ops.rs index 12fae3224108a66feed5e593e8de50bb97c8a9cf..59dbc8302c2e56e280fe274de2af4e82c5e7b632 100644 --- a/client/service/src/chain_ops.rs +++ b/client/service/src/chain_ops.rs @@ -27,7 +27,7 @@ use sp_runtime::traits::{ }; use sp_runtime::generic::{BlockId, SignedBlock}; use codec::{Decode, Encode, IoReader}; -use sc_client::{Client, LocalCallExecutor}; +use crate::client::{Client, LocalCallExecutor}; use sp_consensus::{ BlockOrigin, import_queue::{IncomingBlock, Link, BlockImportError, BlockImportResult, ImportQueue}, diff --git a/client/src/block_rules.rs b/client/service/src/client/block_rules.rs similarity index 100% rename from client/src/block_rules.rs rename to client/service/src/client/block_rules.rs diff --git a/client/src/call_executor.rs b/client/service/src/client/call_executor.rs similarity index 91% rename from client/src/call_executor.rs rename to client/service/src/client/call_executor.rs index 1160449eeef88c251673251d873c76855d7bcab3..229e7478e939f818977c268e2b57e9fae9422a5f 100644 --- a/client/src/call_executor.rs +++ b/client/service/src/client/call_executor.rs @@ -25,9 +25,10 @@ use sp_state_machine::{ }; use sc_executor::{RuntimeVersion, RuntimeInfo, NativeVersion}; use sp_externalities::Extensions; -use sp_core::{NativeOrEncoded, NeverNativeValue, traits::CodeExecutor}; +use sp_core::{NativeOrEncoded, NeverNativeValue, traits::CodeExecutor, offchain::storage::OffchainOverlayedChanges}; use sp_api::{ProofRecorder, InitializeBlock, StorageTransactionCache}; use sc_client_api::{backend, call_executor::CallExecutor, CloneableSpawn}; +use super::client::ClientConfig; /// Call executor that executes methods locally, querying all required /// data from local backend. @@ -35,6 +36,7 @@ pub struct LocalCallExecutor { backend: Arc, executor: E, spawn_handle: Box, + client_config: ClientConfig, } impl LocalCallExecutor { @@ -43,11 +45,13 @@ impl LocalCallExecutor { backend: Arc, executor: E, spawn_handle: Box, + client_config: ClientConfig, ) -> Self { LocalCallExecutor { backend, executor, spawn_handle, + client_config, } } } @@ -58,6 +62,7 @@ impl Clone for LocalCallExecutor where E: Clone { backend: self.backend.clone(), executor: self.executor.clone(), spawn_handle: self.spawn_handle.clone(), + client_config: self.client_config.clone(), } } } @@ -81,6 +86,11 @@ where extensions: Option, ) -> sp_blockchain::Result> { let mut changes = OverlayedChanges::default(); + let mut offchain_changes = if self.client_config.offchain_indexing_api { + OffchainOverlayedChanges::enabled() + } else { + OffchainOverlayedChanges::disabled() + }; let changes_trie = backend::changes_tries_state_at_block( id, self.backend.changes_trie_storage() )?; @@ -90,6 +100,7 @@ where &state, changes_trie, &mut changes, + &mut offchain_changes, &self.executor, method, call_data, @@ -120,6 +131,7 @@ where method: &str, call_data: &[u8], changes: &RefCell, + offchain_changes: &RefCell, storage_transaction_cache: Option<&RefCell< StorageTransactionCache >>, @@ -143,6 +155,9 @@ where let mut state = self.backend.state_at(*at)?; + let changes = &mut *changes.borrow_mut(); + let offchain_changes = &mut *offchain_changes.borrow_mut(); + match recorder { Some(recorder) => { let trie_state = state.as_trie_backend() @@ -160,11 +175,11 @@ where recorder.clone(), ); - let changes = &mut *changes.borrow_mut(); let mut state_machine = StateMachine::new( &backend, changes_trie_state, changes, + offchain_changes, &self.executor, method, call_data, @@ -179,11 +194,11 @@ where None => { let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state); let runtime_code = state_runtime_code.runtime_code()?; - let changes = &mut *changes.borrow_mut(); let mut state_machine = StateMachine::new( &state, changes_trie_state, changes, + offchain_changes, &self.executor, method, call_data, @@ -198,6 +213,7 @@ where fn runtime_version(&self, id: &BlockId) -> sp_blockchain::Result { let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); let changes_trie_state = backend::changes_tries_state_at_block( id, self.backend.changes_trie_storage(), @@ -206,6 +222,7 @@ where let mut cache = StorageTransactionCache::::default(); let mut ext = Ext::new( &mut overlay, + &mut offchain_overlay, &mut cache, &state, changes_trie_state, diff --git a/client/src/client.rs b/client/service/src/client/client.rs similarity index 50% rename from client/src/client.rs rename to client/service/src/client/client.rs index 13d654a7c2ab66beb48125edb2fb9af925fa6c0e..9758bbe01e7edca5295fd3d99a66aeabfea1d75a 100644 --- a/client/src/client.rs +++ b/client/service/src/client/client.rs @@ -17,24 +17,25 @@ //! Substrate Client use std::{ - marker::PhantomData, collections::{HashSet, BTreeMap, HashMap}, sync::Arc, panic::UnwindSafe, - result, + marker::PhantomData, + collections::{HashSet, BTreeMap, HashMap}, + sync::Arc, panic::UnwindSafe, result, }; use log::{info, trace, warn}; use parking_lot::{Mutex, RwLock}; use codec::{Encode, Decode}; use hash_db::Prefix; use sp_core::{ - ChangesTrieConfiguration, convert_hash, traits::CodeExecutor, - NativeOrEncoded, storage::{StorageKey, StorageData, well_known_keys, ChildInfo}, + ChangesTrieConfiguration, convert_hash, traits::CodeExecutor, NativeOrEncoded, + storage::{StorageKey, PrefixedStorageKey, StorageData, well_known_keys, ChildInfo}, }; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; use sp_runtime::{ Justification, BuildStorage, generic::{BlockId, SignedBlock, DigestItem}, traits::{ - Block as BlockT, Header as HeaderT, Zero, NumberFor, HashFor, SaturatedConversion, One, - DigestFor, + Block as BlockT, Header as HeaderT, Zero, NumberFor, + HashFor, SaturatedConversion, One, DigestFor, }, }; use sp_state_machine::{ @@ -44,61 +45,56 @@ use sp_state_machine::{ }; use sc_executor::{RuntimeVersion, RuntimeInfo}; use sp_consensus::{ - Error as ConsensusError, BlockStatus, BlockImportParams, BlockCheckParams, ImportResult, - BlockOrigin, ForkChoiceStrategy, SelectChain, RecordProof, + Error as ConsensusError, BlockStatus, BlockImportParams, BlockCheckParams, + ImportResult, BlockOrigin, ForkChoiceStrategy, RecordProof, }; -use sp_blockchain::{self as blockchain, +use sp_blockchain::{ + self as blockchain, Backend as ChainBackend, HeaderBackend as ChainHeaderBackend, ProvideCache, Cache, well_known_cache_keys::Id as CacheKeyId, HeaderMetadata, CachedHeaderMetadata, }; use sp_trie::StorageProof; - use sp_api::{ CallApiAt, ConstructRuntimeApi, Core as CoreApi, ApiExt, ApiRef, ProvideRuntimeApi, CallApiAtParams, }; use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider}; - -pub use sc_client_api::{ - backend::{ - self, BlockImportOperation, PrunableStateChangesTrieStorage, - ClientImportOperation, Finalizer, ImportSummary, NewBlockState, - changes_tries_state_at_block, StorageProvider, - LockImportRun, - }, - client::{ - ImportNotifications, FinalityNotification, FinalityNotifications, BlockImportNotification, - ClientInfo, BlockchainEvents, BlockBackend, ProvideUncles, BadBlocks, ForkBlocks, - BlockOf, - }, - execution_extensions::{ExecutionExtensions, ExecutionStrategies}, - notifications::{StorageNotifications, StorageEventStream}, - CallExecutor, ExecutorProvider, ProofProvider, CloneableSpawn, -}; -use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; +use sc_client_api::{backend::{ + self, BlockImportOperation, PrunableStateChangesTrieStorage, + ClientImportOperation, Finalizer, ImportSummary, NewBlockState, + changes_tries_state_at_block, StorageProvider, + LockImportRun, apply_aux, +}, client::{ + ImportNotifications, FinalityNotification, FinalityNotifications, BlockImportNotification, + ClientInfo, BlockchainEvents, BlockBackend, ProvideUncles, BadBlocks, ForkBlocks, + BlockOf, +}, execution_extensions::ExecutionExtensions, notifications::{StorageNotifications, StorageEventStream}, KeyIterator, CallExecutor, ExecutorProvider, ProofProvider, CloneableSpawn, cht, in_mem, UsageProvider}; +use sp_utils::mpsc::tracing_unbounded; use sp_blockchain::Error; use prometheus_endpoint::Registry; - -use crate::{ - call_executor::LocalCallExecutor, +use super::{ + genesis, call_executor::LocalCallExecutor, light::{call_executor::prove_execution, fetcher::ChangesProof}, - in_mem, genesis, cht, block_rules::{BlockRules, LookupResult as BlockLookupResult}, + block_rules::{BlockRules, LookupResult as BlockLookupResult}, }; -use crate::client::backend::KeyIterator; +use futures::channel::mpsc; + +type NotificationSinks = Mutex>>; /// Substrate Client pub struct Client where Block: BlockT { backend: Arc, executor: E, storage_notifications: Mutex>, - import_notification_sinks: Mutex>>>, - finality_notification_sinks: Mutex>>>, + import_notification_sinks: NotificationSinks>, + finality_notification_sinks: NotificationSinks>, // holds the block hash currently being imported. TODO: replace this with block queue importing_block: RwLock>, block_rules: BlockRules, execution_extensions: ExecutionExtensions, + config: ClientConfig, _phantom: PhantomData, } @@ -136,6 +132,7 @@ pub fn new_in_mem( keystore: Option, prometheus_registry: Option, spawn_handle: Box, + config: ClientConfig, ) -> sp_blockchain::Result, LocalCallExecutor, E>, @@ -146,7 +143,24 @@ pub fn new_in_mem( S: BuildStorage, Block: BlockT, { - new_with_backend(Arc::new(in_mem::Backend::new()), executor, genesis_storage, keystore, spawn_handle, prometheus_registry) + new_with_backend( + Arc::new(in_mem::Backend::new()), + executor, + genesis_storage, + keystore, + spawn_handle, + prometheus_registry, + config, + ) +} + +/// Relevant client configuration items relevant for the client. +#[derive(Debug,Clone,Default)] +pub struct ClientConfig { + /// Enable the offchain worker db. + pub offchain_worker_enabled: bool, + /// If true, allows access from the runtime to write into offchain worker db. + pub offchain_indexing_api: bool, } /// Create a client with the explicitly provided backend. @@ -158,6 +172,7 @@ pub fn new_with_backend( keystore: Option, spawn_handle: Box, prometheus_registry: Option, + config: ClientConfig, ) -> sp_blockchain::Result, Block, RA>> where E: CodeExecutor + RuntimeInfo, @@ -165,7 +180,7 @@ pub fn new_with_backend( Block: BlockT, B: backend::LocalBackend + 'static, { - let call_executor = LocalCallExecutor::new(backend.clone(), executor, spawn_handle); + let call_executor = LocalCallExecutor::new(backend.clone(), executor, spawn_handle, config.clone()); let extensions = ExecutionExtensions::new(Default::default(), keystore); Client::new( backend, @@ -175,6 +190,7 @@ pub fn new_with_backend( Default::default(), extensions, prometheus_registry, + config, ) } @@ -243,6 +259,7 @@ impl Client where B: backend::Backend, E: CallExecutor, Block: BlockT, + Block::Header: Clone, { /// Creates new Substrate Client with given blockchain and code executor. pub fn new( @@ -252,7 +269,8 @@ impl Client where fork_blocks: ForkBlocks, bad_blocks: BadBlocks, execution_extensions: ExecutionExtensions, - _prometheus_registry: Option, + prometheus_registry: Option, + config: ClientConfig, ) -> sp_blockchain::Result { if backend.blockchain().header(BlockId::Number(Zero::zero()))?.is_none() { let genesis_storage = build_genesis_storage.build_storage()?; @@ -276,16 +294,29 @@ impl Client where Ok(Client { backend, executor, - storage_notifications: Default::default(), + storage_notifications: Mutex::new(StorageNotifications::new(prometheus_registry)), import_notification_sinks: Default::default(), finality_notification_sinks: Default::default(), importing_block: Default::default(), block_rules: BlockRules::new(fork_blocks, bad_blocks), execution_extensions, + config, _phantom: Default::default(), }) } + /// returns a reference to the block import notification sinks + /// useful for test environments. + pub fn import_notification_sinks(&self) -> &NotificationSinks> { + &self.import_notification_sinks + } + + /// returns a reference to the finality notification sinks + /// useful for test environments. + pub fn finality_notification_sinks(&self) -> &NotificationSinks> { + &self.finality_notification_sinks + } + /// Get a reference to the state at a given block. pub fn state_at(&self, block: &BlockId) -> sp_blockchain::Result { self.backend.state_at(*block) @@ -344,7 +375,7 @@ impl Client where last: Block::Hash, min: Block::Hash, max: Block::Hash, - storage_key: Option<&StorageKey>, + storage_key: Option<&PrefixedStorageKey>, key: &StorageKey, cht_size: NumberFor, ) -> sp_blockchain::Result> { @@ -393,7 +424,7 @@ impl Client where fn with_cached_changed_keys( &self, root: &Block::Hash, - functor: &mut dyn FnMut(&HashMap>, HashSet>>), + functor: &mut dyn FnMut(&HashMap, HashSet>>), ) -> bool { self.storage.with_cached_changed_keys(root, functor) } @@ -438,7 +469,7 @@ impl Client where number: last_number, }, max_number, - storage_key.as_ref().map(|x| &x.0[..]), + storage_key, &key.0, ) .map_err(|err| sp_blockchain::Error::ChangesTrieAccessFailed(err))?; @@ -613,11 +644,20 @@ impl Client where if let Ok(ImportResult::Imported(ref aux)) = result { if aux.is_new_best { - telemetry!(SUBSTRATE_INFO; "block.import"; - "height" => height, - "best" => ?hash, - "origin" => ?origin - ); + use rand::Rng; + + // don't send telemetry block import events during initial sync for every + // block to avoid spamming the telemetry server, these events will be randomly + // sent at a rate of 1/10. + if origin != BlockOrigin::NetworkInitialSync || + rand::thread_rng().gen_bool(0.1) + { + telemetry!(SUBSTRATE_INFO; "block.import"; + "height" => height, + "best" => ?hash, + "origin" => ?origin + ); + } } } @@ -687,7 +727,22 @@ impl Client where operation.op.update_cache(new_cache); - let (main_sc, child_sc, tx, _, changes_trie_tx) = storage_changes.into_inner(); + let ( + main_sc, + child_sc, + offchain_sc, + tx, _, + changes_trie_tx, + ) = storage_changes.into_inner(); + + if self.config.offchain_indexing_api { + // if let Some(mut offchain_storage) = self.backend.offchain_storage() { + // offchain_sc.iter().for_each(|(k,v)| { + // offchain_storage.set(b"block-import-info", k,v) + // }); + // } + operation.op.update_offchain_storage(offchain_sc)?; + } operation.op.update_db_storage(tx)?; operation.op.update_storage(main_sc.clone(), child_sc.clone())?; @@ -991,14 +1046,6 @@ impl Client where Ok(self.backend.revert(n, true)?) } - /// Get usage info about current client. - pub fn usage_info(&self) -> ClientInfo { - ClientInfo { - chain: self.chain_info(), - usage: self.backend.usage_info(), - } - } - /// Get blockchain info. pub fn chain_info(&self) -> blockchain::Info { self.backend.blockchain().info() @@ -1082,6 +1129,20 @@ impl Client where } } +impl UsageProvider for Client where + B: backend::Backend, + E: CallExecutor, + Block: BlockT, +{ + /// Get usage info about current client. + fn usage_info(&self) -> ClientInfo { + ClientInfo { + chain: self.chain_info(), + usage: self.backend.usage_info(), + } + } +} + impl ProofProvider for Client where B: backend::Backend, E: CallExecutor, @@ -1100,12 +1161,11 @@ impl ProofProvider for Client where fn read_child_proof( &self, id: &BlockId, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, keys: &mut dyn Iterator, ) -> sp_blockchain::Result { self.state_at(id) - .and_then(|state| prove_child_read(state, storage_key, child_info, keys) + .and_then(|state| prove_child_read(state, child_info, keys) .map_err(Into::into)) } @@ -1147,7 +1207,7 @@ impl ProofProvider for Client where last: Block::Hash, min: Block::Hash, max: Block::Hash, - storage_key: Option<&StorageKey>, + storage_key: Option<&PrefixedStorageKey>, key: &StorageKey, ) -> sp_blockchain::Result> { self.key_changes_proof_with_cht_size( @@ -1277,46 +1337,40 @@ impl StorageProvider for Client wher ) } - fn child_storage_keys( &self, id: &BlockId, - child_storage_key: &StorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key_prefix: &StorageKey ) -> sp_blockchain::Result> { let keys = self.state_at(id)? - .child_keys(&child_storage_key.0, child_info, &key_prefix.0) + .child_keys(child_info, &key_prefix.0) .into_iter() .map(StorageKey) .collect(); Ok(keys) } - fn child_storage( &self, id: &BlockId, - storage_key: &StorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: &StorageKey ) -> sp_blockchain::Result> { Ok(self.state_at(id)? - .child_storage(&storage_key.0, child_info, &key.0) + .child_storage(child_info, &key.0) .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? .map(StorageData)) } - fn child_storage_hash( &self, id: &BlockId, - storage_key: &StorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: &StorageKey ) -> sp_blockchain::Result> { Ok(self.state_at(id)? - .child_storage_hash(&storage_key.0, child_info, &key.0) + .child_storage_hash(child_info, &key.0) .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? ) } @@ -1352,7 +1406,7 @@ impl StorageProvider for Client wher &self, first: NumberFor, last: BlockId, - storage_key: Option<&StorageKey>, + storage_key: Option<&PrefixedStorageKey>, key: &StorageKey ) -> sp_blockchain::Result, u32)>> { let last_number = self.backend.blockchain().expect_block_number_from_id(&last)?; @@ -1383,7 +1437,7 @@ impl StorageProvider for Client wher range_first, &range_anchor, best_number, - storage_key.as_ref().map(|x| &x.0[..]), + storage_key, &key.0) .and_then(|r| r.map(|r| r.map(|(block, tx)| (block, tx))).collect::>()) .map_err(|err| sp_blockchain::Error::ChangesTrieAccessFailed(err))?; @@ -1552,6 +1606,7 @@ impl CallApiAt for Client where params.function, ¶ms.arguments, params.overlayed_changes, + params.offchain_changes, Some(params.storage_transaction_cache), params.initialize_block, manager, @@ -1785,82 +1840,6 @@ where } } -/// Implement Longest Chain Select implementation -/// where 'longest' is defined as the highest number of blocks -pub struct LongestChain { - backend: Arc, - _phantom: PhantomData -} - -impl Clone for LongestChain { - fn clone(&self) -> Self { - let backend = self.backend.clone(); - LongestChain { - backend, - _phantom: Default::default() - } - } -} - -impl LongestChain -where - B: backend::Backend, - Block: BlockT, -{ - /// Instantiate a new LongestChain for Backend B - pub fn new(backend: Arc) -> Self { - LongestChain { - backend, - _phantom: Default::default() - } - } - - fn best_block_header(&self) -> sp_blockchain::Result<::Header> { - let info = self.backend.blockchain().info(); - let import_lock = self.backend.get_import_lock(); - let best_hash = self.backend - .blockchain() - .best_containing(info.best_hash, None, import_lock)? - .unwrap_or(info.best_hash); - - Ok(self.backend.blockchain().header(BlockId::Hash(best_hash))? - .expect("given block hash was fetched from block in db; qed")) - } - - fn leaves(&self) -> Result::Hash>, sp_blockchain::Error> { - self.backend.blockchain().leaves() - } -} - -impl SelectChain for LongestChain -where - B: backend::Backend, - Block: BlockT, -{ - - fn leaves(&self) -> Result::Hash>, ConsensusError> { - LongestChain::leaves(self) - .map_err(|e| ConsensusError::ChainLookup(e.to_string()).into()) - } - - fn best_chain(&self) - -> Result<::Header, ConsensusError> - { - LongestChain::best_block_header(&self) - .map_err(|e| ConsensusError::ChainLookup(e.to_string()).into()) - } - - fn finality_target( - &self, - target_hash: Block::Hash, - maybe_max_number: Option> - ) -> Result, ConsensusError> { - let import_lock = self.backend.get_import_lock(); - self.backend.blockchain().best_containing(target_hash, maybe_max_number, import_lock) - .map_err(|e| ConsensusError::ChainLookup(e.to_string()).into()) - } -} - impl BlockBackend for Client where B: backend::Backend, @@ -1964,26 +1943,6 @@ impl backend::AuxStore for &Client } } - -/// Helper function to apply auxiliary data insertion into an operation. -pub fn apply_aux<'a, 'b: 'a, 'c: 'a, B, Block, D, I>( - operation: &mut ClientImportOperation, - insert: I, - delete: D, -) -> sp_blockchain::Result<()> -where - Block: BlockT, - B: backend::Backend, - I: IntoIterator, - D: IntoIterator, -{ - operation.op.insert_aux( - insert.into_iter() - .map(|(k, v)| (k.to_vec(), Some(v.to_vec()))) - .chain(delete.into_iter().map(|k| (k.to_vec(), None))) - ) -} - impl sp_consensus::block_validation::Chain for Client where BE: backend::Backend, E: CallExecutor, @@ -1996,1555 +1955,3 @@ impl sp_consensus::block_validation::Chain for Client) } } - -#[cfg(test)] -pub(crate) mod tests { - use std::collections::HashMap; - use super::*; - use sp_core::{blake2_256, H256}; - use sp_runtime::DigestItem; - use sp_consensus::{BlockOrigin, SelectChain, BlockImport}; - use substrate_test_runtime_client::{ - prelude::*, - client_ext::ClientExt, - sc_client_db::{Backend, DatabaseSettings, DatabaseSettingsSrc, PruningMode}, - runtime::{self, Block, Transfer, RuntimeApi, TestAPI}, - }; - use hex_literal::hex; - - /// Returns tuple, consisting of: - /// 1) test client pre-filled with blocks changing balances; - /// 2) roots of changes tries for these blocks - /// 3) test cases in form (begin, end, key, vec![(block, extrinsic)]) that are required to pass - pub fn prepare_client_with_key_changes() -> ( - substrate_test_runtime_client::sc_client::Client, - Vec, - Vec<(u64, u64, Vec, Vec<(u64, u32)>)>, - ) { - // prepare block structure - let blocks_transfers = vec![ - vec![(AccountKeyring::Alice, AccountKeyring::Dave), (AccountKeyring::Bob, AccountKeyring::Dave)], - vec![(AccountKeyring::Charlie, AccountKeyring::Eve)], - vec![], - vec![(AccountKeyring::Alice, AccountKeyring::Dave)], - ]; - - // prepare client ang import blocks - let mut local_roots = Vec::new(); - let config = Some(ChangesTrieConfiguration::new(4, 2)); - let mut remote_client = TestClientBuilder::new().changes_trie_config(config).build(); - let mut nonces: HashMap<_, u64> = Default::default(); - for (i, block_transfers) in blocks_transfers.into_iter().enumerate() { - let mut builder = remote_client.new_block(Default::default()).unwrap(); - for (from, to) in block_transfers { - builder.push_transfer(Transfer { - from: from.into(), - to: to.into(), - amount: 1, - nonce: *nonces.entry(from).and_modify(|n| { *n = *n + 1 }).or_default(), - }).unwrap(); - } - let block = builder.build().unwrap().block; - remote_client.import(BlockOrigin::Own, block).unwrap(); - - let header = remote_client.header(&BlockId::Number(i as u64 + 1)).unwrap().unwrap(); - let trie_root = header.digest().log(DigestItem::as_changes_trie_root) - .map(|root| H256::from_slice(root.as_ref())) - .unwrap(); - local_roots.push(trie_root); - } - - // prepare test cases - let alice = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Alice.into())).to_vec(); - let bob = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Bob.into())).to_vec(); - let charlie = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Charlie.into())).to_vec(); - let dave = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec(); - let eve = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Eve.into())).to_vec(); - let ferdie = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Ferdie.into())).to_vec(); - let test_cases = vec![ - (1, 4, alice.clone(), vec![(4, 0), (1, 0)]), - (1, 3, alice.clone(), vec![(1, 0)]), - (2, 4, alice.clone(), vec![(4, 0)]), - (2, 3, alice.clone(), vec![]), - - (1, 4, bob.clone(), vec![(1, 1)]), - (1, 1, bob.clone(), vec![(1, 1)]), - (2, 4, bob.clone(), vec![]), - - (1, 4, charlie.clone(), vec![(2, 0)]), - - (1, 4, dave.clone(), vec![(4, 0), (1, 1), (1, 0)]), - (1, 1, dave.clone(), vec![(1, 1), (1, 0)]), - (3, 4, dave.clone(), vec![(4, 0)]), - - (1, 4, eve.clone(), vec![(2, 0)]), - (1, 1, eve.clone(), vec![]), - (3, 4, eve.clone(), vec![]), - - (1, 4, ferdie.clone(), vec![]), - ]; - - (remote_client, local_roots, test_cases) - } - - #[test] - fn client_initializes_from_genesis_ok() { - let client = substrate_test_runtime_client::new(); - - assert_eq!( - client.runtime_api().balance_of( - &BlockId::Number(client.chain_info().best_number), - AccountKeyring::Alice.into() - ).unwrap(), - 1000 - ); - assert_eq!( - client.runtime_api().balance_of( - &BlockId::Number(client.chain_info().best_number), - AccountKeyring::Ferdie.into() - ).unwrap(), - 0 - ); - } - - #[test] - fn block_builder_works_with_no_transactions() { - let mut client = substrate_test_runtime_client::new(); - - let block = client.new_block(Default::default()).unwrap().build().unwrap().block; - - client.import(BlockOrigin::Own, block).unwrap(); - - assert_eq!(client.chain_info().best_number, 1); - } - - #[test] - fn block_builder_works_with_transactions() { - let mut client = substrate_test_runtime_client::new(); - - let mut builder = client.new_block(Default::default()).unwrap(); - - builder.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Ferdie.into(), - amount: 42, - nonce: 0, - }).unwrap(); - - let block = builder.build().unwrap().block; - client.import(BlockOrigin::Own, block).unwrap(); - - assert_eq!(client.chain_info().best_number, 1); - assert_ne!( - client.state_at(&BlockId::Number(1)).unwrap().pairs(), - client.state_at(&BlockId::Number(0)).unwrap().pairs() - ); - assert_eq!( - client.runtime_api().balance_of( - &BlockId::Number(client.chain_info().best_number), - AccountKeyring::Alice.into() - ).unwrap(), - 958 - ); - assert_eq!( - client.runtime_api().balance_of( - &BlockId::Number(client.chain_info().best_number), - AccountKeyring::Ferdie.into() - ).unwrap(), - 42 - ); - } - - #[test] - fn block_builder_does_not_include_invalid() { - let mut client = substrate_test_runtime_client::new(); - - let mut builder = client.new_block(Default::default()).unwrap(); - - builder.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Ferdie.into(), - amount: 42, - nonce: 0, - }).unwrap(); - - assert!( - builder.push_transfer(Transfer { - from: AccountKeyring::Eve.into(), - to: AccountKeyring::Alice.into(), - amount: 42, - nonce: 0, - }).is_err() - ); - - let block = builder.build().unwrap().block; - client.import(BlockOrigin::Own, block).unwrap(); - - assert_eq!(client.chain_info().best_number, 1); - assert_ne!( - client.state_at(&BlockId::Number(1)).unwrap().pairs(), - client.state_at(&BlockId::Number(0)).unwrap().pairs() - ); - assert_eq!(client.body(&BlockId::Number(1)).unwrap().unwrap().len(), 1) - } - - #[test] - fn best_containing_with_genesis_block() { - // block tree: - // G - - let (client, longest_chain_select) = TestClientBuilder::new().build_with_longest_chain(); - - let genesis_hash = client.chain_info().genesis_hash; - - assert_eq!( - genesis_hash.clone(), - longest_chain_select.finality_target(genesis_hash.clone(), None).unwrap().unwrap() - ); - } - - #[test] - fn best_containing_with_hash_not_found() { - // block tree: - // G - - let (client, longest_chain_select) = TestClientBuilder::new().build_with_longest_chain(); - - let uninserted_block = client.new_block(Default::default()).unwrap().build().unwrap().block; - - assert_eq!( - None, - longest_chain_select.finality_target(uninserted_block.hash().clone(), None).unwrap() - ); - } - - #[test] - fn uncles_with_only_ancestors() { - // block tree: - // G -> A1 -> A2 - let mut client = substrate_test_runtime_client::new(); - - // G -> A1 - let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a1.clone()).unwrap(); - - // A1 -> A2 - let a2 = client.new_block(Default::default()).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a2.clone()).unwrap(); - let v: Vec = Vec::new(); - assert_eq!(v, client.uncles(a2.hash(), 3).unwrap()); - } - - #[test] - fn uncles_with_multiple_forks() { - // block tree: - // G -> A1 -> A2 -> A3 -> A4 -> A5 - // A1 -> B2 -> B3 -> B4 - // B2 -> C3 - // A1 -> D2 - let mut client = substrate_test_runtime_client::new(); - - // G -> A1 - let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a1.clone()).unwrap(); - - // A1 -> A2 - let a2 = client.new_block_at( - &BlockId::Hash(a1.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a2.clone()).unwrap(); - - // A2 -> A3 - let a3 = client.new_block_at( - &BlockId::Hash(a2.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a3.clone()).unwrap(); - - // A3 -> A4 - let a4 = client.new_block_at( - &BlockId::Hash(a3.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a4.clone()).unwrap(); - - // A4 -> A5 - let a5 = client.new_block_at( - &BlockId::Hash(a4.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a5.clone()).unwrap(); - - // A1 -> B2 - let mut builder = client.new_block_at( - &BlockId::Hash(a1.hash()), - Default::default(), - false, - ).unwrap(); - // this push is required as otherwise B2 has the same hash as A2 and won't get imported - builder.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Ferdie.into(), - amount: 41, - nonce: 0, - }).unwrap(); - let b2 = builder.build().unwrap().block; - client.import(BlockOrigin::Own, b2.clone()).unwrap(); - - // B2 -> B3 - let b3 = client.new_block_at( - &BlockId::Hash(b2.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, b3.clone()).unwrap(); - - // B3 -> B4 - let b4 = client.new_block_at( - &BlockId::Hash(b3.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, b4.clone()).unwrap(); - - // // B2 -> C3 - let mut builder = client.new_block_at( - &BlockId::Hash(b2.hash()), - Default::default(), - false, - ).unwrap(); - // this push is required as otherwise C3 has the same hash as B3 and won't get imported - builder.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Ferdie.into(), - amount: 1, - nonce: 1, - }).unwrap(); - let c3 = builder.build().unwrap().block; - client.import(BlockOrigin::Own, c3.clone()).unwrap(); - - // A1 -> D2 - let mut builder = client.new_block_at( - &BlockId::Hash(a1.hash()), - Default::default(), - false, - ).unwrap(); - // this push is required as otherwise D2 has the same hash as B2 and won't get imported - builder.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Ferdie.into(), - amount: 1, - nonce: 0, - }).unwrap(); - let d2 = builder.build().unwrap().block; - client.import(BlockOrigin::Own, d2.clone()).unwrap(); - - let genesis_hash = client.chain_info().genesis_hash; - - let uncles1 = client.uncles(a4.hash(), 10).unwrap(); - assert_eq!(vec![b2.hash(), d2.hash()], uncles1); - - let uncles2 = client.uncles(a4.hash(), 0).unwrap(); - assert_eq!(0, uncles2.len()); - - let uncles3 = client.uncles(a1.hash(), 10).unwrap(); - assert_eq!(0, uncles3.len()); - - let uncles4 = client.uncles(genesis_hash, 10).unwrap(); - assert_eq!(0, uncles4.len()); - - let uncles5 = client.uncles(d2.hash(), 10).unwrap(); - assert_eq!(vec![a2.hash(), b2.hash()], uncles5); - - let uncles6 = client.uncles(b3.hash(), 1).unwrap(); - assert_eq!(vec![c3.hash()], uncles6); - } - - #[test] - fn best_containing_on_longest_chain_with_single_chain_3_blocks() { - // block tree: - // G -> A1 -> A2 - - let (mut client, longest_chain_select) = TestClientBuilder::new().build_with_longest_chain(); - - // G -> A1 - let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a1.clone()).unwrap(); - - // A1 -> A2 - let a2 = client.new_block(Default::default()).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a2.clone()).unwrap(); - - let genesis_hash = client.chain_info().genesis_hash; - - assert_eq!(a2.hash(), longest_chain_select.finality_target(genesis_hash, None).unwrap().unwrap()); - assert_eq!(a2.hash(), longest_chain_select.finality_target(a1.hash(), None).unwrap().unwrap()); - assert_eq!(a2.hash(), longest_chain_select.finality_target(a2.hash(), None).unwrap().unwrap()); - } - - #[test] - fn best_containing_on_longest_chain_with_multiple_forks() { - // block tree: - // G -> A1 -> A2 -> A3 -> A4 -> A5 - // A1 -> B2 -> B3 -> B4 - // B2 -> C3 - // A1 -> D2 - let (mut client, longest_chain_select) = TestClientBuilder::new().build_with_longest_chain(); - - // G -> A1 - let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a1.clone()).unwrap(); - - // A1 -> A2 - let a2 = client.new_block_at( - &BlockId::Hash(a1.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a2.clone()).unwrap(); - - // A2 -> A3 - let a3 = client.new_block_at( - &BlockId::Hash(a2.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a3.clone()).unwrap(); - - // A3 -> A4 - let a4 = client.new_block_at( - &BlockId::Hash(a3.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a4.clone()).unwrap(); - - // A4 -> A5 - let a5 = client.new_block_at( - &BlockId::Hash(a4.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a5.clone()).unwrap(); - - // A1 -> B2 - let mut builder = client.new_block_at( - &BlockId::Hash(a1.hash()), - Default::default(), - false, - ).unwrap(); - // this push is required as otherwise B2 has the same hash as A2 and won't get imported - builder.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Ferdie.into(), - amount: 41, - nonce: 0, - }).unwrap(); - let b2 = builder.build().unwrap().block; - client.import(BlockOrigin::Own, b2.clone()).unwrap(); - - // B2 -> B3 - let b3 = client.new_block_at( - &BlockId::Hash(b2.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, b3.clone()).unwrap(); - - // B3 -> B4 - let b4 = client.new_block_at( - &BlockId::Hash(b3.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, b4.clone()).unwrap(); - - // // B2 -> C3 - let mut builder = client.new_block_at( - &BlockId::Hash(b2.hash()), - Default::default(), - false, - ).unwrap(); - // this push is required as otherwise C3 has the same hash as B3 and won't get imported - builder.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Ferdie.into(), - amount: 1, - nonce: 1, - }).unwrap(); - let c3 = builder.build().unwrap().block; - client.import(BlockOrigin::Own, c3.clone()).unwrap(); - - // A1 -> D2 - let mut builder = client.new_block_at( - &BlockId::Hash(a1.hash()), - Default::default(), - false, - ).unwrap(); - // this push is required as otherwise D2 has the same hash as B2 and won't get imported - builder.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Ferdie.into(), - amount: 1, - nonce: 0, - }).unwrap(); - let d2 = builder.build().unwrap().block; - client.import(BlockOrigin::Own, d2.clone()).unwrap(); - - assert_eq!(client.chain_info().best_hash, a5.hash()); - - let genesis_hash = client.chain_info().genesis_hash; - let leaves = longest_chain_select.leaves().unwrap(); - - assert!(leaves.contains(&a5.hash())); - assert!(leaves.contains(&b4.hash())); - assert!(leaves.contains(&c3.hash())); - assert!(leaves.contains(&d2.hash())); - assert_eq!(leaves.len(), 4); - - // search without restriction - - assert_eq!(a5.hash(), longest_chain_select.finality_target( - genesis_hash, None).unwrap().unwrap()); - assert_eq!(a5.hash(), longest_chain_select.finality_target( - a1.hash(), None).unwrap().unwrap()); - assert_eq!(a5.hash(), longest_chain_select.finality_target( - a2.hash(), None).unwrap().unwrap()); - assert_eq!(a5.hash(), longest_chain_select.finality_target( - a3.hash(), None).unwrap().unwrap()); - assert_eq!(a5.hash(), longest_chain_select.finality_target( - a4.hash(), None).unwrap().unwrap()); - assert_eq!(a5.hash(), longest_chain_select.finality_target( - a5.hash(), None).unwrap().unwrap()); - - assert_eq!(b4.hash(), longest_chain_select.finality_target( - b2.hash(), None).unwrap().unwrap()); - assert_eq!(b4.hash(), longest_chain_select.finality_target( - b3.hash(), None).unwrap().unwrap()); - assert_eq!(b4.hash(), longest_chain_select.finality_target( - b4.hash(), None).unwrap().unwrap()); - - assert_eq!(c3.hash(), longest_chain_select.finality_target( - c3.hash(), None).unwrap().unwrap()); - - assert_eq!(d2.hash(), longest_chain_select.finality_target( - d2.hash(), None).unwrap().unwrap()); - - - // search only blocks with number <= 5. equivalent to without restriction for this scenario - - assert_eq!(a5.hash(), longest_chain_select.finality_target( - genesis_hash, Some(5)).unwrap().unwrap()); - assert_eq!(a5.hash(), longest_chain_select.finality_target( - a1.hash(), Some(5)).unwrap().unwrap()); - assert_eq!(a5.hash(), longest_chain_select.finality_target( - a2.hash(), Some(5)).unwrap().unwrap()); - assert_eq!(a5.hash(), longest_chain_select.finality_target( - a3.hash(), Some(5)).unwrap().unwrap()); - assert_eq!(a5.hash(), longest_chain_select.finality_target( - a4.hash(), Some(5)).unwrap().unwrap()); - assert_eq!(a5.hash(), longest_chain_select.finality_target( - a5.hash(), Some(5)).unwrap().unwrap()); - - assert_eq!(b4.hash(), longest_chain_select.finality_target( - b2.hash(), Some(5)).unwrap().unwrap()); - assert_eq!(b4.hash(), longest_chain_select.finality_target( - b3.hash(), Some(5)).unwrap().unwrap()); - assert_eq!(b4.hash(), longest_chain_select.finality_target( - b4.hash(), Some(5)).unwrap().unwrap()); - - assert_eq!(c3.hash(), longest_chain_select.finality_target( - c3.hash(), Some(5)).unwrap().unwrap()); - - assert_eq!(d2.hash(), longest_chain_select.finality_target( - d2.hash(), Some(5)).unwrap().unwrap()); - - - // search only blocks with number <= 4 - - assert_eq!(a4.hash(), longest_chain_select.finality_target( - genesis_hash, Some(4)).unwrap().unwrap()); - assert_eq!(a4.hash(), longest_chain_select.finality_target( - a1.hash(), Some(4)).unwrap().unwrap()); - assert_eq!(a4.hash(), longest_chain_select.finality_target( - a2.hash(), Some(4)).unwrap().unwrap()); - assert_eq!(a4.hash(), longest_chain_select.finality_target( - a3.hash(), Some(4)).unwrap().unwrap()); - assert_eq!(a4.hash(), longest_chain_select.finality_target( - a4.hash(), Some(4)).unwrap().unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - a5.hash(), Some(4)).unwrap()); - - assert_eq!(b4.hash(), longest_chain_select.finality_target( - b2.hash(), Some(4)).unwrap().unwrap()); - assert_eq!(b4.hash(), longest_chain_select.finality_target( - b3.hash(), Some(4)).unwrap().unwrap()); - assert_eq!(b4.hash(), longest_chain_select.finality_target( - b4.hash(), Some(4)).unwrap().unwrap()); - - assert_eq!(c3.hash(), longest_chain_select.finality_target( - c3.hash(), Some(4)).unwrap().unwrap()); - - assert_eq!(d2.hash(), longest_chain_select.finality_target( - d2.hash(), Some(4)).unwrap().unwrap()); - - - // search only blocks with number <= 3 - - assert_eq!(a3.hash(), longest_chain_select.finality_target( - genesis_hash, Some(3)).unwrap().unwrap()); - assert_eq!(a3.hash(), longest_chain_select.finality_target( - a1.hash(), Some(3)).unwrap().unwrap()); - assert_eq!(a3.hash(), longest_chain_select.finality_target( - a2.hash(), Some(3)).unwrap().unwrap()); - assert_eq!(a3.hash(), longest_chain_select.finality_target( - a3.hash(), Some(3)).unwrap().unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - a4.hash(), Some(3)).unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - a5.hash(), Some(3)).unwrap()); - - assert_eq!(b3.hash(), longest_chain_select.finality_target( - b2.hash(), Some(3)).unwrap().unwrap()); - assert_eq!(b3.hash(), longest_chain_select.finality_target( - b3.hash(), Some(3)).unwrap().unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - b4.hash(), Some(3)).unwrap()); - - assert_eq!(c3.hash(), longest_chain_select.finality_target( - c3.hash(), Some(3)).unwrap().unwrap()); - - assert_eq!(d2.hash(), longest_chain_select.finality_target( - d2.hash(), Some(3)).unwrap().unwrap()); - - - // search only blocks with number <= 2 - - assert_eq!(a2.hash(), longest_chain_select.finality_target( - genesis_hash, Some(2)).unwrap().unwrap()); - assert_eq!(a2.hash(), longest_chain_select.finality_target( - a1.hash(), Some(2)).unwrap().unwrap()); - assert_eq!(a2.hash(), longest_chain_select.finality_target( - a2.hash(), Some(2)).unwrap().unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - a3.hash(), Some(2)).unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - a4.hash(), Some(2)).unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - a5.hash(), Some(2)).unwrap()); - - assert_eq!(b2.hash(), longest_chain_select.finality_target( - b2.hash(), Some(2)).unwrap().unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - b3.hash(), Some(2)).unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - b4.hash(), Some(2)).unwrap()); - - assert_eq!(None, longest_chain_select.finality_target( - c3.hash(), Some(2)).unwrap()); - - assert_eq!(d2.hash(), longest_chain_select.finality_target( - d2.hash(), Some(2)).unwrap().unwrap()); - - - // search only blocks with number <= 1 - - assert_eq!(a1.hash(), longest_chain_select.finality_target( - genesis_hash, Some(1)).unwrap().unwrap()); - assert_eq!(a1.hash(), longest_chain_select.finality_target( - a1.hash(), Some(1)).unwrap().unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - a2.hash(), Some(1)).unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - a3.hash(), Some(1)).unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - a4.hash(), Some(1)).unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - a5.hash(), Some(1)).unwrap()); - - assert_eq!(None, longest_chain_select.finality_target( - b2.hash(), Some(1)).unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - b3.hash(), Some(1)).unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - b4.hash(), Some(1)).unwrap()); - - assert_eq!(None, longest_chain_select.finality_target( - c3.hash(), Some(1)).unwrap()); - - assert_eq!(None, longest_chain_select.finality_target( - d2.hash(), Some(1)).unwrap()); - - // search only blocks with number <= 0 - - assert_eq!(genesis_hash, longest_chain_select.finality_target( - genesis_hash, Some(0)).unwrap().unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - a1.hash(), Some(0)).unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - a2.hash(), Some(0)).unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - a3.hash(), Some(0)).unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - a4.hash(), Some(0)).unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - a5.hash(), Some(0)).unwrap()); - - assert_eq!(None, longest_chain_select.finality_target( - b2.hash(), Some(0)).unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - b3.hash(), Some(0)).unwrap()); - assert_eq!(None, longest_chain_select.finality_target( - b4.hash(), Some(0)).unwrap()); - - assert_eq!(None, longest_chain_select.finality_target( - c3.hash().clone(), Some(0)).unwrap()); - - assert_eq!(None, longest_chain_select.finality_target( - d2.hash().clone(), Some(0)).unwrap()); - } - - #[test] - fn best_containing_on_longest_chain_with_max_depth_higher_than_best() { - // block tree: - // G -> A1 -> A2 - - let (mut client, longest_chain_select) = TestClientBuilder::new().build_with_longest_chain(); - - // G -> A1 - let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a1.clone()).unwrap(); - - // A1 -> A2 - let a2 = client.new_block(Default::default()).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a2.clone()).unwrap(); - - let genesis_hash = client.chain_info().genesis_hash; - - assert_eq!(a2.hash(), longest_chain_select.finality_target(genesis_hash, Some(10)).unwrap().unwrap()); - } - - #[test] - fn key_changes_works() { - let (client, _, test_cases) = prepare_client_with_key_changes(); - - for (index, (begin, end, key, expected_result)) in test_cases.into_iter().enumerate() { - let end = client.block_hash(end).unwrap().unwrap(); - let actual_result = client.key_changes( - begin, - BlockId::Hash(end), - None, - &StorageKey(key), - ).unwrap(); - match actual_result == expected_result { - true => (), - false => panic!(format!("Failed test {}: actual = {:?}, expected = {:?}", - index, actual_result, expected_result)), - } - } - } - - #[test] - fn import_with_justification() { - let mut client = substrate_test_runtime_client::new(); - - // G -> A1 - let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a1.clone()).unwrap(); - - // A1 -> A2 - let a2 = client.new_block_at( - &BlockId::Hash(a1.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a2.clone()).unwrap(); - - // A2 -> A3 - let justification = vec![1, 2, 3]; - let a3 = client.new_block_at( - &BlockId::Hash(a2.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import_justified(BlockOrigin::Own, a3.clone(), justification.clone()).unwrap(); - - assert_eq!( - client.chain_info().finalized_hash, - a3.hash(), - ); - - assert_eq!( - client.justification(&BlockId::Hash(a3.hash())).unwrap(), - Some(justification), - ); - - assert_eq!( - client.justification(&BlockId::Hash(a1.hash())).unwrap(), - None, - ); - - assert_eq!( - client.justification(&BlockId::Hash(a2.hash())).unwrap(), - None, - ); - } - - #[test] - fn importing_diverged_finalized_block_should_trigger_reorg() { - let mut client = substrate_test_runtime_client::new(); - - // G -> A1 -> A2 - // \ - // -> B1 - let a1 = client.new_block_at( - &BlockId::Number(0), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a1.clone()).unwrap(); - - let a2 = client.new_block_at( - &BlockId::Hash(a1.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a2.clone()).unwrap(); - - let mut b1 = client.new_block_at( - &BlockId::Number(0), - Default::default(), - false, - ).unwrap(); - // needed to make sure B1 gets a different hash from A1 - b1.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Ferdie.into(), - amount: 1, - nonce: 0, - }).unwrap(); - // create but don't import B1 just yet - let b1 = b1.build().unwrap().block; - - // A2 is the current best since it's the longest chain - assert_eq!( - client.chain_info().best_hash, - a2.hash(), - ); - - // importing B1 as finalized should trigger a re-org and set it as new best - let justification = vec![1, 2, 3]; - client.import_justified(BlockOrigin::Own, b1.clone(), justification).unwrap(); - - assert_eq!( - client.chain_info().best_hash, - b1.hash(), - ); - - assert_eq!( - client.chain_info().finalized_hash, - b1.hash(), - ); - } - - #[test] - fn finalizing_diverged_block_should_trigger_reorg() { - - let (mut client, select_chain) = TestClientBuilder::new().build_with_longest_chain(); - - // G -> A1 -> A2 - // \ - // -> B1 -> B2 - let a1 = client.new_block_at( - &BlockId::Number(0), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a1.clone()).unwrap(); - - let a2 = client.new_block_at( - &BlockId::Hash(a1.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a2.clone()).unwrap(); - - let mut b1 = client.new_block_at( - &BlockId::Number(0), - Default::default(), - false, - ).unwrap(); - // needed to make sure B1 gets a different hash from A1 - b1.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Ferdie.into(), - amount: 1, - nonce: 0, - }).unwrap(); - let b1 = b1.build().unwrap().block; - client.import(BlockOrigin::Own, b1.clone()).unwrap(); - - let b2 = client.new_block_at( - &BlockId::Hash(b1.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, b2.clone()).unwrap(); - - // A2 is the current best since it's the longest chain - assert_eq!( - client.chain_info().best_hash, - a2.hash(), - ); - - // we finalize block B1 which is on a different branch from current best - // which should trigger a re-org. - ClientExt::finalize_block(&client, BlockId::Hash(b1.hash()), None).unwrap(); - - // B1 should now be the latest finalized - assert_eq!( - client.chain_info().finalized_hash, - b1.hash(), - ); - - // and B1 should be the new best block (`finalize_block` as no way of - // knowing about B2) - assert_eq!( - client.chain_info().best_hash, - b1.hash(), - ); - - // `SelectChain` should report B2 as best block though - assert_eq!( - select_chain.best_chain().unwrap().hash(), - b2.hash(), - ); - - // after we build B3 on top of B2 and import it - // it should be the new best block, - let b3 = client.new_block_at( - &BlockId::Hash(b2.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, b3.clone()).unwrap(); - - assert_eq!( - client.chain_info().best_hash, - b3.hash(), - ); - } - - #[test] - fn get_header_by_block_number_doesnt_panic() { - let client = substrate_test_runtime_client::new(); - - // backend uses u32 for block numbers, make sure we don't panic when - // trying to convert - let id = BlockId::::Number(72340207214430721); - client.header(&id).expect_err("invalid block number overflows u32"); - } - - #[test] - fn state_reverted_on_reorg() { - let _ = env_logger::try_init(); - let mut client = substrate_test_runtime_client::new(); - - let current_balance = |client: &substrate_test_runtime_client::TestClient| - client.runtime_api().balance_of( - &BlockId::number(client.chain_info().best_number), AccountKeyring::Alice.into() - ).unwrap(); - - // G -> A1 -> A2 - // \ - // -> B1 - let mut a1 = client.new_block_at( - &BlockId::Number(0), - Default::default(), - false, - ).unwrap(); - a1.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Bob.into(), - amount: 10, - nonce: 0, - }).unwrap(); - let a1 = a1.build().unwrap().block; - client.import(BlockOrigin::Own, a1.clone()).unwrap(); - - let mut b1 = client.new_block_at( - &BlockId::Number(0), - Default::default(), - false, - ).unwrap(); - b1.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Ferdie.into(), - amount: 50, - nonce: 0, - }).unwrap(); - let b1 = b1.build().unwrap().block; - // Reorg to B1 - client.import_as_best(BlockOrigin::Own, b1.clone()).unwrap(); - - assert_eq!(950, current_balance(&client)); - let mut a2 = client.new_block_at( - &BlockId::Hash(a1.hash()), - Default::default(), - false, - ).unwrap(); - a2.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Charlie.into(), - amount: 10, - nonce: 1, - }).unwrap(); - let a2 = a2.build().unwrap().block; - // Re-org to A2 - client.import_as_best(BlockOrigin::Own, a2).unwrap(); - assert_eq!(980, current_balance(&client)); - } - - #[test] - fn doesnt_import_blocks_that_revert_finality() { - let _ = env_logger::try_init(); - let tmp = tempfile::tempdir().unwrap(); - - // we need to run with archive pruning to avoid pruning non-canonical - // states - let backend = Arc::new(Backend::new( - DatabaseSettings { - state_cache_size: 1 << 20, - state_cache_child_ratio: None, - pruning: PruningMode::ArchiveAll, - source: DatabaseSettingsSrc::Path { - path: tmp.path().into(), - cache_size: None, - } - }, - u64::max_value(), - ).unwrap()); - - let mut client = TestClientBuilder::with_backend(backend).build(); - - // -> C1 - // / - // G -> A1 -> A2 - // \ - // -> B1 -> B2 -> B3 - - let a1 = client.new_block_at( - &BlockId::Number(0), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a1.clone()).unwrap(); - - let a2 = client.new_block_at( - &BlockId::Hash(a1.hash()), - Default::default(), - false, - ).unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, a2.clone()).unwrap(); - - let mut b1 = client.new_block_at(&BlockId::Number(0), Default::default(), false).unwrap(); - - // needed to make sure B1 gets a different hash from A1 - b1.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Ferdie.into(), - amount: 1, - nonce: 0, - }).unwrap(); - let b1 = b1.build().unwrap().block; - client.import(BlockOrigin::Own, b1.clone()).unwrap(); - - let b2 = client.new_block_at(&BlockId::Hash(b1.hash()), Default::default(), false) - .unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, b2.clone()).unwrap(); - - // prepare B3 before we finalize A2, because otherwise we won't be able to - // read changes trie configuration after A2 is finalized - let b3 = client.new_block_at(&BlockId::Hash(b2.hash()), Default::default(), false) - .unwrap().build().unwrap().block; - - // we will finalize A2 which should make it impossible to import a new - // B3 at the same height but that doesn't include it - ClientExt::finalize_block(&client, BlockId::Hash(a2.hash()), None).unwrap(); - - let import_err = client.import(BlockOrigin::Own, b3).err().unwrap(); - let expected_err = ConsensusError::ClientImport( - sp_blockchain::Error::NotInFinalizedChain.to_string() - ); - - assert_eq!( - import_err.to_string(), - expected_err.to_string(), - ); - - // adding a C1 block which is lower than the last finalized should also - // fail (with a cheaper check that doesn't require checking ancestry). - let mut c1 = client.new_block_at(&BlockId::Number(0), Default::default(), false).unwrap(); - - // needed to make sure C1 gets a different hash from A1 and B1 - c1.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Ferdie.into(), - amount: 2, - nonce: 0, - }).unwrap(); - let c1 = c1.build().unwrap().block; - - let import_err = client.import(BlockOrigin::Own, c1).err().unwrap(); - let expected_err = ConsensusError::ClientImport( - sp_blockchain::Error::NotInFinalizedChain.to_string() - ); - - assert_eq!( - import_err.to_string(), - expected_err.to_string(), - ); - } - - - #[test] - fn respects_block_rules() { - - fn run_test( - record_only: bool, - known_bad: &mut HashSet, - fork_rules: &mut Vec<(u64, H256)>, - ) { - let mut client = if record_only { - TestClientBuilder::new().build() - } else { - TestClientBuilder::new() - .set_block_rules( - Some(fork_rules.clone()), - Some(known_bad.clone()), - ) - .build() - }; - - let block_ok = client.new_block_at(&BlockId::Number(0), Default::default(), false) - .unwrap().build().unwrap().block; - - let params = BlockCheckParams { - hash: block_ok.hash().clone(), - number: 0, - parent_hash: block_ok.header().parent_hash().clone(), - allow_missing_state: false, - import_existing: false, - }; - assert_eq!(client.check_block(params).unwrap(), ImportResult::imported(false)); - - // this is 0x0d6d6612a10485370d9e085aeea7ec427fb3f34d961c6a816cdbe5cde2278864 - let mut block_not_ok = client.new_block_at(&BlockId::Number(0), Default::default(), false) - .unwrap(); - block_not_ok.push_storage_change(vec![0], Some(vec![1])).unwrap(); - let block_not_ok = block_not_ok.build().unwrap().block; - - let params = BlockCheckParams { - hash: block_not_ok.hash().clone(), - number: 0, - parent_hash: block_not_ok.header().parent_hash().clone(), - allow_missing_state: false, - import_existing: false, - }; - if record_only { - known_bad.insert(block_not_ok.hash()); - } else { - assert_eq!(client.check_block(params).unwrap(), ImportResult::KnownBad); - } - - // Now going to the fork - client.import_as_final(BlockOrigin::Own, block_ok).unwrap(); - - // And check good fork - let mut block_ok = client.new_block_at(&BlockId::Number(1), Default::default(), false) - .unwrap(); - block_ok.push_storage_change(vec![0], Some(vec![2])).unwrap(); - let block_ok = block_ok.build().unwrap().block; - - let params = BlockCheckParams { - hash: block_ok.hash().clone(), - number: 1, - parent_hash: block_ok.header().parent_hash().clone(), - allow_missing_state: false, - import_existing: false, - }; - if record_only { - fork_rules.push((1, block_ok.hash().clone())); - } - assert_eq!(client.check_block(params).unwrap(), ImportResult::imported(false)); - - // And now try bad fork - let mut block_not_ok = client.new_block_at(&BlockId::Number(1), Default::default(), false) - .unwrap(); - block_not_ok.push_storage_change(vec![0], Some(vec![3])).unwrap(); - let block_not_ok = block_not_ok.build().unwrap().block; - - let params = BlockCheckParams { - hash: block_not_ok.hash().clone(), - number: 1, - parent_hash: block_not_ok.header().parent_hash().clone(), - allow_missing_state: false, - import_existing: false, - }; - - if !record_only { - assert_eq!(client.check_block(params).unwrap(), ImportResult::KnownBad); - } - } - - let mut known_bad = HashSet::new(); - let mut fork_rules = Vec::new(); - - // records what bad_blocks and fork_blocks hashes should be - run_test(true, &mut known_bad, &mut fork_rules); - - // enforces rules and actually makes assertions - run_test(false, &mut known_bad, &mut fork_rules); - } - - #[test] - fn returns_status_for_pruned_blocks() { - let _ = env_logger::try_init(); - let tmp = tempfile::tempdir().unwrap(); - - // set to prune after 1 block - // states - let backend = Arc::new(Backend::new( - DatabaseSettings { - state_cache_size: 1 << 20, - state_cache_child_ratio: None, - pruning: PruningMode::keep_blocks(1), - source: DatabaseSettingsSrc::Path { - path: tmp.path().into(), - cache_size: None, - } - }, - u64::max_value(), - ).unwrap()); - - let mut client = TestClientBuilder::with_backend(backend).build(); - - let a1 = client.new_block_at(&BlockId::Number(0), Default::default(), false) - .unwrap().build().unwrap().block; - - let mut b1 = client.new_block_at(&BlockId::Number(0), Default::default(), false).unwrap(); - - // b1 is created, but not imported - b1.push_transfer(Transfer { - from: AccountKeyring::Alice.into(), - to: AccountKeyring::Ferdie.into(), - amount: 1, - nonce: 0, - }).unwrap(); - let b1 = b1.build().unwrap().block; - - let check_block_a1 = BlockCheckParams { - hash: a1.hash().clone(), - number: 0, - parent_hash: a1.header().parent_hash().clone(), - allow_missing_state: false, - import_existing: false, - }; - - assert_eq!(client.check_block(check_block_a1.clone()).unwrap(), ImportResult::imported(false)); - assert_eq!(client.block_status(&BlockId::hash(check_block_a1.hash)).unwrap(), BlockStatus::Unknown); - - client.import_as_final(BlockOrigin::Own, a1.clone()).unwrap(); - - assert_eq!(client.check_block(check_block_a1.clone()).unwrap(), ImportResult::AlreadyInChain); - assert_eq!(client.block_status(&BlockId::hash(check_block_a1.hash)).unwrap(), BlockStatus::InChainWithState); - - let a2 = client.new_block_at(&BlockId::Hash(a1.hash()), Default::default(), false) - .unwrap().build().unwrap().block; - client.import_as_final(BlockOrigin::Own, a2.clone()).unwrap(); - - let check_block_a2 = BlockCheckParams { - hash: a2.hash().clone(), - number: 1, - parent_hash: a1.header().parent_hash().clone(), - allow_missing_state: false, - import_existing: false, - }; - - assert_eq!(client.check_block(check_block_a1.clone()).unwrap(), ImportResult::AlreadyInChain); - assert_eq!(client.block_status(&BlockId::hash(check_block_a1.hash)).unwrap(), BlockStatus::InChainPruned); - assert_eq!(client.check_block(check_block_a2.clone()).unwrap(), ImportResult::AlreadyInChain); - assert_eq!(client.block_status(&BlockId::hash(check_block_a2.hash)).unwrap(), BlockStatus::InChainWithState); - - let a3 = client.new_block_at(&BlockId::Hash(a2.hash()), Default::default(), false) - .unwrap().build().unwrap().block; - - client.import_as_final(BlockOrigin::Own, a3.clone()).unwrap(); - let check_block_a3 = BlockCheckParams { - hash: a3.hash().clone(), - number: 2, - parent_hash: a2.header().parent_hash().clone(), - allow_missing_state: false, - import_existing: false, - }; - - // a1 and a2 are both pruned at this point - assert_eq!(client.check_block(check_block_a1.clone()).unwrap(), ImportResult::AlreadyInChain); - assert_eq!(client.block_status(&BlockId::hash(check_block_a1.hash)).unwrap(), BlockStatus::InChainPruned); - assert_eq!(client.check_block(check_block_a2.clone()).unwrap(), ImportResult::AlreadyInChain); - assert_eq!(client.block_status(&BlockId::hash(check_block_a2.hash)).unwrap(), BlockStatus::InChainPruned); - assert_eq!(client.check_block(check_block_a3.clone()).unwrap(), ImportResult::AlreadyInChain); - assert_eq!(client.block_status(&BlockId::hash(check_block_a3.hash)).unwrap(), BlockStatus::InChainWithState); - - let mut check_block_b1 = BlockCheckParams { - hash: b1.hash().clone(), - number: 0, - parent_hash: b1.header().parent_hash().clone(), - allow_missing_state: false, - import_existing: false, - }; - assert_eq!(client.check_block(check_block_b1.clone()).unwrap(), ImportResult::MissingState); - check_block_b1.allow_missing_state = true; - assert_eq!(client.check_block(check_block_b1.clone()).unwrap(), ImportResult::imported(false)); - check_block_b1.parent_hash = H256::random(); - assert_eq!(client.check_block(check_block_b1.clone()).unwrap(), ImportResult::UnknownParent); - } - - #[test] - fn imports_blocks_with_changes_tries_config_change() { - // create client with initial 4^2 configuration - let mut client = TestClientBuilder::with_default_backend() - .changes_trie_config(Some(ChangesTrieConfiguration { - digest_interval: 4, - digest_levels: 2, - })).build(); - - // =================================================================== - // blocks 1,2,3,4,5,6,7,8,9,10 are empty - // block 11 changes the key - // block 12 is the L1 digest that covers this change - // blocks 13,14,15,16,17,18,19,20,21,22 are empty - // block 23 changes the configuration to 5^1 AND is skewed digest - // =================================================================== - // blocks 24,25 are changing the key - // block 26 is empty - // block 27 changes the key - // block 28 is the L1 digest (NOT SKEWED!!!) that covers changes AND changes configuration to 3^1 - // =================================================================== - // block 29 is empty - // block 30 changes the key - // block 31 is L1 digest that covers this change - // =================================================================== - (1..11).for_each(|number| { - let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, block).unwrap(); - }); - (11..12).for_each(|number| { - let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap(); - block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap(); - let block = block.build().unwrap().block; - client.import(BlockOrigin::Own, block).unwrap(); - }); - (12..23).for_each(|number| { - let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, block).unwrap(); - }); - (23..24).for_each(|number| { - let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap(); - block.push_changes_trie_configuration_update(Some(ChangesTrieConfiguration { - digest_interval: 5, - digest_levels: 1, - })).unwrap(); - let block = block.build().unwrap().block; - client.import(BlockOrigin::Own, block).unwrap(); - }); - (24..26).for_each(|number| { - let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap(); - block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap(); - let block = block.build().unwrap().block; - client.import(BlockOrigin::Own, block).unwrap(); - }); - (26..27).for_each(|number| { - let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, block).unwrap(); - }); - (27..28).for_each(|number| { - let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap(); - block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap(); - let block = block.build().unwrap().block; - client.import(BlockOrigin::Own, block).unwrap(); - }); - (28..29).for_each(|number| { - let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap(); - block.push_changes_trie_configuration_update(Some(ChangesTrieConfiguration { - digest_interval: 3, - digest_levels: 1, - })).unwrap(); - let block = block.build().unwrap().block; - client.import(BlockOrigin::Own, block).unwrap(); - }); - (29..30).for_each(|number| { - let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, block).unwrap(); - }); - (30..31).for_each(|number| { - let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap(); - block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap(); - let block = block.build().unwrap().block; - client.import(BlockOrigin::Own, block).unwrap(); - }); - (31..32).for_each(|number| { - let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap().build().unwrap().block; - client.import(BlockOrigin::Own, block).unwrap(); - }); - - // now check that configuration cache works - assert_eq!( - client.key_changes(1, BlockId::Number(31), None, &StorageKey(vec![42])).unwrap(), - vec![(30, 0), (27, 0), (25, 0), (24, 0), (11, 0)] - ); - } - - #[test] - fn storage_keys_iter_prefix_and_start_key_works() { - let client = substrate_test_runtime_client::new(); - - let prefix = StorageKey(hex!("3a").to_vec()); - - let res: Vec<_> = client.storage_keys_iter(&BlockId::Number(0), Some(&prefix), None) - .unwrap() - .map(|x| x.0) - .collect(); - assert_eq!(res, [hex!("3a636f6465").to_vec(), hex!("3a686561707061676573").to_vec()]); - - let res: Vec<_> = client.storage_keys_iter(&BlockId::Number(0), Some(&prefix), Some(&StorageKey(hex!("3a636f6465").to_vec()))) - .unwrap() - .map(|x| x.0) - .collect(); - assert_eq!(res, [hex!("3a686561707061676573").to_vec()]); - - let res: Vec<_> = client.storage_keys_iter(&BlockId::Number(0), Some(&prefix), Some(&StorageKey(hex!("3a686561707061676573").to_vec()))) - .unwrap() - .map(|x| x.0) - .collect(); - assert_eq!(res, Vec::>::new()); - } - - #[test] - fn storage_keys_iter_works() { - let client = substrate_test_runtime_client::new(); - - let prefix = StorageKey(hex!("").to_vec()); - - let res: Vec<_> = client.storage_keys_iter(&BlockId::Number(0), Some(&prefix), None) - .unwrap() - .take(2) - .map(|x| x.0) - .collect(); - assert_eq!(res, [hex!("0befda6e1ca4ef40219d588a727f1271").to_vec(), hex!("3a636f6465").to_vec()]); - - let res: Vec<_> = client.storage_keys_iter(&BlockId::Number(0), Some(&prefix), Some(&StorageKey(hex!("3a636f6465").to_vec()))) - .unwrap() - .take(3) - .map(|x| x.0) - .collect(); - assert_eq!(res, [ - hex!("3a686561707061676573").to_vec(), - hex!("6644b9b8bc315888ac8e41a7968dc2b4141a5403c58acdf70b7e8f7e07bf5081").to_vec(), - hex!("79c07e2b1d2e2abfd4855b936617eeff5e0621c4869aa60c02be9adcc98a0d1d").to_vec(), - ]); - - let res: Vec<_> = client.storage_keys_iter(&BlockId::Number(0), Some(&prefix), Some(&StorageKey(hex!("79c07e2b1d2e2abfd4855b936617eeff5e0621c4869aa60c02be9adcc98a0d1d").to_vec()))) - .unwrap() - .take(1) - .map(|x| x.0) - .collect(); - assert_eq!(res, [hex!("cf722c0832b5231d35e29f319ff27389f5032bfc7bfc3ba5ed7839f2042fb99f").to_vec()]); - } - - #[test] - fn cleans_up_closed_notification_sinks_on_block_import() { - use substrate_test_runtime_client::GenesisInit; - - // NOTE: we need to build the client here instead of using the client - // provided by test_runtime_client otherwise we can't access the private - // `import_notification_sinks` and `finality_notification_sinks` fields. - let mut client = - new_in_mem::< - _, - substrate_test_runtime_client::runtime::Block, - _, - substrate_test_runtime_client::runtime::RuntimeApi - >( - substrate_test_runtime_client::new_native_executor(), - &substrate_test_runtime_client::GenesisParameters::default().genesis_storage(), - None, - None, - sp_core::tasks::executor(), - ) - .unwrap(); - - type TestClient = Client< - in_mem::Backend, - LocalCallExecutor, sc_executor::NativeExecutor>, - substrate_test_runtime_client::runtime::Block, - substrate_test_runtime_client::runtime::RuntimeApi, - >; - - let import_notif1 = client.import_notification_stream(); - let import_notif2 = client.import_notification_stream(); - let finality_notif1 = client.finality_notification_stream(); - let finality_notif2 = client.finality_notification_stream(); - - // for some reason I can't seem to use `ClientBlockImportExt` - let bake_and_import_block = |client: &mut TestClient, origin| { - let block = client - .new_block(Default::default()) - .unwrap() - .build() - .unwrap() - .block; - - let (header, extrinsics) = block.deconstruct(); - let mut import = BlockImportParams::new(origin, header); - import.body = Some(extrinsics); - import.fork_choice = Some(ForkChoiceStrategy::LongestChain); - client.import_block(import, Default::default()).unwrap(); - }; - - // after importing a block we should still have 4 notification sinks - // (2 import + 2 finality) - bake_and_import_block(&mut client, BlockOrigin::Own); - assert_eq!(client.import_notification_sinks.lock().len(), 2); - assert_eq!(client.finality_notification_sinks.lock().len(), 2); - - // if we drop one import notification receiver and one finality - // notification receiver - drop(import_notif2); - drop(finality_notif2); - - // the sinks should be cleaned up after block import - bake_and_import_block(&mut client, BlockOrigin::Own); - assert_eq!(client.import_notification_sinks.lock().len(), 1); - assert_eq!(client.finality_notification_sinks.lock().len(), 1); - - // the same thing should happen if block import happens during initial - // sync - drop(import_notif1); - drop(finality_notif1); - - bake_and_import_block(&mut client, BlockOrigin::NetworkInitialSync); - assert_eq!(client.import_notification_sinks.lock().len(), 0); - assert_eq!(client.finality_notification_sinks.lock().len(), 0); - } -} diff --git a/client/service/src/client/genesis.rs b/client/service/src/client/genesis.rs new file mode 100644 index 0000000000000000000000000000000000000000..41dbccc517390ff0a3884a59b35989327e546e06 --- /dev/null +++ b/client/service/src/client/genesis.rs @@ -0,0 +1,41 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Tool for creating the genesis block. + +use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Hash as HashT, Zero}; + +/// Create a genesis block, given the initial storage. +pub fn construct_genesis_block< + Block: BlockT +> ( + state_root: Block::Hash +) -> Block { + let extrinsics_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( + Vec::new(), + ); + + Block::new( + <::Header as HeaderT>::new( + Zero::zero(), + extrinsics_root, + state_root, + Default::default(), + Default::default() + ), + Default::default() + ) +} diff --git a/client/src/light/backend.rs b/client/service/src/client/light/backend.rs similarity index 82% rename from client/src/light/backend.rs rename to client/service/src/client/light/backend.rs index 0b334d48b7a251a6173f292e9416e0fa7925d6da..78f3938aaa8285bc2dfb32829342794f133e567c 100644 --- a/client/src/light/backend.rs +++ b/client/service/src/client/light/backend.rs @@ -24,7 +24,7 @@ use parking_lot::RwLock; use codec::{Decode, Encode}; use sp_core::ChangesTrieConfiguration; -use sp_core::storage::{well_known_keys, ChildInfo, OwnedChildInfo}; +use sp_core::storage::{well_known_keys, ChildInfo}; use sp_core::offchain::storage::InMemOffchainStorage; use sp_state_machine::{ Backend as StateBackend, TrieBackend, InMemoryBackend, ChangesTrieTransaction, @@ -32,7 +32,6 @@ use sp_state_machine::{ }; use sp_runtime::{generic::BlockId, Justification, Storage}; use sp_runtime::traits::{Block as BlockT, NumberFor, Zero, Header, HashFor}; -use crate::in_mem::check_genesis_storage; use sp_blockchain::{Error as ClientError, Result as ClientResult}; use sc_client_api::{ backend::{ @@ -43,9 +42,10 @@ use sc_client_api::{ HeaderBackend as BlockchainHeaderBackend, well_known_cache_keys, }, light::Storage as BlockchainStorage, + in_mem::check_genesis_storage, UsageInfo, }; -use crate::light::blockchain::Blockchain; +use super::blockchain::Blockchain; use hash_db::Hasher; const IN_MEMORY_EXPECT_PROOF: &str = "InMemory state backend has Void error type and always succeeds; qed"; @@ -251,7 +251,7 @@ where .unwrap_or(false) } - fn remote_blockchain(&self) -> Arc> { + fn remote_blockchain(&self) -> Arc> { self.blockchain.clone() } } @@ -312,17 +312,17 @@ impl BlockImportOperation for ImportOperation self.changes_trie_config_update = Some(changes_trie_config); // this is only called when genesis block is imported => shouldn't be performance bottleneck - let mut storage: HashMap, OwnedChildInfo)>, _> = HashMap::new(); + let mut storage: HashMap, _> = HashMap::new(); storage.insert(None, input.top); // create a list of children keys to re-compute roots for - let child_delta = input.children.iter() - .map(|(storage_key, storage_child)| (storage_key.clone(), None, storage_child.child_info.clone())) + let child_delta = input.children_default.iter() + .map(|(_storage_key, storage_child)| (storage_child.child_info.clone(), None)) .collect::>(); // make sure to persist the child storage - for (child_key, storage_child) in input.children { - storage.insert(Some((child_key, storage_child.child_info)), storage_child.data); + for (_child_key, storage_child) in input.children_default { + storage.insert(Some(storage_child.child_info), storage_child.data); } let storage_update = InMemoryBackend::from(storage); @@ -386,13 +386,12 @@ impl StateBackend for GenesisOrUnavailableState fn child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> ClientResult>> { match *self { GenesisOrUnavailableState::Genesis(ref state) => - Ok(state.child_storage(storage_key, child_info, key).expect(IN_MEMORY_EXPECT_PROOF)), + Ok(state.child_storage(child_info, key).expect(IN_MEMORY_EXPECT_PROOF)), GenesisOrUnavailableState::Unavailable => Err(ClientError::NotAvailableOnLightClient), } } @@ -407,13 +406,12 @@ impl StateBackend for GenesisOrUnavailableState fn next_child_storage_key( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result>, Self::Error> { match *self { GenesisOrUnavailableState::Genesis(ref state) => Ok( - state.next_child_storage_key(storage_key, child_info, key) + state.next_child_storage_key(child_info, key) .expect(IN_MEMORY_EXPECT_PROOF) ), GenesisOrUnavailableState::Unavailable => Err(ClientError::NotAvailableOnLightClient), @@ -436,27 +434,25 @@ impl StateBackend for GenesisOrUnavailableState fn for_keys_in_child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, action: A, ) { match *self { GenesisOrUnavailableState::Genesis(ref state) => - state.for_keys_in_child_storage(storage_key, child_info, action), + state.for_keys_in_child_storage(child_info, action), GenesisOrUnavailableState::Unavailable => (), } } fn for_child_keys_with_prefix( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], action: A, ) { match *self { GenesisOrUnavailableState::Genesis(ref state) => - state.for_child_keys_with_prefix(storage_key, child_info, prefix, action), + state.for_child_keys_with_prefix(child_info, prefix, action), GenesisOrUnavailableState::Unavailable => (), } } @@ -474,8 +470,7 @@ impl StateBackend for GenesisOrUnavailableState fn child_storage_root( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, delta: I, ) -> (H::Out, bool, Self::Transaction) where @@ -483,7 +478,7 @@ impl StateBackend for GenesisOrUnavailableState { match *self { GenesisOrUnavailableState::Genesis(ref state) => { - let (root, is_equal, _) = state.child_storage_root(storage_key, child_info, delta); + let (root, is_equal, _) = state.child_storage_root(child_info, delta); (root, is_equal, Default::default()) }, GenesisOrUnavailableState::Unavailable => @@ -518,53 +513,3 @@ impl StateBackend for GenesisOrUnavailableState } } } - -#[cfg(test)] -mod tests { - use substrate_test_runtime_client::{self, runtime::Block}; - use sc_client_api::backend::NewBlockState; - use crate::light::blockchain::tests::{DummyBlockchain, DummyStorage}; - use sp_runtime::traits::BlakeTwo256; - use super::*; - - #[test] - fn local_state_is_created_when_genesis_state_is_available() { - let def = Default::default(); - let header0 = substrate_test_runtime_client::runtime::Header::new(0, def, def, def, Default::default()); - - let backend: Backend<_, BlakeTwo256> = Backend::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - ); - let mut op = backend.begin_operation().unwrap(); - op.set_block_data(header0, None, None, NewBlockState::Final).unwrap(); - op.reset_storage(Default::default()).unwrap(); - backend.commit_operation(op).unwrap(); - - match backend.state_at(BlockId::Number(0)).unwrap() { - GenesisOrUnavailableState::Genesis(_) => (), - _ => panic!("unexpected state"), - } - } - - #[test] - fn unavailable_state_is_created_when_genesis_state_is_unavailable() { - let backend: Backend<_, BlakeTwo256> = Backend::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - ); - - match backend.state_at(BlockId::Number(0)).unwrap() { - GenesisOrUnavailableState::Unavailable => (), - _ => panic!("unexpected state"), - } - } - - #[test] - fn light_aux_store_is_updated_via_non_importing_op() { - let backend = Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new()))); - let mut op = ClientBackend::::begin_operation(&backend).unwrap(); - BlockImportOperation::::insert_aux(&mut op, vec![(vec![1], Some(vec![2]))]).unwrap(); - ClientBackend::::commit_operation(&backend, op).unwrap(); - - assert_eq!(AuxStore::get_aux(&backend, &[1]).unwrap(), Some(vec![2])); - } -} diff --git a/client/src/light/blockchain.rs b/client/service/src/client/light/blockchain.rs similarity index 54% rename from client/src/light/blockchain.rs rename to client/service/src/client/light/blockchain.rs index 756147c941b39eaebe4f0e6257a88a33327f31fb..b6ccb4744b55907bff952cf6eba0753d72144394 100644 --- a/client/src/light/blockchain.rs +++ b/client/service/src/client/light/blockchain.rs @@ -17,7 +17,6 @@ //! Light client blockchain backend. Only stores headers and justifications of recent //! blocks. CHT roots are stored for headers of ancient blocks. -use std::future::Future; use std::sync::Arc; use sp_runtime::{Justification, generic::BlockId}; @@ -38,10 +37,10 @@ pub use sc_client_api::{ }, light::{ RemoteBlockchain, LocalOrRemote, Storage - } + }, + cht, }; -use crate::cht; -use crate::light::fetcher::{Fetcher, RemoteHeaderRequest}; +use super::fetcher::RemoteHeaderRequest; /// Light client blockchain. pub struct Blockchain { @@ -173,154 +172,3 @@ impl RemoteBlockchain for Blockchain })) } } - -/// Returns future that resolves header either locally, or remotely. -pub fn future_header>( - blockchain: &dyn RemoteBlockchain, - fetcher: &F, - id: BlockId, -) -> impl Future, ClientError>> { - use futures::future::{ready, Either, FutureExt}; - - match blockchain.header(id) { - Ok(LocalOrRemote::Remote(request)) => Either::Left( - fetcher - .remote_header(request) - .then(|header| ready(header.map(Some))) - ), - Ok(LocalOrRemote::Unknown) => Either::Right(ready(Ok(None))), - Ok(LocalOrRemote::Local(local_header)) => Either::Right(ready(Ok(Some(local_header)))), - Err(err) => Either::Right(ready(Err(err))), - } -} - -#[cfg(test)] -pub mod tests { - use std::collections::HashMap; - use parking_lot::Mutex; - use substrate_test_runtime_client::runtime::{Hash, Block, Header}; - use sc_client_api::blockchain::Info; - use super::*; - - pub type DummyBlockchain = Blockchain; - - pub struct DummyStorage { - pub changes_tries_cht_roots: HashMap, - pub aux_store: Mutex, Vec>>, - } - - impl DummyStorage { - pub fn new() -> Self { - DummyStorage { - changes_tries_cht_roots: HashMap::new(), - aux_store: Mutex::new(HashMap::new()), - } - } - } - - impl BlockchainHeaderBackend for DummyStorage { - fn header(&self, _id: BlockId) -> ClientResult> { - Err(ClientError::Backend("Test error".into())) - } - - fn info(&self) -> Info { - panic!("Test error") - } - - fn status(&self, _id: BlockId) -> ClientResult { - Err(ClientError::Backend("Test error".into())) - } - - fn number(&self, hash: Hash) -> ClientResult>> { - if hash == Default::default() { - Ok(Some(Default::default())) - } else { - Err(ClientError::Backend("Test error".into())) - } - } - - fn hash(&self, number: u64) -> ClientResult> { - if number == 0 { - Ok(Some(Default::default())) - } else { - Err(ClientError::Backend("Test error".into())) - } - } - } - - impl HeaderMetadata for DummyStorage { - type Error = ClientError; - - fn header_metadata(&self, hash: Hash) -> Result, Self::Error> { - self.header(BlockId::hash(hash))?.map(|header| CachedHeaderMetadata::from(&header)) - .ok_or(ClientError::UnknownBlock("header not found".to_owned())) - } - fn insert_header_metadata(&self, _hash: Hash, _metadata: CachedHeaderMetadata) {} - fn remove_header_metadata(&self, _hash: Hash) {} - } - - impl AuxStore for DummyStorage { - fn insert_aux< - 'a, - 'b: 'a, - 'c: 'a, - I: IntoIterator, - D: IntoIterator, - >(&self, insert: I, _delete: D) -> ClientResult<()> { - for (k, v) in insert.into_iter() { - self.aux_store.lock().insert(k.to_vec(), v.to_vec()); - } - Ok(()) - } - - fn get_aux(&self, key: &[u8]) -> ClientResult>> { - Ok(self.aux_store.lock().get(key).cloned()) - } - } - - impl Storage for DummyStorage { - fn import_header( - &self, - _header: Header, - _cache: HashMap>, - _state: NewBlockState, - _aux_ops: Vec<(Vec, Option>)>, - ) -> ClientResult<()> { - Ok(()) - } - - fn set_head(&self, _block: BlockId) -> ClientResult<()> { - Err(ClientError::Backend("Test error".into())) - } - - fn finalize_header(&self, _block: BlockId) -> ClientResult<()> { - Err(ClientError::Backend("Test error".into())) - } - - fn last_finalized(&self) -> ClientResult { - Err(ClientError::Backend("Test error".into())) - } - - fn header_cht_root(&self, _cht_size: u64, _block: u64) -> ClientResult> { - Err(ClientError::Backend("Test error".into())) - } - - fn changes_trie_cht_root(&self, cht_size: u64, block: u64) -> ClientResult> { - cht::block_to_cht_number(cht_size, block) - .and_then(|cht_num| self.changes_tries_cht_roots.get(&cht_num)) - .cloned() - .ok_or_else(|| ClientError::Backend( - format!("Test error: CHT for block #{} not found", block) - ).into()) - .map(Some) - } - - fn cache(&self) -> Option>> { - None - } - - fn usage_info(&self) -> Option { - None - } - } -} diff --git a/client/src/light/call_executor.rs b/client/service/src/client/light/call_executor.rs similarity index 52% rename from client/src/light/call_executor.rs rename to client/service/src/client/light/call_executor.rs index b439a268d2fe18ef4fd1722e7ec2566d4065bea8..54fcf8e8f7f4c3d6b0e33c32e03d49a5f511eb79 100644 --- a/client/src/light/call_executor.rs +++ b/client/service/src/client/light/call_executor.rs @@ -21,7 +21,7 @@ use std::{ }; use codec::{Encode, Decode}; -use sp_core::{convert_hash, NativeOrEncoded, traits::CodeExecutor}; +use sp_core::{convert_hash, NativeOrEncoded, traits::CodeExecutor, offchain::storage::OffchainOverlayedChanges}; use sp_runtime::{ generic::BlockId, traits::{One, Block as BlockT, Header as HeaderT, HashFor}, }; @@ -108,6 +108,7 @@ impl CallExecutor for method: &str, call_data: &[u8], changes: &RefCell, + offchain_changes: &RefCell, _: Option<&RefCell>>, initialize_block: InitializeBlock<'a, Block>, _manager: ExecutionManager, @@ -134,6 +135,7 @@ impl CallExecutor for method, call_data, changes, + offchain_changes, None, initialize_block, ExecutionManager::NativeWhenPossible, @@ -241,7 +243,11 @@ pub fn check_execution_proof( ) } -fn check_execution_proof_with_make_header Header>( +/// Check remote contextual execution proof using given backend and header factory. +/// +/// Method is executed using passed header as environment' current block. +/// Proof should include both environment preparation proof and method execution proof. +pub fn check_execution_proof_with_make_header( executor: &E, spawn_handle: Box, request: &RemoteCallRequest

, @@ -249,10 +255,11 @@ fn check_execution_proof_with_make_header ClientResult> where - Header: HeaderT, E: CodeExecutor + Clone + 'static, H: Hasher, + Header: HeaderT, H::Out: Ord + codec::Codec + 'static, + MakeNextHeader: Fn(&Header) -> Header, { let local_state_root = request.header.state_root(); let root: H::Out = convert_hash(&local_state_root); @@ -288,231 +295,3 @@ fn check_execution_proof_with_make_header for DummyCallExecutor { - type Error = ClientError; - - type Backend = substrate_test_runtime_client::Backend; - - fn call( - &self, - _id: &BlockId, - _method: &str, - _call_data: &[u8], - _strategy: ExecutionStrategy, - _extensions: Option, - ) -> Result, ClientError> { - Ok(vec![42]) - } - - fn contextual_call< - 'a, - IB: Fn() -> ClientResult<()>, - EM: Fn( - Result, Self::Error>, - Result, Self::Error> - ) -> Result, Self::Error>, - R: Encode + Decode + PartialEq, - NC: FnOnce() -> result::Result + UnwindSafe, - >( - &self, - _initialize_block_fn: IB, - _at: &BlockId, - _method: &str, - _call_data: &[u8], - _changes: &RefCell, - _storage_transaction_cache: Option<&RefCell< - StorageTransactionCache< - Block, - >::State, - > - >>, - _initialize_block: InitializeBlock<'a, Block>, - _execution_manager: ExecutionManager, - _native_call: Option, - _proof_recorder: &Option>, - _extensions: Option, - ) -> ClientResult> where ExecutionManager: Clone { - unreachable!() - } - - fn runtime_version(&self, _id: &BlockId) -> Result { - unreachable!() - } - - fn prove_at_trie_state>>( - &self, - _trie_state: &sp_state_machine::TrieBackend>, - _overlay: &mut OverlayedChanges, - _method: &str, - _call_data: &[u8] - ) -> Result<(Vec, StorageProof), ClientError> { - unreachable!() - } - - fn native_runtime_version(&self) -> Option<&NativeVersion> { - unreachable!() - } - } - - fn local_executor() -> NativeExecutor { - NativeExecutor::new(WasmExecutionMethod::Interpreted, None, 8) - } - - #[test] - fn execution_proof_is_generated_and_checked() { - fn execute(remote_client: &TestClient, at: u64, method: &'static str) -> (Vec, Vec) { - let remote_block_id = BlockId::Number(at); - let remote_header = remote_client.header(&remote_block_id).unwrap().unwrap(); - - // 'fetch' execution proof from remote node - let (remote_result, remote_execution_proof) = remote_client.execution_proof( - &remote_block_id, - method, - &[] - ).unwrap(); - - // check remote execution proof locally - let local_result = check_execution_proof::<_, _, BlakeTwo256>( - &local_executor(), - tasks_executor(), - &RemoteCallRequest { - block: substrate_test_runtime_client::runtime::Hash::default(), - header: remote_header, - method: method.into(), - call_data: vec![], - retry_count: None, - }, - remote_execution_proof, - ).unwrap(); - - (remote_result, local_result) - } - - fn execute_with_proof_failure(remote_client: &TestClient, at: u64, method: &'static str) { - let remote_block_id = BlockId::Number(at); - let remote_header = remote_client.header(&remote_block_id).unwrap().unwrap(); - - // 'fetch' execution proof from remote node - let (_, remote_execution_proof) = remote_client.execution_proof( - &remote_block_id, - method, - &[] - ).unwrap(); - - // check remote execution proof locally - let execution_result = check_execution_proof_with_make_header::<_, _, BlakeTwo256, _>( - &local_executor(), - tasks_executor(), - &RemoteCallRequest { - block: substrate_test_runtime_client::runtime::Hash::default(), - header: remote_header, - method: method.into(), - call_data: vec![], - retry_count: None, - }, - remote_execution_proof, - |header|
::new( - at + 1, - Default::default(), - Default::default(), - header.hash(), - header.digest().clone(), // this makes next header wrong - ), - ); - match execution_result { - Err(sp_blockchain::Error::Execution(_)) => (), - _ => panic!("Unexpected execution result: {:?}", execution_result), - } - } - - // prepare remote client - let mut remote_client = substrate_test_runtime_client::new(); - for i in 1u32..3u32 { - let mut digest = Digest::default(); - digest.push(sp_runtime::generic::DigestItem::Other::(i.to_le_bytes().to_vec())); - remote_client.import_justified( - BlockOrigin::Own, - remote_client.new_block(digest).unwrap().build().unwrap().block, - Default::default(), - ).unwrap(); - } - - // check method that doesn't requires environment - let (remote, local) = execute(&remote_client, 0, "Core_version"); - assert_eq!(remote, local); - - let (remote, local) = execute(&remote_client, 2, "Core_version"); - assert_eq!(remote, local); - - // check method that requires environment - let (_, block) = execute(&remote_client, 0, "BlockBuilder_finalize_block"); - let local_block: Header = Decode::decode(&mut &block[..]).unwrap(); - assert_eq!(local_block.number, 1); - - let (_, block) = execute(&remote_client, 2, "BlockBuilder_finalize_block"); - let local_block: Header = Decode::decode(&mut &block[..]).unwrap(); - assert_eq!(local_block.number, 3); - - // check that proof check doesn't panic even if proof is incorrect AND no panic handler is set - execute_with_proof_failure(&remote_client, 2, "Core_version"); - - // check that proof check doesn't panic even if proof is incorrect AND panic handler is set - sp_panic_handler::set("TEST", "1.2.3"); - execute_with_proof_failure(&remote_client, 2, "Core_version"); - } - - #[test] - fn code_is_executed_at_genesis_only() { - let backend = Arc::new(InMemBackend::::new()); - let def = H256::default(); - let header0 = substrate_test_runtime_client::runtime::Header::new(0, def, def, def, Default::default()); - let hash0 = header0.hash(); - let header1 = substrate_test_runtime_client::runtime::Header::new(1, def, def, hash0, Default::default()); - let hash1 = header1.hash(); - backend.blockchain().insert(hash0, header0, None, None, NewBlockState::Final).unwrap(); - backend.blockchain().insert(hash1, header1, None, None, NewBlockState::Final).unwrap(); - - let genesis_executor = GenesisCallExecutor::new(backend, DummyCallExecutor); - assert_eq!( - genesis_executor.call( - &BlockId::Number(0), - "test_method", - &[], - ExecutionStrategy::NativeElseWasm, - None, - ).unwrap(), - vec![42], - ); - - let call_on_unavailable = genesis_executor.call( - &BlockId::Number(1), - "test_method", - &[], - ExecutionStrategy::NativeElseWasm, - None, - ); - - match call_on_unavailable { - Err(ClientError::NotAvailableOnLightClient) => (), - _ => unreachable!("unexpected result: {:?}", call_on_unavailable), - } - } -} diff --git a/client/service/src/client/light/fetcher.rs b/client/service/src/client/light/fetcher.rs new file mode 100644 index 0000000000000000000000000000000000000000..ae64565a504d1e3e90d2a3f4892ca69ac0b11069 --- /dev/null +++ b/client/service/src/client/light/fetcher.rs @@ -0,0 +1,341 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Light client data fetcher. Fetches requested data from remote full nodes. + +use std::sync::Arc; +use std::collections::{BTreeMap, HashMap}; +use std::marker::PhantomData; + +use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; +use codec::{Decode, Encode}; +use sp_core::{convert_hash, traits::CodeExecutor}; +use sp_core::storage::{ChildInfo, ChildType}; +use sp_runtime::traits::{ + Block as BlockT, Header as HeaderT, Hash, HashFor, NumberFor, + AtLeast32Bit, CheckedConversion, +}; +use sp_state_machine::{ + ChangesTrieRootsStorage, ChangesTrieAnchorBlockId, ChangesTrieConfigurationRange, + InMemoryChangesTrieStorage, TrieBackend, read_proof_check, key_changes_proof_check_with_db, + read_child_proof_check, CloneableSpawn, +}; +pub use sp_state_machine::StorageProof; +use sp_blockchain::{Error as ClientError, Result as ClientResult}; + +pub use sc_client_api::{ + light::{ + RemoteCallRequest, RemoteHeaderRequest, RemoteReadRequest, RemoteReadChildRequest, + RemoteChangesRequest, ChangesProof, RemoteBodyRequest, Fetcher, FetchChecker, + Storage as BlockchainStorage, + }, + cht, +}; +use super::blockchain::{Blockchain}; +use super::call_executor::check_execution_proof; + +/// Remote data checker. +pub struct LightDataChecker> { + blockchain: Arc>, + executor: E, + spawn_handle: Box, + _hasher: PhantomData<(B, H)>, +} + +impl> LightDataChecker { + /// Create new light data checker. + pub fn new(blockchain: Arc>, executor: E, spawn_handle: Box) -> Self { + Self { + blockchain, executor, spawn_handle, _hasher: PhantomData + } + } + + /// Check remote changes query proof assuming that CHT-s are of given size. + pub fn check_changes_proof_with_cht_size( + &self, + request: &RemoteChangesRequest, + remote_proof: ChangesProof, + cht_size: NumberFor, + ) -> ClientResult, u32)>> + where + H: Hasher, + H::Out: Ord + codec::Codec, + { + // since we need roots of all changes tries for the range begin..max + // => remote node can't use max block greater that one that we have passed + if remote_proof.max_block > request.max_block.0 || remote_proof.max_block < request.last_block.0 { + return Err(ClientError::ChangesTrieAccessFailed(format!( + "Invalid max_block used by the remote node: {}. Local: {}..{}..{}", + remote_proof.max_block, request.first_block.0, request.last_block.0, request.max_block.0, + )).into()); + } + + // check if remote node has responded with extra changes trie roots proofs + // all changes tries roots must be in range [request.first_block.0; request.tries_roots.0) + let is_extra_first_root = remote_proof.roots.keys().next() + .map(|first_root| *first_root < request.first_block.0 + || *first_root >= request.tries_roots.0) + .unwrap_or(false); + let is_extra_last_root = remote_proof.roots.keys().next_back() + .map(|last_root| *last_root >= request.tries_roots.0) + .unwrap_or(false); + if is_extra_first_root || is_extra_last_root { + return Err(ClientError::ChangesTrieAccessFailed(format!( + "Extra changes tries roots proofs provided by the remote node: [{:?}..{:?}]. Expected in range: [{}; {})", + remote_proof.roots.keys().next(), remote_proof.roots.keys().next_back(), + request.first_block.0, request.tries_roots.0, + )).into()); + } + + // if request has been composed when some required headers were already pruned + // => remote node has sent us CHT-based proof of required changes tries roots + // => check that this proof is correct before proceeding with changes proof + let remote_max_block = remote_proof.max_block; + let remote_roots = remote_proof.roots; + let remote_roots_proof = remote_proof.roots_proof; + let remote_proof = remote_proof.proof; + if !remote_roots.is_empty() { + self.check_changes_tries_proof( + cht_size, + &remote_roots, + remote_roots_proof, + )?; + } + + // and now check the key changes proof + get the changes + let mut result = Vec::new(); + let proof_storage = InMemoryChangesTrieStorage::with_proof(remote_proof); + for config_range in &request.changes_trie_configs { + let result_range = key_changes_proof_check_with_db::( + ChangesTrieConfigurationRange { + config: config_range.config.as_ref().ok_or(ClientError::ChangesTriesNotSupported)?, + zero: config_range.zero.0, + end: config_range.end.map(|(n, _)| n), + }, + &RootsStorage { + roots: (request.tries_roots.0, &request.tries_roots.2), + prev_roots: &remote_roots, + }, + &proof_storage, + request.first_block.0, + &ChangesTrieAnchorBlockId { + hash: convert_hash(&request.last_block.1), + number: request.last_block.0, + }, + remote_max_block, + request.storage_key.as_ref(), + &request.key) + .map_err(|err| ClientError::ChangesTrieAccessFailed(err))?; + result.extend(result_range); + } + + Ok(result) + } + + /// Check CHT-based proof for changes tries roots. + pub fn check_changes_tries_proof( + &self, + cht_size: NumberFor, + remote_roots: &BTreeMap, B::Hash>, + remote_roots_proof: StorageProof, + ) -> ClientResult<()> + where + H: Hasher, + H::Out: Ord + codec::Codec, + { + // all the checks are sharing the same storage + let storage = remote_roots_proof.into_memory_db(); + + // remote_roots.keys() are sorted => we can use this to group changes tries roots + // that are belongs to the same CHT + let blocks = remote_roots.keys().cloned(); + cht::for_each_cht_group::(cht_size, blocks, |mut storage, _, cht_blocks| { + // get local changes trie CHT root for given CHT + // it should be there, because it is never pruned AND request has been composed + // when required header has been pruned (=> replaced with CHT) + let first_block = cht_blocks.first().cloned() + .expect("for_each_cht_group never calls callback with empty groups"); + let local_cht_root = self.blockchain.storage().changes_trie_cht_root(cht_size, first_block)? + .ok_or(ClientError::InvalidCHTProof)?; + + // check changes trie root for every block within CHT range + for block in cht_blocks { + // check if the proofs storage contains the root + // normally this happens in when the proving backend is created, but since + // we share the storage for multiple checks, do it here + let mut cht_root = H::Out::default(); + cht_root.as_mut().copy_from_slice(local_cht_root.as_ref()); + if !storage.contains(&cht_root, EMPTY_PREFIX) { + return Err(ClientError::InvalidCHTProof.into()); + } + + // check proof for single changes trie root + let proving_backend = TrieBackend::new(storage, cht_root); + let remote_changes_trie_root = remote_roots[&block]; + cht::check_proof_on_proving_backend::( + local_cht_root, + block, + remote_changes_trie_root, + &proving_backend, + )?; + + // and return the storage to use in following checks + storage = proving_backend.into_storage(); + } + + Ok(storage) + }, storage) + } +} + +impl FetchChecker for LightDataChecker + where + Block: BlockT, + E: CodeExecutor + Clone + 'static, + H: Hasher, + H::Out: Ord + codec::Codec + 'static, + S: BlockchainStorage, +{ + fn check_header_proof( + &self, + request: &RemoteHeaderRequest, + remote_header: Option, + remote_proof: StorageProof, + ) -> ClientResult { + let remote_header = remote_header.ok_or_else(|| + ClientError::from(ClientError::InvalidCHTProof))?; + let remote_header_hash = remote_header.hash(); + cht::check_proof::( + request.cht_root, + request.block, + remote_header_hash, + remote_proof, + ).map(|_| remote_header) + } + + fn check_read_proof( + &self, + request: &RemoteReadRequest, + remote_proof: StorageProof, + ) -> ClientResult, Option>>> { + read_proof_check::( + convert_hash(request.header.state_root()), + remote_proof, + request.keys.iter(), + ).map_err(Into::into) + } + + fn check_read_child_proof( + &self, + request: &RemoteReadChildRequest, + remote_proof: StorageProof, + ) -> ClientResult, Option>>> { + let child_info = match ChildType::from_prefixed_key(&request.storage_key) { + Some((ChildType::ParentKeyId, storage_key)) => ChildInfo::new_default(storage_key), + None => return Err("Invalid child type".into()), + }; + read_child_proof_check::( + convert_hash(request.header.state_root()), + remote_proof, + &child_info, + request.keys.iter(), + ).map_err(Into::into) + } + + fn check_execution_proof( + &self, + request: &RemoteCallRequest, + remote_proof: StorageProof, + ) -> ClientResult> { + check_execution_proof::<_, _, H>( + &self.executor, + self.spawn_handle.clone(), + request, + remote_proof, + ) + } + + fn check_changes_proof( + &self, + request: &RemoteChangesRequest, + remote_proof: ChangesProof + ) -> ClientResult, u32)>> { + self.check_changes_proof_with_cht_size(request, remote_proof, cht::size()) + } + + fn check_body_proof( + &self, + request: &RemoteBodyRequest, + body: Vec + ) -> ClientResult> { + // TODO: #2621 + let extrinsics_root = HashFor::::ordered_trie_root( + body.iter().map(Encode::encode).collect(), + ); + if *request.header.extrinsics_root() == extrinsics_root { + Ok(body) + } else { + Err(format!("RemoteBodyRequest: invalid extrinsics root expected: {} but got {}", + *request.header.extrinsics_root(), + extrinsics_root, + ).into()) + } + + } +} + +/// A view of BTreeMap as a changes trie roots storage. +struct RootsStorage<'a, Number: AtLeast32Bit, Hash: 'a> { + roots: (Number, &'a [Hash]), + prev_roots: &'a BTreeMap, +} + +impl<'a, H, Number, Hash> ChangesTrieRootsStorage for RootsStorage<'a, Number, Hash> + where + H: Hasher, + Number: std::fmt::Display + std::hash::Hash + Clone + AtLeast32Bit + Encode + Decode + Send + Sync + 'static, + Hash: 'a + Send + Sync + Clone + AsRef<[u8]>, +{ + fn build_anchor( + &self, + _hash: H::Out, + ) -> Result, String> { + Err("build_anchor is only called when building block".into()) + } + + fn root( + &self, + _anchor: &ChangesTrieAnchorBlockId, + block: Number, + ) -> Result, String> { + // we can't ask for roots from parallel forks here => ignore anchor + let root = if block < self.roots.0 { + self.prev_roots.get(&Number::unique_saturated_from(block)).cloned() + } else { + let index: Option = block.checked_sub(&self.roots.0).and_then(|index| index.checked_into()); + match index { + Some(index) => self.roots.1.get(index as usize).cloned(), + None => None, + } + }; + + Ok(root.map(|root| { + let mut hasher_root: H::Out = Default::default(); + hasher_root.as_mut().copy_from_slice(root.as_ref()); + hasher_root + })) + } +} diff --git a/client/src/light/mod.rs b/client/service/src/client/light/mod.rs similarity index 89% rename from client/src/light/mod.rs rename to client/service/src/client/light/mod.rs index 2bb6c85376859af3ab0bfb07b8863d86bd544f5c..9b3c3f5b29042a16ce402505a80a989f44269382 100644 --- a/client/src/light/mod.rs +++ b/client/service/src/client/light/mod.rs @@ -30,15 +30,15 @@ use sp_runtime::traits::{Block as BlockT, HashFor}; use sp_blockchain::Result as ClientResult; use prometheus_endpoint::Registry; -use crate::call_executor::LocalCallExecutor; -use crate::client::Client; +use super::call_executor::LocalCallExecutor; +use super::client::{Client,ClientConfig}; use sc_client_api::{ light::Storage as BlockchainStorage, CloneableSpawn, }; -use crate::light::backend::Backend; -use crate::light::blockchain::Blockchain; -use crate::light::call_executor::GenesisCallExecutor; -use crate::light::fetcher::LightDataChecker; +use self::backend::Backend; +use self::blockchain::Blockchain; +use self::call_executor::GenesisCallExecutor; +use self::fetcher::LightDataChecker; /// Create an instance of light client blockchain backend. pub fn new_light_blockchain>(storage: S) -> Arc> { @@ -77,7 +77,7 @@ pub fn new_light( S: BlockchainStorage + 'static, E: CodeExecutor + RuntimeInfo + Clone + 'static, { - let local_executor = LocalCallExecutor::new(backend.clone(), code_executor, spawn_handle.clone()); + let local_executor = LocalCallExecutor::new(backend.clone(), code_executor, spawn_handle.clone(), ClientConfig::default()); let executor = GenesisCallExecutor::new(backend.clone(), local_executor); Client::new( backend, @@ -87,6 +87,7 @@ pub fn new_light( Default::default(), Default::default(), prometheus_registry, + ClientConfig::default(), ) } diff --git a/client/src/lib.rs b/client/service/src/client/mod.rs similarity index 53% rename from client/src/lib.rs rename to client/service/src/client/mod.rs index 20a3ed058aa7cb5fd562e87e03aac6ae078d40ab..079b9d243f982aec3ac1b105af2600df64d8624c 100644 --- a/client/src/lib.rs +++ b/client/service/src/client/mod.rs @@ -41,68 +41,14 @@ //! Additionally, the fourth generic parameter of the `Client` is a marker type representing //! the ways in which the runtime can interface with the outside. Any code that builds a `Client` //! is responsible for putting the right marker. -//! -//! ## Example -//! -//! ``` -//! use std::sync::Arc; -//! use sc_client::{Client, in_mem::Backend, LocalCallExecutor}; -//! use sp_runtime::Storage; -//! use sc_executor::{NativeExecutor, WasmExecutionMethod}; -//! -//! // In this example, we're using the `Block` and `RuntimeApi` types from the -//! // `substrate-test-runtime-client` crate. These types are automatically generated when -//! // compiling a runtime. In a typical use-case, these types would have been to be generated -//! // from your runtime. -//! use substrate_test_runtime_client::{LocalExecutor, runtime::Block, runtime::RuntimeApi}; -//! -//! let backend = Arc::new(Backend::::new()); -//! let client = Client::<_, _, _, RuntimeApi>::new( -//! backend.clone(), -//! LocalCallExecutor::new( -//! backend.clone(), -//! NativeExecutor::::new(WasmExecutionMethod::Interpreted, None, 8), -//! sp_core::tasks::executor(), -//! ), -//! // This parameter provides the storage for the chain genesis. -//! &::default(), -//! Default::default(), -//! Default::default(), -//! Default::default(), -//! None, -//! ); -//! ``` -//! - -#![warn(missing_docs)] -#![recursion_limit="128"] -pub mod cht; -pub mod in_mem; pub mod genesis; pub mod light; -pub mod leaves; mod call_executor; mod client; mod block_rules; -pub use sc_client_api::{ - blockchain, - blockchain::well_known_cache_keys, - blockchain::Info as ChainInfo, - notifications::{StorageEventStream, StorageChangeSet}, - call_executor::CallExecutor, - utils, -}; -pub use crate::{ +pub use self::{ call_executor::LocalCallExecutor, - client::{ - new_with_backend, - new_in_mem, - BlockBackend, ImportNotifications, FinalityNotifications, BlockchainEvents, LockImportRun, - BlockImportNotification, Client, ClientInfo, ExecutionStrategies, FinalityNotification, - LongestChain, BlockOf, ProvideUncles, BadBlocks, ForkBlocks, apply_aux, - }, - leaves::LeafSet, + client::{new_with_backend, new_in_mem, Client, ClientConfig}, }; -pub use sp_state_machine::{ExecutionStrategy, StorageProof, StateMachine}; diff --git a/client/service/src/config.rs b/client/service/src/config.rs index 109ff1bfd50b56f9c828d4a54fb55ec5a3742754..4654158cb36163fa97dc82dbe6fc6c6066029211 100644 --- a/client/service/src/config.rs +++ b/client/service/src/config.rs @@ -16,62 +16,37 @@ //! Service configuration. -pub use sc_client::ExecutionStrategies; -pub use sc_client_db::{kvdb::KeyValueDB, PruningMode}; -pub use sc_network::{Multiaddr, config::{MultiaddrWithPeerId, ExtTransport, NetworkConfiguration, Role}}; +pub use sc_client_db::{Database, PruningMode, DatabaseSettingsSrc as DatabaseConfig}; +pub use sc_network::Multiaddr; +pub use sc_network::config::{ExtTransport, MultiaddrWithPeerId, NetworkConfiguration, Role, NodeKeyConfig}; pub use sc_executor::WasmExecutionMethod; +use sc_client_api::execution_extensions::ExecutionStrategies; use std::{future::Future, path::{PathBuf, Path}, pin::Pin, net::SocketAddr, sync::Arc}; pub use sc_transaction_pool::txpool::Options as TransactionPoolOptions; use sc_chain_spec::ChainSpec; use sp_core::crypto::Protected; -use target_info::Target; -use sc_telemetry::TelemetryEndpoints; +pub use sc_telemetry::TelemetryEndpoints; use prometheus_endpoint::Registry; -/// Executable version. Used to pass version information from the root crate. -#[derive(Clone)] -pub struct VersionInfo { - /// Implementation name. - pub name: &'static str, - /// Implementation version. - pub version: &'static str, - /// SCM Commit hash. - pub commit: &'static str, - /// Executable file name. - pub executable_name: &'static str, - /// Executable file description. - pub description: &'static str, - /// Executable file author. - pub author: &'static str, - /// Support URL. - pub support_url: &'static str, - /// Copyright starting year (x-current year) - pub copyright_start_year: i32, -} - /// Service configuration. pub struct Configuration { /// Implementation name pub impl_name: &'static str, - /// Implementation version + /// Implementation version (see sc-cli to see an example of format) pub impl_version: &'static str, - /// Git commit if any. - pub impl_commit: &'static str, /// Node role. pub role: Role, /// How to spawn background tasks. Mandatory, otherwise creating a `Service` will error. - pub task_executor: Option + Send>>) + Send + Sync>>, + pub task_executor: Arc + Send>>) + Send + Sync>, /// Extrinsic pool configuration. pub transaction_pool: TransactionPoolOptions, /// Network configuration. pub network: NetworkConfiguration, - /// Path to the base configuration directory. - pub config_dir: Option, /// Configuration for the keystore. pub keystore: KeystoreConfig, /// Configuration for the database. - pub database: Option, + pub database: DatabaseConfig, /// Size of internal state cache in Bytes pub state_cache_size: usize, /// Size in percent of cache size dedicated to child tries @@ -79,13 +54,13 @@ pub struct Configuration { /// Pruning settings. pub pruning: PruningMode, /// Chain configuration. - pub chain_spec: Option>, - /// Node name. - pub name: String, + pub chain_spec: Box, /// Wasm execution method. pub wasm_method: WasmExecutionMethod, /// Execution strategies. pub execution_strategies: ExecutionStrategies, + /// Whether potentially unsafe RPC is considered safe to be exposed. + pub unsafe_rpc_expose: bool, /// RPC over HTTP binding address. `None` if disabled. pub rpc_http: Option, /// RPC over Websockets binding address. `None` if disabled. @@ -104,7 +79,7 @@ pub struct Configuration { /// The default number of 64KB pages to allocate for Wasm execution pub default_heap_pages: Option, /// Should offchain workers be executed. - pub offchain_worker: bool, + pub offchain_worker: OffchainWorkerConfig, /// Enable authoring even when offline. pub force_authoring: bool, /// Disable GRANDPA when running in validator mode @@ -130,8 +105,6 @@ pub struct Configuration { /// Configuration of the client keystore. #[derive(Clone)] pub enum KeystoreConfig { - /// No config supplied. - None, /// Keystore at a path on-disk. Recommended for native nodes. Path { /// The path of the keystore. @@ -147,25 +120,18 @@ impl KeystoreConfig { /// Returns the path for the keystore. pub fn path(&self) -> Option<&Path> { match self { - Self::Path { path, .. } => Some(&path), - Self::None | Self::InMemory => None, + Self::Path { path, .. } => Some(path), + Self::InMemory => None, } } } - /// Configuration of the database of the client. -#[derive(Clone)] -pub enum DatabaseConfig { - /// Database file at a specific path. Recommended for most uses. - Path { - /// Path to the database. - path: PathBuf, - /// Cache Size for internal database in MiB - cache_size: Option, - }, - - /// A custom implementation of an already-open database. - Custom(Arc), +#[derive(Clone, Default)] +pub struct OffchainWorkerConfig { + /// If this is allowed. + pub enabled: bool, + /// allow writes from the runtime to the offchain worker database. + pub indexing_enabled: bool, } /// Configuration of the Prometheus endpoint. @@ -190,123 +156,9 @@ impl PrometheusConfig { } } -impl Default for Configuration { - /// Create a default config - fn default() -> Self { - Configuration { - impl_name: "parity-substrate", - impl_version: "0.0.0", - impl_commit: "", - chain_spec: None, - config_dir: None, - name: Default::default(), - role: Role::Full, - task_executor: None, - transaction_pool: Default::default(), - network: Default::default(), - keystore: KeystoreConfig::None, - database: None, - state_cache_size: Default::default(), - state_cache_child_ratio: Default::default(), - pruning: PruningMode::default(), - wasm_method: WasmExecutionMethod::Interpreted, - execution_strategies: Default::default(), - rpc_http: None, - rpc_ws: None, - rpc_ws_max_connections: None, - rpc_cors: Some(vec![]), - prometheus_config: None, - telemetry_endpoints: None, - telemetry_external_transport: None, - default_heap_pages: None, - offchain_worker: Default::default(), - force_authoring: false, - disable_grandpa: false, - dev_key_seed: None, - tracing_targets: Default::default(), - tracing_receiver: Default::default(), - max_runtime_instances: 8, - announce_block: true, - } - } -} - impl Configuration { - /// Create a default config using `VersionInfo` - pub fn from_version(version: &VersionInfo) -> Self { - let mut config = Configuration::default(); - config.impl_name = version.name; - config.impl_version = version.version; - config.impl_commit = version.commit; - - config - } - - /// Returns full version string of this configuration. - pub fn full_version(&self) -> String { - full_version_from_strs(self.impl_version, self.impl_commit) - } - - /// Implementation id and version. - pub fn client_id(&self) -> String { - format!("{}/v{}", self.impl_name, self.full_version()) - } - - /// Generate a PathBuf to sub in the chain configuration directory - /// if given - pub fn in_chain_config_dir(&self, sub: &str) -> Option { - self.config_dir.clone().map(|mut path| { - path.push("chains"); - path.push(self.expect_chain_spec().id()); - path.push(sub); - path - }) - } - - /// Return a reference to the `ChainSpec` of this `Configuration`. - /// - /// ### Panics - /// - /// This method panic if the `chain_spec` is `None` - pub fn expect_chain_spec(&self) -> &dyn ChainSpec { - &**self.chain_spec.as_ref().expect("chain_spec must be specified") - } - - /// Return a reference to the `DatabaseConfig` of this `Configuration`. - /// - /// ### Panics - /// - /// This method panic if the `database` is `None` - pub fn expect_database(&self) -> &DatabaseConfig { - self.database.as_ref().expect("database must be specified") - } - /// Returns a string displaying the node role. pub fn display_role(&self) -> String { self.role.to_string() } - - /// Use in memory keystore config when it is not required at all. - /// - /// This function returns an error if the keystore is already set to something different than - /// `KeystoreConfig::None`. - pub fn use_in_memory_keystore(&mut self) -> Result<(), String> { - match &mut self.keystore { - cfg @ KeystoreConfig::None => { *cfg = KeystoreConfig::InMemory; Ok(()) }, - _ => Err("Keystore config specified when it should not be!".into()), - } - } -} - -/// Returns platform info -pub fn platform() -> String { - let env = Target::env(); - let env_dash = if env.is_empty() { "" } else { "-" }; - format!("{}-{}{}{}", Target::arch(), Target::os(), env_dash, env) -} - -/// Returns full version string, using supplied version and commit. -pub fn full_version_from_strs(impl_version: &str, impl_commit: &str) -> String { - let commit_dash = if impl_commit.is_empty() { "" } else { "-" }; - format!("{}{}{}-{}", impl_version, commit_dash, impl_commit, platform()) } diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 0ceb310d07c80c78125d7a3a2aec62c40ff86042..8e8c9e1e37bb346604e9c80e3b0741fc9216b226 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -18,6 +18,7 @@ //! Manages communication between them. #![warn(missing_docs)] +#![recursion_limit="128"] pub mod config; #[macro_use] @@ -26,10 +27,14 @@ pub mod error; mod metrics; mod builder; +#[cfg(feature = "test-helpers")] +pub mod client; +#[cfg(not(feature = "test-helpers"))] +mod client; mod status_sinks; mod task_manager; -use std::{borrow::Cow, io, pin::Pin}; +use std::{io, pin::Pin}; use std::marker::PhantomData; use std::net::SocketAddr; use std::collections::HashMap; @@ -38,7 +43,7 @@ use wasm_timer::Instant; use std::task::{Poll, Context}; use parking_lot::Mutex; -use sc_client::Client; +use client::Client; use futures::{ Future, FutureExt, Stream, StreamExt, compat::*, @@ -49,31 +54,43 @@ use sc_network::{NetworkService, network_state::NetworkState, PeerId, ReportHand use log::{log, warn, debug, error, Level}; use codec::{Encode, Decode}; use sp_runtime::generic::BlockId; -use sp_runtime::traits::{NumberFor, Block as BlockT}; +use sp_runtime::traits::{NumberFor, Block as BlockT, BlockIdTo}; use parity_util_mem::MallocSizeOf; use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; pub use self::error::Error; pub use self::builder::{ - new_full_client, + new_full_client, new_client, ServiceBuilder, ServiceBuilderCommand, TFullClient, TLightClient, TFullBackend, TLightBackend, TFullCallExecutor, TLightCallExecutor, }; pub use config::{Configuration, Role, PruningMode, DatabaseConfig}; pub use sc_chain_spec::{ - ChainSpec, GenericChainSpec, Properties, RuntimeGenesis, Extension as ChainSpecExtension + ChainSpec, GenericChainSpec, Properties, RuntimeGenesis, Extension as ChainSpecExtension, + NoExtension, ChainType, }; pub use sp_transaction_pool::{TransactionPool, InPoolTransaction, error::IntoPoolError}; pub use sc_transaction_pool::txpool::Options as TransactionPoolOptions; -pub use sc_client::FinalityNotifications; pub use sc_rpc::Metadata as RpcMetadata; pub use sc_executor::NativeExecutionDispatch; #[doc(hidden)] pub use std::{ops::Deref, result::Result, sync::Arc}; #[doc(hidden)] pub use sc_network::config::{FinalityProofProvider, OnDemand, BoxFinalityProofRequestBuilder}; -pub use task_manager::{TaskManagerBuilder, SpawnTaskHandle}; +pub use sc_tracing::TracingReceiver; +pub use task_manager::SpawnTaskHandle; use task_manager::TaskManager; +use sp_blockchain::{HeaderBackend, HeaderMetadata, ProvideCache}; +use sp_api::{ProvideRuntimeApi, CallApiAt, ApiExt, ConstructRuntimeApi, ApiErrorExt}; +use sc_client_api::{ + LockImportRun, Backend as BackendT, ProofProvider, ProvideUncles, + StorageProvider, ExecutorProvider, Finalizer, AuxStore, Backend, + BlockBackend, BlockchainEvents, CallExecutor, TransactionFor, + UsageProvider, +}; +use sc_block_builder::BlockBuilderProvider; +use sp_consensus::{block_validation::Chain, BlockImport}; +use sp_block_builder::BlockBuilder; const DEFAULT_PROTOCOL_ID: &str = "sup"; @@ -114,21 +131,104 @@ pub struct Service { impl Unpin for Service {} +/// Client super trait, use this instead of the concrete Client type. +pub trait ClientProvider< + Block: BlockT, + Backend: BackendT, + Executor: CallExecutor, + Runtime: ConstructRuntimeApi, +>: + HeaderBackend + + ProvideRuntimeApi< + Block, + Api = >::RuntimeApi + > + + LockImportRun + + ProofProvider + + BlockBuilderProvider + + ProvideUncles + + StorageProvider + + Chain + + HeaderMetadata + + ExecutorProvider + + ProvideCache + + BlockIdTo + + CallApiAt< + Block, + Error = sp_blockchain::Error, + StateBackend = >::State + > + + BlockImport< + Block, + Error = sp_consensus::Error, + Transaction = TransactionFor + > + + Finalizer + + BlockchainEvents + + BlockBackend + + UsageProvider + + AuxStore +{} + +impl ClientProvider + for + Client + where + Block: BlockT, + Backend: BackendT, + Executor: CallExecutor, + Runtime: ConstructRuntimeApi, + Self: HeaderBackend + + ProvideRuntimeApi< + Block, + Api = >::RuntimeApi + > + + LockImportRun + + ProofProvider + + BlockBuilderProvider + + ProvideUncles + + StorageProvider + + Chain + + HeaderMetadata + + ExecutorProvider + + ProvideCache + + BlockIdTo + + CallApiAt< + Block, + Error = sp_blockchain::Error, + StateBackend = >::State + > + + BlockImport< + Block, + Error = sp_consensus::Error, + Transaction = TransactionFor + > + + Finalizer + + BlockchainEvents + + BlockBackend + + UsageProvider + + AuxStore +{} + /// Abstraction over a Substrate service. -pub trait AbstractService: 'static + Future> + - Spawn + Send + Unpin { +pub trait AbstractService: Future> + Send + Unpin + Spawn + 'static { /// Type of block of this chain. type Block: BlockT; /// Backend storage for the client. - type Backend: 'static + sc_client_api::backend::Backend; + type Backend: 'static + BackendT; /// How to execute calls towards the runtime. - type CallExecutor: 'static + sc_client::CallExecutor + Send + Sync + Clone; + type CallExecutor: 'static + CallExecutor + Send + Sync + Clone; /// API that the runtime provides. type RuntimeApi: Send + Sync; /// Chain selection algorithm. type SelectChain: sp_consensus::SelectChain; /// Transaction pool. type TransactionPool: TransactionPool + MallocSizeOfWasm; + /// The generic Client type, the bounds here are the ones specifically required by + /// internal crates like sc_informant. + type Client: + HeaderMetadata + UsageProvider + + BlockchainEvents + HeaderBackend + Send + Sync; /// Get event stream for telemetry connection established events. fn telemetry_on_connect_stream(&self) -> TracingUnboundedReceiver<()>; @@ -137,12 +237,18 @@ pub trait AbstractService: 'static + Future> + fn telemetry(&self) -> Option; /// Spawns a task in the background that runs the future passed as parameter. - fn spawn_task(&self, name: impl Into>, task: impl Future + Send + 'static); + /// + /// Information about this task will be reported to Prometheus. + /// + /// The task name is a `&'static str` as opposed to a `String`. The reason for that is that + /// in order to avoid memory consumption issues with the Prometheus metrics, the set of + /// possible task names has to be bounded. + fn spawn_task(&self, name: &'static str, task: impl Future + Send + 'static); /// Spawns a task in the background that runs the future passed as /// parameter. The given task is considered essential, i.e. if it errors we /// trigger a service exit. - fn spawn_essential_task(&self, name: impl Into>, task: impl Future + Send + 'static); + fn spawn_essential_task(&self, name: &'static str, task: impl Future + Send + 'static); /// Returns a handle for spawning tasks. fn spawn_task_handle(&self) -> SpawnTaskHandle; @@ -162,7 +268,7 @@ pub trait AbstractService: 'static + Future> + fn rpc_query(&self, mem: &RpcSession, request: &str) -> Pin> + Send>>; /// Get shared client instance. - fn client(&self) -> Arc>; + fn client(&self) -> Arc; /// Get clone of select chain. fn select_chain(&self) -> Option; @@ -190,9 +296,14 @@ impl AbstractService for NetworkService, TExPool, TOc> where TBl: BlockT, - TBackend: 'static + sc_client_api::backend::Backend, - TExec: 'static + sc_client::CallExecutor + Send + Sync + Clone, - TRtApi: 'static + Send + Sync, + TBackend: 'static + Backend, + TExec: 'static + CallExecutor + Send + Sync + Clone, + TRtApi: 'static + Send + Sync + ConstructRuntimeApi>, + >>::RuntimeApi: + sp_api::Core + + ApiExt + + ApiErrorExt + + BlockBuilder, TSc: sp_consensus::SelectChain + 'static + Clone + Send + Unpin, TExPool: 'static + TransactionPool + MallocSizeOfWasm, TOc: 'static + Send + Sync, @@ -203,6 +314,7 @@ where type RuntimeApi = TRtApi; type SelectChain = TSc; type TransactionPool = TExPool; + type Client = Client; fn telemetry_on_connect_stream(&self) -> TracingUnboundedReceiver<()> { let (sink, stream) = tracing_unbounded("mpsc_telemetry_on_connect"); @@ -218,11 +330,11 @@ where self.keystore.clone() } - fn spawn_task(&self, name: impl Into>, task: impl Future + Send + 'static) { + fn spawn_task(&self, name: &'static str, task: impl Future + Send + 'static) { self.task_manager.spawn(name, task) } - fn spawn_essential_task(&self, name: impl Into>, task: impl Future + Send + 'static) { + fn spawn_essential_task(&self, name: &'static str, task: impl Future + Send + 'static) { let mut essential_failed = self.essential_failed_tx.clone(); let essential_task = std::panic::AssertUnwindSafe(task) .catch_unwind() @@ -246,7 +358,7 @@ where ) } - fn client(&self) -> Arc> { + fn client(&self) -> Arc { self.client.clone() } @@ -296,8 +408,6 @@ impl Future for } } - this.task_manager.process_receiver(cx); - // The service future never ends. Poll::Pending } @@ -310,8 +420,8 @@ impl Spawn for &self, future: FutureObj<'static, ()> ) -> Result<(), SpawnError> { - self.task_manager.scheduler().unbounded_send((Box::pin(future), From::from("unnamed"))) - .map_err(|_| SpawnError::shutdown()) + self.task_manager.spawn_handle().spawn("unnamed", future); + Ok(()) } } @@ -320,7 +430,7 @@ impl Spawn for /// The `status_sink` contain a list of senders to send a periodic network status to. fn build_network_future< B: BlockT, - C: sc_client::BlockchainEvents, + C: BlockchainEvents, H: sc_network::ExHashT > ( role: Role, @@ -365,6 +475,17 @@ fn build_network_future< should_have_peers, }); }, + sc_rpc::system::Request::LocalPeerId(sender) => { + let _ = sender.send(network.local_peer_id().to_base58()); + }, + sc_rpc::system::Request::LocalListenAddresses(sender) => { + let peer_id = network.local_peer_id().clone().into(); + let p2p_proto_suffix = sc_network::multiaddr::Protocol::P2p(peer_id); + let addresses = network.listen_addresses() + .map(|addr| addr.clone().with(p2p_proto_suffix.clone()).to_string()) + .collect(); + let _ = sender.send(addresses); + }, sc_rpc::system::Request::Peers(sender) => { let _ = sender.send(network.peers_debug_info().into_iter().map(|(peer_id, p)| sc_rpc::system::PeerInfo { @@ -492,7 +613,7 @@ mod waiting { /// Starts RPC servers that run in their own thread, and returns an opaque object that keeps them alive. #[cfg(not(target_os = "unknown"))] -fn start_rpc_servers sc_rpc_server::RpcHandler>( +fn start_rpc_servers sc_rpc_server::RpcHandler>( config: &Configuration, mut gen_handler: H ) -> Result, error::Error> { @@ -514,10 +635,23 @@ fn start_rpc_servers sc_rpc_server::RpcHandler>( }) } + fn deny_unsafe(addr: &Option, unsafe_rpc_expose: bool) -> sc_rpc::DenyUnsafe { + let is_exposed_addr = addr.map(|x| x.ip().is_loopback()).unwrap_or(false); + if is_exposed_addr && !unsafe_rpc_expose { + sc_rpc::DenyUnsafe::Yes + } else { + sc_rpc::DenyUnsafe::No + } + } + Ok(Box::new(( maybe_start_server( config.rpc_http, - |address| sc_rpc_server::start_http(address, config.rpc_cors.as_ref(), gen_handler()), + |address| sc_rpc_server::start_http( + address, + config.rpc_cors.as_ref(), + gen_handler(deny_unsafe(&config.rpc_http, config.unsafe_rpc_expose)), + ), )?.map(|s| waiting::HttpServer(Some(s))), maybe_start_server( config.rpc_ws, @@ -525,15 +659,15 @@ fn start_rpc_servers sc_rpc_server::RpcHandler>( address, config.rpc_ws_max_connections, config.rpc_cors.as_ref(), - gen_handler(), + gen_handler(deny_unsafe(&config.rpc_ws, config.unsafe_rpc_expose)), ), - )?.map(|s| waiting::WsServer(Some(s))).map(Mutex::new), + )?.map(|s| waiting::WsServer(Some(s))), ))) } /// Starts RPC servers that run in their own thread, and returns an opaque object that keeps them alive. #[cfg(target_os = "unknown")] -fn start_rpc_servers sc_rpc_server::RpcHandler>( +fn start_rpc_servers sc_rpc_server::RpcHandler>( _: &Configuration, _: H ) -> Result, error::Error> { @@ -677,6 +811,7 @@ mod tests { let pool = Arc::new(BasicPool::new( Default::default(), Arc::new(FullChainApi::new(client.clone())), + None, ).0); let source = sp_runtime::transaction_validity::TransactionSource::External; let best = longest_chain.best_chain().unwrap(); diff --git a/client/service/src/metrics.rs b/client/service/src/metrics.rs index 740a795edae5bd200466cc174239ea0628dee69c..6456f9b1ee0be5df1b113574647d629f77bb6af3 100644 --- a/client/service/src/metrics.rs +++ b/client/service/src/metrics.rs @@ -14,27 +14,26 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +use std::convert::TryFrom; + use crate::NetworkStatus; use prometheus_endpoint::{register, Gauge, U64, F64, Registry, PrometheusError, Opts, GaugeVec}; -use sc_client::ClientInfo; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; -use std::convert::TryFrom; use sp_runtime::traits::{NumberFor, Block, SaturatedConversion, UniqueSaturatedInto}; use sp_transaction_pool::PoolStatus; use sp_utils::metrics::register_globals; +use sc_client_api::ClientInfo; -#[cfg(any(windows, unix))] use sysinfo::{self, ProcessExt, SystemExt}; -#[cfg(any(unix, windows))] -use netstat2::{TcpState, ProtocolSocketInfo, iterate_sockets_info, AddressFamilyFlags, ProtocolFlags}; - -#[cfg(target_os = "linux")] -use procfs; +#[cfg(all(any(unix, windows), not(target_os = "android")))] +use netstat2::{ + TcpState, ProtocolSocketInfo, iterate_sockets_info, AddressFamilyFlags, ProtocolFlags, +}; struct PrometheusMetrics { // system - #[cfg(any(unix, windows))] + #[cfg(all(any(unix, windows), not(target_os = "android")))] load_avg: GaugeVec, // process @@ -42,8 +41,8 @@ struct PrometheusMetrics { memory_usage_bytes: Gauge, threads: Gauge, open_files: GaugeVec, - - #[cfg(any(unix, windows))] + + #[cfg(all(any(unix, windows), not(target_os = "android")))] netstat: GaugeVec, // -- inner counters @@ -71,16 +70,16 @@ impl PrometheusMetrics { .const_label("name", name) .const_label("version", version) )?, ®istry)?.set(1); - + register(Gauge::::new( "node_roles", "The roles the node is running as", )?, ®istry)?.set(roles); register_globals(registry)?; - + Ok(Self { // system - #[cfg(any(unix, windows))] + #[cfg(all(any(unix, windows), not(target_os = "android")))] load_avg: register(GaugeVec::new( Opts::new("load_avg", "System load average"), &["over"] @@ -95,7 +94,7 @@ impl PrometheusMetrics { "cpu_usage_percentage", "Node CPU usage", )?, registry)?, - #[cfg(any(unix, windows))] + #[cfg(all(any(unix, windows), not(target_os = "android")))] netstat: register(GaugeVec::new( Opts::new("netstat_tcp", "Current TCP connections "), &["status"] @@ -113,7 +112,6 @@ impl PrometheusMetrics { // --- internal // generic internals - block_height: register(GaugeVec::new( Opts::new("block_height", "Block height info of the chain"), &["status"] @@ -128,7 +126,6 @@ impl PrometheusMetrics { )?, registry)?, // I/ O - network_per_sec_bytes: register(GaugeVec::new( Opts::new("network_per_sec_bytes", "Networking bytes per second"), &["direction"] @@ -147,7 +144,7 @@ impl PrometheusMetrics { } } -#[cfg(any(unix, windows))] +#[cfg(all(any(unix, windows), not(target_os = "android")))] #[derive(Default)] struct ConnectionsCount { listen: u64, @@ -179,9 +176,9 @@ struct ProcessInfo { pub struct MetricsService { metrics: Option, - #[cfg(any(windows, unix))] + #[cfg(all(any(unix, windows), not(target_os = "android")))] system: sysinfo::System, - pid: Option, + pid: Option, } #[cfg(target_os = "linux")] @@ -196,12 +193,14 @@ impl MetricsService { pid: Some(process.pid), } } + fn process_info(&mut self) -> ProcessInfo { let pid = self.pid.clone().expect("unix always has a pid. qed"); - let mut info = self._process_info_for(&pid); + let mut info = self.process_info_for(&pid); let process = procfs::process::Process::new(pid).expect("Our process exists. qed."); info.threads = process.stat().ok().map(|s| - u64::try_from(s.num_threads).expect("There are no negative thread counts. qed")); + u64::try_from(s.num_threads).expect("There are no negative thread counts. qed"), + ); info.open_fd = process.fd().ok().map(|i| i.into_iter().fold(FdCounter::default(), |mut f, info| { match info.target { @@ -218,33 +217,33 @@ impl MetricsService { ); info } - } -#[cfg(all(any(unix, windows), not(target_os = "linux")))] +#[cfg(all(any(unix, windows), not(target_os = "android"), not(target_os = "linux")))] impl MetricsService { fn inner_new(metrics: Option) -> Self { Self { metrics, system: sysinfo::System::new(), - pid: sysinfo::get_current_pid().ok() + pid: sysinfo::get_current_pid().ok(), } } - + fn process_info(&mut self) -> ProcessInfo { - self.pid.map(|pid| self._process_info_for(&pid)).unwrap_or_else(ProcessInfo::default) + self.pid.map(|pid| self.process_info_for(&pid)).unwrap_or_default() } } -#[cfg(not(any(unix, windows)))] + +#[cfg(not(all(any(unix, windows), not(target_os = "android"))))] impl MetricsService { fn inner_new(metrics: Option) -> Self { Self { metrics, - pid: None + pid: None, } } - + fn process_info(&mut self) -> ProcessInfo { ProcessInfo::default() } @@ -252,7 +251,6 @@ impl MetricsService { impl MetricsService { - pub fn with_prometheus(registry: &Registry, name: &str, version: &str, roles: u64) -> Result { @@ -265,8 +263,8 @@ impl MetricsService { Self::inner_new(None) } - #[cfg(any(windows, unix))] - fn _process_info_for(&mut self, pid: &i32) -> ProcessInfo { + #[cfg(all(any(unix, windows), not(target_os = "android")))] + fn process_info_for(&mut self, pid: &sysinfo::Pid) -> ProcessInfo { let mut info = ProcessInfo::default(); if self.system.refresh_process(*pid) { let prc = self.system.get_process(*pid) @@ -277,7 +275,7 @@ impl MetricsService { info } - #[cfg(any(unix, windows))] + #[cfg(all(any(unix, windows), not(target_os = "android")))] fn connections_info(&self) -> Option { self.pid.as_ref().and_then(|pid| { let af_flags = AddressFamilyFlags::IPV4 | AddressFamilyFlags::IPV6; @@ -285,15 +283,12 @@ impl MetricsService { let netstat_pid = *pid as u32; iterate_sockets_info(af_flags, proto_flags).ok().map(|iter| - iter.filter_map(|r| + iter.filter_map(|r| r.ok().and_then(|s| { - if s.associated_pids.contains(&netstat_pid) { - match s.protocol_socket_info { - ProtocolSocketInfo::Tcp(info) => Some(info.state), - _ => None - } - } else { - None + match s.protocol_socket_info { + ProtocolSocketInfo::Tcp(info) + if s.associated_pids.contains(&netstat_pid) => Some(info.state), + _ => None } }) ).fold(ConnectionsCount::default(), |mut counter, socket_state| { @@ -317,7 +312,7 @@ impl MetricsService { &mut self, info: &ClientInfo, txpool_status: &PoolStatus, - net_status: &NetworkStatus + net_status: &NetworkStatus, ) { let best_number = info.chain.best_number.saturated_into::(); @@ -377,8 +372,12 @@ impl MetricsService { } - metrics.network_per_sec_bytes.with_label_values(&["download"]).set(net_status.average_download_per_sec); - metrics.network_per_sec_bytes.with_label_values(&["upload"]).set(net_status.average_upload_per_sec); + metrics.network_per_sec_bytes.with_label_values(&["download"]).set( + net_status.average_download_per_sec, + ); + metrics.network_per_sec_bytes.with_label_values(&["upload"]).set( + net_status.average_upload_per_sec, + ); metrics.block_height.with_label_values(&["finalized"]).set(finalized_number); metrics.block_height.with_label_values(&["best"]).set(best_number); @@ -396,14 +395,18 @@ impl MetricsService { metrics.database_cache.set(info.memory.database_cache.as_bytes() as u64); metrics.state_cache.set(info.memory.state_cache.as_bytes() as u64); - metrics.state_db.with_label_values(&["non_canonical"]).set(info.memory.state_db.non_canonical.as_bytes() as u64); + metrics.state_db.with_label_values(&["non_canonical"]).set( + info.memory.state_db.non_canonical.as_bytes() as u64, + ); if let Some(pruning) = info.memory.state_db.pruning { metrics.state_db.with_label_values(&["pruning"]).set(pruning.as_bytes() as u64); } - metrics.state_db.with_label_values(&["pinned"]).set(info.memory.state_db.pinned.as_bytes() as u64); + metrics.state_db.with_label_values(&["pinned"]).set( + info.memory.state_db.pinned.as_bytes() as u64, + ); } - #[cfg(any(unix, windows))] + #[cfg(all(any(unix, windows), not(target_os = "android")))] { let load = self.system.get_load_average(); metrics.load_avg.with_label_values(&["1min"]).set(load.one); diff --git a/client/service/src/task_manager.rs b/client/service/src/task_manager.rs index 7c5862e8535c4bcacdb846f9f228c630ccfc0c3d..e6847d08811204abb08fd9ee4b25e4cc1669a068 100644 --- a/client/service/src/task_manager.rs +++ b/client/service/src/task_manager.rs @@ -14,109 +14,81 @@ //! Substrate service tasks management module. use std::{ - result::Result, sync::Arc, - task::{Poll, Context}, - borrow::Cow, pin::Pin, + pin::Pin, + result::Result, sync::Arc }; use exit_future::Signal; -use log::{debug, error}; +use log::{debug}; use futures::{ - Future, FutureExt, Stream, + Future, FutureExt, future::select, compat::*, task::{Spawn, FutureObj, SpawnError}, }; +use prometheus_endpoint::{ + exponential_buckets, register, + PrometheusError, + CounterVec, HistogramOpts, HistogramVec, Opts, Registry, U64 +}; use sc_client_api::CloneableSpawn; -use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver}; + +mod prometheus_future; /// Type alias for service task executor (usually runtime). pub type ServiceTaskExecutor = Arc + Send>>) + Send + Sync>; -/// Type alias for the task scheduler. -pub type TaskScheduler = TracingUnboundedSender<(Pin + Send>>, Cow<'static, str>)>; - -/// Helper struct to setup background tasks execution for service. -pub struct TaskManagerBuilder { - /// A future that resolves when the service has exited, this is useful to - /// make sure any internally spawned futures stop when the service does. - on_exit: exit_future::Exit, - /// A signal that makes the exit future above resolve, fired on service drop. - signal: Option, - /// Sender for futures that must be spawned as background tasks. - to_spawn_tx: TaskScheduler, - /// Receiver for futures that must be spawned as background tasks. - to_spawn_rx: TracingUnboundedReceiver<(Pin + Send>>, Cow<'static, str>)>, -} - -impl TaskManagerBuilder { - /// New asynchronous task manager setup. - pub fn new() -> Self { - let (signal, on_exit) = exit_future::signal(); - let (to_spawn_tx, to_spawn_rx) = tracing_unbounded("mpsc_task_manager"); - Self { - on_exit, - signal: Some(signal), - to_spawn_tx, - to_spawn_rx, - } - } - - /// Get spawn handle. - /// - /// Tasks spawned through this handle will get scheduled once - /// service is up and running. - pub fn spawn_handle(&self) -> SpawnTaskHandle { - SpawnTaskHandle { - on_exit: self.on_exit.clone(), - sender: self.to_spawn_tx.clone(), - } - } - - /// Convert into actual task manager from initial setup. - pub(crate) fn into_task_manager(self, executor: ServiceTaskExecutor) -> TaskManager { - let TaskManagerBuilder { - on_exit, - signal, - to_spawn_rx, - to_spawn_tx - } = self; - TaskManager { - on_exit, - signal, - to_spawn_tx, - to_spawn_rx, - executor, - } - } -} - /// An handle for spawning tasks in the service. #[derive(Clone)] pub struct SpawnTaskHandle { - sender: TaskScheduler, on_exit: exit_future::Exit, + executor: ServiceTaskExecutor, + metrics: Option, } impl SpawnTaskHandle { /// Spawns the given task with the given name. - pub fn spawn(&self, name: impl Into>, task: impl Future + Send + 'static) { + /// + /// Note that the `name` is a `&'static str`. The reason for this choice is that statistics + /// about this task are getting reported to the Prometheus endpoint (if enabled), and that + /// therefore the set of possible task names must be bounded. + /// + /// In other words, it would be a bad idea for someone to do for example + /// `spawn(format!("{:?}", some_public_key))`. + pub fn spawn(&self, name: &'static str, task: impl Future + Send + 'static) { let on_exit = self.on_exit.clone(); + let metrics = self.metrics.clone(); + + // Note that we increase the started counter here and not within the future. This way, + // we could properly visualize on Prometheus situations where the spawning doesn't work. + if let Some(metrics) = &self.metrics { + metrics.tasks_spawned.with_label_values(&[name]).inc(); + // We do a dummy increase in order for the task to show up in metrics. + metrics.tasks_ended.with_label_values(&[name]).inc_by(0); + } + let future = async move { - futures::pin_mut!(task); - let _ = select(on_exit, task).await; + if let Some(metrics) = metrics { + let poll_duration = metrics.poll_duration.with_label_values(&[name]); + let poll_start = metrics.poll_start.with_label_values(&[name]); + let task = prometheus_future::with_poll_durations(poll_duration, poll_start, task); + futures::pin_mut!(task); + let _ = select(on_exit, task).await; + metrics.tasks_ended.with_label_values(&[name]).inc(); + } else { + futures::pin_mut!(task); + let _ = select(on_exit, task).await; + } }; - if self.sender.unbounded_send((Box::pin(future), name.into())).is_err() { - error!("Failed to send task to spawn over channel"); - } + + (self.executor)(Box::pin(future)); } } impl Spawn for SpawnTaskHandle { fn spawn_obj(&self, future: FutureObj<'static, ()>) -> Result<(), SpawnError> { - let future = select(self.on_exit.clone(), future).map(drop); - self.sender.unbounded_send((Box::pin(future), From::from("unnamed"))) - .map_err(|_| SpawnError::shutdown()) + self.spawn("unamed", future); + Ok(()) } } @@ -142,43 +114,43 @@ pub struct TaskManager { on_exit: exit_future::Exit, /// A signal that makes the exit future above resolve, fired on service drop. signal: Option, - /// Sender for futures that must be spawned as background tasks. - to_spawn_tx: TaskScheduler, - /// Receiver for futures that must be spawned as background tasks. - to_spawn_rx: TracingUnboundedReceiver<(Pin + Send>>, Cow<'static, str>)>, /// How to spawn background tasks. executor: ServiceTaskExecutor, + /// Prometheus metric where to report the polling times. + metrics: Option, } impl TaskManager { + /// If a Prometheus registry is passed, it will be used to report statistics about the + /// service tasks. + pub(super) fn new( + executor: ServiceTaskExecutor, + prometheus_registry: Option<&Registry> + ) -> Result { + let (signal, on_exit) = exit_future::signal(); + + let metrics = prometheus_registry.map(Metrics::register).transpose()?; + + Ok(Self { + on_exit, + signal: Some(signal), + executor, + metrics, + }) + } + /// Spawn background/async task, which will be aware on exit signal. - pub(super) fn spawn(&self, name: impl Into>, task: impl Future + Send + 'static) { - let on_exit = self.on_exit.clone(); - let future = async move { - futures::pin_mut!(task); - let _ = select(on_exit, task).await; - }; - if self.to_spawn_tx.unbounded_send((Box::pin(future), name.into())).is_err() { - error!("Failed to send task to spawn over channel"); - } + /// + /// See also the documentation of [`SpawnTaskHandler::spawn`]. + pub(super) fn spawn(&self, name: &'static str, task: impl Future + Send + 'static) { + self.spawn_handle().spawn(name, task) } pub(super) fn spawn_handle(&self) -> SpawnTaskHandle { SpawnTaskHandle { on_exit: self.on_exit.clone(), - sender: self.to_spawn_tx.clone(), - } - } - - /// Get sender where background/async tasks can be sent. - pub(super) fn scheduler(&self) -> TaskScheduler { - self.to_spawn_tx.clone() - } - - /// Process background task receiver. - pub(super) fn process_receiver(&mut self, cx: &mut Context) { - while let Poll::Ready(Some((task_to_spawn, name))) = Pin::new(&mut self.to_spawn_rx).poll_next(cx) { - (self.executor)(Box::pin(futures_diagnose::diagnose(name, task_to_spawn))); + executor: self.executor.clone(), + metrics: self.metrics.clone(), } } @@ -196,3 +168,51 @@ impl Drop for TaskManager { } } } + +#[derive(Clone)] +struct Metrics { + // This list is ordered alphabetically + poll_duration: HistogramVec, + poll_start: CounterVec, + tasks_spawned: CounterVec, + tasks_ended: CounterVec, +} + +impl Metrics { + fn register(registry: &Registry) -> Result { + Ok(Self { + poll_duration: register(HistogramVec::new( + HistogramOpts { + common_opts: Opts::new( + "tasks_polling_duration", + "Duration in seconds of each invocation of Future::poll" + ), + buckets: exponential_buckets(0.001, 4.0, 9) + .expect("function parameters are constant and always valid; qed"), + }, + &["task_name"] + )?, registry)?, + poll_start: register(CounterVec::new( + Opts::new( + "tasks_polling_started_total", + "Total number of times we started invoking Future::poll" + ), + &["task_name"] + )?, registry)?, + tasks_spawned: register(CounterVec::new( + Opts::new( + "tasks_spawned_total", + "Total number of tasks that have been spawned on the Service" + ), + &["task_name"] + )?, registry)?, + tasks_ended: register(CounterVec::new( + Opts::new( + "tasks_ended_total", + "Total number of tasks for which Future::poll has returned Ready(())" + ), + &["task_name"] + )?, registry)?, + }) + } +} diff --git a/client/service/src/task_manager/prometheus_future.rs b/client/service/src/task_manager/prometheus_future.rs new file mode 100644 index 0000000000000000000000000000000000000000..53bd59aa7a507ed52e240c4f44f718efa1567c6c --- /dev/null +++ b/client/service/src/task_manager/prometheus_future.rs @@ -0,0 +1,69 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +//! Wrapper around a `Future` that reports statistics about when the `Future` is polled. + +use futures::prelude::*; +use prometheus_endpoint::{Counter, Histogram, U64}; +use std::{fmt, pin::Pin, task::{Context, Poll}}; + +/// Wraps around a `Future`. Report the polling duration to the `Histogram` and when the polling +/// starts to the `Counter`. +pub fn with_poll_durations( + poll_duration: Histogram, + poll_start: Counter, + inner: T +) -> PrometheusFuture { + PrometheusFuture { + inner, + poll_duration, + poll_start, + } +} + +/// Wraps around `Future` and adds diagnostics to it. +#[pin_project::pin_project] +#[derive(Clone)] +pub struct PrometheusFuture { + /// The inner future doing the actual work. + #[pin] + inner: T, + poll_duration: Histogram, + poll_start: Counter, +} + +impl Future for PrometheusFuture +where + T: Future, +{ + type Output = T::Output; + + fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { + let this = self.project(); + + this.poll_start.inc(); + let _timer = this.poll_duration.start_timer(); + Future::poll(this.inner, cx) + + // `_timer` is dropped here and will observe the duration + } +} + +impl fmt::Debug for PrometheusFuture +where + T: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self.inner, f) + } +} diff --git a/client/service/test/Cargo.toml b/client/service/test/Cargo.toml index 39c17420bf571fc7688b728999f3e302bf6a7f23..0a270a8eac55705021bdc11edcab6fe7d29ca924 100644 --- a/client/service/test/Cargo.toml +++ b/client/service/test/Cargo.toml @@ -9,20 +9,35 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" [dependencies] +hex-literal = "0.2.1" tempfile = "3.1.0" tokio = "0.1.22" futures01 = { package = "futures", version = "0.1.29" } log = "0.4.8" env_logger = "0.7.0" fdlimit = "0.1.4" +parking_lot = "0.10.0" +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } +sp-api = { version = "2.0.0-dev", path = "../../../primitives/api" } +sp-state-machine = { version = "0.8.0-dev", path = "../../../primitives/state-machine" } +sp-externalities = { version = "0.8.0-dev", path = "../../../primitives/externalities" } +sp-trie = { version = "2.0.0-dev", path = "../../../primitives/trie" } +sp-storage = { version = "2.0.0-dev", path = "../../../primitives/storage" } +sc-client-db = { version = "0.8.0-dev", default-features = false, path = "../../db" } futures = { version = "0.3.1", features = ["compat"] } -sc-service = { version = "0.8.0-alpha.5", default-features = false, path = "../../service" } -sc-network = { version = "0.8.0-alpha.5", path = "../../network" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } -sc-client = { version = "0.8.0-alpha.5", path = "../../" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../primitives/transaction-pool" } +sc-service = { version = "0.8.0-dev", default-features = false, features = ["test-helpers"], path = "../../service" } +sc-network = { version = "0.8.0-dev", path = "../../network" } +sp-consensus = { version = "0.8.0-dev", path = "../../../primitives/consensus/common" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-transaction-pool = { version = "2.0.0-dev", path = "../../../primitives/transaction-pool" } +substrate-test-runtime = { version = "2.0.0-dev", path = "../../../test-utils/runtime" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } +sc-client-api = { version = "2.0.0-dev", path = "../../api" } +sc-block-builder = { version = "0.8.0-dev", path = "../../block-builder" } +sc-executor = { version = "0.8.0-dev", path = "../../executor" } +sp-panic-handler = { version = "2.0.0-dev", path = "../../../primitives/panic-handler" } +parity-scale-codec = "1.3.0" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/service/test/src/client/db.rs b/client/service/test/src/client/db.rs new file mode 100644 index 0000000000000000000000000000000000000000..bc175652c9f79f36fd3b67c18b96beb68781dedb --- /dev/null +++ b/client/service/test/src/client/db.rs @@ -0,0 +1,55 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use sp_core::offchain::{OffchainStorage, storage::InMemOffchainStorage}; +use std::sync::Arc; + +type TestBackend = sc_client_api::in_mem::Backend; + +#[test] +fn test_leaves_with_complex_block_tree() { + let backend = Arc::new(TestBackend::new()); + + substrate_test_runtime_client::trait_tests::test_leaves_for_backend(backend); +} + +#[test] +fn test_blockchain_query_by_number_gets_canonical() { + let backend = Arc::new(TestBackend::new()); + + substrate_test_runtime_client::trait_tests::test_blockchain_query_by_number_gets_canonical(backend); +} + +#[test] +fn in_memory_offchain_storage() { + + let mut storage = InMemOffchainStorage::default(); + assert_eq!(storage.get(b"A", b"B"), None); + assert_eq!(storage.get(b"B", b"A"), None); + + storage.set(b"A", b"B", b"C"); + assert_eq!(storage.get(b"A", b"B"), Some(b"C".to_vec())); + assert_eq!(storage.get(b"B", b"A"), None); + + storage.compare_and_set(b"A", b"B", Some(b"X"), b"D"); + assert_eq!(storage.get(b"A", b"B"), Some(b"C".to_vec())); + storage.compare_and_set(b"A", b"B", Some(b"C"), b"D"); + assert_eq!(storage.get(b"A", b"B"), Some(b"D".to_vec())); + + assert!(!storage.compare_and_set(b"B", b"A", Some(b""), b"Y")); + assert!(storage.compare_and_set(b"B", b"A", None, b"X")); + assert_eq!(storage.get(b"B", b"A"), Some(b"X".to_vec())); +} \ No newline at end of file diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs new file mode 100644 index 0000000000000000000000000000000000000000..76e48828ee429c1128b6d7b448d1d3d95fd8c817 --- /dev/null +++ b/client/service/test/src/client/light.rs @@ -0,0 +1,896 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . +use sc_service::client::light::{ + call_executor::{ + GenesisCallExecutor, + check_execution_proof, + check_execution_proof_with_make_header, + }, + fetcher::LightDataChecker, + blockchain::{BlockchainCache, Blockchain}, + backend::{Backend, GenesisOrUnavailableState}, +}; +use std::sync::Arc; +use sp_runtime::{ + traits::{BlakeTwo256, HashFor, NumberFor}, + generic::BlockId, traits::{Block as _, Header as HeaderT}, Digest, +}; +use std::collections::HashMap; +use parking_lot::Mutex; +use substrate_test_runtime_client::{ + runtime::{Hash, Block, Header}, TestClient, ClientBlockImportExt, +}; +use sp_api::{InitializeBlock, StorageTransactionCache, ProofRecorder, OffchainOverlayedChanges}; +use sp_consensus::{BlockOrigin}; +use sc_executor::{NativeExecutor, WasmExecutionMethod, RuntimeVersion, NativeVersion}; +use sp_core::{H256, tasks::executor as tasks_executor, NativeOrEncoded}; +use sc_client_api::{blockchain::Info, backend::NewBlockState, Backend as ClientBackend, ProofProvider, in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, AuxStore, Storage, CallExecutor, cht, ExecutionStrategy, StorageProof, BlockImportOperation, RemoteCallRequest, StorageProvider, ChangesProof, RemoteBodyRequest, RemoteReadRequest, RemoteChangesRequest, FetchChecker, RemoteReadChildRequest, RemoteHeaderRequest}; +use sp_externalities::Extensions; +use sc_block_builder::BlockBuilderProvider; +use sp_blockchain::{ + BlockStatus, Result as ClientResult, Error as ClientError, CachedHeaderMetadata, + HeaderBackend, well_known_cache_keys +}; +use std::panic::UnwindSafe; +use std::cell::RefCell; +use sp_state_machine::{OverlayedChanges, ExecutionManager}; +use parity_scale_codec::{Decode, Encode}; +use super::prepare_client_with_key_changes; +use substrate_test_runtime_client::{ + AccountKeyring, runtime::{self, Extrinsic}, +}; + +use sp_core::{blake2_256, ChangesTrieConfiguration, storage::{well_known_keys, StorageKey, ChildInfo}}; +use sp_state_machine::Backend as _; + +pub type DummyBlockchain = Blockchain; + +pub struct DummyStorage { + pub changes_tries_cht_roots: HashMap, + pub aux_store: Mutex, Vec>>, +} + +impl DummyStorage { + pub fn new() -> Self { + DummyStorage { + changes_tries_cht_roots: HashMap::new(), + aux_store: Mutex::new(HashMap::new()), + } + } +} + +impl sp_blockchain::HeaderBackend for DummyStorage { + fn header(&self, _id: BlockId) -> ClientResult> { + Err(ClientError::Backend("Test error".into())) + } + + fn info(&self) -> Info { + panic!("Test error") + } + + fn status(&self, _id: BlockId) -> ClientResult { + Err(ClientError::Backend("Test error".into())) + } + + fn number(&self, hash: Hash) -> ClientResult>> { + if hash == Default::default() { + Ok(Some(Default::default())) + } else { + Err(ClientError::Backend("Test error".into())) + } + } + + fn hash(&self, number: u64) -> ClientResult> { + if number == 0 { + Ok(Some(Default::default())) + } else { + Err(ClientError::Backend("Test error".into())) + } + } +} + +impl sp_blockchain::HeaderMetadata for DummyStorage { + type Error = ClientError; + + fn header_metadata(&self, hash: Hash) -> Result, Self::Error> { + self.header(BlockId::hash(hash))?.map(|header| CachedHeaderMetadata::from(&header)) + .ok_or(ClientError::UnknownBlock("header not found".to_owned())) + } + fn insert_header_metadata(&self, _hash: Hash, _metadata: CachedHeaderMetadata) {} + fn remove_header_metadata(&self, _hash: Hash) {} +} + +impl AuxStore for DummyStorage { + fn insert_aux< + 'a, + 'b: 'a, + 'c: 'a, + I: IntoIterator, + D: IntoIterator, + >(&self, insert: I, _delete: D) -> ClientResult<()> { + for (k, v) in insert.into_iter() { + self.aux_store.lock().insert(k.to_vec(), v.to_vec()); + } + Ok(()) + } + + fn get_aux(&self, key: &[u8]) -> ClientResult>> { + Ok(self.aux_store.lock().get(key).cloned()) + } +} + +impl Storage for DummyStorage { + fn import_header( + &self, + _header: Header, + _cache: HashMap>, + _state: NewBlockState, + _aux_ops: Vec<(Vec, Option>)>, + ) -> ClientResult<()> { + Ok(()) + } + + fn set_head(&self, _block: BlockId) -> ClientResult<()> { + Err(ClientError::Backend("Test error".into())) + } + + fn finalize_header(&self, _block: BlockId) -> ClientResult<()> { + Err(ClientError::Backend("Test error".into())) + } + + fn last_finalized(&self) -> ClientResult { + Err(ClientError::Backend("Test error".into())) + } + + fn header_cht_root(&self, _cht_size: u64, _block: u64) -> ClientResult> { + Err(ClientError::Backend("Test error".into())) + } + + fn changes_trie_cht_root(&self, cht_size: u64, block: u64) -> ClientResult> { + cht::block_to_cht_number(cht_size, block) + .and_then(|cht_num| self.changes_tries_cht_roots.get(&cht_num)) + .cloned() + .ok_or_else(|| ClientError::Backend( + format!("Test error: CHT for block #{} not found", block) + ).into()) + .map(Some) + } + + fn cache(&self) -> Option>> { + None + } + + fn usage_info(&self) -> Option { + None + } +} + +struct DummyCallExecutor; + +impl CallExecutor for DummyCallExecutor { + type Error = ClientError; + + type Backend = substrate_test_runtime_client::Backend; + + fn call( + &self, + _id: &BlockId, + _method: &str, + _call_data: &[u8], + _strategy: ExecutionStrategy, + _extensions: Option, + ) -> Result, ClientError> { + Ok(vec![42]) + } + + fn contextual_call< + 'a, + IB: Fn() -> ClientResult<()>, + EM: Fn( + Result, Self::Error>, + Result, Self::Error> + ) -> Result, Self::Error>, + R: Encode + Decode + PartialEq, + NC: FnOnce() -> Result + UnwindSafe, + >( + &self, + _initialize_block_fn: IB, + _at: &BlockId, + _method: &str, + _call_data: &[u8], + _changes: &RefCell, + _offchain_changes: &RefCell, + _storage_transaction_cache: Option<&RefCell< + StorageTransactionCache< + Block, + >::State, + > + >>, + _initialize_block: InitializeBlock<'a, Block>, + _execution_manager: ExecutionManager, + _native_call: Option, + _proof_recorder: &Option>, + _extensions: Option, + ) -> ClientResult> where ExecutionManager: Clone { + unreachable!() + } + + fn runtime_version(&self, _id: &BlockId) -> Result { + unreachable!() + } + + fn prove_at_trie_state>>( + &self, + _trie_state: &sp_state_machine::TrieBackend>, + _overlay: &mut OverlayedChanges, + _method: &str, + _call_data: &[u8] + ) -> Result<(Vec, StorageProof), ClientError> { + unreachable!() + } + + fn native_runtime_version(&self) -> Option<&NativeVersion> { + unreachable!() + } +} + +fn local_executor() -> NativeExecutor { + NativeExecutor::new(WasmExecutionMethod::Interpreted, None, 8) +} + +#[test] +fn local_state_is_created_when_genesis_state_is_available() { + let def = Default::default(); + let header0 = substrate_test_runtime_client::runtime::Header::new(0, def, def, def, Default::default()); + + let backend: Backend<_, BlakeTwo256> = Backend::new( + Arc::new(DummyBlockchain::new(DummyStorage::new())), + ); + let mut op = backend.begin_operation().unwrap(); + op.set_block_data(header0, None, None, NewBlockState::Final).unwrap(); + op.reset_storage(Default::default()).unwrap(); + backend.commit_operation(op).unwrap(); + + match backend.state_at(BlockId::Number(0)).unwrap() { + GenesisOrUnavailableState::Genesis(_) => (), + _ => panic!("unexpected state"), + } +} + +#[test] +fn unavailable_state_is_created_when_genesis_state_is_unavailable() { + let backend: Backend<_, BlakeTwo256> = Backend::new( + Arc::new(DummyBlockchain::new(DummyStorage::new())), + ); + + match backend.state_at(BlockId::Number(0)).unwrap() { + GenesisOrUnavailableState::Unavailable => (), + _ => panic!("unexpected state"), + } +} + +#[test] +fn light_aux_store_is_updated_via_non_importing_op() { + let backend = Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new()))); + let mut op = ClientBackend::::begin_operation(&backend).unwrap(); + BlockImportOperation::::insert_aux(&mut op, vec![(vec![1], Some(vec![2]))]).unwrap(); + ClientBackend::::commit_operation(&backend, op).unwrap(); + + assert_eq!(AuxStore::get_aux(&backend, &[1]).unwrap(), Some(vec![2])); +} + +#[test] +fn execution_proof_is_generated_and_checked() { + fn execute(remote_client: &TestClient, at: u64, method: &'static str) -> (Vec, Vec) { + let remote_block_id = BlockId::Number(at); + let remote_header = remote_client.header(&remote_block_id).unwrap().unwrap(); + + // 'fetch' execution proof from remote node + let (remote_result, remote_execution_proof) = remote_client.execution_proof( + &remote_block_id, + method, + &[] + ).unwrap(); + + // check remote execution proof locally + let local_result = check_execution_proof::<_, _, BlakeTwo256>( + &local_executor(), + tasks_executor(), + &RemoteCallRequest { + block: substrate_test_runtime_client::runtime::Hash::default(), + header: remote_header, + method: method.into(), + call_data: vec![], + retry_count: None, + }, + remote_execution_proof, + ).unwrap(); + + (remote_result, local_result) + } + + fn execute_with_proof_failure(remote_client: &TestClient, at: u64, method: &'static str) { + let remote_block_id = BlockId::Number(at); + let remote_header = remote_client.header(&remote_block_id).unwrap().unwrap(); + + // 'fetch' execution proof from remote node + let (_, remote_execution_proof) = remote_client.execution_proof( + &remote_block_id, + method, + &[] + ).unwrap(); + + // check remote execution proof locally + let execution_result = check_execution_proof_with_make_header::<_, _, BlakeTwo256, _>( + &local_executor(), + tasks_executor(), + &RemoteCallRequest { + block: substrate_test_runtime_client::runtime::Hash::default(), + header: remote_header, + method: method.into(), + call_data: vec![], + retry_count: None, + }, + remote_execution_proof, + |header|
::new( + at + 1, + Default::default(), + Default::default(), + header.hash(), + header.digest().clone(), // this makes next header wrong + ), + ); + match execution_result { + Err(sp_blockchain::Error::Execution(_)) => (), + _ => panic!("Unexpected execution result: {:?}", execution_result), + } + } + + // prepare remote client + let mut remote_client = substrate_test_runtime_client::new(); + for i in 1u32..3u32 { + let mut digest = Digest::default(); + digest.push(sp_runtime::generic::DigestItem::Other::(i.to_le_bytes().to_vec())); + remote_client.import_justified( + BlockOrigin::Own, + remote_client.new_block(digest).unwrap().build().unwrap().block, + Default::default(), + ).unwrap(); + } + + // check method that doesn't requires environment + let (remote, local) = execute(&remote_client, 0, "Core_version"); + assert_eq!(remote, local); + + let (remote, local) = execute(&remote_client, 2, "Core_version"); + assert_eq!(remote, local); + + // check method that requires environment + let (_, block) = execute(&remote_client, 0, "BlockBuilder_finalize_block"); + let local_block: Header = Decode::decode(&mut &block[..]).unwrap(); + assert_eq!(local_block.number, 1); + + let (_, block) = execute(&remote_client, 2, "BlockBuilder_finalize_block"); + let local_block: Header = Decode::decode(&mut &block[..]).unwrap(); + assert_eq!(local_block.number, 3); + + // check that proof check doesn't panic even if proof is incorrect AND no panic handler is set + execute_with_proof_failure(&remote_client, 2, "Core_version"); + + // check that proof check doesn't panic even if proof is incorrect AND panic handler is set + sp_panic_handler::set("TEST", "1.2.3"); + execute_with_proof_failure(&remote_client, 2, "Core_version"); +} + +#[test] +fn code_is_executed_at_genesis_only() { + let backend = Arc::new(InMemBackend::::new()); + let def = H256::default(); + let header0 = substrate_test_runtime_client::runtime::Header::new(0, def, def, def, Default::default()); + let hash0 = header0.hash(); + let header1 = substrate_test_runtime_client::runtime::Header::new(1, def, def, hash0, Default::default()); + let hash1 = header1.hash(); + backend.blockchain().insert(hash0, header0, None, None, NewBlockState::Final).unwrap(); + backend.blockchain().insert(hash1, header1, None, None, NewBlockState::Final).unwrap(); + + let genesis_executor = GenesisCallExecutor::new(backend, DummyCallExecutor); + assert_eq!( + genesis_executor.call( + &BlockId::Number(0), + "test_method", + &[], + ExecutionStrategy::NativeElseWasm, + None, + ).unwrap(), + vec![42], + ); + + let call_on_unavailable = genesis_executor.call( + &BlockId::Number(1), + "test_method", + &[], + ExecutionStrategy::NativeElseWasm, + None, + ); + + match call_on_unavailable { + Err(ClientError::NotAvailableOnLightClient) => (), + _ => unreachable!("unexpected result: {:?}", call_on_unavailable), + } +} + + +type TestChecker = LightDataChecker< + NativeExecutor, + BlakeTwo256, + Block, + DummyStorage, +>; + +fn prepare_for_read_proof_check() -> (TestChecker, Header, StorageProof, u32) { + // prepare remote client + let remote_client = substrate_test_runtime_client::new(); + let remote_block_id = BlockId::Number(0); + let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); + let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap(); + remote_block_header.state_root = remote_client.state_at(&remote_block_id).unwrap() + .storage_root(::std::iter::empty()).0.into(); + + // 'fetch' read proof from remote node + let heap_pages = remote_client.storage(&remote_block_id, &StorageKey(well_known_keys::HEAP_PAGES.to_vec())) + .unwrap() + .and_then(|v| Decode::decode(&mut &v.0[..]).ok()).unwrap(); + let remote_read_proof = remote_client.read_proof( + &remote_block_id, + &mut std::iter::once(well_known_keys::HEAP_PAGES), + ).unwrap(); + + // check remote read proof locally + let local_storage = InMemoryBlockchain::::new(); + local_storage.insert( + remote_block_hash, + remote_block_header.clone(), + None, + None, + NewBlockState::Final, + ).unwrap(); + let local_checker = LightDataChecker::new( + Arc::new(DummyBlockchain::new(DummyStorage::new())), + local_executor(), + tasks_executor(), + ); + (local_checker, remote_block_header, remote_read_proof, heap_pages) +} + +fn prepare_for_read_child_proof_check() -> (TestChecker, Header, StorageProof, Vec) { + use substrate_test_runtime_client::DefaultTestClientBuilderExt; + use substrate_test_runtime_client::TestClientBuilderExt; + let child_info = ChildInfo::new_default(b"child1"); + let child_info = &child_info; + // prepare remote client + let remote_client = substrate_test_runtime_client::TestClientBuilder::new() + .add_extra_child_storage( + child_info, + b"key1".to_vec(), + b"value1".to_vec(), + ).build(); + let remote_block_id = BlockId::Number(0); + let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); + let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap(); + remote_block_header.state_root = remote_client.state_at(&remote_block_id).unwrap() + .storage_root(::std::iter::empty()).0.into(); + + // 'fetch' child read proof from remote node + let child_value = remote_client.child_storage( + &remote_block_id, + child_info, + &StorageKey(b"key1".to_vec()), + ).unwrap().unwrap().0; + assert_eq!(b"value1"[..], child_value[..]); + let remote_read_proof = remote_client.read_child_proof( + &remote_block_id, + child_info, + &mut std::iter::once("key1".as_bytes()), + ).unwrap(); + + // check locally + let local_storage = InMemoryBlockchain::::new(); + local_storage.insert( + remote_block_hash, + remote_block_header.clone(), + None, + None, + NewBlockState::Final, + ).unwrap(); + let local_checker = LightDataChecker::new( + Arc::new(DummyBlockchain::new(DummyStorage::new())), + local_executor(), + tasks_executor(), + ); + (local_checker, remote_block_header, remote_read_proof, child_value) +} + +fn prepare_for_header_proof_check(insert_cht: bool) -> (TestChecker, Hash, Header, StorageProof) { + // prepare remote client + let mut remote_client = substrate_test_runtime_client::new(); + let mut local_headers_hashes = Vec::new(); + for i in 0..4 { + let block = remote_client.new_block(Default::default()).unwrap().build().unwrap().block; + remote_client.import(BlockOrigin::Own, block).unwrap(); + local_headers_hashes.push( + remote_client.block_hash(i + 1) + .map_err(|_| ClientError::Backend("TestError".into())) + ); + } + + // 'fetch' header proof from remote node + let remote_block_id = BlockId::Number(1); + let (remote_block_header, remote_header_proof) = remote_client.header_proof_with_cht_size(&remote_block_id, 4).unwrap(); + + // check remote read proof locally + let local_storage = InMemoryBlockchain::::new(); + let local_cht_root = cht::compute_root::(4, 0, local_headers_hashes).unwrap(); + if insert_cht { + local_storage.insert_cht_root(1, local_cht_root); + } + let local_checker = LightDataChecker::new( + Arc::new(DummyBlockchain::new(DummyStorage::new())), + local_executor(), + tasks_executor(), + ); + (local_checker, local_cht_root, remote_block_header, remote_header_proof) +} + +fn header_with_computed_extrinsics_root(extrinsics: Vec) -> Header { + use sp_trie::{TrieConfiguration, trie_types::Layout}; + let iter = extrinsics.iter().map(Encode::encode); + let extrinsics_root = Layout::::ordered_trie_root(iter); + + // only care about `extrinsics_root` + Header::new(0, extrinsics_root, H256::zero(), H256::zero(), Default::default()) +} + +#[test] +fn storage_read_proof_is_generated_and_checked() { + let (local_checker, remote_block_header, remote_read_proof, heap_pages) = prepare_for_read_proof_check(); + assert_eq!((&local_checker as &dyn FetchChecker).check_read_proof(&RemoteReadRequest::
{ + block: remote_block_header.hash(), + header: remote_block_header, + keys: vec![well_known_keys::HEAP_PAGES.to_vec()], + retry_count: None, + }, remote_read_proof).unwrap().remove(well_known_keys::HEAP_PAGES).unwrap().unwrap()[0], heap_pages as u8); +} + +#[test] +fn storage_child_read_proof_is_generated_and_checked() { + let child_info = ChildInfo::new_default(&b"child1"[..]); + let ( + local_checker, + remote_block_header, + remote_read_proof, + result, + ) = prepare_for_read_child_proof_check(); + assert_eq!((&local_checker as &dyn FetchChecker).check_read_child_proof( + &RemoteReadChildRequest::
{ + block: remote_block_header.hash(), + header: remote_block_header, + storage_key: child_info.prefixed_storage_key(), + keys: vec![b"key1".to_vec()], + retry_count: None, + }, + remote_read_proof + ).unwrap().remove(b"key1".as_ref()).unwrap().unwrap(), result); +} + +#[test] +fn header_proof_is_generated_and_checked() { + let (local_checker, local_cht_root, remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true); + assert_eq!((&local_checker as &dyn FetchChecker).check_header_proof(&RemoteHeaderRequest::
{ + cht_root: local_cht_root, + block: 1, + retry_count: None, + }, Some(remote_block_header.clone()), remote_header_proof).unwrap(), remote_block_header); +} + +#[test] +fn check_header_proof_fails_if_cht_root_is_invalid() { + let (local_checker, _, mut remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true); + remote_block_header.number = 100; + assert!((&local_checker as &dyn FetchChecker).check_header_proof(&RemoteHeaderRequest::
{ + cht_root: Default::default(), + block: 1, + retry_count: None, + }, Some(remote_block_header.clone()), remote_header_proof).is_err()); +} + +#[test] +fn check_header_proof_fails_if_invalid_header_provided() { + let (local_checker, local_cht_root, mut remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true); + remote_block_header.number = 100; + assert!((&local_checker as &dyn FetchChecker).check_header_proof(&RemoteHeaderRequest::
{ + cht_root: local_cht_root, + block: 1, + retry_count: None, + }, Some(remote_block_header.clone()), remote_header_proof).is_err()); +} + +#[test] +fn changes_proof_is_generated_and_checked_when_headers_are_not_pruned() { + let (remote_client, local_roots, test_cases) = prepare_client_with_key_changes(); + let local_checker = TestChecker::new( + Arc::new(DummyBlockchain::new(DummyStorage::new())), + local_executor(), + tasks_executor(), + ); + let local_checker = &local_checker as &dyn FetchChecker; + let max = remote_client.chain_info().best_number; + let max_hash = remote_client.chain_info().best_hash; + + for (index, (begin, end, key, expected_result)) in test_cases.into_iter().enumerate() { + let begin_hash = remote_client.block_hash(begin).unwrap().unwrap(); + let end_hash = remote_client.block_hash(end).unwrap().unwrap(); + + // 'fetch' changes proof from remote node + let key = StorageKey(key); + let remote_proof = remote_client.key_changes_proof( + begin_hash, end_hash, begin_hash, max_hash, None, &key + ).unwrap(); + + // check proof on local client + let local_roots_range = local_roots.clone()[(begin - 1) as usize..].to_vec(); + let config = ChangesTrieConfiguration::new(4, 2); + let request = RemoteChangesRequest::
{ + changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { + zero: (0, Default::default()), + end: None, + config: Some(config), + }], + first_block: (begin, begin_hash), + last_block: (end, end_hash), + max_block: (max, max_hash), + tries_roots: (begin, begin_hash, local_roots_range), + key: key.0, + storage_key: None, + retry_count: None, + }; + let local_result = local_checker.check_changes_proof(&request, ChangesProof { + max_block: remote_proof.max_block, + proof: remote_proof.proof, + roots: remote_proof.roots, + roots_proof: remote_proof.roots_proof, + }).unwrap(); + + // ..and ensure that result is the same as on remote node + match local_result == expected_result { + true => (), + false => panic!(format!("Failed test {}: local = {:?}, expected = {:?}", + index, local_result, expected_result)), + } + } +} + +#[test] +fn changes_proof_is_generated_and_checked_when_headers_are_pruned() { + // we're testing this test case here: + // (1, 4, dave.clone(), vec![(4, 0), (1, 1), (1, 0)]), + let (remote_client, remote_roots, _) = prepare_client_with_key_changes(); + let dave = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec(); + let dave = StorageKey(dave); + + // 'fetch' changes proof from remote node: + // we're fetching changes for range b1..b4 + // we do not know changes trie roots before b3 (i.e. we only know b3+b4) + // but we have changes trie CHT root for b1...b4 + let b1 = remote_client.block_hash_from_id(&BlockId::Number(1)).unwrap().unwrap(); + let b3 = remote_client.block_hash_from_id(&BlockId::Number(3)).unwrap().unwrap(); + let b4 = remote_client.block_hash_from_id(&BlockId::Number(4)).unwrap().unwrap(); + let remote_proof = remote_client.key_changes_proof_with_cht_size( + b1, b4, b3, b4, None, &dave, 4 + ).unwrap(); + + // prepare local checker, having a root of changes trie CHT#0 + let local_cht_root = cht::compute_root::(4, 0, remote_roots.iter().cloned().map(|ct| Ok(Some(ct)))).unwrap(); + let mut local_storage = DummyStorage::new(); + local_storage.changes_tries_cht_roots.insert(0, local_cht_root); + let local_checker = TestChecker::new( + Arc::new(DummyBlockchain::new(local_storage)), + local_executor(), + tasks_executor(), + ); + + // check proof on local client + let config = ChangesTrieConfiguration::new(4, 2); + let request = RemoteChangesRequest::
{ + changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { + zero: (0, Default::default()), + end: None, + config: Some(config), + }], + first_block: (1, b1), + last_block: (4, b4), + max_block: (4, b4), + tries_roots: (3, b3, vec![remote_roots[2].clone(), remote_roots[3].clone()]), + storage_key: None, + key: dave.0, + retry_count: None, + }; + let local_result = local_checker.check_changes_proof_with_cht_size(&request, ChangesProof { + max_block: remote_proof.max_block, + proof: remote_proof.proof, + roots: remote_proof.roots, + roots_proof: remote_proof.roots_proof, + }, 4).unwrap(); + + assert_eq!(local_result, vec![(4, 0), (1, 1), (1, 0)]); +} + +#[test] +fn check_changes_proof_fails_if_proof_is_wrong() { + let (remote_client, local_roots, test_cases) = prepare_client_with_key_changes(); + let local_checker = TestChecker::new( + Arc::new(DummyBlockchain::new(DummyStorage::new())), + local_executor(), + tasks_executor(), + ); + let local_checker = &local_checker as &dyn FetchChecker; + let max = remote_client.chain_info().best_number; + let max_hash = remote_client.chain_info().best_hash; + + let (begin, end, key, _) = test_cases[0].clone(); + let begin_hash = remote_client.block_hash(begin).unwrap().unwrap(); + let end_hash = remote_client.block_hash(end).unwrap().unwrap(); + + // 'fetch' changes proof from remote node + let key = StorageKey(key); + let remote_proof = remote_client.key_changes_proof( + begin_hash, end_hash, begin_hash, max_hash, None, &key).unwrap(); + + let local_roots_range = local_roots.clone()[(begin - 1) as usize..].to_vec(); + let config = ChangesTrieConfiguration::new(4, 2); + let request = RemoteChangesRequest::
{ + changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { + zero: (0, Default::default()), + end: None, + config: Some(config), + }], + first_block: (begin, begin_hash), + last_block: (end, end_hash), + max_block: (max, max_hash), + tries_roots: (begin, begin_hash, local_roots_range.clone()), + storage_key: None, + key: key.0, + retry_count: None, + }; + + // check proof on local client using max from the future + assert!(local_checker.check_changes_proof(&request, ChangesProof { + max_block: remote_proof.max_block + 1, + proof: remote_proof.proof.clone(), + roots: remote_proof.roots.clone(), + roots_proof: remote_proof.roots_proof.clone(), + }).is_err()); + + // check proof on local client using broken proof + assert!(local_checker.check_changes_proof(&request, ChangesProof { + max_block: remote_proof.max_block, + proof: local_roots_range.clone().into_iter().map(|v| v.as_ref().to_vec()).collect(), + roots: remote_proof.roots, + roots_proof: remote_proof.roots_proof, + }).is_err()); + + // extra roots proofs are provided + assert!(local_checker.check_changes_proof(&request, ChangesProof { + max_block: remote_proof.max_block, + proof: remote_proof.proof.clone(), + roots: vec![(begin - 1, Default::default())].into_iter().collect(), + roots_proof: StorageProof::empty(), + }).is_err()); + assert!(local_checker.check_changes_proof(&request, ChangesProof { + max_block: remote_proof.max_block, + proof: remote_proof.proof.clone(), + roots: vec![(end + 1, Default::default())].into_iter().collect(), + roots_proof: StorageProof::empty(), + }).is_err()); +} + +#[test] +fn check_changes_tries_proof_fails_if_proof_is_wrong() { + // we're testing this test case here: + // (1, 4, dave.clone(), vec![(4, 0), (1, 1), (1, 0)]), + let (remote_client, remote_roots, _) = prepare_client_with_key_changes(); + let local_cht_root = cht::compute_root::( + 4, 0, remote_roots.iter().cloned().map(|ct| Ok(Some(ct)))).unwrap(); + let dave = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec(); + let dave = StorageKey(dave); + + // 'fetch' changes proof from remote node: + // we're fetching changes for range b1..b4 + // we do not know changes trie roots before b3 (i.e. we only know b3+b4) + // but we have changes trie CHT root for b1...b4 + let b1 = remote_client.block_hash_from_id(&BlockId::Number(1)).unwrap().unwrap(); + let b3 = remote_client.block_hash_from_id(&BlockId::Number(3)).unwrap().unwrap(); + let b4 = remote_client.block_hash_from_id(&BlockId::Number(4)).unwrap().unwrap(); + let remote_proof = remote_client.key_changes_proof_with_cht_size( + b1, b4, b3, b4, None, &dave, 4 + ).unwrap(); + + // fails when changes trie CHT is missing from the local db + let local_checker = TestChecker::new( + Arc::new(DummyBlockchain::new(DummyStorage::new())), + local_executor(), + tasks_executor(), + ); + assert!(local_checker.check_changes_tries_proof(4, &remote_proof.roots, + remote_proof.roots_proof.clone()).is_err()); + + // fails when proof is broken + let mut local_storage = DummyStorage::new(); + local_storage.changes_tries_cht_roots.insert(0, local_cht_root); + let local_checker = TestChecker::new( + Arc::new(DummyBlockchain::new(local_storage)), + local_executor(), + tasks_executor(), + ); + let result = local_checker.check_changes_tries_proof( + 4, &remote_proof.roots, StorageProof::empty() + ); + assert!(result.is_err()); +} + +#[test] +fn check_body_proof_faulty() { + let header = header_with_computed_extrinsics_root( + vec![Extrinsic::IncludeData(vec![1, 2, 3, 4])] + ); + let block = Block::new(header.clone(), Vec::new()); + + let local_checker = TestChecker::new( + Arc::new(DummyBlockchain::new(DummyStorage::new())), + local_executor(), + tasks_executor(), + ); + + let body_request = RemoteBodyRequest { + header: header.clone(), + retry_count: None, + }; + + assert!( + local_checker.check_body_proof(&body_request, block.extrinsics).is_err(), + "vec![1, 2, 3, 4] != vec![]" + ); +} + +#[test] +fn check_body_proof_of_same_data_should_succeed() { + let extrinsics = vec![Extrinsic::IncludeData(vec![1, 2, 3, 4, 5, 6, 7, 8, 255])]; + + let header = header_with_computed_extrinsics_root(extrinsics.clone()); + let block = Block::new(header.clone(), extrinsics); + + let local_checker = TestChecker::new( + Arc::new(DummyBlockchain::new(DummyStorage::new())), + local_executor(), + tasks_executor(), + ); + + let body_request = RemoteBodyRequest { + header: header.clone(), + retry_count: None, + }; + + assert!(local_checker.check_body_proof(&body_request, block.extrinsics).is_ok()); +} diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..e81d1ebb5364bc404cec0bb80539a0c385b77ee7 --- /dev/null +++ b/client/service/test/src/client/mod.rs @@ -0,0 +1,1802 @@ +// Copyright 2018-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use parity_scale_codec::{Encode, Decode, Joiner}; +use sc_executor::native_executor_instance; +use sp_state_machine::{StateMachine, OverlayedChanges, ExecutionStrategy, InMemoryBackend}; +use substrate_test_runtime_client::{ + prelude::*, + runtime::{ + self, genesismap::{GenesisConfig, insert_genesis_block}, + Hash, Transfer, Block, BlockNumber, Header, Digest, RuntimeApi, + }, + AccountKeyring, Sr25519Keyring, TestClientBuilder, ClientBlockImportExt, + BlockBuilderExt, DefaultTestClientBuilderExt, TestClientBuilderExt, ClientExt, +}; +use sc_client_api::{ + StorageProvider, BlockBackend, in_mem, BlockchainEvents, +}; +use sc_client_db::{Backend, DatabaseSettings, DatabaseSettingsSrc, PruningMode}; +use sc_block_builder::BlockBuilderProvider; +use sc_service::client::{self, Client, LocalCallExecutor, new_in_mem}; +use sp_runtime::traits::{ + BlakeTwo256, Block as BlockT, Header as HeaderT, +}; +use substrate_test_runtime::TestAPI; +use sp_state_machine::backend::Backend as _; +use sp_api::{ProvideRuntimeApi, OffchainOverlayedChanges}; +use sp_core::tasks::executor as tasks_executor; +use sp_core::{H256, ChangesTrieConfiguration, blake2_256}; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use sp_consensus::{ + BlockOrigin, SelectChain, BlockImport, Error as ConsensusError, BlockCheckParams, ImportResult, + BlockStatus, BlockImportParams, ForkChoiceStrategy, +}; +use sp_storage::StorageKey; +use sp_trie::{TrieConfiguration, trie_types::Layout}; +use sp_runtime::{generic::BlockId, DigestItem}; +use hex_literal::hex; + +mod light; +mod db; + +native_executor_instance!( + Executor, + substrate_test_runtime_client::runtime::api::dispatch, + substrate_test_runtime_client::runtime::native_version, +); + +fn executor() -> sc_executor::NativeExecutor { + sc_executor::NativeExecutor::new( + sc_executor::WasmExecutionMethod::Interpreted, + None, + 8, + ) +} + +pub fn prepare_client_with_key_changes() -> ( + client::Client< + substrate_test_runtime_client::Backend, + substrate_test_runtime_client::Executor, + Block, + RuntimeApi + >, + Vec, + Vec<(u64, u64, Vec, Vec<(u64, u32)>)>, +) { + // prepare block structure + let blocks_transfers = vec![ + vec![(AccountKeyring::Alice, AccountKeyring::Dave), (AccountKeyring::Bob, AccountKeyring::Dave)], + vec![(AccountKeyring::Charlie, AccountKeyring::Eve)], + vec![], + vec![(AccountKeyring::Alice, AccountKeyring::Dave)], + ]; + + // prepare client ang import blocks + let mut local_roots = Vec::new(); + let config = Some(ChangesTrieConfiguration::new(4, 2)); + let mut remote_client = TestClientBuilder::new().changes_trie_config(config).build(); + let mut nonces: HashMap<_, u64> = Default::default(); + for (i, block_transfers) in blocks_transfers.into_iter().enumerate() { + let mut builder = remote_client.new_block(Default::default()).unwrap(); + for (from, to) in block_transfers { + builder.push_transfer(Transfer { + from: from.into(), + to: to.into(), + amount: 1, + nonce: *nonces.entry(from).and_modify(|n| { *n = *n + 1 }).or_default(), + }).unwrap(); + } + let block = builder.build().unwrap().block; + remote_client.import(BlockOrigin::Own, block).unwrap(); + + let header = remote_client.header(&BlockId::Number(i as u64 + 1)).unwrap().unwrap(); + let trie_root = header.digest().log(DigestItem::as_changes_trie_root) + .map(|root| H256::from_slice(root.as_ref())) + .unwrap(); + local_roots.push(trie_root); + } + + // prepare test cases + let alice = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Alice.into())).to_vec(); + let bob = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Bob.into())).to_vec(); + let charlie = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Charlie.into())).to_vec(); + let dave = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec(); + let eve = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Eve.into())).to_vec(); + let ferdie = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Ferdie.into())).to_vec(); + let test_cases = vec![ + (1, 4, alice.clone(), vec![(4, 0), (1, 0)]), + (1, 3, alice.clone(), vec![(1, 0)]), + (2, 4, alice.clone(), vec![(4, 0)]), + (2, 3, alice.clone(), vec![]), + (1, 4, bob.clone(), vec![(1, 1)]), + (1, 1, bob.clone(), vec![(1, 1)]), + (2, 4, bob.clone(), vec![]), + (1, 4, charlie.clone(), vec![(2, 0)]), + (1, 4, dave.clone(), vec![(4, 0), (1, 1), (1, 0)]), + (1, 1, dave.clone(), vec![(1, 1), (1, 0)]), + (3, 4, dave.clone(), vec![(4, 0)]), + (1, 4, eve.clone(), vec![(2, 0)]), + (1, 1, eve.clone(), vec![]), + (3, 4, eve.clone(), vec![]), + (1, 4, ferdie.clone(), vec![]), + ]; + + (remote_client, local_roots, test_cases) +} + +fn construct_block( + backend: &InMemoryBackend, + number: BlockNumber, + parent_hash: Hash, + state_root: Hash, + txs: Vec, +) -> (Vec, Hash) { + let transactions = txs.into_iter().map(|tx| tx.into_signed_tx()).collect::>(); + + let iter = transactions.iter().map(Encode::encode); + let extrinsics_root = Layout::::ordered_trie_root(iter).into(); + + let mut header = Header { + parent_hash, + number, + state_root, + extrinsics_root, + digest: Digest { logs: vec![] }, + }; + let hash = header.hash(); + let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); + let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); + let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); + + StateMachine::new( + backend, + sp_state_machine::disabled_changes_trie_state::<_, u64>(), + &mut overlay, + &mut offchain_overlay, + &executor(), + "Core_initialize_block", + &header.encode(), + Default::default(), + &runtime_code, + tasks_executor(), + ).execute( + ExecutionStrategy::NativeElseWasm, + ).unwrap(); + + for tx in transactions.iter() { + StateMachine::new( + backend, + sp_state_machine::disabled_changes_trie_state::<_, u64>(), + &mut overlay, + &mut offchain_overlay, + &executor(), + "BlockBuilder_apply_extrinsic", + &tx.encode(), + Default::default(), + &runtime_code, + tasks_executor(), + ).execute( + ExecutionStrategy::NativeElseWasm, + ).unwrap(); + } + + let ret_data = StateMachine::new( + backend, + sp_state_machine::disabled_changes_trie_state::<_, u64>(), + &mut overlay, + &mut offchain_overlay, + &executor(), + "BlockBuilder_finalize_block", + &[], + Default::default(), + &runtime_code, + tasks_executor(), + ).execute( + ExecutionStrategy::NativeElseWasm, + ).unwrap(); + header = Header::decode(&mut &ret_data[..]).unwrap(); + + (vec![].and(&Block { header, extrinsics: transactions }), hash) +} + +fn block1(genesis_hash: Hash, backend: &InMemoryBackend) -> (Vec, Hash) { + construct_block( + backend, + 1, + genesis_hash, + hex!("25e5b37074063ab75c889326246640729b40d0c86932edc527bc80db0e04fe5c").into(), + vec![Transfer { + from: AccountKeyring::One.into(), + to: AccountKeyring::Two.into(), + amount: 69, + nonce: 0, + }], + ) +} + +#[test] +fn construct_genesis_should_work_with_native() { + let mut storage = GenesisConfig::new( + None, + vec![Sr25519Keyring::One.public().into(), Sr25519Keyring::Two.public().into()], + vec![AccountKeyring::One.into(), AccountKeyring::Two.into()], + 1000, + None, + Default::default(), + ).genesis_map(); + let genesis_hash = insert_genesis_block(&mut storage); + + let backend = InMemoryBackend::from(storage); + let (b1data, _b1hash) = block1(genesis_hash, &backend); + let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); + let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); + + let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); + + let _ = StateMachine::new( + &backend, + sp_state_machine::disabled_changes_trie_state::<_, u64>(), + &mut overlay, + &mut offchain_overlay, + &executor(), + "Core_execute_block", + &b1data, + Default::default(), + &runtime_code, + tasks_executor(), + ).execute( + ExecutionStrategy::NativeElseWasm, + ).unwrap(); +} + +#[test] +fn construct_genesis_should_work_with_wasm() { + let mut storage = GenesisConfig::new( + None, + vec![Sr25519Keyring::One.public().into(), Sr25519Keyring::Two.public().into()], + vec![AccountKeyring::One.into(), AccountKeyring::Two.into()], + 1000, + None, + Default::default(), + ).genesis_map(); + let genesis_hash = insert_genesis_block(&mut storage); + + let backend = InMemoryBackend::from(storage); + let (b1data, _b1hash) = block1(genesis_hash, &backend); + let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); + let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); + + let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); + + let _ = StateMachine::new( + &backend, + sp_state_machine::disabled_changes_trie_state::<_, u64>(), + &mut overlay, + &mut offchain_overlay, + &executor(), + "Core_execute_block", + &b1data, + Default::default(), + &runtime_code, + tasks_executor(), + ).execute( + ExecutionStrategy::AlwaysWasm, + ).unwrap(); +} + +#[test] +fn construct_genesis_with_bad_transaction_should_panic() { + let mut storage = GenesisConfig::new( + None, + vec![Sr25519Keyring::One.public().into(), Sr25519Keyring::Two.public().into()], + vec![AccountKeyring::One.into(), AccountKeyring::Two.into()], + 68, + None, + Default::default(), + ).genesis_map(); + let genesis_hash = insert_genesis_block(&mut storage); + + let backend = InMemoryBackend::from(storage); + let (b1data, _b1hash) = block1(genesis_hash, &backend); + let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); + let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); + + let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); + + let r = StateMachine::new( + &backend, + sp_state_machine::disabled_changes_trie_state::<_, u64>(), + &mut overlay, + &mut offchain_overlay, + &executor(), + "Core_execute_block", + &b1data, + Default::default(), + &runtime_code, + tasks_executor(), + ).execute( + ExecutionStrategy::NativeElseWasm, + ); + assert!(r.is_err()); +} + + +#[test] +fn client_initializes_from_genesis_ok() { + let client = substrate_test_runtime_client::new(); + + assert_eq!( + client.runtime_api().balance_of( + &BlockId::Number(client.chain_info().best_number), + AccountKeyring::Alice.into(), + ).unwrap(), + 1000 + ); + assert_eq!( + client.runtime_api().balance_of( + &BlockId::Number(client.chain_info().best_number), + AccountKeyring::Ferdie.into(), + ).unwrap(), + 0 + ); +} + +#[test] +fn block_builder_works_with_no_transactions() { + let mut client = substrate_test_runtime_client::new(); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + + client.import(BlockOrigin::Own, block).unwrap(); + + assert_eq!(client.chain_info().best_number, 1); +} + +#[test] +fn block_builder_works_with_transactions() { + let mut client = substrate_test_runtime_client::new(); + + let mut builder = client.new_block(Default::default()).unwrap(); + + builder.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 42, + nonce: 0, + }).unwrap(); + + let block = builder.build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + + assert_eq!(client.chain_info().best_number, 1); + assert_ne!( + client.state_at(&BlockId::Number(1)).unwrap().pairs(), + client.state_at(&BlockId::Number(0)).unwrap().pairs() + ); + assert_eq!( + client.runtime_api().balance_of( + &BlockId::Number(client.chain_info().best_number), + AccountKeyring::Alice.into(), + ).unwrap(), + 958 + ); + assert_eq!( + client.runtime_api().balance_of( + &BlockId::Number(client.chain_info().best_number), + AccountKeyring::Ferdie.into(), + ).unwrap(), + 42 + ); +} + +#[test] +fn block_builder_does_not_include_invalid() { + let mut client = substrate_test_runtime_client::new(); + + let mut builder = client.new_block(Default::default()).unwrap(); + + builder.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 42, + nonce: 0, + }).unwrap(); + + assert!( + builder.push_transfer(Transfer { + from: AccountKeyring::Eve.into(), + to: AccountKeyring::Alice.into(), + amount: 42, + nonce: 0, + }).is_err() + ); + + let block = builder.build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + + assert_eq!(client.chain_info().best_number, 1); + assert_ne!( + client.state_at(&BlockId::Number(1)).unwrap().pairs(), + client.state_at(&BlockId::Number(0)).unwrap().pairs() + ); + assert_eq!(client.body(&BlockId::Number(1)).unwrap().unwrap().len(), 1) +} + +#[test] +fn best_containing_with_genesis_block() { + // block tree: + // G + + let (client, longest_chain_select) = TestClientBuilder::new().build_with_longest_chain(); + + let genesis_hash = client.chain_info().genesis_hash; + + assert_eq!( + genesis_hash.clone(), + longest_chain_select.finality_target(genesis_hash.clone(), None).unwrap().unwrap() + ); +} + +#[test] +fn best_containing_with_hash_not_found() { + // block tree: + // G + + let (client, longest_chain_select) = TestClientBuilder::new().build_with_longest_chain(); + + let uninserted_block = client.new_block(Default::default()).unwrap().build().unwrap().block; + + assert_eq!( + None, + longest_chain_select.finality_target(uninserted_block.hash().clone(), None).unwrap() + ); +} + +#[test] +fn uncles_with_only_ancestors() { + // block tree: + // G -> A1 -> A2 + let mut client = substrate_test_runtime_client::new(); + + // G -> A1 + let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a1.clone()).unwrap(); + + // A1 -> A2 + let a2 = client.new_block(Default::default()).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a2.clone()).unwrap(); + let v: Vec = Vec::new(); + assert_eq!(v, client.uncles(a2.hash(), 3).unwrap()); +} + +#[test] +fn uncles_with_multiple_forks() { + // block tree: + // G -> A1 -> A2 -> A3 -> A4 -> A5 + // A1 -> B2 -> B3 -> B4 + // B2 -> C3 + // A1 -> D2 + let mut client = substrate_test_runtime_client::new(); + + // G -> A1 + let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a1.clone()).unwrap(); + + // A1 -> A2 + let a2 = client.new_block_at( + &BlockId::Hash(a1.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a2.clone()).unwrap(); + + // A2 -> A3 + let a3 = client.new_block_at( + &BlockId::Hash(a2.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a3.clone()).unwrap(); + + // A3 -> A4 + let a4 = client.new_block_at( + &BlockId::Hash(a3.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a4.clone()).unwrap(); + + // A4 -> A5 + let a5 = client.new_block_at( + &BlockId::Hash(a4.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a5.clone()).unwrap(); + + // A1 -> B2 + let mut builder = client.new_block_at( + &BlockId::Hash(a1.hash()), + Default::default(), + false, + ).unwrap(); + // this push is required as otherwise B2 has the same hash as A2 and won't get imported + builder.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 41, + nonce: 0, + }).unwrap(); + let b2 = builder.build().unwrap().block; + client.import(BlockOrigin::Own, b2.clone()).unwrap(); + + // B2 -> B3 + let b3 = client.new_block_at( + &BlockId::Hash(b2.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, b3.clone()).unwrap(); + + // B3 -> B4 + let b4 = client.new_block_at( + &BlockId::Hash(b3.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, b4.clone()).unwrap(); + + // // B2 -> C3 + let mut builder = client.new_block_at( + &BlockId::Hash(b2.hash()), + Default::default(), + false, + ).unwrap(); + // this push is required as otherwise C3 has the same hash as B3 and won't get imported + builder.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 1, + nonce: 1, + }).unwrap(); + let c3 = builder.build().unwrap().block; + client.import(BlockOrigin::Own, c3.clone()).unwrap(); + + // A1 -> D2 + let mut builder = client.new_block_at( + &BlockId::Hash(a1.hash()), + Default::default(), + false, + ).unwrap(); + // this push is required as otherwise D2 has the same hash as B2 and won't get imported + builder.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 1, + nonce: 0, + }).unwrap(); + let d2 = builder.build().unwrap().block; + client.import(BlockOrigin::Own, d2.clone()).unwrap(); + + let genesis_hash = client.chain_info().genesis_hash; + + let uncles1 = client.uncles(a4.hash(), 10).unwrap(); + assert_eq!(vec![b2.hash(), d2.hash()], uncles1); + + let uncles2 = client.uncles(a4.hash(), 0).unwrap(); + assert_eq!(0, uncles2.len()); + + let uncles3 = client.uncles(a1.hash(), 10).unwrap(); + assert_eq!(0, uncles3.len()); + + let uncles4 = client.uncles(genesis_hash, 10).unwrap(); + assert_eq!(0, uncles4.len()); + + let uncles5 = client.uncles(d2.hash(), 10).unwrap(); + assert_eq!(vec![a2.hash(), b2.hash()], uncles5); + + let uncles6 = client.uncles(b3.hash(), 1).unwrap(); + assert_eq!(vec![c3.hash()], uncles6); +} + +#[test] +fn best_containing_on_longest_chain_with_single_chain_3_blocks() { + // block tree: + // G -> A1 -> A2 + + let (mut client, longest_chain_select) = TestClientBuilder::new().build_with_longest_chain(); + + // G -> A1 + let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a1.clone()).unwrap(); + + // A1 -> A2 + let a2 = client.new_block(Default::default()).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a2.clone()).unwrap(); + + let genesis_hash = client.chain_info().genesis_hash; + + assert_eq!(a2.hash(), longest_chain_select.finality_target(genesis_hash, None).unwrap().unwrap()); + assert_eq!(a2.hash(), longest_chain_select.finality_target(a1.hash(), None).unwrap().unwrap()); + assert_eq!(a2.hash(), longest_chain_select.finality_target(a2.hash(), None).unwrap().unwrap()); +} + +#[test] +fn best_containing_on_longest_chain_with_multiple_forks() { + // block tree: + // G -> A1 -> A2 -> A3 -> A4 -> A5 + // A1 -> B2 -> B3 -> B4 + // B2 -> C3 + // A1 -> D2 + let (mut client, longest_chain_select) = TestClientBuilder::new().build_with_longest_chain(); + + // G -> A1 + let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a1.clone()).unwrap(); + + // A1 -> A2 + let a2 = client.new_block_at( + &BlockId::Hash(a1.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a2.clone()).unwrap(); + + // A2 -> A3 + let a3 = client.new_block_at( + &BlockId::Hash(a2.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a3.clone()).unwrap(); + + // A3 -> A4 + let a4 = client.new_block_at( + &BlockId::Hash(a3.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a4.clone()).unwrap(); + + // A4 -> A5 + let a5 = client.new_block_at( + &BlockId::Hash(a4.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a5.clone()).unwrap(); + + // A1 -> B2 + let mut builder = client.new_block_at( + &BlockId::Hash(a1.hash()), + Default::default(), + false, + ).unwrap(); + // this push is required as otherwise B2 has the same hash as A2 and won't get imported + builder.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 41, + nonce: 0, + }).unwrap(); + let b2 = builder.build().unwrap().block; + client.import(BlockOrigin::Own, b2.clone()).unwrap(); + + // B2 -> B3 + let b3 = client.new_block_at( + &BlockId::Hash(b2.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, b3.clone()).unwrap(); + + // B3 -> B4 + let b4 = client.new_block_at( + &BlockId::Hash(b3.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, b4.clone()).unwrap(); + + // // B2 -> C3 + let mut builder = client.new_block_at( + &BlockId::Hash(b2.hash()), + Default::default(), + false, + ).unwrap(); + // this push is required as otherwise C3 has the same hash as B3 and won't get imported + builder.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 1, + nonce: 1, + }).unwrap(); + let c3 = builder.build().unwrap().block; + client.import(BlockOrigin::Own, c3.clone()).unwrap(); + + // A1 -> D2 + let mut builder = client.new_block_at( + &BlockId::Hash(a1.hash()), + Default::default(), + false, + ).unwrap(); + // this push is required as otherwise D2 has the same hash as B2 and won't get imported + builder.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 1, + nonce: 0, + }).unwrap(); + let d2 = builder.build().unwrap().block; + client.import(BlockOrigin::Own, d2.clone()).unwrap(); + + assert_eq!(client.chain_info().best_hash, a5.hash()); + + let genesis_hash = client.chain_info().genesis_hash; + let leaves = longest_chain_select.leaves().unwrap(); + + assert!(leaves.contains(&a5.hash())); + assert!(leaves.contains(&b4.hash())); + assert!(leaves.contains(&c3.hash())); + assert!(leaves.contains(&d2.hash())); + assert_eq!(leaves.len(), 4); + + // search without restriction + + assert_eq!(a5.hash(), longest_chain_select.finality_target( + genesis_hash, None).unwrap().unwrap()); + assert_eq!(a5.hash(), longest_chain_select.finality_target( + a1.hash(), None).unwrap().unwrap()); + assert_eq!(a5.hash(), longest_chain_select.finality_target( + a2.hash(), None).unwrap().unwrap()); + assert_eq!(a5.hash(), longest_chain_select.finality_target( + a3.hash(), None).unwrap().unwrap()); + assert_eq!(a5.hash(), longest_chain_select.finality_target( + a4.hash(), None).unwrap().unwrap()); + assert_eq!(a5.hash(), longest_chain_select.finality_target( + a5.hash(), None).unwrap().unwrap()); + + assert_eq!(b4.hash(), longest_chain_select.finality_target( + b2.hash(), None).unwrap().unwrap()); + assert_eq!(b4.hash(), longest_chain_select.finality_target( + b3.hash(), None).unwrap().unwrap()); + assert_eq!(b4.hash(), longest_chain_select.finality_target( + b4.hash(), None).unwrap().unwrap()); + + assert_eq!(c3.hash(), longest_chain_select.finality_target( + c3.hash(), None).unwrap().unwrap()); + + assert_eq!(d2.hash(), longest_chain_select.finality_target( + d2.hash(), None).unwrap().unwrap()); + + + // search only blocks with number <= 5. equivalent to without restriction for this scenario + + assert_eq!(a5.hash(), longest_chain_select.finality_target( + genesis_hash, Some(5)).unwrap().unwrap()); + assert_eq!(a5.hash(), longest_chain_select.finality_target( + a1.hash(), Some(5)).unwrap().unwrap()); + assert_eq!(a5.hash(), longest_chain_select.finality_target( + a2.hash(), Some(5)).unwrap().unwrap()); + assert_eq!(a5.hash(), longest_chain_select.finality_target( + a3.hash(), Some(5)).unwrap().unwrap()); + assert_eq!(a5.hash(), longest_chain_select.finality_target( + a4.hash(), Some(5)).unwrap().unwrap()); + assert_eq!(a5.hash(), longest_chain_select.finality_target( + a5.hash(), Some(5)).unwrap().unwrap()); + + assert_eq!(b4.hash(), longest_chain_select.finality_target( + b2.hash(), Some(5)).unwrap().unwrap()); + assert_eq!(b4.hash(), longest_chain_select.finality_target( + b3.hash(), Some(5)).unwrap().unwrap()); + assert_eq!(b4.hash(), longest_chain_select.finality_target( + b4.hash(), Some(5)).unwrap().unwrap()); + + assert_eq!(c3.hash(), longest_chain_select.finality_target( + c3.hash(), Some(5)).unwrap().unwrap()); + + assert_eq!(d2.hash(), longest_chain_select.finality_target( + d2.hash(), Some(5)).unwrap().unwrap()); + + + // search only blocks with number <= 4 + + assert_eq!(a4.hash(), longest_chain_select.finality_target( + genesis_hash, Some(4)).unwrap().unwrap()); + assert_eq!(a4.hash(), longest_chain_select.finality_target( + a1.hash(), Some(4)).unwrap().unwrap()); + assert_eq!(a4.hash(), longest_chain_select.finality_target( + a2.hash(), Some(4)).unwrap().unwrap()); + assert_eq!(a4.hash(), longest_chain_select.finality_target( + a3.hash(), Some(4)).unwrap().unwrap()); + assert_eq!(a4.hash(), longest_chain_select.finality_target( + a4.hash(), Some(4)).unwrap().unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + a5.hash(), Some(4)).unwrap()); + + assert_eq!(b4.hash(), longest_chain_select.finality_target( + b2.hash(), Some(4)).unwrap().unwrap()); + assert_eq!(b4.hash(), longest_chain_select.finality_target( + b3.hash(), Some(4)).unwrap().unwrap()); + assert_eq!(b4.hash(), longest_chain_select.finality_target( + b4.hash(), Some(4)).unwrap().unwrap()); + + assert_eq!(c3.hash(), longest_chain_select.finality_target( + c3.hash(), Some(4)).unwrap().unwrap()); + + assert_eq!(d2.hash(), longest_chain_select.finality_target( + d2.hash(), Some(4)).unwrap().unwrap()); + + + // search only blocks with number <= 3 + + assert_eq!(a3.hash(), longest_chain_select.finality_target( + genesis_hash, Some(3)).unwrap().unwrap()); + assert_eq!(a3.hash(), longest_chain_select.finality_target( + a1.hash(), Some(3)).unwrap().unwrap()); + assert_eq!(a3.hash(), longest_chain_select.finality_target( + a2.hash(), Some(3)).unwrap().unwrap()); + assert_eq!(a3.hash(), longest_chain_select.finality_target( + a3.hash(), Some(3)).unwrap().unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + a4.hash(), Some(3)).unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + a5.hash(), Some(3)).unwrap()); + + assert_eq!(b3.hash(), longest_chain_select.finality_target( + b2.hash(), Some(3)).unwrap().unwrap()); + assert_eq!(b3.hash(), longest_chain_select.finality_target( + b3.hash(), Some(3)).unwrap().unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + b4.hash(), Some(3)).unwrap()); + + assert_eq!(c3.hash(), longest_chain_select.finality_target( + c3.hash(), Some(3)).unwrap().unwrap()); + + assert_eq!(d2.hash(), longest_chain_select.finality_target( + d2.hash(), Some(3)).unwrap().unwrap()); + + + // search only blocks with number <= 2 + + assert_eq!(a2.hash(), longest_chain_select.finality_target( + genesis_hash, Some(2)).unwrap().unwrap()); + assert_eq!(a2.hash(), longest_chain_select.finality_target( + a1.hash(), Some(2)).unwrap().unwrap()); + assert_eq!(a2.hash(), longest_chain_select.finality_target( + a2.hash(), Some(2)).unwrap().unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + a3.hash(), Some(2)).unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + a4.hash(), Some(2)).unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + a5.hash(), Some(2)).unwrap()); + + assert_eq!(b2.hash(), longest_chain_select.finality_target( + b2.hash(), Some(2)).unwrap().unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + b3.hash(), Some(2)).unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + b4.hash(), Some(2)).unwrap()); + + assert_eq!(None, longest_chain_select.finality_target( + c3.hash(), Some(2)).unwrap()); + + assert_eq!(d2.hash(), longest_chain_select.finality_target( + d2.hash(), Some(2)).unwrap().unwrap()); + + + // search only blocks with number <= 1 + + assert_eq!(a1.hash(), longest_chain_select.finality_target( + genesis_hash, Some(1)).unwrap().unwrap()); + assert_eq!(a1.hash(), longest_chain_select.finality_target( + a1.hash(), Some(1)).unwrap().unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + a2.hash(), Some(1)).unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + a3.hash(), Some(1)).unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + a4.hash(), Some(1)).unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + a5.hash(), Some(1)).unwrap()); + + assert_eq!(None, longest_chain_select.finality_target( + b2.hash(), Some(1)).unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + b3.hash(), Some(1)).unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + b4.hash(), Some(1)).unwrap()); + + assert_eq!(None, longest_chain_select.finality_target( + c3.hash(), Some(1)).unwrap()); + + assert_eq!(None, longest_chain_select.finality_target( + d2.hash(), Some(1)).unwrap()); + + // search only blocks with number <= 0 + + assert_eq!(genesis_hash, longest_chain_select.finality_target( + genesis_hash, Some(0)).unwrap().unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + a1.hash(), Some(0)).unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + a2.hash(), Some(0)).unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + a3.hash(), Some(0)).unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + a4.hash(), Some(0)).unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + a5.hash(), Some(0)).unwrap()); + + assert_eq!(None, longest_chain_select.finality_target( + b2.hash(), Some(0)).unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + b3.hash(), Some(0)).unwrap()); + assert_eq!(None, longest_chain_select.finality_target( + b4.hash(), Some(0)).unwrap()); + + assert_eq!(None, longest_chain_select.finality_target( + c3.hash().clone(), Some(0)).unwrap()); + + assert_eq!(None, longest_chain_select.finality_target( + d2.hash().clone(), Some(0)).unwrap()); +} + +#[test] +fn best_containing_on_longest_chain_with_max_depth_higher_than_best() { + // block tree: + // G -> A1 -> A2 + + let (mut client, longest_chain_select) = TestClientBuilder::new().build_with_longest_chain(); + + // G -> A1 + let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a1.clone()).unwrap(); + + // A1 -> A2 + let a2 = client.new_block(Default::default()).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a2.clone()).unwrap(); + + let genesis_hash = client.chain_info().genesis_hash; + + assert_eq!(a2.hash(), longest_chain_select.finality_target(genesis_hash, Some(10)).unwrap().unwrap()); +} + +#[test] +fn key_changes_works() { + let (client, _, test_cases) = prepare_client_with_key_changes(); + + for (index, (begin, end, key, expected_result)) in test_cases.into_iter().enumerate() { + let end = client.block_hash(end).unwrap().unwrap(); + let actual_result = client.key_changes( + begin, + BlockId::Hash(end), + None, + &StorageKey(key), + ).unwrap(); + match actual_result == expected_result { + true => (), + false => panic!(format!("Failed test {}: actual = {:?}, expected = {:?}", + index, actual_result, expected_result)), + } + } +} + +#[test] +fn import_with_justification() { + let mut client = substrate_test_runtime_client::new(); + + // G -> A1 + let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a1.clone()).unwrap(); + + // A1 -> A2 + let a2 = client.new_block_at( + &BlockId::Hash(a1.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a2.clone()).unwrap(); + + // A2 -> A3 + let justification = vec![1, 2, 3]; + let a3 = client.new_block_at( + &BlockId::Hash(a2.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import_justified(BlockOrigin::Own, a3.clone(), justification.clone()).unwrap(); + + assert_eq!( + client.chain_info().finalized_hash, + a3.hash(), + ); + + assert_eq!( + client.justification(&BlockId::Hash(a3.hash())).unwrap(), + Some(justification), + ); + + assert_eq!( + client.justification(&BlockId::Hash(a1.hash())).unwrap(), + None, + ); + + assert_eq!( + client.justification(&BlockId::Hash(a2.hash())).unwrap(), + None, + ); +} + +#[test] +fn importing_diverged_finalized_block_should_trigger_reorg() { + let mut client = substrate_test_runtime_client::new(); + + // G -> A1 -> A2 + // \ + // -> B1 + let a1 = client.new_block_at( + &BlockId::Number(0), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a1.clone()).unwrap(); + + let a2 = client.new_block_at( + &BlockId::Hash(a1.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a2.clone()).unwrap(); + + let mut b1 = client.new_block_at( + &BlockId::Number(0), + Default::default(), + false, + ).unwrap(); + // needed to make sure B1 gets a different hash from A1 + b1.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 1, + nonce: 0, + }).unwrap(); + // create but don't import B1 just yet + let b1 = b1.build().unwrap().block; + + // A2 is the current best since it's the longest chain + assert_eq!( + client.chain_info().best_hash, + a2.hash(), + ); + + // importing B1 as finalized should trigger a re-org and set it as new best + let justification = vec![1, 2, 3]; + client.import_justified(BlockOrigin::Own, b1.clone(), justification).unwrap(); + + assert_eq!( + client.chain_info().best_hash, + b1.hash(), + ); + + assert_eq!( + client.chain_info().finalized_hash, + b1.hash(), + ); +} + +#[test] +fn finalizing_diverged_block_should_trigger_reorg() { + let (mut client, select_chain) = TestClientBuilder::new().build_with_longest_chain(); + + // G -> A1 -> A2 + // \ + // -> B1 -> B2 + let a1 = client.new_block_at( + &BlockId::Number(0), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a1.clone()).unwrap(); + + let a2 = client.new_block_at( + &BlockId::Hash(a1.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a2.clone()).unwrap(); + + let mut b1 = client.new_block_at( + &BlockId::Number(0), + Default::default(), + false, + ).unwrap(); + // needed to make sure B1 gets a different hash from A1 + b1.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 1, + nonce: 0, + }).unwrap(); + let b1 = b1.build().unwrap().block; + client.import(BlockOrigin::Own, b1.clone()).unwrap(); + + let b2 = client.new_block_at( + &BlockId::Hash(b1.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, b2.clone()).unwrap(); + + // A2 is the current best since it's the longest chain + assert_eq!( + client.chain_info().best_hash, + a2.hash(), + ); + + // we finalize block B1 which is on a different branch from current best + // which should trigger a re-org. + ClientExt::finalize_block(&client, BlockId::Hash(b1.hash()), None).unwrap(); + + // B1 should now be the latest finalized + assert_eq!( + client.chain_info().finalized_hash, + b1.hash(), + ); + + // and B1 should be the new best block (`finalize_block` as no way of + // knowing about B2) + assert_eq!( + client.chain_info().best_hash, + b1.hash(), + ); + + // `SelectChain` should report B2 as best block though + assert_eq!( + select_chain.best_chain().unwrap().hash(), + b2.hash(), + ); + + // after we build B3 on top of B2 and import it + // it should be the new best block, + let b3 = client.new_block_at( + &BlockId::Hash(b2.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, b3.clone()).unwrap(); + + assert_eq!( + client.chain_info().best_hash, + b3.hash(), + ); +} + +#[test] +fn get_header_by_block_number_doesnt_panic() { + let client = substrate_test_runtime_client::new(); + + // backend uses u32 for block numbers, make sure we don't panic when + // trying to convert + let id = BlockId::::Number(72340207214430721); + client.header(&id).expect_err("invalid block number overflows u32"); +} + +#[test] +fn state_reverted_on_reorg() { + let _ = env_logger::try_init(); + let mut client = substrate_test_runtime_client::new(); + + let current_balance = |client: &substrate_test_runtime_client::TestClient| + client.runtime_api().balance_of( + &BlockId::number(client.chain_info().best_number), AccountKeyring::Alice.into(), + ).unwrap(); + + // G -> A1 -> A2 + // \ + // -> B1 + let mut a1 = client.new_block_at( + &BlockId::Number(0), + Default::default(), + false, + ).unwrap(); + a1.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Bob.into(), + amount: 10, + nonce: 0, + }).unwrap(); + let a1 = a1.build().unwrap().block; + client.import(BlockOrigin::Own, a1.clone()).unwrap(); + + let mut b1 = client.new_block_at( + &BlockId::Number(0), + Default::default(), + false, + ).unwrap(); + b1.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 50, + nonce: 0, + }).unwrap(); + let b1 = b1.build().unwrap().block; + // Reorg to B1 + client.import_as_best(BlockOrigin::Own, b1.clone()).unwrap(); + + assert_eq!(950, current_balance(&client)); + let mut a2 = client.new_block_at( + &BlockId::Hash(a1.hash()), + Default::default(), + false, + ).unwrap(); + a2.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Charlie.into(), + amount: 10, + nonce: 1, + }).unwrap(); + let a2 = a2.build().unwrap().block; + // Re-org to A2 + client.import_as_best(BlockOrigin::Own, a2).unwrap(); + assert_eq!(980, current_balance(&client)); +} + +#[test] +fn doesnt_import_blocks_that_revert_finality() { + let _ = env_logger::try_init(); + let tmp = tempfile::tempdir().unwrap(); + + // we need to run with archive pruning to avoid pruning non-canonical + // states + let backend = Arc::new(Backend::new( + DatabaseSettings { + state_cache_size: 1 << 20, + state_cache_child_ratio: None, + pruning: PruningMode::ArchiveAll, + source: DatabaseSettingsSrc::RocksDb { + path: tmp.path().into(), + cache_size: 1024, + }, + }, + u64::max_value(), + ).unwrap()); + + let mut client = TestClientBuilder::with_backend(backend).build(); + + // -> C1 + // / + // G -> A1 -> A2 + // \ + // -> B1 -> B2 -> B3 + + let a1 = client.new_block_at( + &BlockId::Number(0), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a1.clone()).unwrap(); + + let a2 = client.new_block_at( + &BlockId::Hash(a1.hash()), + Default::default(), + false, + ).unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, a2.clone()).unwrap(); + + let mut b1 = client.new_block_at(&BlockId::Number(0), Default::default(), false).unwrap(); + + // needed to make sure B1 gets a different hash from A1 + b1.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 1, + nonce: 0, + }).unwrap(); + let b1 = b1.build().unwrap().block; + client.import(BlockOrigin::Own, b1.clone()).unwrap(); + + let b2 = client.new_block_at(&BlockId::Hash(b1.hash()), Default::default(), false) + .unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, b2.clone()).unwrap(); + + // prepare B3 before we finalize A2, because otherwise we won't be able to + // read changes trie configuration after A2 is finalized + let b3 = client.new_block_at(&BlockId::Hash(b2.hash()), Default::default(), false) + .unwrap().build().unwrap().block; + + // we will finalize A2 which should make it impossible to import a new + // B3 at the same height but that doesn't include it + ClientExt::finalize_block(&client, BlockId::Hash(a2.hash()), None).unwrap(); + + let import_err = client.import(BlockOrigin::Own, b3).err().unwrap(); + let expected_err = ConsensusError::ClientImport( + sp_blockchain::Error::NotInFinalizedChain.to_string() + ); + + assert_eq!( + import_err.to_string(), + expected_err.to_string(), + ); + + // adding a C1 block which is lower than the last finalized should also + // fail (with a cheaper check that doesn't require checking ancestry). + let mut c1 = client.new_block_at(&BlockId::Number(0), Default::default(), false).unwrap(); + + // needed to make sure C1 gets a different hash from A1 and B1 + c1.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 2, + nonce: 0, + }).unwrap(); + let c1 = c1.build().unwrap().block; + + let import_err = client.import(BlockOrigin::Own, c1).err().unwrap(); + let expected_err = ConsensusError::ClientImport( + sp_blockchain::Error::NotInFinalizedChain.to_string() + ); + + assert_eq!( + import_err.to_string(), + expected_err.to_string(), + ); +} + + +#[test] +fn respects_block_rules() { + fn run_test( + record_only: bool, + known_bad: &mut HashSet, + fork_rules: &mut Vec<(u64, H256)>, + ) { + let mut client = if record_only { + TestClientBuilder::new().build() + } else { + TestClientBuilder::new() + .set_block_rules( + Some(fork_rules.clone()), + Some(known_bad.clone()), + ) + .build() + }; + + let block_ok = client.new_block_at(&BlockId::Number(0), Default::default(), false) + .unwrap().build().unwrap().block; + + let params = BlockCheckParams { + hash: block_ok.hash().clone(), + number: 0, + parent_hash: block_ok.header().parent_hash().clone(), + allow_missing_state: false, + import_existing: false, + }; + assert_eq!(client.check_block(params).unwrap(), ImportResult::imported(false)); + + // this is 0x0d6d6612a10485370d9e085aeea7ec427fb3f34d961c6a816cdbe5cde2278864 + let mut block_not_ok = client.new_block_at(&BlockId::Number(0), Default::default(), false) + .unwrap(); + block_not_ok.push_storage_change(vec![0], Some(vec![1])).unwrap(); + let block_not_ok = block_not_ok.build().unwrap().block; + + let params = BlockCheckParams { + hash: block_not_ok.hash().clone(), + number: 0, + parent_hash: block_not_ok.header().parent_hash().clone(), + allow_missing_state: false, + import_existing: false, + }; + if record_only { + known_bad.insert(block_not_ok.hash()); + } else { + assert_eq!(client.check_block(params).unwrap(), ImportResult::KnownBad); + } + + // Now going to the fork + client.import_as_final(BlockOrigin::Own, block_ok).unwrap(); + + // And check good fork + let mut block_ok = client.new_block_at(&BlockId::Number(1), Default::default(), false) + .unwrap(); + block_ok.push_storage_change(vec![0], Some(vec![2])).unwrap(); + let block_ok = block_ok.build().unwrap().block; + + let params = BlockCheckParams { + hash: block_ok.hash().clone(), + number: 1, + parent_hash: block_ok.header().parent_hash().clone(), + allow_missing_state: false, + import_existing: false, + }; + if record_only { + fork_rules.push((1, block_ok.hash().clone())); + } + assert_eq!(client.check_block(params).unwrap(), ImportResult::imported(false)); + + // And now try bad fork + let mut block_not_ok = client.new_block_at(&BlockId::Number(1), Default::default(), false) + .unwrap(); + block_not_ok.push_storage_change(vec![0], Some(vec![3])).unwrap(); + let block_not_ok = block_not_ok.build().unwrap().block; + + let params = BlockCheckParams { + hash: block_not_ok.hash().clone(), + number: 1, + parent_hash: block_not_ok.header().parent_hash().clone(), + allow_missing_state: false, + import_existing: false, + }; + + if !record_only { + assert_eq!(client.check_block(params).unwrap(), ImportResult::KnownBad); + } + } + + let mut known_bad = HashSet::new(); + let mut fork_rules = Vec::new(); + + // records what bad_blocks and fork_blocks hashes should be + run_test(true, &mut known_bad, &mut fork_rules); + + // enforces rules and actually makes assertions + run_test(false, &mut known_bad, &mut fork_rules); +} + +#[test] +fn returns_status_for_pruned_blocks() { + let _ = env_logger::try_init(); + let tmp = tempfile::tempdir().unwrap(); + + // set to prune after 1 block + // states + let backend = Arc::new(Backend::new( + DatabaseSettings { + state_cache_size: 1 << 20, + state_cache_child_ratio: None, + pruning: PruningMode::keep_blocks(1), + source: DatabaseSettingsSrc::RocksDb { + path: tmp.path().into(), + cache_size: 1024, + }, + }, + u64::max_value(), + ).unwrap()); + + let mut client = TestClientBuilder::with_backend(backend).build(); + + let a1 = client.new_block_at(&BlockId::Number(0), Default::default(), false) + .unwrap().build().unwrap().block; + + let mut b1 = client.new_block_at(&BlockId::Number(0), Default::default(), false).unwrap(); + + // b1 is created, but not imported + b1.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 1, + nonce: 0, + }).unwrap(); + let b1 = b1.build().unwrap().block; + + let check_block_a1 = BlockCheckParams { + hash: a1.hash().clone(), + number: 0, + parent_hash: a1.header().parent_hash().clone(), + allow_missing_state: false, + import_existing: false, + }; + + assert_eq!(client.check_block(check_block_a1.clone()).unwrap(), ImportResult::imported(false)); + assert_eq!(client.block_status(&BlockId::hash(check_block_a1.hash)).unwrap(), BlockStatus::Unknown); + + client.import_as_final(BlockOrigin::Own, a1.clone()).unwrap(); + + assert_eq!(client.check_block(check_block_a1.clone()).unwrap(), ImportResult::AlreadyInChain); + assert_eq!(client.block_status(&BlockId::hash(check_block_a1.hash)).unwrap(), BlockStatus::InChainWithState); + + let a2 = client.new_block_at(&BlockId::Hash(a1.hash()), Default::default(), false) + .unwrap().build().unwrap().block; + client.import_as_final(BlockOrigin::Own, a2.clone()).unwrap(); + + let check_block_a2 = BlockCheckParams { + hash: a2.hash().clone(), + number: 1, + parent_hash: a1.header().parent_hash().clone(), + allow_missing_state: false, + import_existing: false, + }; + + assert_eq!(client.check_block(check_block_a1.clone()).unwrap(), ImportResult::AlreadyInChain); + assert_eq!(client.block_status(&BlockId::hash(check_block_a1.hash)).unwrap(), BlockStatus::InChainPruned); + assert_eq!(client.check_block(check_block_a2.clone()).unwrap(), ImportResult::AlreadyInChain); + assert_eq!(client.block_status(&BlockId::hash(check_block_a2.hash)).unwrap(), BlockStatus::InChainWithState); + + let a3 = client.new_block_at(&BlockId::Hash(a2.hash()), Default::default(), false) + .unwrap().build().unwrap().block; + + client.import_as_final(BlockOrigin::Own, a3.clone()).unwrap(); + let check_block_a3 = BlockCheckParams { + hash: a3.hash().clone(), + number: 2, + parent_hash: a2.header().parent_hash().clone(), + allow_missing_state: false, + import_existing: false, + }; + + // a1 and a2 are both pruned at this point + assert_eq!(client.check_block(check_block_a1.clone()).unwrap(), ImportResult::AlreadyInChain); + assert_eq!(client.block_status(&BlockId::hash(check_block_a1.hash)).unwrap(), BlockStatus::InChainPruned); + assert_eq!(client.check_block(check_block_a2.clone()).unwrap(), ImportResult::AlreadyInChain); + assert_eq!(client.block_status(&BlockId::hash(check_block_a2.hash)).unwrap(), BlockStatus::InChainPruned); + assert_eq!(client.check_block(check_block_a3.clone()).unwrap(), ImportResult::AlreadyInChain); + assert_eq!(client.block_status(&BlockId::hash(check_block_a3.hash)).unwrap(), BlockStatus::InChainWithState); + + let mut check_block_b1 = BlockCheckParams { + hash: b1.hash().clone(), + number: 0, + parent_hash: b1.header().parent_hash().clone(), + allow_missing_state: false, + import_existing: false, + }; + assert_eq!(client.check_block(check_block_b1.clone()).unwrap(), ImportResult::MissingState); + check_block_b1.allow_missing_state = true; + assert_eq!(client.check_block(check_block_b1.clone()).unwrap(), ImportResult::imported(false)); + check_block_b1.parent_hash = H256::random(); + assert_eq!(client.check_block(check_block_b1.clone()).unwrap(), ImportResult::UnknownParent); +} + +#[test] +fn imports_blocks_with_changes_tries_config_change() { + // create client with initial 4^2 configuration + let mut client = TestClientBuilder::with_default_backend() + .changes_trie_config(Some(ChangesTrieConfiguration { + digest_interval: 4, + digest_levels: 2, + })).build(); + + // =================================================================== + // blocks 1,2,3,4,5,6,7,8,9,10 are empty + // block 11 changes the key + // block 12 is the L1 digest that covers this change + // blocks 13,14,15,16,17,18,19,20,21,22 are empty + // block 23 changes the configuration to 5^1 AND is skewed digest + // =================================================================== + // blocks 24,25 are changing the key + // block 26 is empty + // block 27 changes the key + // block 28 is the L1 digest (NOT SKEWED!!!) that covers changes AND changes configuration to 3^1 + // =================================================================== + // block 29 is empty + // block 30 changes the key + // block 31 is L1 digest that covers this change + // =================================================================== + (1..11).for_each(|number| { + let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false) + .unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + }); + (11..12).for_each(|number| { + let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap(); + block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap(); + let block = block.build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + }); + (12..23).for_each(|number| { + let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false) + .unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + }); + (23..24).for_each(|number| { + let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap(); + block.push_changes_trie_configuration_update(Some(ChangesTrieConfiguration { + digest_interval: 5, + digest_levels: 1, + })).unwrap(); + let block = block.build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + }); + (24..26).for_each(|number| { + let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap(); + block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap(); + let block = block.build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + }); + (26..27).for_each(|number| { + let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false) + .unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + }); + (27..28).for_each(|number| { + let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap(); + block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap(); + let block = block.build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + }); + (28..29).for_each(|number| { + let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap(); + block.push_changes_trie_configuration_update(Some(ChangesTrieConfiguration { + digest_interval: 3, + digest_levels: 1, + })).unwrap(); + let block = block.build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + }); + (29..30).for_each(|number| { + let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false) + .unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + }); + (30..31).for_each(|number| { + let mut block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false).unwrap(); + block.push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())).unwrap(); + let block = block.build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + }); + (31..32).for_each(|number| { + let block = client.new_block_at(&BlockId::Number(number - 1), Default::default(), false) + .unwrap().build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + }); + + // now check that configuration cache works + assert_eq!( + client.key_changes(1, BlockId::Number(31), None, &StorageKey(vec![42])).unwrap(), + vec![(30, 0), (27, 0), (25, 0), (24, 0), (11, 0)] + ); +} + +#[test] +fn storage_keys_iter_prefix_and_start_key_works() { + let client = substrate_test_runtime_client::new(); + + let prefix = StorageKey(hex!("3a").to_vec()); + + let res: Vec<_> = client.storage_keys_iter(&BlockId::Number(0), Some(&prefix), None) + .unwrap() + .map(|x| x.0) + .collect(); + assert_eq!(res, [hex!("3a636f6465").to_vec(), hex!("3a686561707061676573").to_vec()]); + + let res: Vec<_> = client.storage_keys_iter(&BlockId::Number(0), Some(&prefix), Some(&StorageKey(hex!("3a636f6465").to_vec()))) + .unwrap() + .map(|x| x.0) + .collect(); + assert_eq!(res, [hex!("3a686561707061676573").to_vec()]); + + let res: Vec<_> = client.storage_keys_iter(&BlockId::Number(0), Some(&prefix), Some(&StorageKey(hex!("3a686561707061676573").to_vec()))) + .unwrap() + .map(|x| x.0) + .collect(); + assert_eq!(res, Vec::>::new()); +} + +#[test] +fn storage_keys_iter_works() { + let client = substrate_test_runtime_client::new(); + + let prefix = StorageKey(hex!("").to_vec()); + + let res: Vec<_> = client.storage_keys_iter(&BlockId::Number(0), Some(&prefix), None) + .unwrap() + .take(2) + .map(|x| x.0) + .collect(); + assert_eq!(res, [hex!("0befda6e1ca4ef40219d588a727f1271").to_vec(), hex!("3a636f6465").to_vec()]); + + let res: Vec<_> = client.storage_keys_iter(&BlockId::Number(0), Some(&prefix), Some(&StorageKey(hex!("3a636f6465").to_vec()))) + .unwrap() + .take(3) + .map(|x| x.0) + .collect(); + assert_eq!(res, [ + hex!("3a686561707061676573").to_vec(), + hex!("6644b9b8bc315888ac8e41a7968dc2b4141a5403c58acdf70b7e8f7e07bf5081").to_vec(), + hex!("79c07e2b1d2e2abfd4855b936617eeff5e0621c4869aa60c02be9adcc98a0d1d").to_vec(), + ]); + + let res: Vec<_> = client.storage_keys_iter(&BlockId::Number(0), Some(&prefix), Some(&StorageKey(hex!("79c07e2b1d2e2abfd4855b936617eeff5e0621c4869aa60c02be9adcc98a0d1d").to_vec()))) + .unwrap() + .take(1) + .map(|x| x.0) + .collect(); + assert_eq!(res, [hex!("cf722c0832b5231d35e29f319ff27389f5032bfc7bfc3ba5ed7839f2042fb99f").to_vec()]); +} + +#[test] +fn cleans_up_closed_notification_sinks_on_block_import() { + use substrate_test_runtime_client::GenesisInit; + + // NOTE: we need to build the client here instead of using the client + // provided by test_runtime_client otherwise we can't access the private + // `import_notification_sinks` and `finality_notification_sinks` fields. + let mut client = + new_in_mem::< + _, + substrate_test_runtime_client::runtime::Block, + _, + substrate_test_runtime_client::runtime::RuntimeApi + >( + substrate_test_runtime_client::new_native_executor(), + &substrate_test_runtime_client::GenesisParameters::default().genesis_storage(), + None, + None, + sp_core::tasks::executor(), + Default::default(), + ) + .unwrap(); + + type TestClient = Client< + in_mem::Backend, + LocalCallExecutor, sc_executor::NativeExecutor>, + substrate_test_runtime_client::runtime::Block, + substrate_test_runtime_client::runtime::RuntimeApi, + >; + + let import_notif1 = client.import_notification_stream(); + let import_notif2 = client.import_notification_stream(); + let finality_notif1 = client.finality_notification_stream(); + let finality_notif2 = client.finality_notification_stream(); + + // for some reason I can't seem to use `ClientBlockImportExt` + let bake_and_import_block = |client: &mut TestClient, origin| { + let block = client + .new_block(Default::default()) + .unwrap() + .build() + .unwrap() + .block; + + let (header, extrinsics) = block.deconstruct(); + let mut import = BlockImportParams::new(origin, header); + import.body = Some(extrinsics); + import.fork_choice = Some(ForkChoiceStrategy::LongestChain); + client.import_block(import, Default::default()).unwrap(); + }; + + // after importing a block we should still have 4 notification sinks + // (2 import + 2 finality) + bake_and_import_block(&mut client, BlockOrigin::Own); + assert_eq!(client.import_notification_sinks().lock().len(), 2); + assert_eq!(client.finality_notification_sinks().lock().len(), 2); + + // if we drop one import notification receiver and one finality + // notification receiver + drop(import_notif2); + drop(finality_notif2); + + // the sinks should be cleaned up after block import + bake_and_import_block(&mut client, BlockOrigin::Own); + assert_eq!(client.import_notification_sinks().lock().len(), 1); + assert_eq!(client.finality_notification_sinks().lock().len(), 1); + + // the same thing should happen if block import happens during initial + // sync + drop(import_notif1); + drop(finality_notif1); + + bake_and_import_block(&mut client, BlockOrigin::NetworkInitialSync); + assert_eq!(client.import_notification_sinks().lock().len(), 0); + assert_eq!(client.finality_notification_sinks().lock().len(), 0); +} + diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index d63fd4009ec2e7c31987d019d1eb0dd6d8cb1422..fd451ffc8d55f7bd490eaaa10a57d16471161939 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -37,11 +37,15 @@ use sc_service::{ Role, Error, }; -use sc_network::{multiaddr, Multiaddr, NetworkStateInfo}; -use sc_network::config::{NetworkConfiguration, TransportConfig, NodeKeyConfig, Secret, NonReservedPeerMode}; +use sp_blockchain::HeaderBackend; +use sc_network::{multiaddr, Multiaddr}; +use sc_network::config::{NetworkConfiguration, TransportConfig}; use sp_runtime::{generic::BlockId, traits::Block as BlockT}; use sp_transaction_pool::TransactionPool; +#[cfg(test)] +mod client; + /// Maximum duration of single wait call. const MAX_WAIT_TIME: Duration = Duration::from_secs(60 * 3); @@ -143,59 +147,50 @@ fn node_config TestNet where let service = SyncService::from(service); executor.spawn(service.clone().map_err(|_| ())); - let addr = addr.with(multiaddr::Protocol::P2p(service.get().network().local_peer_id().into())); + let addr = addr.with(multiaddr::Protocol::P2p(service.get().network().local_peer_id().clone().into())); self.authority_nodes.push((self.nodes, service, user_data, addr)); self.nodes += 1; } @@ -292,7 +287,7 @@ impl TestNet where let service = SyncService::from(service); executor.spawn(service.clone().map_err(|_| ())); - let addr = addr.with(multiaddr::Protocol::P2p(service.get().network().local_peer_id().into())); + let addr = addr.with(multiaddr::Protocol::P2p(service.get().network().local_peer_id().clone().into())); self.full_nodes.push((self.nodes, service, user_data, addr)); self.nodes += 1; } @@ -307,7 +302,7 @@ impl TestNet where let service = SyncService::from(light(node_config).expect("Error creating test node service")); executor.spawn(service.clone().map_err(|_| ())); - let addr = addr.with(multiaddr::Protocol::P2p(service.get().network().local_peer_id().into())); + let addr = addr.with(multiaddr::Protocol::P2p(service.get().network().local_peer_id().clone().into())); self.light_nodes.push((self.nodes, service, addr)); self.nodes += 1; } @@ -471,15 +466,15 @@ pub fn sync( } network.run_until_all_full( |_index, service| - service.get().client().chain_info().best_number == (NUM_BLOCKS as u32).into(), + service.get().client().info().best_number == (NUM_BLOCKS as u32).into(), |_index, service| - service.get().client().chain_info().best_number == (NUM_BLOCKS as u32).into(), + service.get().client().info().best_number == (NUM_BLOCKS as u32).into(), ); info!("Checking extrinsic propagation"); let first_service = network.full_nodes[0].1.clone(); let first_user_data = &network.full_nodes[0].2; - let best_block = BlockId::number(first_service.get().client().chain_info().best_number); + let best_block = BlockId::number(first_service.get().client().info().best_number); let extrinsic = extrinsic_factory(&first_service.get(), first_user_data); let source = sp_transaction_pool::TransactionSource::External; @@ -532,9 +527,9 @@ pub fn consensus( } network.run_until_all_full( |_index, service| - service.get().client().chain_info().finalized_number >= (NUM_BLOCKS as u32 / 2).into(), + service.get().client().info().finalized_number >= (NUM_BLOCKS as u32 / 2).into(), |_index, service| - service.get().client().chain_info().best_number >= (NUM_BLOCKS as u32 / 2).into(), + service.get().client().info().best_number >= (NUM_BLOCKS as u32 / 2).into(), ); info!("Adding more peers"); @@ -554,8 +549,8 @@ pub fn consensus( } network.run_until_all_full( |_index, service| - service.get().client().chain_info().finalized_number >= (NUM_BLOCKS as u32).into(), + service.get().client().info().finalized_number >= (NUM_BLOCKS as u32).into(), |_index, service| - service.get().client().chain_info().best_number >= (NUM_BLOCKS as u32).into(), + service.get().client().info().best_number >= (NUM_BLOCKS as u32).into(), ); } diff --git a/client/src/genesis.rs b/client/src/genesis.rs index 2c84ff1e4331af066d43e314d31059740adcce6e..c5dd44404c9b10fdd37a1ad00a4d6783ada62223 100644 --- a/client/src/genesis.rs +++ b/client/src/genesis.rs @@ -55,6 +55,7 @@ mod tests { }; use sp_runtime::traits::BlakeTwo256; use sp_core::tasks::executor as tasks_executor; + use sp_core::offchain::storage::OffchainOverlayedChanges; use hex_literal::*; native_executor_instance!( @@ -90,6 +91,7 @@ mod tests { }; let hash = header.hash(); let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); @@ -97,6 +99,7 @@ mod tests { backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, + &mut offchain_overlay, &executor(), "Core_initialize_block", &header.encode(), @@ -112,6 +115,7 @@ mod tests { backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, + &mut offchain_overlay, &executor(), "BlockBuilder_apply_extrinsic", &tx.encode(), @@ -127,6 +131,7 @@ mod tests { backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, + &mut offchain_overlay, &executor(), "BlockBuilder_finalize_block", &[], @@ -174,10 +179,13 @@ mod tests { let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); + let _ = StateMachine::new( &backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, + &mut offchain_overlay, &executor(), "Core_execute_block", &b1data, @@ -206,10 +214,13 @@ mod tests { let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); + let _ = StateMachine::new( &backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, + &mut offchain_overlay, &executor(), "Core_execute_block", &b1data, @@ -238,10 +249,13 @@ mod tests { let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); + let r = StateMachine::new( &backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, + &mut offchain_overlay, &executor(), "Core_execute_block", &b1data, diff --git a/client/src/light/fetcher.rs b/client/src/light/fetcher.rs deleted file mode 100644 index 0ae0e68e0c8781d0a08e84aceeea40817a1d8873..0000000000000000000000000000000000000000 --- a/client/src/light/fetcher.rs +++ /dev/null @@ -1,833 +0,0 @@ -// Copyright 2017-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Light client data fetcher. Fetches requested data from remote full nodes. - -use std::sync::Arc; -use std::collections::{BTreeMap, HashMap}; -use std::marker::PhantomData; - -use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; -use codec::{Decode, Encode}; -use sp_core::{convert_hash, traits::CodeExecutor}; -use sp_runtime::traits::{ - Block as BlockT, Header as HeaderT, Hash, HashFor, NumberFor, - AtLeast32Bit, CheckedConversion, -}; -use sp_state_machine::{ - ChangesTrieRootsStorage, ChangesTrieAnchorBlockId, ChangesTrieConfigurationRange, - InMemoryChangesTrieStorage, TrieBackend, read_proof_check, key_changes_proof_check_with_db, - read_child_proof_check, CloneableSpawn, -}; -pub use sp_state_machine::StorageProof; -use sp_blockchain::{Error as ClientError, Result as ClientResult}; - -use crate::cht; -pub use sc_client_api::{ - light::{ - RemoteCallRequest, RemoteHeaderRequest, RemoteReadRequest, RemoteReadChildRequest, - RemoteChangesRequest, ChangesProof, RemoteBodyRequest, Fetcher, FetchChecker, - Storage as BlockchainStorage, - }, -}; -use crate::light::blockchain::{Blockchain}; -use crate::light::call_executor::check_execution_proof; - -/// Remote data checker. -pub struct LightDataChecker> { - blockchain: Arc>, - executor: E, - spawn_handle: Box, - _hasher: PhantomData<(B, H)>, -} - -impl> LightDataChecker { - /// Create new light data checker. - pub fn new(blockchain: Arc>, executor: E, spawn_handle: Box) -> Self { - Self { - blockchain, executor, spawn_handle, _hasher: PhantomData - } - } - - /// Check remote changes query proof assuming that CHT-s are of given size. - fn check_changes_proof_with_cht_size( - &self, - request: &RemoteChangesRequest, - remote_proof: ChangesProof, - cht_size: NumberFor, - ) -> ClientResult, u32)>> - where - H: Hasher, - H::Out: Ord + codec::Codec, - { - // since we need roots of all changes tries for the range begin..max - // => remote node can't use max block greater that one that we have passed - if remote_proof.max_block > request.max_block.0 || remote_proof.max_block < request.last_block.0 { - return Err(ClientError::ChangesTrieAccessFailed(format!( - "Invalid max_block used by the remote node: {}. Local: {}..{}..{}", - remote_proof.max_block, request.first_block.0, request.last_block.0, request.max_block.0, - )).into()); - } - - // check if remote node has responded with extra changes trie roots proofs - // all changes tries roots must be in range [request.first_block.0; request.tries_roots.0) - let is_extra_first_root = remote_proof.roots.keys().next() - .map(|first_root| *first_root < request.first_block.0 - || *first_root >= request.tries_roots.0) - .unwrap_or(false); - let is_extra_last_root = remote_proof.roots.keys().next_back() - .map(|last_root| *last_root >= request.tries_roots.0) - .unwrap_or(false); - if is_extra_first_root || is_extra_last_root { - return Err(ClientError::ChangesTrieAccessFailed(format!( - "Extra changes tries roots proofs provided by the remote node: [{:?}..{:?}]. Expected in range: [{}; {})", - remote_proof.roots.keys().next(), remote_proof.roots.keys().next_back(), - request.first_block.0, request.tries_roots.0, - )).into()); - } - - // if request has been composed when some required headers were already pruned - // => remote node has sent us CHT-based proof of required changes tries roots - // => check that this proof is correct before proceeding with changes proof - let remote_max_block = remote_proof.max_block; - let remote_roots = remote_proof.roots; - let remote_roots_proof = remote_proof.roots_proof; - let remote_proof = remote_proof.proof; - if !remote_roots.is_empty() { - self.check_changes_tries_proof( - cht_size, - &remote_roots, - remote_roots_proof, - )?; - } - - // and now check the key changes proof + get the changes - let mut result = Vec::new(); - let proof_storage = InMemoryChangesTrieStorage::with_proof(remote_proof); - for config_range in &request.changes_trie_configs { - let result_range = key_changes_proof_check_with_db::( - ChangesTrieConfigurationRange { - config: config_range.config.as_ref().ok_or(ClientError::ChangesTriesNotSupported)?, - zero: config_range.zero.0, - end: config_range.end.map(|(n, _)| n), - }, - &RootsStorage { - roots: (request.tries_roots.0, &request.tries_roots.2), - prev_roots: &remote_roots, - }, - &proof_storage, - request.first_block.0, - &ChangesTrieAnchorBlockId { - hash: convert_hash(&request.last_block.1), - number: request.last_block.0, - }, - remote_max_block, - request.storage_key.as_ref().map(Vec::as_slice), - &request.key) - .map_err(|err| ClientError::ChangesTrieAccessFailed(err))?; - result.extend(result_range); - } - - Ok(result) - } - - /// Check CHT-based proof for changes tries roots. - fn check_changes_tries_proof( - &self, - cht_size: NumberFor, - remote_roots: &BTreeMap, B::Hash>, - remote_roots_proof: StorageProof, - ) -> ClientResult<()> - where - H: Hasher, - H::Out: Ord + codec::Codec, - { - // all the checks are sharing the same storage - let storage = remote_roots_proof.into_memory_db(); - - // remote_roots.keys() are sorted => we can use this to group changes tries roots - // that are belongs to the same CHT - let blocks = remote_roots.keys().cloned(); - cht::for_each_cht_group::(cht_size, blocks, |mut storage, _, cht_blocks| { - // get local changes trie CHT root for given CHT - // it should be there, because it is never pruned AND request has been composed - // when required header has been pruned (=> replaced with CHT) - let first_block = cht_blocks.first().cloned() - .expect("for_each_cht_group never calls callback with empty groups"); - let local_cht_root = self.blockchain.storage().changes_trie_cht_root(cht_size, first_block)? - .ok_or(ClientError::InvalidCHTProof)?; - - // check changes trie root for every block within CHT range - for block in cht_blocks { - // check if the proofs storage contains the root - // normally this happens in when the proving backend is created, but since - // we share the storage for multiple checks, do it here - let mut cht_root = H::Out::default(); - cht_root.as_mut().copy_from_slice(local_cht_root.as_ref()); - if !storage.contains(&cht_root, EMPTY_PREFIX) { - return Err(ClientError::InvalidCHTProof.into()); - } - - // check proof for single changes trie root - let proving_backend = TrieBackend::new(storage, cht_root); - let remote_changes_trie_root = remote_roots[&block]; - cht::check_proof_on_proving_backend::( - local_cht_root, - block, - remote_changes_trie_root, - &proving_backend, - )?; - - // and return the storage to use in following checks - storage = proving_backend.into_storage(); - } - - Ok(storage) - }, storage) - } -} - -impl FetchChecker for LightDataChecker - where - Block: BlockT, - E: CodeExecutor + Clone + 'static, - H: Hasher, - H::Out: Ord + codec::Codec + 'static, - S: BlockchainStorage, -{ - fn check_header_proof( - &self, - request: &RemoteHeaderRequest, - remote_header: Option, - remote_proof: StorageProof, - ) -> ClientResult { - let remote_header = remote_header.ok_or_else(|| - ClientError::from(ClientError::InvalidCHTProof))?; - let remote_header_hash = remote_header.hash(); - cht::check_proof::( - request.cht_root, - request.block, - remote_header_hash, - remote_proof, - ).map(|_| remote_header) - } - - fn check_read_proof( - &self, - request: &RemoteReadRequest, - remote_proof: StorageProof, - ) -> ClientResult, Option>>> { - read_proof_check::( - convert_hash(request.header.state_root()), - remote_proof, - request.keys.iter(), - ).map_err(Into::into) - } - - fn check_read_child_proof( - &self, - request: &RemoteReadChildRequest, - remote_proof: StorageProof, - ) -> ClientResult, Option>>> { - read_child_proof_check::( - convert_hash(request.header.state_root()), - remote_proof, - &request.storage_key, - request.keys.iter(), - ).map_err(Into::into) - } - - fn check_execution_proof( - &self, - request: &RemoteCallRequest, - remote_proof: StorageProof, - ) -> ClientResult> { - check_execution_proof::<_, _, H>( - &self.executor, - self.spawn_handle.clone(), - request, - remote_proof, - ) - } - - fn check_changes_proof( - &self, - request: &RemoteChangesRequest, - remote_proof: ChangesProof - ) -> ClientResult, u32)>> { - self.check_changes_proof_with_cht_size(request, remote_proof, cht::size()) - } - - fn check_body_proof( - &self, - request: &RemoteBodyRequest, - body: Vec - ) -> ClientResult> { - // TODO: #2621 - let extrinsics_root = HashFor::::ordered_trie_root( - body.iter().map(Encode::encode).collect(), - ); - if *request.header.extrinsics_root() == extrinsics_root { - Ok(body) - } else { - Err(format!("RemoteBodyRequest: invalid extrinsics root expected: {} but got {}", - *request.header.extrinsics_root(), - extrinsics_root, - ).into()) - } - - } -} - -/// A view of BTreeMap as a changes trie roots storage. -struct RootsStorage<'a, Number: AtLeast32Bit, Hash: 'a> { - roots: (Number, &'a [Hash]), - prev_roots: &'a BTreeMap, -} - -impl<'a, H, Number, Hash> ChangesTrieRootsStorage for RootsStorage<'a, Number, Hash> - where - H: Hasher, - Number: std::fmt::Display + std::hash::Hash + Clone + AtLeast32Bit + Encode + Decode + Send + Sync + 'static, - Hash: 'a + Send + Sync + Clone + AsRef<[u8]>, -{ - fn build_anchor( - &self, - _hash: H::Out, - ) -> Result, String> { - Err("build_anchor is only called when building block".into()) - } - - fn root( - &self, - _anchor: &ChangesTrieAnchorBlockId, - block: Number, - ) -> Result, String> { - // we can't ask for roots from parallel forks here => ignore anchor - let root = if block < self.roots.0 { - self.prev_roots.get(&Number::unique_saturated_from(block)).cloned() - } else { - let index: Option = block.checked_sub(&self.roots.0).and_then(|index| index.checked_into()); - match index { - Some(index) => self.roots.1.get(index as usize).cloned(), - None => None, - } - }; - - Ok(root.map(|root| { - let mut hasher_root: H::Out = Default::default(); - hasher_root.as_mut().copy_from_slice(root.as_ref()); - hasher_root - })) - } -} - -#[cfg(test)] -pub mod tests { - use codec::Decode; - use crate::client::tests::prepare_client_with_key_changes; - use sc_executor::{NativeExecutor, WasmExecutionMethod}; - use sp_blockchain::Error as ClientError; - use sc_client_api::backend::NewBlockState; - use substrate_test_runtime_client::{ - blockchain::HeaderBackend, AccountKeyring, ClientBlockImportExt, - runtime::{self, Hash, Block, Header, Extrinsic}, - tasks_executor, - }; - use sp_consensus::BlockOrigin; - - use crate::in_mem::Blockchain as InMemoryBlockchain; - use crate::light::fetcher::{FetchChecker, LightDataChecker, RemoteHeaderRequest}; - use crate::light::blockchain::tests::{DummyStorage, DummyBlockchain}; - use sp_core::{blake2_256, ChangesTrieConfiguration, H256}; - use sp_core::storage::{well_known_keys, StorageKey, ChildInfo}; - use sp_runtime::{generic::BlockId, traits::BlakeTwo256}; - use sp_state_machine::Backend; - use super::*; - use sc_client_api::{StorageProvider, ProofProvider}; - use sc_block_builder::BlockBuilderProvider; - - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); - - type TestChecker = LightDataChecker< - NativeExecutor, - BlakeTwo256, - Block, - DummyStorage, - >; - - fn local_executor() -> NativeExecutor { - NativeExecutor::new(WasmExecutionMethod::Interpreted, None, 8) - } - - fn prepare_for_read_proof_check() -> (TestChecker, Header, StorageProof, u32) { - // prepare remote client - let remote_client = substrate_test_runtime_client::new(); - let remote_block_id = BlockId::Number(0); - let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); - let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap(); - remote_block_header.state_root = remote_client.state_at(&remote_block_id).unwrap() - .storage_root(::std::iter::empty()).0.into(); - - // 'fetch' read proof from remote node - let heap_pages = remote_client.storage(&remote_block_id, &StorageKey(well_known_keys::HEAP_PAGES.to_vec())) - .unwrap() - .and_then(|v| Decode::decode(&mut &v.0[..]).ok()).unwrap(); - let remote_read_proof = remote_client.read_proof( - &remote_block_id, - &mut std::iter::once(well_known_keys::HEAP_PAGES), - ).unwrap(); - - // check remote read proof locally - let local_storage = InMemoryBlockchain::::new(); - local_storage.insert( - remote_block_hash, - remote_block_header.clone(), - None, - None, - NewBlockState::Final, - ).unwrap(); - let local_checker = LightDataChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - tasks_executor(), - ); - (local_checker, remote_block_header, remote_read_proof, heap_pages) - } - - fn prepare_for_read_child_proof_check() -> (TestChecker, Header, StorageProof, Vec) { - use substrate_test_runtime_client::DefaultTestClientBuilderExt; - use substrate_test_runtime_client::TestClientBuilderExt; - // prepare remote client - let remote_client = substrate_test_runtime_client::TestClientBuilder::new() - .add_extra_child_storage( - b":child_storage:default:child1".to_vec(), - CHILD_INFO_1, - b"key1".to_vec(), - b"value1".to_vec(), - ).build(); - let remote_block_id = BlockId::Number(0); - let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); - let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap(); - remote_block_header.state_root = remote_client.state_at(&remote_block_id).unwrap() - .storage_root(::std::iter::empty()).0.into(); - - // 'fetch' child read proof from remote node - let child_value = remote_client.child_storage( - &remote_block_id, - &StorageKey(b":child_storage:default:child1".to_vec()), - CHILD_INFO_1, - &StorageKey(b"key1".to_vec()), - ).unwrap().unwrap().0; - assert_eq!(b"value1"[..], child_value[..]); - let remote_read_proof = remote_client.read_child_proof( - &remote_block_id, - b":child_storage:default:child1", - CHILD_INFO_1, - &mut std::iter::once("key1".as_bytes()), - ).unwrap(); - - // check locally - let local_storage = InMemoryBlockchain::::new(); - local_storage.insert( - remote_block_hash, - remote_block_header.clone(), - None, - None, - NewBlockState::Final, - ).unwrap(); - let local_checker = LightDataChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - tasks_executor(), - ); - (local_checker, remote_block_header, remote_read_proof, child_value) - } - - fn prepare_for_header_proof_check(insert_cht: bool) -> (TestChecker, Hash, Header, StorageProof) { - // prepare remote client - let mut remote_client = substrate_test_runtime_client::new(); - let mut local_headers_hashes = Vec::new(); - for i in 0..4 { - let block = remote_client.new_block(Default::default()).unwrap().build().unwrap().block; - remote_client.import(BlockOrigin::Own, block).unwrap(); - local_headers_hashes.push( - remote_client.block_hash(i + 1) - .map_err(|_| ClientError::Backend("TestError".into())) - ); - } - - // 'fetch' header proof from remote node - let remote_block_id = BlockId::Number(1); - let (remote_block_header, remote_header_proof) = remote_client.header_proof_with_cht_size(&remote_block_id, 4).unwrap(); - - // check remote read proof locally - let local_storage = InMemoryBlockchain::::new(); - let local_cht_root = cht::compute_root::(4, 0, local_headers_hashes).unwrap(); - if insert_cht { - local_storage.insert_cht_root(1, local_cht_root); - } - let local_checker = LightDataChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - tasks_executor(), - ); - (local_checker, local_cht_root, remote_block_header, remote_header_proof) - } - - fn header_with_computed_extrinsics_root(extrinsics: Vec) -> Header { - use sp_trie::{TrieConfiguration, trie_types::Layout}; - let iter = extrinsics.iter().map(Encode::encode); - let extrinsics_root = Layout::::ordered_trie_root(iter); - - // only care about `extrinsics_root` - Header::new(0, extrinsics_root, H256::zero(), H256::zero(), Default::default()) - } - - #[test] - fn storage_read_proof_is_generated_and_checked() { - let (local_checker, remote_block_header, remote_read_proof, heap_pages) = prepare_for_read_proof_check(); - assert_eq!((&local_checker as &dyn FetchChecker).check_read_proof(&RemoteReadRequest::
{ - block: remote_block_header.hash(), - header: remote_block_header, - keys: vec![well_known_keys::HEAP_PAGES.to_vec()], - retry_count: None, - }, remote_read_proof).unwrap().remove(well_known_keys::HEAP_PAGES).unwrap().unwrap()[0], heap_pages as u8); - } - - #[test] - fn storage_child_read_proof_is_generated_and_checked() { - let ( - local_checker, - remote_block_header, - remote_read_proof, - result, - ) = prepare_for_read_child_proof_check(); - let child_infos = CHILD_INFO_1.info(); - assert_eq!((&local_checker as &dyn FetchChecker).check_read_child_proof( - &RemoteReadChildRequest::
{ - block: remote_block_header.hash(), - header: remote_block_header, - storage_key: b":child_storage:default:child1".to_vec(), - child_info: child_infos.0.to_vec(), - child_type: child_infos.1, - keys: vec![b"key1".to_vec()], - retry_count: None, - }, - remote_read_proof - ).unwrap().remove(b"key1".as_ref()).unwrap().unwrap(), result); - } - - #[test] - fn header_proof_is_generated_and_checked() { - let (local_checker, local_cht_root, remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true); - assert_eq!((&local_checker as &dyn FetchChecker).check_header_proof(&RemoteHeaderRequest::
{ - cht_root: local_cht_root, - block: 1, - retry_count: None, - }, Some(remote_block_header.clone()), remote_header_proof).unwrap(), remote_block_header); - } - - #[test] - fn check_header_proof_fails_if_cht_root_is_invalid() { - let (local_checker, _, mut remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true); - remote_block_header.number = 100; - assert!((&local_checker as &dyn FetchChecker).check_header_proof(&RemoteHeaderRequest::
{ - cht_root: Default::default(), - block: 1, - retry_count: None, - }, Some(remote_block_header.clone()), remote_header_proof).is_err()); - } - - #[test] - fn check_header_proof_fails_if_invalid_header_provided() { - let (local_checker, local_cht_root, mut remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true); - remote_block_header.number = 100; - assert!((&local_checker as &dyn FetchChecker).check_header_proof(&RemoteHeaderRequest::
{ - cht_root: local_cht_root, - block: 1, - retry_count: None, - }, Some(remote_block_header.clone()), remote_header_proof).is_err()); - } - - #[test] - fn changes_proof_is_generated_and_checked_when_headers_are_not_pruned() { - let (remote_client, local_roots, test_cases) = prepare_client_with_key_changes(); - let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - tasks_executor(), - ); - let local_checker = &local_checker as &dyn FetchChecker; - let max = remote_client.chain_info().best_number; - let max_hash = remote_client.chain_info().best_hash; - - for (index, (begin, end, key, expected_result)) in test_cases.into_iter().enumerate() { - let begin_hash = remote_client.block_hash(begin).unwrap().unwrap(); - let end_hash = remote_client.block_hash(end).unwrap().unwrap(); - - // 'fetch' changes proof from remote node - let key = StorageKey(key); - let remote_proof = remote_client.key_changes_proof( - begin_hash, end_hash, begin_hash, max_hash, None, &key - ).unwrap(); - - // check proof on local client - let local_roots_range = local_roots.clone()[(begin - 1) as usize..].to_vec(); - let config = ChangesTrieConfiguration::new(4, 2); - let request = RemoteChangesRequest::
{ - changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { - zero: (0, Default::default()), - end: None, - config: Some(config), - }], - first_block: (begin, begin_hash), - last_block: (end, end_hash), - max_block: (max, max_hash), - tries_roots: (begin, begin_hash, local_roots_range), - key: key.0, - storage_key: None, - retry_count: None, - }; - let local_result = local_checker.check_changes_proof(&request, ChangesProof { - max_block: remote_proof.max_block, - proof: remote_proof.proof, - roots: remote_proof.roots, - roots_proof: remote_proof.roots_proof, - }).unwrap(); - - // ..and ensure that result is the same as on remote node - match local_result == expected_result { - true => (), - false => panic!(format!("Failed test {}: local = {:?}, expected = {:?}", - index, local_result, expected_result)), - } - } - } - - #[test] - fn changes_proof_is_generated_and_checked_when_headers_are_pruned() { - // we're testing this test case here: - // (1, 4, dave.clone(), vec![(4, 0), (1, 1), (1, 0)]), - let (remote_client, remote_roots, _) = prepare_client_with_key_changes(); - let dave = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec(); - let dave = StorageKey(dave); - - // 'fetch' changes proof from remote node: - // we're fetching changes for range b1..b4 - // we do not know changes trie roots before b3 (i.e. we only know b3+b4) - // but we have changes trie CHT root for b1...b4 - let b1 = remote_client.block_hash_from_id(&BlockId::Number(1)).unwrap().unwrap(); - let b3 = remote_client.block_hash_from_id(&BlockId::Number(3)).unwrap().unwrap(); - let b4 = remote_client.block_hash_from_id(&BlockId::Number(4)).unwrap().unwrap(); - let remote_proof = remote_client.key_changes_proof_with_cht_size( - b1, b4, b3, b4, None, &dave, 4 - ).unwrap(); - - // prepare local checker, having a root of changes trie CHT#0 - let local_cht_root = cht::compute_root::(4, 0, remote_roots.iter().cloned().map(|ct| Ok(Some(ct)))).unwrap(); - let mut local_storage = DummyStorage::new(); - local_storage.changes_tries_cht_roots.insert(0, local_cht_root); - let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(local_storage)), - local_executor(), - tasks_executor(), - ); - - // check proof on local client - let config = ChangesTrieConfiguration::new(4, 2); - let request = RemoteChangesRequest::
{ - changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { - zero: (0, Default::default()), - end: None, - config: Some(config), - }], - first_block: (1, b1), - last_block: (4, b4), - max_block: (4, b4), - tries_roots: (3, b3, vec![remote_roots[2].clone(), remote_roots[3].clone()]), - storage_key: None, - key: dave.0, - retry_count: None, - }; - let local_result = local_checker.check_changes_proof_with_cht_size(&request, ChangesProof { - max_block: remote_proof.max_block, - proof: remote_proof.proof, - roots: remote_proof.roots, - roots_proof: remote_proof.roots_proof, - }, 4).unwrap(); - - assert_eq!(local_result, vec![(4, 0), (1, 1), (1, 0)]); - } - - #[test] - fn check_changes_proof_fails_if_proof_is_wrong() { - let (remote_client, local_roots, test_cases) = prepare_client_with_key_changes(); - let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - tasks_executor(), - ); - let local_checker = &local_checker as &dyn FetchChecker; - let max = remote_client.chain_info().best_number; - let max_hash = remote_client.chain_info().best_hash; - - let (begin, end, key, _) = test_cases[0].clone(); - let begin_hash = remote_client.block_hash(begin).unwrap().unwrap(); - let end_hash = remote_client.block_hash(end).unwrap().unwrap(); - - // 'fetch' changes proof from remote node - let key = StorageKey(key); - let remote_proof = remote_client.key_changes_proof( - begin_hash, end_hash, begin_hash, max_hash, None, &key).unwrap(); - - let local_roots_range = local_roots.clone()[(begin - 1) as usize..].to_vec(); - let config = ChangesTrieConfiguration::new(4, 2); - let request = RemoteChangesRequest::
{ - changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { - zero: (0, Default::default()), - end: None, - config: Some(config), - }], - first_block: (begin, begin_hash), - last_block: (end, end_hash), - max_block: (max, max_hash), - tries_roots: (begin, begin_hash, local_roots_range.clone()), - storage_key: None, - key: key.0, - retry_count: None, - }; - - // check proof on local client using max from the future - assert!(local_checker.check_changes_proof(&request, ChangesProof { - max_block: remote_proof.max_block + 1, - proof: remote_proof.proof.clone(), - roots: remote_proof.roots.clone(), - roots_proof: remote_proof.roots_proof.clone(), - }).is_err()); - - // check proof on local client using broken proof - assert!(local_checker.check_changes_proof(&request, ChangesProof { - max_block: remote_proof.max_block, - proof: local_roots_range.clone().into_iter().map(|v| v.as_ref().to_vec()).collect(), - roots: remote_proof.roots, - roots_proof: remote_proof.roots_proof, - }).is_err()); - - // extra roots proofs are provided - assert!(local_checker.check_changes_proof(&request, ChangesProof { - max_block: remote_proof.max_block, - proof: remote_proof.proof.clone(), - roots: vec![(begin - 1, Default::default())].into_iter().collect(), - roots_proof: StorageProof::empty(), - }).is_err()); - assert!(local_checker.check_changes_proof(&request, ChangesProof { - max_block: remote_proof.max_block, - proof: remote_proof.proof.clone(), - roots: vec![(end + 1, Default::default())].into_iter().collect(), - roots_proof: StorageProof::empty(), - }).is_err()); - } - - #[test] - fn check_changes_tries_proof_fails_if_proof_is_wrong() { - // we're testing this test case here: - // (1, 4, dave.clone(), vec![(4, 0), (1, 1), (1, 0)]), - let (remote_client, remote_roots, _) = prepare_client_with_key_changes(); - let local_cht_root = cht::compute_root::( - 4, 0, remote_roots.iter().cloned().map(|ct| Ok(Some(ct)))).unwrap(); - let dave = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec(); - let dave = StorageKey(dave); - - // 'fetch' changes proof from remote node: - // we're fetching changes for range b1..b4 - // we do not know changes trie roots before b3 (i.e. we only know b3+b4) - // but we have changes trie CHT root for b1...b4 - let b1 = remote_client.block_hash_from_id(&BlockId::Number(1)).unwrap().unwrap(); - let b3 = remote_client.block_hash_from_id(&BlockId::Number(3)).unwrap().unwrap(); - let b4 = remote_client.block_hash_from_id(&BlockId::Number(4)).unwrap().unwrap(); - let remote_proof = remote_client.key_changes_proof_with_cht_size( - b1, b4, b3, b4, None, &dave, 4 - ).unwrap(); - - // fails when changes trie CHT is missing from the local db - let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - tasks_executor(), - ); - assert!(local_checker.check_changes_tries_proof(4, &remote_proof.roots, - remote_proof.roots_proof.clone()).is_err()); - - // fails when proof is broken - let mut local_storage = DummyStorage::new(); - local_storage.changes_tries_cht_roots.insert(0, local_cht_root); - let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(local_storage)), - local_executor(), - tasks_executor(), - ); - let result = local_checker.check_changes_tries_proof( - 4, &remote_proof.roots, StorageProof::empty() - ); - assert!(result.is_err()); - } - - #[test] - fn check_body_proof_faulty() { - let header = header_with_computed_extrinsics_root( - vec![Extrinsic::IncludeData(vec![1, 2, 3, 4])] - ); - let block = Block::new(header.clone(), Vec::new()); - - let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - tasks_executor(), - ); - - let body_request = RemoteBodyRequest { - header: header.clone(), - retry_count: None, - }; - - assert!( - local_checker.check_body_proof(&body_request, block.extrinsics).is_err(), - "vec![1, 2, 3, 4] != vec![]" - ); - } - - #[test] - fn check_body_proof_of_same_data_should_succeed() { - let extrinsics = vec![Extrinsic::IncludeData(vec![1, 2, 3, 4, 5, 6, 7, 8, 255])]; - - let header = header_with_computed_extrinsics_root(extrinsics.clone()); - let block = Block::new(header.clone(), extrinsics); - - let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - tasks_executor(), - ); - - let body_request = RemoteBodyRequest { - header: header.clone(), - retry_count: None, - }; - - assert!(local_checker.check_body_proof(&body_request, block.extrinsics).is_ok()); - } -} diff --git a/client/state-db/Cargo.toml b/client/state-db/Cargo.toml index 9da3c4e1274cd6065999b2a0214280b4b44ce87d..c2d2f2eb0badf5a60a8db5c5af44790eb6e125a8 100644 --- a/client/state-db/Cargo.toml +++ b/client/state-db/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-state-db" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,17 +8,17 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "State database maintenance. Handles canonicalization and pruning in the database." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] parking_lot = "0.10.0" log = "0.4.8" -sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } +sc-client-api = { version = "2.0.0-dev", path = "../api" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } -parity-util-mem = "0.6" +parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } parity-util-mem-derive = "0.1.0" [dev-dependencies] env_logger = "0.7.0" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/state-db/src/lib.rs b/client/state-db/src/lib.rs index 49b1a59285e11eb332bb099b7839f8b7383f5439..94d51c89126485167228606da10db2aff4df1cc3 100644 --- a/client/state-db/src/lib.rs +++ b/client/state-db/src/lib.rs @@ -201,9 +201,10 @@ struct StateDbSync { impl StateDbSync { fn new( mode: PruningMode, + ref_counting: bool, db: &D, ) -> Result, Error> { - trace!(target: "state-db", "StateDb settings: {:?}", mode); + trace!(target: "state-db", "StateDb settings: {:?}. Ref-counting: {}", mode, ref_counting); // Check that settings match Self::check_meta(&mode, db)?; @@ -214,7 +215,7 @@ impl StateDbSync unimplemented!(), - PruningMode::Constrained(_) => Some(RefWindow::new(db)?), + PruningMode::Constrained(_) => Some(RefWindow::new(db, ref_counting)?), PruningMode::ArchiveAll | PruningMode::ArchiveCanonical => None, }; @@ -387,8 +388,11 @@ impl StateDbSync(&self, key: &Key, db: &D) -> Result, Error> - where Key: AsRef + pub fn get(&self, key: &Q, db: &D) -> Result, Error> + where + Q: AsRef, + Key: std::borrow::Borrow, + Q: std::hash::Hash + Eq, { if let Some(value) = self.non_canonical.get(key) { return Ok(Some(value)); @@ -438,10 +442,11 @@ impl StateDb( mode: PruningMode, + ref_counting: bool, db: &D, ) -> Result, Error> { Ok(StateDb { - db: RwLock::new(StateDbSync::new(mode, db)?) + db: RwLock::new(StateDbSync::new(mode, ref_counting, db)?) }) } @@ -475,8 +480,11 @@ impl StateDb(&self, key: &Key, db: &D) -> Result, Error> - where Key: AsRef + pub fn get(&self, key: &Q, db: &D) -> Result, Error> + where + Q: AsRef, + Key: std::borrow::Borrow, + Q: std::hash::Hash + Eq, { self.db.read().get(key, db) } @@ -523,7 +531,7 @@ mod tests { fn make_test_db(settings: PruningMode) -> (TestDb, StateDb) { let mut db = make_db(&[91, 921, 922, 93, 94]); - let state_db = StateDb::new(settings, &db).unwrap(); + let state_db = StateDb::new(settings, false, &db).unwrap(); db.commit( &state_db @@ -638,7 +646,7 @@ mod tests { #[test] fn detects_incompatible_mode() { let mut db = make_db(&[]); - let state_db = StateDb::new(PruningMode::ArchiveAll, &db).unwrap(); + let state_db = StateDb::new(PruningMode::ArchiveAll, false, &db).unwrap(); db.commit( &state_db .insert_block::( @@ -650,7 +658,7 @@ mod tests { .unwrap(), ); let new_mode = PruningMode::Constrained(Constraints { max_blocks: Some(2), max_mem: None }); - let state_db: Result, _> = StateDb::new(new_mode, &db); + let state_db: Result, _> = StateDb::new(new_mode, false, &db); assert!(state_db.is_err()); } } diff --git a/client/state-db/src/noncanonical.rs b/client/state-db/src/noncanonical.rs index 6a34523b66fff9e510518a4db4a884b5a693daf4..6a743e7d45913bc684961a300494b5e4154339f6 100644 --- a/client/state-db/src/noncanonical.rs +++ b/client/state-db/src/noncanonical.rs @@ -40,7 +40,7 @@ pub struct NonCanonicalOverlay { values: HashMap, //ref counted //would be deleted but kept around because block is pinned, ref counted. pinned: HashMap, - pinned_insertions: HashMap>, + pinned_insertions: HashMap, u32)>, } #[derive(Encode, Decode)] @@ -90,25 +90,44 @@ fn discard_values(values: &mut HashMap, inserted } fn discard_descendants( - levels: &mut VecDeque>>, + levels: &mut (&mut [Vec>], &mut [Vec>]), mut values: &mut HashMap, - index: usize, parents: &mut HashMap, pinned: &HashMap, - pinned_insertions: &mut HashMap>, + pinned_insertions: &mut HashMap, u32)>, hash: &BlockHash, -) { - let mut discarded = Vec::new(); - if let Some(level) = levels.get_mut(index) { +) -> u32 { + let (first, mut remainder) = if let Some((first, rest)) = levels.0.split_first_mut() { + (Some(first), (rest, &mut levels.1[..])) + } else { + if let Some((first, rest)) = levels.1.split_first_mut() { + (Some(first), (&mut levels.0[..], rest)) + } else { + (None, (&mut levels.0[..], &mut levels.1[..])) + } + }; + let mut pinned_children = 0; + if let Some(level) = first { *level = level.drain(..).filter_map(|overlay| { let parent = parents.get(&overlay.hash) .expect("there is a parent entry for each entry in levels; qed"); if parent == hash { - discarded.push(overlay.hash.clone()); + let mut num_pinned = discard_descendants( + &mut remainder, + values, + parents, + pinned, + pinned_insertions, + &overlay.hash + ); if pinned.contains_key(&overlay.hash) { + num_pinned += 1; + } + if num_pinned != 0 { // save to be discarded later. - pinned_insertions.insert(overlay.hash.clone(), overlay.inserted); + pinned_insertions.insert(overlay.hash.clone(), (overlay.inserted, num_pinned)); + pinned_children += num_pinned; } else { // discard immediately. parents.remove(&overlay.hash); @@ -120,9 +139,7 @@ fn discard_descendants( } }).collect(); } - for hash in discarded { - discard_descendants(levels, values, index + 1, parents, pinned, pinned_insertions, &hash); - } + pinned_children } impl NonCanonicalOverlay { @@ -346,19 +363,23 @@ impl NonCanonicalOverlay { // discard unfinalized overlays and values for (i, overlay) in level.into_iter().enumerate() { - if i != index { + let mut pinned_children = if i != index { discard_descendants( - &mut self.levels, + &mut self.levels.as_mut_slices(), &mut self.values, - 0, &mut self.parents, &self.pinned, &mut self.pinned_insertions, &overlay.hash, - ); - } + ) + } else { + 0 + }; if self.pinned.contains_key(&overlay.hash) { - self.pinned_insertions.insert(overlay.hash.clone(), overlay.inserted); + pinned_children += 1; + } + if pinned_children != 0 { + self.pinned_insertions.insert(overlay.hash.clone(), (overlay.inserted, pinned_children)); } else { self.parents.remove(&overlay.hash); discard_values(&mut self.values, overlay.inserted); @@ -372,7 +393,11 @@ impl NonCanonicalOverlay { } /// Get a value from the node overlay. This searches in every existing changeset. - pub fn get(&self, key: &Key) -> Option { + pub fn get(&self, key: &Q) -> Option + where + Key: std::borrow::Borrow, + Q: std::hash::Hash + Eq, + { if let Some((_, value)) = self.values.get(&key) { return Some(value.clone()); } @@ -435,37 +460,47 @@ impl NonCanonicalOverlay { debug_assert!(false, "Trying to pin pending state"); return; } - // Also pin all parents - let mut parent = Some(hash); - while let Some(hash) = parent { - let refs = self.pinned.entry(hash.clone()).or_default(); - if *refs == 0 { - trace!(target: "state-db-pin", "Pinned non-canon block: {:?}", hash); - } - *refs += 1; - parent = self.parents.get(hash); + let refs = self.pinned.entry(hash.clone()).or_default(); + if *refs == 0 { + trace!(target: "state-db-pin", "Pinned non-canon block: {:?}", hash); } + *refs += 1; } /// Discard pinned state pub fn unpin(&mut self, hash: &BlockHash) { - // Also unpin all parents - let mut parent = Some(hash.clone()); - while let Some(hash) = parent { - parent = self.parents.get(&hash).cloned(); - match self.pinned.entry(hash.clone()) { - Entry::Occupied(mut entry) => { - *entry.get_mut() -= 1; - if *entry.get() == 0 { - entry.remove(); - if let Some(inserted) = self.pinned_insertions.remove(&hash) { + let removed = match self.pinned.entry(hash.clone()) { + Entry::Occupied(mut entry) => { + *entry.get_mut() -= 1; + if *entry.get() == 0 { + entry.remove(); + true + } else { + false + } + }, + Entry::Vacant(_) => false, + }; + + if removed { + let mut parent = Some(hash.clone()); + while let Some(hash) = parent { + parent = self.parents.get(&hash).cloned(); + match self.pinned_insertions.entry(hash.clone()) { + Entry::Occupied(mut entry) => { + entry.get_mut().1 -= 1; + if entry.get().1 == 0 { + let (inserted, _) = entry.remove(); trace!(target: "state-db-pin", "Discarding unpinned non-canon block: {:?}", hash); discard_values(&mut self.values, inserted); self.parents.remove(&hash); + true + } else { + false } - } - }, - Entry::Vacant(_) => {}, + }, + Entry::Vacant(_) => break, + }; } } } diff --git a/client/state-db/src/pruning.rs b/client/state-db/src/pruning.rs index 6cf5f260060f500432ff306eb2e3f3c6a69f9089..1d15e617a1839d2b36b6f53c9c18aa750c30272b 100644 --- a/client/state-db/src/pruning.rs +++ b/client/state-db/src/pruning.rs @@ -45,6 +45,10 @@ pub struct RefWindow { /// Number of calls of `prune_one` after /// last call `apply_pending` or `revert_pending` pending_prunings: usize, + /// Keep track of re-inserted keys and do not delete them when pruning. + /// Setting this to false requires backend that supports reference + /// counting. + count_insertions: bool, } #[derive(Debug, PartialEq, Eq, parity_util_mem_derive::MallocSizeOf)] @@ -66,7 +70,7 @@ fn to_journal_key(block: u64) -> Vec { } impl RefWindow { - pub fn new(db: &D) -> Result, Error> { + pub fn new(db: &D, count_insertions: bool) -> Result, Error> { let last_pruned = db.get_meta(&to_meta_key(LAST_PRUNED, &())) .map_err(|e| Error::Db(e))?; let pending_number: u64 = match last_pruned { @@ -80,6 +84,7 @@ impl RefWindow { pending_number: pending_number, pending_canonicalizations: 0, pending_prunings: 0, + count_insertions, }; // read the journal trace!(target: "state-db", "Reading pruning journal. Pending #{}", pending_number); @@ -99,17 +104,19 @@ impl RefWindow { } fn import>(&mut self, hash: &BlockHash, journal_key: Vec, inserted: I, deleted: Vec) { - // remove all re-inserted keys from death rows - for k in inserted { - if let Some(block) = self.death_index.remove(&k) { - self.death_rows[(block - self.pending_number) as usize].deleted.remove(&k); + if self.count_insertions { + // remove all re-inserted keys from death rows + for k in inserted { + if let Some(block) = self.death_index.remove(&k) { + self.death_rows[(block - self.pending_number) as usize].deleted.remove(&k); + } } - } - // add new keys - let imported_block = self.pending_number + self.death_rows.len() as u64; - for k in deleted.iter() { - self.death_index.insert(k.clone(), imported_block); + // add new keys + let imported_block = self.pending_number + self.death_rows.len() as u64; + for k in deleted.iter() { + self.death_index.insert(k.clone(), imported_block); + } } self.death_rows.push_back( DeathRow { @@ -157,7 +164,11 @@ impl RefWindow { /// Add a change set to the window. Creates a journal record and pushes it to `commit` pub fn note_canonical(&mut self, hash: &BlockHash, commit: &mut CommitSet) { trace!(target: "state-db", "Adding to pruning window: {:?} ({} inserted, {} deleted)", hash, commit.data.inserted.len(), commit.data.deleted.len()); - let inserted = commit.data.inserted.iter().map(|(k, _)| k.clone()).collect(); + let inserted = if self.count_insertions { + commit.data.inserted.iter().map(|(k, _)| k.clone()).collect() + } else { + Default::default() + }; let deleted = ::std::mem::replace(&mut commit.data.deleted, Vec::new()); let journal_record = JournalRecord { hash: hash.clone(), @@ -177,8 +188,10 @@ impl RefWindow { for _ in 0 .. self.pending_prunings { let pruned = self.death_rows.pop_front().expect("pending_prunings is always < death_rows.len()"); trace!(target: "state-db", "Applying pruning {:?} ({} deleted)", pruned.hash, pruned.deleted.len()); - for k in pruned.deleted.iter() { - self.death_index.remove(&k); + if self.count_insertions { + for k in pruned.deleted.iter() { + self.death_index.remove(&k); + } } self.pending_number += 1; } @@ -192,8 +205,10 @@ impl RefWindow { // We don't bother to track and revert that for now. This means that a few nodes might end up no being // deleted in case transaction fails and `revert_pending` is called. self.death_rows.truncate(self.death_rows.len() - self.pending_canonicalizations); - let new_max_block = self.death_rows.len() as u64 + self.pending_number; - self.death_index.retain(|_, block| *block < new_max_block); + if self.count_insertions { + let new_max_block = self.death_rows.len() as u64 + self.pending_number; + self.death_index.retain(|_, block| *block < new_max_block); + } self.pending_canonicalizations = 0; self.pending_prunings = 0; } @@ -207,7 +222,7 @@ mod tests { use crate::test::{make_db, make_commit, TestDb}; fn check_journal(pruning: &RefWindow, db: &TestDb) { - let restored: RefWindow = RefWindow::new(db).unwrap(); + let restored: RefWindow = RefWindow::new(db, pruning.count_insertions).unwrap(); assert_eq!(pruning.pending_number, restored.pending_number); assert_eq!(pruning.death_rows, restored.death_rows); assert_eq!(pruning.death_index, restored.death_index); @@ -216,7 +231,7 @@ mod tests { #[test] fn created_from_empty_db() { let db = make_db(&[]); - let pruning: RefWindow = RefWindow::new(&db).unwrap(); + let pruning: RefWindow = RefWindow::new(&db, true).unwrap(); assert_eq!(pruning.pending_number, 0); assert!(pruning.death_rows.is_empty()); assert!(pruning.death_index.is_empty()); @@ -225,7 +240,7 @@ mod tests { #[test] fn prune_empty() { let db = make_db(&[]); - let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); + let mut pruning: RefWindow = RefWindow::new(&db, true).unwrap(); let mut commit = CommitSet::default(); pruning.prune_one(&mut commit); assert_eq!(pruning.pending_number, 0); @@ -238,7 +253,7 @@ mod tests { #[test] fn prune_one() { let mut db = make_db(&[1, 2, 3]); - let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); + let mut pruning: RefWindow = RefWindow::new(&db, true).unwrap(); let mut commit = make_commit(&[4, 5], &[1, 3]); let h = H256::random(); pruning.note_canonical(&h, &mut commit); @@ -267,7 +282,7 @@ mod tests { #[test] fn prune_two() { let mut db = make_db(&[1, 2, 3]); - let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); + let mut pruning: RefWindow = RefWindow::new(&db, true).unwrap(); let mut commit = make_commit(&[4], &[1]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); @@ -295,7 +310,7 @@ mod tests { #[test] fn prune_two_pending() { let mut db = make_db(&[1, 2, 3]); - let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); + let mut pruning: RefWindow = RefWindow::new(&db, true).unwrap(); let mut commit = make_commit(&[4], &[1]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); @@ -318,7 +333,7 @@ mod tests { #[test] fn reinserted_survives() { let mut db = make_db(&[1, 2, 3]); - let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); + let mut pruning: RefWindow = RefWindow::new(&db, true).unwrap(); let mut commit = make_commit(&[], &[2]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); @@ -351,7 +366,7 @@ mod tests { #[test] fn reinserted_survive_pending() { let mut db = make_db(&[1, 2, 3]); - let mut pruning: RefWindow = RefWindow::new(&db).unwrap(); + let mut pruning: RefWindow = RefWindow::new(&db, true).unwrap(); let mut commit = make_commit(&[], &[2]); pruning.note_canonical(&H256::random(), &mut commit); db.commit(&commit); @@ -377,4 +392,30 @@ mod tests { pruning.apply_pending(); assert_eq!(pruning.pending_number, 3); } + + #[test] + fn reinserted_ignores() { + let mut db = make_db(&[1, 2, 3]); + let mut pruning: RefWindow = RefWindow::new(&db, false).unwrap(); + let mut commit = make_commit(&[], &[2]); + pruning.note_canonical(&H256::random(), &mut commit); + db.commit(&commit); + let mut commit = make_commit(&[2], &[]); + pruning.note_canonical(&H256::random(), &mut commit); + db.commit(&commit); + let mut commit = make_commit(&[], &[2]); + pruning.note_canonical(&H256::random(), &mut commit); + db.commit(&commit); + assert!(db.data_eq(&make_db(&[1, 2, 3]))); + pruning.apply_pending(); + + check_journal(&pruning, &db); + + let mut commit = CommitSet::default(); + pruning.prune_one(&mut commit); + db.commit(&commit); + assert!(db.data_eq(&make_db(&[1, 3]))); + assert!(pruning.death_index.is_empty()); + } + } diff --git a/client/telemetry/Cargo.toml b/client/telemetry/Cargo.toml index bb7e7e510372bc253ba6fa2130c6f0e1b7d4a31b..0928d1d2a151822643e58781b8da23178acc6f9c 100644 --- a/client/telemetry/Cargo.toml +++ b/client/telemetry/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-telemetry" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] description = "Telemetry utils" edition = "2018" @@ -9,6 +9,9 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sc-telemetry" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] bytes = "0.5" @@ -16,7 +19,7 @@ parking_lot = "0.10.0" futures = "0.3.4" futures-timer = "3.0.1" wasm-timer = "0.2.0" -libp2p = { version = "0.16.2", default-features = false, features = ["libp2p-websocket"] } +libp2p = { version = "0.18.1", default-features = false, features = ["websocket", "wasm-ext", "tcp", "dns"] } log = "0.4.8" pin-project = "0.4.6" rand = "0.7.2" @@ -26,6 +29,3 @@ slog-json = { version = "2.3.0", features = ["nested-values"] } slog-scope = "4.1.2" take_mut = "0.2.2" void = "1.0.2" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/tracing/Cargo.toml b/client/tracing/Cargo.toml index 319526b61044858240908719d233aaef5a134f98..23f44fd057570e4aae988a2d316e51aea01e3150 100644 --- a/client/tracing/Cargo.toml +++ b/client/tracing/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-tracing" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" @@ -8,6 +8,9 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Instrumentation implementation for substrate." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] erased-serde = "0.3.9" log = { version = "0.4.8" } @@ -17,10 +20,7 @@ serde_json = "1.0.41" slog = { version = "2.5.2", features = ["nested-values"] } tracing-core = "0.1.7" -sc-telemetry = { version = "2.0.0-alpha.5", path = "../telemetry" } +sc-telemetry = { version = "2.0.0-dev", path = "../telemetry" } [dev-dependencies] tracing = "0.1.10" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/tracing/src/lib.rs b/client/tracing/src/lib.rs index c00bca9275eec7fa2bf99050b6413c99f846bc78..d450700ed3c49f9f41d27999aad6753fbe5f84bf 100644 --- a/client/tracing/src/lib.rs +++ b/client/tracing/src/lib.rs @@ -20,20 +20,8 @@ //! //! # Usage //! -//! Monitor performance throughout the codebase via the creation of `Span`s. -//! A span is set in the following way: -//! ``` -//! let span = tracing::span!(tracing::Level::INFO, "my_span_name"); -//! let _guard = span.enter(); -//! ``` -//! To begin timing, a span must be entered. When the span is dropped, the execution time -//! is recorded and details sent to the `Receiver` which defines how to process it. +//! See `sp-tracing` for examples on how to use tracing. //! -//! It's possible to record values with each span in the following way: -//! ``` -//! let span = tracing::span!(tracing::Level::INFO, "my_span_name", my_number = 10, a_key = "a value"); -//! let _guard = span.enter(); -//! ``` //! Currently we provide `Log` (default), `Telemetry` variants for `Receiver` use std::collections::HashMap; diff --git a/client/transaction-pool/Cargo.toml b/client/transaction-pool/Cargo.toml index 29b80698424e1a60a8d4fc542d6dea3354706910..6d4f69676c6308858ff30006d6136ab04abe16b5 100644 --- a/client/transaction-pool/Cargo.toml +++ b/client/transaction-pool/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-transaction-pool" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,31 +8,33 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Substrate transaction pool implementation." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0" } derive_more = "0.99.2" futures = { version = "0.3.1", features = ["compat"] } futures-diagnose = "1.0" +intervalier = "0.4.0" log = "0.4.8" +parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } parking_lot = "0.10.0" +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-dev"} +sc-client-api = { version = "2.0.0-dev", path = "../api" } +sc-transaction-graph = { version = "2.0.0-dev", path = "./graph" } +sp-api = { version = "2.0.0-dev", path = "../../primitives/api" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } +sp-tracing = { version = "2.0.0-dev", path = "../../primitives/tracing" } +sp-transaction-pool = { version = "2.0.0-dev", path = "../../primitives/transaction-pool" } +sp-blockchain = { version = "2.0.0-dev", path = "../../primitives/blockchain" } +sp-utils = { version = "2.0.0-dev", path = "../../primitives/utils" } wasm-timer = "0.2" -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } -sp-utils = { version = "2.0.0-alpha.5", path = "../../primitives/utils" } -sc-transaction-graph = { version = "2.0.0-alpha.5", path = "./graph" } -sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../primitives/transaction-pool" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../api" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } -intervalier = "0.3.1" -parity-util-mem = { version = "0.6.0", default-features = false, features = ["primitive-types"] } [dev-dependencies] assert_matches = "1.3.0" hex = "0.4" -sp-keyring = { version = "2.0.0-alpha.5", path = "../../primitives/keyring" } +sp-keyring = { version = "2.0.0-dev", path = "../../primitives/keyring" } substrate-test-runtime-transaction-pool = { version = "2.0.0-dev", path = "../../test-utils/runtime/transaction-pool" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/transaction-pool/graph/Cargo.toml b/client/transaction-pool/graph/Cargo.toml index df2fd8546a2650453d273d558bf6b17355509b5c..842d54f920a71acc70ef31ce2fa2d474099808a7 100644 --- a/client/transaction-pool/graph/Cargo.toml +++ b/client/transaction-pool/graph/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sc-transaction-graph" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,6 +8,9 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Generic Transaction Pool" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] derive_more = "0.99.2" futures = "0.3.4" @@ -15,12 +18,12 @@ log = "0.4.8" parking_lot = "0.10.0" serde = { version = "1.0.101", features = ["derive"] } wasm-timer = "0.2" -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } -sp-utils = { version = "2.0.0-alpha.5", path = "../../../primitives/utils" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../primitives/transaction-pool" } -parity-util-mem = { version = "0.6.0", default-features = false, features = ["primitive-types"] } +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } +sp-utils = { version = "2.0.0-dev", path = "../../../primitives/utils" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-transaction-pool = { version = "2.0.0-dev", path = "../../../primitives/transaction-pool" } +parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } linked-hash-map = "0.5.2" [dev-dependencies] @@ -32,6 +35,3 @@ criterion = "0.3" [[bench]] name = "basics" harness = false - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/client/transaction-pool/src/api.rs b/client/transaction-pool/src/api.rs index bf104c8d78191c231b7102e97d31ccdf1237fefa..bd7e11a3a69520d03b027309532e48697f0d5db2 100644 --- a/client/transaction-pool/src/api.rs +++ b/client/transaction-pool/src/api.rs @@ -89,12 +89,17 @@ impl sc_transaction_graph::ChainApi for FullChainApi, _>( &at, |v| v >= 2, ) - .unwrap_or_default(); + .unwrap_or_default() + }; + + sp_tracing::enter_span!("runtime::validate_transaction"); let res = if has_v2 { runtime_api.validate_transaction(&at, source, uxt) } else { diff --git a/client/transaction-pool/src/lib.rs b/client/transaction-pool/src/lib.rs index c50d9dbbb45acdf82945ba9bfc6923c0e9817d04..e095191c574fad94fb8f3bec7d2ce8c5e5f55a03 100644 --- a/client/transaction-pool/src/lib.rs +++ b/client/transaction-pool/src/lib.rs @@ -21,8 +21,10 @@ #![warn(unused_extern_crates)] mod api; -pub mod error; mod revalidation; +mod metrics; + +pub mod error; #[cfg(any(feature = "test-helpers", test))] pub mod testing; @@ -45,6 +47,9 @@ use sp_transaction_pool::{ }; use wasm_timer::Instant; +use prometheus_endpoint::Registry as PrometheusRegistry; +use crate::metrics::MetricsLink as PrometheusMetrics; + type BoxedReadyIterator = Box>> + Send>; type ReadyIteratorFor = BoxedReadyIterator, sc_transaction_graph::ExtrinsicFor>; @@ -62,6 +67,7 @@ pub struct BasicPool revalidation_strategy: Arc>>>, revalidation_queue: Arc>, ready_poll: Arc, Block>>>, + metrics: PrometheusMetrics, } struct ReadyPoll { @@ -147,8 +153,9 @@ impl BasicPool pub fn new( options: sc_transaction_graph::Options, pool_api: Arc, + prometheus: Option<&PrometheusRegistry>, ) -> (Self, Option + Send>>>) { - Self::with_revalidation_type(options, pool_api, RevalidationType::Full) + Self::with_revalidation_type(options, pool_api, prometheus, RevalidationType::Full) } /// Create new basic transaction pool with provided api, for tests. @@ -166,6 +173,7 @@ impl BasicPool revalidation_queue: Arc::new(revalidation_queue), revalidation_strategy: Arc::new(Mutex::new(RevalidationStrategy::Always)), ready_poll: Default::default(), + metrics: Default::default(), }, background_task, notifier, @@ -177,6 +185,7 @@ impl BasicPool pub fn with_revalidation_type( options: sc_transaction_graph::Options, pool_api: Arc, + prometheus: Option<&PrometheusRegistry>, revalidation_type: RevalidationType, ) -> (Self, Option + Send>>>) { let pool = Arc::new(sc_transaction_graph::Pool::new(options, pool_api.clone())); @@ -187,6 +196,7 @@ impl BasicPool (queue, Some(background)) }, }; + ( BasicPool { api: pool_api, @@ -199,6 +209,7 @@ impl BasicPool } )), ready_poll: Default::default(), + metrics: PrometheusMetrics::new(prometheus), }, background_task, ) @@ -228,8 +239,15 @@ impl TransactionPool for BasicPool ) -> PoolFuture, Self::Error>>, Self::Error> { let pool = self.pool.clone(); let at = *at; + + self.metrics.report(|metrics| metrics.validations_scheduled.inc_by(xts.len() as u64)); + + let metrics = self.metrics.clone(); async move { - pool.submit_at(&at, source, xts, false).await + let tx_count = xts.len(); + let res = pool.submit_at(&at, source, xts, false).await; + metrics.report(|metrics| metrics.validations_finished.inc_by(tx_count as u64)); + res }.boxed() } @@ -241,8 +259,16 @@ impl TransactionPool for BasicPool ) -> PoolFuture, Self::Error> { let pool = self.pool.clone(); let at = *at; + + self.metrics.report(|metrics| metrics.validations_scheduled.inc()); + + let metrics = self.metrics.clone(); async move { - pool.submit_one(&at, source, xt).await + let res = pool.submit_one(&at, source, xt).await; + + metrics.report(|metrics| metrics.validations_finished.inc()); + res + }.boxed() } @@ -255,10 +281,17 @@ impl TransactionPool for BasicPool let at = *at; let pool = self.pool.clone(); + self.metrics.report(|metrics| metrics.validations_scheduled.inc()); + + let metrics = self.metrics.clone(); async move { - pool.submit_and_watch(&at, source, xt) + let result = pool.submit_and_watch(&at, source, xt) .map(|result| result.map(|watcher| Box::new(watcher.into_stream()) as _)) - .await + .await; + + metrics.report(|metrics| metrics.validations_finished.inc()); + + result }.boxed() } diff --git a/client/transaction-pool/src/metrics.rs b/client/transaction-pool/src/metrics.rs new file mode 100644 index 0000000000000000000000000000000000000000..78e49b3ca53af732d28bd48ea65dc65b74835264 --- /dev/null +++ b/client/transaction-pool/src/metrics.rs @@ -0,0 +1,69 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Transaction pool Prometheus metrics. + +use std::sync::Arc; + +use prometheus_endpoint::{register, Counter, PrometheusError, Registry, U64}; + +#[derive(Clone, Default)] +pub struct MetricsLink(Arc>); + +impl MetricsLink { + pub fn new(registry: Option<&Registry>) -> Self { + Self(Arc::new( + registry.and_then(|registry| + Metrics::register(registry) + .map_err(|err| { log::warn!("Failed to register prometheus metrics: {}", err); }) + .ok() + ) + )) + } + + pub fn report(&self, do_this: impl FnOnce(&Metrics)) { + if let Some(metrics) = self.0.as_ref() { + do_this(metrics); + } + } +} + +/// Transaction pool Prometheus metrics. +pub struct Metrics { + pub validations_scheduled: Counter, + pub validations_finished: Counter, +} + +impl Metrics { + pub fn register(registry: &Registry) -> Result { + Ok(Self { + validations_scheduled: register( + Counter::new( + "sub_txpool_validations_scheduled", + "Total number of transactions scheduled for validation", + )?, + registry, + )?, + validations_finished: register( + Counter::new( + "sub_txpool_validations_finished", + "Total number of transactions that finished validation", + )?, + registry, + )?, + }) + } +} diff --git a/client/transaction-pool/src/revalidation.rs b/client/transaction-pool/src/revalidation.rs index 9bcb2dac395816a73ff9799f4fe533845c06205a..f203bf08a0cf0321d084de6ff7efaf96905f466b 100644 --- a/client/transaction-pool/src/revalidation.rs +++ b/client/transaction-pool/src/revalidation.rs @@ -30,7 +30,7 @@ use std::time::Duration; #[cfg(not(test))] const BACKGROUND_REVALIDATION_INTERVAL: Duration = Duration::from_millis(200); #[cfg(test)] -pub const BACKGROUND_REVALIDATION_INTERVAL: Duration = Duration::from_millis(5); +pub const BACKGROUND_REVALIDATION_INTERVAL: Duration = Duration::from_millis(1); const BACKGROUND_REVALIDATION_BATCH_SIZE: usize = 20; @@ -214,12 +214,21 @@ impl RevalidationWorker { loop { futures::select! { - _ = interval.next() => { + _guard = interval.next() => { let next_batch = this.prepare_batch(); let batch_len = next_batch.len(); batch_revalidate(this.pool.clone(), this.api.clone(), this.best_block, next_batch).await; + #[cfg(test)] + { + use intervalier::Guard; + // only trigger test events if something was processed + if batch_len == 0 { + _guard.expect("Always some() in tests").skip(); + } + } + if batch_len > 0 || this.len() > 0 { log::debug!( target: "txpool", diff --git a/client/transaction-pool/src/testing/pool.rs b/client/transaction-pool/src/testing/pool.rs index c6c54d97209445f17b2daad50377555cdd0906e4..45fb6f42c321fab9dfe038506d60fb79ec527ee6 100644 --- a/client/transaction-pool/src/testing/pool.rs +++ b/client/transaction-pool/src/testing/pool.rs @@ -20,7 +20,7 @@ use futures::executor::block_on; use txpool::{self, Pool}; use sp_runtime::{ generic::BlockId, - transaction_validity::{ValidTransaction, InvalidTransaction, TransactionSource}, + transaction_validity::{ValidTransaction, TransactionSource, InvalidTransaction}, }; use substrate_test_runtime_client::{ runtime::{Block, Hash, Index, Header, Extrinsic, Transfer}, @@ -221,7 +221,7 @@ fn should_revalidate_during_maintenance() { block_on(pool.maintain(block_event(1))); assert_eq!(pool.status().ready, 1); - block_on(notifier.next_blocking()); + block_on(notifier.next()); // test that pool revalidated transaction that left ready and not included in the block assert_eq!(pool.api.validation_requests().len(), 3); @@ -263,9 +263,8 @@ fn should_not_retain_invalid_hashes_from_retracted() { let event = block_event_with_retracted(1, vec![retracted_hash]); block_on(pool.maintain(event)); - // maintenance is in background - block_on(notifier.next_blocking()); - + block_on(notifier.next()); + assert_eq!(pool.status().ready, 0); } @@ -281,7 +280,6 @@ fn should_revalidate_transaction_multiple_times() { pool.api.push_block(1, vec![xt.clone()]); block_on(pool.maintain(block_event(1))); - block_on(notifier.next_blocking()); block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); @@ -290,7 +288,7 @@ fn should_revalidate_transaction_multiple_times() { pool.api.add_invalid(&xt); block_on(pool.maintain(block_event(2))); - block_on(notifier.next_blocking()); + block_on(notifier.next()); assert_eq!(pool.status().ready, 0); } @@ -309,14 +307,14 @@ fn should_revalidate_across_many_blocks() { pool.api.push_block(1, vec![]); block_on(pool.maintain(block_event(1))); - block_on(notifier.next_blocking()); + block_on(notifier.next()); block_on(pool.submit_one(&BlockId::number(2), SOURCE, xt3.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 3); pool.api.push_block(2, vec![xt1.clone()]); block_on(pool.maintain(block_event(2))); - block_on(notifier.next_blocking()); + block_on(notifier.next()); assert_eq!(pool.status().ready, 2); // xt1 and xt2 validated twice, then xt3 once, then xt2 and xt3 again @@ -361,7 +359,7 @@ fn should_push_watchers_during_maintaince() { // clear timer events if any block_on(pool.maintain(block_event(0))); - block_on(notifier.next_blocking()); + block_on(notifier.next()); // then // hash3 is now invalid @@ -417,7 +415,7 @@ fn finalization() { let xt = uxt(Alice, 209); let api = TestApi::with_alice_nonce(209); api.push_block(1, vec![]); - let (pool, _background) = BasicPool::new(Default::default(), api.into()); + let (pool, _background, _) = BasicPool::new_test(api.into()); let watcher = block_on( pool.submit_and_watch(&BlockId::number(1), SOURCE, xt.clone()) ).expect("1. Imported"); @@ -448,7 +446,7 @@ fn fork_aware_finalization() { // starting block A1 (last finalized.) api.push_block(1, vec![]); - let (pool, _background) = BasicPool::new(Default::default(), api.into()); + let (pool, _background, _) = BasicPool::new_test(api.into()); let mut canon_watchers = vec![]; let from_alice = uxt(Alice, 1); @@ -679,7 +677,7 @@ fn should_not_accept_old_signatures() { let client = Arc::new(substrate_test_runtime_client::new()); let pool = Arc::new( - BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client))).0 + BasicPool::new_test(Arc::new(FullChainApi::new(client))).0 ); let transfer = Transfer { @@ -702,6 +700,6 @@ fn should_not_accept_old_signatures() { Err(error::Error::Pool( sp_transaction_pool::error::Error::InvalidTransaction(InvalidTransaction::BadProof) )), - "Should be invalid transactiono with bad proof", + "Should be invalid transaction with bad proof", ); } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6ac08309348f0140a3e59adb48d2a3146845f465..b0b9ca9f6c6a1c9fcfb3df8850e067382aa9ed58 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,6 +6,46 @@ The format is based on [Keep a Changelog]. ## Unreleased + +## 2.0.0-alpha.5 -> 2.0.0-alpha.6 + + +Runtime +------- + +* Unsigned Validation best practices (#5563) +* Generate Unit Tests for Benchmarks (#5527) +* Mandate weight annotation (#5357) +* Make Staking pallet using a proper Time module. (#4662) +* Pass transaction source to validate_transaction (#5366) +* on_initialize return weight consumed and default cost to default DispatchInfo instead of zero (#5382) + +Client +------ + +* Add new RPC method to get the chain type (#5576) +* Reuse wasmtime instances, the PR (#5567) +* Prometheus Metrics: Turn notifications_total counter into notifications_sizes histogram (#5535) +* Make verbosity level mandatory with telemetry opt (#5057) +* Additional Metrics collected and exposed via prometheus (#5414) +* Switch to new light client protocol (#5472) +* client/finality-grandpa: Instrument until-imported queue (#5438) +* Batch benchmarks together with `*` notation. (#5436) +* src/service/src/builder: Fix memory metric exposed in bytes not KiB (#5459) +* Make transactions and block announces use notifications substre… (#5360) +* Adds state_queryStorageAt (#5362) +* Offchain Phragmén BREAKING. (#4517) +* `sc_rpc::system::SystemInfo.impl_version` now returns the full version (2.0.0-alpha.2-b950f731c-x86_64-linux-gnu) instead of the short version (1.0.0) (#5271) + +API +--- + +* Unsigned Validation best practices (#5563) +* Split the Roles in three types (#5520) +* Pass transaction source to validate_transaction (#5366) +* on_initialize return weight consumed and default cost to default DispatchInfo instead of zero (#5382) + + ## 2.0.0-alpha.4 -> 2.0.0-alpha.5 Runtime @@ -51,4 +91,4 @@ API * client/authority-discovery: Instrument code with Prometheus (#5195) * Don't include `:code` by default in storage proofs (#5179) * client/network-gossip: Merge GossipEngine and GossipEngineInner (#5042) -* Introduce `on_runtime_upgrade` (#5058) \ No newline at end of file +* Introduce `on_runtime_upgrade` (#5058) diff --git a/docs/CODEOWNERS b/docs/CODEOWNERS index d4404346294a53abc2520cdfab7f7a4fc582f1f0..ddb776ea22ad2933f857a11560a8af803de44fe2 100644 --- a/docs/CODEOWNERS +++ b/docs/CODEOWNERS @@ -34,8 +34,8 @@ /primitives/core/src/sandbox.rs @pepyakin # Transaction pool -/client/transaction-pool/ @tomusdrw @NikVolf -/primitives/transaction-pool/ @tomusdrw @NikVolf +/client/transaction-pool/ @NikVolf +/primitives/transaction-pool/ @NikVolf # Offchain /client/offchain/ @tomusdrw @@ -57,15 +57,12 @@ /primitives/consensus/pow/ @sorpaas # Contracts -/frame/contracts/ @pepyakin @thiolliere +/frame/contracts/ @pepyakin /frame/contracts/src/wasm/runtime.rs @Robbepop # EVM /frame/evm/ @sorpaas -# Inflation points -/frame/staking/src/inflation.rs @thiolliere - # NPoS and Governance and Phragmén /frame/staking/ @kianenigma /frame/elections/ @kianenigma @@ -73,7 +70,7 @@ /primitives/phragmen/ @kianenigma # Fixed point arithmetic -/primitives/sp-arithmetic/ @kianenigma @thiolliere +/primitives/sp-arithmetic/ @kianenigma # End to end testing of substrate node /bin/node/executor/ @kianenigma @@ -82,7 +79,7 @@ /frame/support/src/weights.rs @kianenigma # Support crates -/frame/support/ @thiolliere @kianenigma +/frame/support/ @kianenigma # Authority discovery /client/authority-discovery/ @mxinden @@ -90,3 +87,7 @@ # Prometheus endpoint /utils/prometheus/ @mxinden + +# CLI API +/client/cli @cecton +/client/cli-derive @cecton diff --git a/docs/PULL_REQUEST_TEMPLATE.md b/docs/PULL_REQUEST_TEMPLATE.md index 9755fa0e40ff9bcef1003d0933b174cc1adee06b..fa2c15e6c02fea0f8f8bba9e27059f0e0617b2d6 100644 --- a/docs/PULL_REQUEST_TEMPLATE.md +++ b/docs/PULL_REQUEST_TEMPLATE.md @@ -9,7 +9,7 @@ Before you submitting, please check that: - [ ] You labeled the PR with appropriate labels if you have permissions to do so. - [ ] You mentioned a related issue if this PR related to it, e.g. `Fixes #228` or `Related #1337`. - [ ] You asked any particular reviewers to review. If you aren't sure, start with GH suggestions. -- [ ] Your PR adheres [the style guide](https://github.com/paritytech/polkadot/wiki/Style-Guide) +- [ ] Your PR adheres [the style guide](https://wiki.parity.io/Substrate-Style-Guide) - In particular, mind the maximal line length. - There is no commented code checked in unless necessary. - Any panickers have a proof or removed. diff --git a/frame/assets/Cargo.toml b/frame/assets/Cargo.toml index 8b242ff0e8ef149e47b86365ba14acf7abe9cc0a..4e09ea8aa51e8f5f39d87e5ea8294fca26f065e4 100644 --- a/frame/assets/Cargo.toml +++ b/frame/assets/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-assets" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,20 +8,23 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME asset management pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } # Needed for various traits. In our case, `OnFinalize`. -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } # Needed for type-safe access to storage DB. -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } # `system` module provides us with all sorts of useful stuff and macros depend on it being around. -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.5", path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", path = "../../primitives/io" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-std = { version = "2.0.0-dev", path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", path = "../../primitives/io" } [features] default = ["std"] @@ -32,6 +35,3 @@ std = [ "frame-support/std", "frame-system/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/assets/src/lib.rs b/frame/assets/src/lib.rs index 388eb7780bd6dc7df7525d9e015c20790b0c4594..1aaedd4c74d79ce948606481249c538de95308f9 100644 --- a/frame/assets/src/lib.rs +++ b/frame/assets/src/lib.rs @@ -157,7 +157,14 @@ decl_module! { /// Issue a new class of fungible assets. There are, and will only ever be, `total` /// such assets and they'll all belong to the `origin` initially. It will have an /// identifier `AssetId` instance: this will be specified in the `Issued` event. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + /// + /// # + /// - `O(1)` + /// - 1 storage mutation (codec `O(1)`). + /// - 2 storage writes (condec `O(1)`). + /// - 1 event. + /// # + #[weight = 0] fn issue(origin, #[compact] total: T::Balance) { let origin = ensure_signed(origin)?; @@ -171,7 +178,14 @@ decl_module! { } /// Move some assets from one holder to another. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + /// + /// # + /// - `O(1)` + /// - 1 static lookup + /// - 2 storage mutations (codec `O(1)`). + /// - 1 event. + /// # + #[weight = 0] fn transfer(origin, #[compact] id: T::AssetId, target: ::Source, @@ -190,7 +204,14 @@ decl_module! { } /// Destroy any assets of `id` owned by `origin`. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + /// + /// # + /// - `O(1)` + /// - 1 storage mutation (codec `O(1)`). + /// - 1 storage deletion (codec `O(1)`). + /// - 1 event. + /// # + #[weight = 0] fn destroy(origin, #[compact] id: T::AssetId) { let origin = ensure_signed(origin)?; let balance = >::take((id, &origin)); @@ -292,6 +313,9 @@ mod tests { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); diff --git a/frame/aura/Cargo.toml b/frame/aura/Cargo.toml index 36e79116618e35c296590c96a1666e03bd349784..623cf80df7b24a26b7b0d0567e3cfc8e6f7272f3 100644 --- a/frame/aura/Cargo.toml +++ b/frame/aura/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-aura" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,21 +8,24 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME AURA consensus pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/application-crypto" } +sp-application-crypto = { version = "2.0.0-dev", default-features = false, path = "../../primitives/application-crypto" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../../primitives/inherents" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } serde = { version = "1.0.101", optional = true } -pallet-session = { version = "2.0.0-alpha.5", default-features = false, path = "../session" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.5"} -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -sp-consensus-aura = { path = "../../primitives/consensus/aura", default-features = false, version = "0.8.0-alpha.5"} -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -sp-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/timestamp" } -pallet-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../timestamp" } +pallet-session = { version = "2.0.0-dev", default-features = false, path = "../session" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-dev"} +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +sp-consensus-aura = { path = "../../primitives/consensus/aura", default-features = false, version = "0.8.0-dev"} +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +sp-timestamp = { version = "2.0.0-dev", default-features = false, path = "../../primitives/timestamp" } +pallet-timestamp = { version = "2.0.0-dev", default-features = false, path = "../timestamp" } [dev-dependencies] @@ -46,6 +49,3 @@ std = [ "sp-timestamp/std", "pallet-timestamp/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/aura/src/mock.rs b/frame/aura/src/mock.rs index 05a161ee49c3d6c9ac7c51028d31f1dd9d193bc8..8154ef4c9f50db3dd99a760e4cfd7e8bee00bac8 100644 --- a/frame/aura/src/mock.rs +++ b/frame/aura/src/mock.rs @@ -57,6 +57,9 @@ impl frame_system::Trait for Test { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); diff --git a/frame/authority-discovery/Cargo.toml b/frame/authority-discovery/Cargo.toml index f67d4ee038ab374a5cd2c75c947f141fc58fdde2..5f267b3159561345bb30d0b734f89b36d25acc6e 100644 --- a/frame/authority-discovery/Cargo.toml +++ b/frame/authority-discovery/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-authority-discovery" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,21 +8,24 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for authority discovery" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-authority-discovery = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/authority-discovery" } -sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/application-crypto" } +sp-authority-discovery = { version = "2.0.0-dev", default-features = false, path = "../../primitives/authority-discovery" } +sp-application-crypto = { version = "2.0.0-dev", default-features = false, path = "../../primitives/application-crypto" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } serde = { version = "1.0.101", optional = true } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -pallet-session = { version = "2.0.0-alpha.5", features = ["historical" ], path = "../session", default-features = false } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +pallet-session = { version = "2.0.0-dev", features = ["historical" ], path = "../session", default-features = false } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-staking = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/staking" } +sp-staking = { version = "2.0.0-dev", default-features = false, path = "../../primitives/staking" } [features] default = ["std"] @@ -39,6 +42,3 @@ std = [ "frame-support/std", "frame-system/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/authority-discovery/src/lib.rs b/frame/authority-discovery/src/lib.rs index b8f28b432ba75f14594ee50e9b6fbfeb0ee73312..ca3e293ae740e69dad63f366cdc34c38023fe046 100644 --- a/frame/authority-discovery/src/lib.rs +++ b/frame/authority-discovery/src/lib.rs @@ -154,6 +154,9 @@ mod tests { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); diff --git a/frame/authorship/Cargo.toml b/frame/authorship/Cargo.toml index fb966113d55ee67fda4dcf946487f82a9185d6a6..98776c0e00d012a35d3d4faa5d82f6b8622b5655 100644 --- a/frame/authorship/Cargo.toml +++ b/frame/authorship/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-authorship" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" description = "Block and Uncle Author tracking for the FRAME" authors = ["Parity Technologies "] edition = "2018" @@ -8,16 +8,19 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" } -sp-authorship = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/authorship" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.5"} +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../../primitives/inherents" } +sp-authorship = { version = "2.0.0-dev", default-features = false, path = "../../primitives/authorship" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-dev"} impl-trait-for-tuples = "0.1.3" [features] @@ -33,6 +36,3 @@ std = [ "sp-io/std", "sp-authorship/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/authorship/src/lib.rs b/frame/authorship/src/lib.rs index e6249849bf40ce08844e2a562832b984af9ef7ec..e799ec2367e84e6627e23396f15b34e620db0bc5 100644 --- a/frame/authorship/src/lib.rs +++ b/frame/authorship/src/lib.rs @@ -27,7 +27,7 @@ use frame_support::traits::{FindAuthor, VerifySeal, Get}; use codec::{Encode, Decode}; use frame_system::ensure_none; use sp_runtime::traits::{Header as HeaderT, One, Zero}; -use frame_support::weights::{Weight, SimpleDispatchInfo, WeighData}; +use frame_support::weights::{Weight, DispatchClass}; use sp_inherents::{InherentIdentifier, ProvideInherent, InherentData}; use sp_authorship::{INHERENT_IDENTIFIER, UnclesInherentData, InherentError}; @@ -197,7 +197,7 @@ decl_module! { T::EventHandler::note_author(Self::author()); - SimpleDispatchInfo::default().weigh_data(()) + 0 } fn on_finalize() { @@ -207,7 +207,7 @@ decl_module! { } /// Provide a set of uncles. - #[weight = SimpleDispatchInfo::FixedMandatory(10_000)] + #[weight = (0, DispatchClass::Mandatory)] fn set_uncles(origin, new_uncles: Vec) -> dispatch::DispatchResult { ensure_none(origin)?; ensure!(new_uncles.len() <= MAX_UNCLES, Error::::TooManyUncles); @@ -429,6 +429,9 @@ mod tests { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); diff --git a/frame/babe/Cargo.toml b/frame/babe/Cargo.toml index 8f885500224053f5ccb8ecb80615b468a34bc514..c94ec75b26ad34dd08b5f292bd5adad1924c6670 100644 --- a/frame/babe/Cargo.toml +++ b/frame/babe/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-babe" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,24 +8,27 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Consensus extension module for BABE consensus. Collects on-chain randomness from VRF outputs and manages epoch transitions." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true } -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-staking = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/staking" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -pallet-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../timestamp" } -sp-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/timestamp" } -pallet-session = { version = "2.0.0-alpha.5", default-features = false, path = "../session" } -sp-consensus-babe = { version = "0.8.0-alpha.5", default-features = false, path = "../../primitives/consensus/babe" } -sp-consensus-vrf = { version = "0.8.0-alpha.5", default-features = false, path = "../../primitives/consensus/vrf" } -sp-io = { path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.5"} +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../../primitives/inherents" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "2.0.0-dev", default-features = false, path = "../../primitives/staking" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +pallet-timestamp = { version = "2.0.0-dev", default-features = false, path = "../timestamp" } +sp-timestamp = { version = "2.0.0-dev", default-features = false, path = "../../primitives/timestamp" } +pallet-session = { version = "2.0.0-dev", default-features = false, path = "../session" } +sp-consensus-babe = { version = "0.8.0-dev", default-features = false, path = "../../primitives/consensus/babe" } +sp-consensus-vrf = { version = "0.8.0-dev", default-features = false, path = "../../primitives/consensus/vrf" } +sp-io = { path = "../../primitives/io", default-features = false , version = "2.0.0-dev"} [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } [features] default = ["std"] @@ -45,6 +48,3 @@ std = [ "pallet-session/std", "sp-io/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/babe/src/lib.rs b/frame/babe/src/lib.rs index 5f85b910880670e646eb7c903688f754fcefa1b3..7357ef75ffaf9ef7de9c5cc10ba3e5b6c6c8bf0c 100644 --- a/frame/babe/src/lib.rs +++ b/frame/babe/src/lib.rs @@ -25,7 +25,7 @@ use pallet_timestamp; use sp_std::{result, prelude::*}; use frame_support::{ decl_storage, decl_module, traits::{FindAuthor, Get, Randomness as RandomnessT}, - weights::{Weight, SimpleDispatchInfo, WeighData}, + weights::Weight, }; use sp_timestamp::OnTimestampSet; use sp_runtime::{generic::DigestItem, ConsensusEngineId, Perbill}; @@ -184,7 +184,7 @@ decl_module! { fn on_initialize(now: T::BlockNumber) -> Weight { Self::do_initialize(now); - SimpleDispatchInfo::default().weigh_data(()) + 0 } /// Block finalization @@ -205,6 +205,21 @@ decl_module! { } impl RandomnessT<::Hash> for Module { + /// Some BABE blocks have VRF outputs where the block producer has exactly one bit of influence, + /// either they make the block or they do not make the block and thus someone else makes the + /// next block. Yet, this randomness is not fresh in all BABE blocks. + /// + /// If that is an insufficient security guarantee then two things can be used to improve this + /// randomness: + /// + /// - Name, in advance, the block number whose random value will be used; ensure your module + /// retains a buffer of previous random values for its subject and then index into these in + /// order to obviate the ability of your user to look up the parent hash and choose when to + /// transact based upon it. + /// - Require your user to first commit to an additional value by first posting its hash. + /// Require them to reveal the value to determine the final result, hashing it with the + /// output of this random function. This reduces the ability of a cabal of block producers + /// from conspiring against individuals. fn random(subject: &[u8]) -> T::Hash { let mut subject = subject.to_vec(); subject.reserve(VRF_OUTPUT_LENGTH); diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index ea802b268e399c6f71577507df49b60288e880f5..9f029fd27f75aa74748eb4feeb0e3750054c96cd 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -68,6 +68,9 @@ impl frame_system::Trait for Test { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type ModuleToIndex = (); diff --git a/frame/balances/Cargo.toml b/frame/balances/Cargo.toml index f8a59ad15476fbd0c4f69fc418c17c67be3a7506..769e68112cedb46f89cf7e68462391ee7092378e 100644 --- a/frame/balances/Cargo.toml +++ b/frame/balances/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-balances" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,19 +8,22 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet to manage balances" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../benchmarking", optional = true } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -pallet-transaction-payment = { version = "2.0.0-alpha.5", path = "../transaction-payment" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +pallet-transaction-payment = { version = "2.0.0-dev", path = "../transaction-payment" } [features] default = ["std"] @@ -35,6 +38,3 @@ std = [ "frame-system/std", ] runtime-benchmarks = ["frame-benchmarking"] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/balances/src/benchmarking.rs b/frame/balances/src/benchmarking.rs index 161fdab96b934a5bf234604406c554b261fb7907..3c2067559fcf21dcae48356f0acf00685c7844f2 100644 --- a/frame/balances/src/benchmarking.rs +++ b/frame/balances/src/benchmarking.rs @@ -51,10 +51,13 @@ benchmarks! { let _ = as Currency<_>>::make_free_balance_be(&caller, balance); // Transfer `e - 1` existential deposits + 1 unit, which guarantees to create one account, and reap this user. - let recipient = account("recipient", u, SEED); - let recipient_lookup: ::Source = T::Lookup::unlookup(recipient); + let recipient: T::AccountId = account("recipient", u, SEED); + let recipient_lookup: ::Source = T::Lookup::unlookup(recipient.clone()); let transfer_amount = existential_deposit.saturating_mul((e - 1).into()) + 1.into(); }: _(RawOrigin::Signed(caller), recipient_lookup, transfer_amount) + verify { + assert_eq!(Balances::::free_balance(&recipient), transfer_amount); + } // Benchmark `transfer` with the best possible condition: // * Both accounts exist and will continue to exist. @@ -119,3 +122,45 @@ benchmarks! { let _ = as Currency<_>>::make_free_balance_be(&user, balance_amount); }: set_balance(RawOrigin::Root, user_lookup, 0.into(), 0.into()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests_composite::{ExtBuilder, Test}; + use frame_support::assert_ok; + + #[test] + fn transfer() { + ExtBuilder::default().build().execute_with(|| { + assert_ok!(test_benchmark_transfer::()); + }); + } + + #[test] + fn transfer_best_case() { + ExtBuilder::default().build().execute_with(|| { + assert_ok!(test_benchmark_transfer_best_case::()); + }); + } + + #[test] + fn transfer_keep_alive() { + ExtBuilder::default().build().execute_with(|| { + assert_ok!(test_benchmark_transfer_keep_alive::()); + }); + } + + #[test] + fn transfer_set_balance() { + ExtBuilder::default().build().execute_with(|| { + assert_ok!(test_benchmark_set_balance::()); + }); + } + + #[test] + fn transfer_set_balance_killing() { + ExtBuilder::default().build().execute_with(|| { + assert_ok!(test_benchmark_set_balance_killing::()); + }); + } +} diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index 39e15b3f4f585d38b3d68752b1803782ae0b3619..94dbd3730f163d7af58e303910c7cb0ea35dd17d 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -159,7 +159,7 @@ use sp_std::{cmp, result, mem, fmt::Debug, ops::BitOr, convert::Infallible}; use codec::{Codec, Encode, Decode}; use frame_support::{ StorageValue, Parameter, decl_event, decl_storage, decl_module, decl_error, ensure, - weights::SimpleDispatchInfo, traits::{ + traits::{ Currency, OnKilledAccount, OnUnbalanced, TryDrop, StoredMap, WithdrawReason, WithdrawReasons, LockIdentifier, LockableCurrency, ExistenceRequirement, Imbalance, SignedImbalance, ReservableCurrency, Get, ExistenceRequirement::KeepAlive, @@ -433,7 +433,7 @@ decl_module! { /// check that the transfer will not kill the origin account. /// /// # - #[weight = SimpleDispatchInfo::FixedNormal(1_000_000)] + #[weight = T::DbWeight::get().reads_writes(1, 1) + 200_000_000] pub fn transfer( origin, dest: ::Source, @@ -457,7 +457,7 @@ decl_module! { /// - Independent of the arguments. /// - Contains a limited number of reads and writes. /// # - #[weight = SimpleDispatchInfo::FixedOperational(50_000)] + #[weight = T::DbWeight::get().reads_writes(1, 1) + 100_000_000] fn set_balance( origin, who: ::Source, @@ -495,7 +495,11 @@ decl_module! { /// Exactly as `transfer`, except the origin must be root and the source account may be /// specified. - #[weight = SimpleDispatchInfo::FixedNormal(1_000_000)] + /// # + /// - Same as transfer, but additional read and write because the source account is + /// not assumed to be in the overlay. + /// # + #[weight = T::DbWeight::get().reads_writes(2, 2) + 200_000_000] pub fn force_transfer( origin, source: ::Source, @@ -514,7 +518,7 @@ decl_module! { /// 99% of the time you want [`transfer`] instead. /// /// [`transfer`]: struct.Module.html#method.transfer - #[weight = SimpleDispatchInfo::FixedNormal(1_000_000)] + #[weight = T::DbWeight::get().reads_writes(1, 1) + 150_000_000] pub fn transfer_keep_alive( origin, dest: ::Source, @@ -842,6 +846,9 @@ impl, I: Instance> frame_system::Trait for ElevatedTrait { type Event = (); type BlockHashCount = T::BlockHashCount; type MaximumBlockWeight = T::MaximumBlockWeight; + type DbWeight = T::DbWeight; + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = T::MaximumBlockLength; type AvailableBlockRatio = T::AvailableBlockRatio; type Version = T::Version; @@ -1094,7 +1101,7 @@ impl, I: Instance> Currency for Module where // equal and opposite cause (returned as an Imbalance), then in the // instance that there's no other accounts on the system at all, we might // underflow the issuance and our arithmetic will be off. - ensure!(value + account.reserved >= ed || !account.total().is_zero(), ()); + ensure!(value.saturating_add(account.reserved) >= ed || !account.total().is_zero(), ()); let imbalance = if account.free <= value { SignedImbalance::Positive(PositiveImbalance::new(value - account.free)) diff --git a/frame/balances/src/tests.rs b/frame/balances/src/tests.rs index 8055c2013eaebc5de75e8d1a1b53c22dd87dc62e..7663b8922bc04efa32b1a6fefcc476590436c9c6 100644 --- a/frame/balances/src/tests.rs +++ b/frame/balances/src/tests.rs @@ -18,12 +18,25 @@ #![cfg(test)] +#[derive(Debug)] +pub struct CallWithDispatchInfo; +impl sp_runtime::traits::Dispatchable for CallWithDispatchInfo { + type Origin = (); + type Trait = (); + type Info = frame_support::weights::DispatchInfo; + type PostInfo = frame_support::weights::PostDispatchInfo; + fn dispatch(self, _origin: Self::Origin) + -> sp_runtime::DispatchResultWithInfo { + panic!("Do not use dummy implementation for dispatch."); + } +} + #[macro_export] macro_rules! decl_tests { ($test:ty, $ext_builder:ty, $existential_deposit:expr) => { use crate::*; - use sp_runtime::{Fixed64, traits::{SignedExtension, BadOrigin}}; + use sp_runtime::{Fixed128, traits::{SignedExtension, BadOrigin}}; use frame_support::{ assert_noop, assert_ok, assert_err, traits::{ @@ -40,11 +53,11 @@ macro_rules! decl_tests { pub type System = frame_system::Module<$test>; pub type Balances = Module<$test>; - pub const CALL: &<$test as frame_system::Trait>::Call = &(); + pub const CALL: &<$test as frame_system::Trait>::Call = &$crate::tests::CallWithDispatchInfo; /// create a transaction info struct from weight. Handy to avoid building the whole struct. pub fn info_from_weight(w: Weight) -> DispatchInfo { - DispatchInfo { weight: w, pays_fee: true, ..Default::default() } + DispatchInfo { weight: w, ..Default::default() } } #[test] @@ -140,7 +153,7 @@ macro_rules! decl_tests { .monied(true) .build() .execute_with(|| { - pallet_transaction_payment::NextFeeMultiplier::put(Fixed64::from_natural(1)); + pallet_transaction_payment::NextFeeMultiplier::put(Fixed128::from_natural(1)); Balances::set_lock(ID_1, &1, 10, WithdrawReason::Reserve.into()); assert_noop!( >::transfer(&1, &2, 1, AllowDeath), @@ -154,14 +167,14 @@ macro_rules! decl_tests { ChargeTransactionPayment::from(1), &1, CALL, - info_from_weight(1), + &info_from_weight(1), 1, ).is_err()); assert!( as SignedExtension>::pre_dispatch( ChargeTransactionPayment::from(0), &1, CALL, - info_from_weight(1), + &info_from_weight(1), 1, ).is_ok()); @@ -172,14 +185,14 @@ macro_rules! decl_tests { ChargeTransactionPayment::from(1), &1, CALL, - info_from_weight(1), + &info_from_weight(1), 1, ).is_err()); assert!( as SignedExtension>::pre_dispatch( ChargeTransactionPayment::from(0), &1, CALL, - info_from_weight(1), + &info_from_weight(1), 1, ).is_err()); }); diff --git a/frame/balances/src/tests_composite.rs b/frame/balances/src/tests_composite.rs index 8935dc4c9a12364d30c6f74a2bc5076306c38223..5d25fdb50ceba6dd8b09e014f904a0e408eba43d 100644 --- a/frame/balances/src/tests_composite.rs +++ b/frame/balances/src/tests_composite.rs @@ -18,14 +18,18 @@ #![cfg(test)] -use sp_runtime::{Perbill, traits::{ConvertInto, IdentityLookup}, testing::Header}; +use sp_runtime::{ + Perbill, + traits::{ConvertInto, IdentityLookup}, + testing::Header, +}; use sp_core::H256; use sp_io; use frame_support::{impl_outer_origin, parameter_types}; use frame_support::traits::Get; use frame_support::weights::{Weight, DispatchInfo}; use std::cell::RefCell; -use crate::{GenesisConfig, Module, Trait, decl_tests}; +use crate::{GenesisConfig, Module, Trait, decl_tests, tests::CallWithDispatchInfo}; use frame_system as system; impl_outer_origin!{ @@ -54,7 +58,7 @@ impl frame_system::Trait for Test { type Origin = Origin; type Index = u64; type BlockNumber = u64; - type Call = (); + type Call = CallWithDispatchInfo; type Hash = H256; type Hashing = ::sp_runtime::traits::BlakeTwo256; type AccountId = u64; @@ -63,6 +67,9 @@ impl frame_system::Trait for Test { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -72,13 +79,11 @@ impl frame_system::Trait for Test { type OnKilledAccount = (); } parameter_types! { - pub const TransactionBaseFee: u64 = 0; pub const TransactionByteFee: u64 = 1; } impl pallet_transaction_payment::Trait for Test { type Currency = Module; type OnTransactionPayment = (); - type TransactionBaseFee = TransactionBaseFee; type TransactionByteFee = TransactionByteFee; type WeightToFee = ConvertInto; type FeeMultiplierUpdate = (); diff --git a/frame/balances/src/tests_local.rs b/frame/balances/src/tests_local.rs index c8a4a298f57fd90abcf0db63ed1e0d0b1323857f..afc0edbae7f161c3c0284f271b63c2f15cfdb3f2 100644 --- a/frame/balances/src/tests_local.rs +++ b/frame/balances/src/tests_local.rs @@ -18,14 +18,18 @@ #![cfg(test)] -use sp_runtime::{Perbill, traits::{ConvertInto, IdentityLookup}, testing::Header}; +use sp_runtime::{ + Perbill, + traits::{ConvertInto, IdentityLookup}, + testing::Header, +}; use sp_core::H256; use sp_io; use frame_support::{impl_outer_origin, parameter_types}; use frame_support::traits::{Get, StorageMapShim}; use frame_support::weights::{Weight, DispatchInfo}; use std::cell::RefCell; -use crate::{GenesisConfig, Module, Trait, decl_tests}; +use crate::{GenesisConfig, Module, Trait, decl_tests, tests::CallWithDispatchInfo}; use frame_system as system; impl_outer_origin!{ @@ -54,7 +58,7 @@ impl frame_system::Trait for Test { type Origin = Origin; type Index = u64; type BlockNumber = u64; - type Call = (); + type Call = CallWithDispatchInfo; type Hash = H256; type Hashing = ::sp_runtime::traits::BlakeTwo256; type AccountId = u64; @@ -63,6 +67,9 @@ impl frame_system::Trait for Test { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -72,13 +79,11 @@ impl frame_system::Trait for Test { type OnKilledAccount = Module; } parameter_types! { - pub const TransactionBaseFee: u64 = 0; pub const TransactionByteFee: u64 = 1; } impl pallet_transaction_payment::Trait for Test { type Currency = Module; type OnTransactionPayment = (); - type TransactionBaseFee = TransactionBaseFee; type TransactionByteFee = TransactionByteFee; type WeightToFee = ConvertInto; type FeeMultiplierUpdate = (); diff --git a/frame/benchmark/Cargo.toml b/frame/benchmark/Cargo.toml index 804a28370449c46056caf5b4e97a515f6e33b9c7..0b506d12ec90e838438b433e74ac8e8df5a413f7 100644 --- a/frame/benchmark/Cargo.toml +++ b/frame/benchmark/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-benchmark" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,15 +8,18 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Patterns to benchmark in a FRAME runtime." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../benchmarking", optional = true } [features] default = ["std"] @@ -31,6 +34,3 @@ std = [ "frame-benchmarking/std", ] runtime-benchmarks = ["frame-benchmarking"] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/benchmark/src/lib.rs b/frame/benchmark/src/lib.rs index b571ffb5b9cab6272908585270a44e86efa48e6d..6f7b074a9e6796414fdd016584cd9380a2aace65 100644 --- a/frame/benchmark/src/lib.rs +++ b/frame/benchmark/src/lib.rs @@ -70,7 +70,7 @@ decl_module! { fn deposit_event() = default; /// Do nothing. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn do_nothing(_origin, input: u32) { if input > 0 { return Ok(()); @@ -82,7 +82,7 @@ decl_module! { /// storage database, however, the `repeat` calls will all pull from the /// storage overlay cache. You must consider this when analyzing the /// results of the benchmark. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn read_value(_origin, repeat: u32) { for _ in 0..repeat { MyValue::get(); @@ -90,7 +90,7 @@ decl_module! { } /// Put a value into a storage value. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn put_value(_origin, repeat: u32) { for r in 0..repeat { MyValue::put(r); @@ -102,7 +102,7 @@ decl_module! { /// storage database, however, the `repeat` calls will all pull from the /// storage overlay cache. You must consider this when analyzing the /// results of the benchmark. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn exists_value(_origin, repeat: u32) { for _ in 0..repeat { MyValue::exists(); @@ -110,7 +110,7 @@ decl_module! { } /// Remove a value from storage `repeat` number of times. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn remove_value(_origin, repeat: u32) { for r in 0..repeat { MyMap::remove(r); @@ -118,7 +118,7 @@ decl_module! { } /// Read a value from storage map `repeat` number of times. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn read_map(_origin, repeat: u32) { for r in 0..repeat { MyMap::get(r); @@ -126,7 +126,7 @@ decl_module! { } /// Insert a value into a map. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn insert_map(_origin, repeat: u32) { for r in 0..repeat { MyMap::insert(r, r); @@ -134,7 +134,7 @@ decl_module! { } /// Check is a map contains a value `repeat` number of times. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn contains_key_map(_origin, repeat: u32) { for r in 0..repeat { MyMap::contains_key(r); @@ -142,7 +142,7 @@ decl_module! { } /// Read a value from storage `repeat` number of times. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn remove_prefix(_origin, repeat: u32) { for r in 0..repeat { MyDoubleMap::remove_prefix(r); @@ -150,21 +150,21 @@ decl_module! { } /// Add user to the list. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn add_member_list(origin) { let who = ensure_signed(origin)?; MyMemberList::::mutate(|x| x.push(who)); } /// Append user to the list. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn append_member_list(origin) { let who = ensure_signed(origin)?; MyMemberList::::append(&[who])?; } /// Encode a vector of accounts to bytes. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn encode_accounts(_origin, accounts: Vec) { let bytes = accounts.encode(); @@ -176,7 +176,7 @@ decl_module! { } /// Decode bytes into a vector of accounts. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn decode_accounts(_origin, bytes: Vec) { let accounts: Vec = Decode::decode(&mut bytes.as_slice()).map_err(|_| "Could not decode")?; diff --git a/frame/benchmarking/Cargo.toml b/frame/benchmarking/Cargo.toml index 7ed60664190ee53bfe21574ad67133aca99de639..8089a2a3661d0ffecc7b5c6c067ce740e350a083 100644 --- a/frame/benchmarking/Cargo.toml +++ b/frame/benchmarking/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-benchmarking" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,16 +8,20 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Macro for benchmarking a FRAME runtime." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] linregress = "0.1" +paste = "0.1" codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-api = { version = "2.0.0-alpha.5", path = "../../primitives/api", default-features = false } -sp-runtime-interface = { version = "2.0.0-alpha.5", path = "../../primitives/runtime-interface", default-features = false } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime", default-features = false } -sp-std = { version = "2.0.0-alpha.5", path = "../../primitives/std", default-features = false } -sp-io = { path = "../../primitives/io", default-features = false, version = "2.0.0-alpha.5"} -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-api = { version = "2.0.0-dev", path = "../../primitives/api", default-features = false } +sp-runtime-interface = { version = "2.0.0-dev", path = "../../primitives/runtime-interface", default-features = false } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime", default-features = false } +sp-std = { version = "2.0.0-dev", path = "../../primitives/std", default-features = false } +sp-io = { path = "../../primitives/io", default-features = false, version = "2.0.0-dev"} +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } [features] default = [ "std" ] @@ -30,6 +34,3 @@ std = [ "frame-support/std", "frame-system/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index b1427b792de3a9b4e4e85a226c4e0035ebf4b35b..8a9df7e4cf3419823bb696b979eda0550aec5e16 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -28,7 +28,8 @@ pub use utils::*; pub use analysis::Analysis; #[doc(hidden)] pub use sp_io::storage::root as storage_root; -pub use sp_runtime::traits::Dispatchable; +pub use sp_runtime::traits::{Dispatchable, Zero}; +pub use paste; /// Construct pallet benchmarks for weighing dispatchables. /// @@ -68,6 +69,9 @@ pub use sp_runtime::traits::Dispatchable; /// (or not) by each arm. Syntax is available to allow for only the range to be drawn upon if /// desired, allowing an alternative instancing expression to be given. /// +/// Note that the ranges are *inclusive* on both sides. This is in contrast to ranges in Rust which +/// are left-inclusive right-exclusive. +/// /// Each arm may also have a block of code which is run prior to any instancing and a block of code /// which is run afterwards. All code blocks may draw upon the specific value of each parameter /// at any time. Local variables are shared between the two pre- and post- code blocks, but do not @@ -80,6 +84,7 @@ pub use sp_runtime::traits::Dispatchable; /// ```ignore /// benchmarks! { /// // common parameter; just one for this example. +/// // will be `1`, `MAX_LENGTH` or any value inbetween /// _ { /// let l in 1 .. MAX_LENGTH => initialize_l(l); /// } @@ -124,6 +129,45 @@ pub use sp_runtime::traits::Dispatchable; /// }: { m.into_iter().collect::() } /// } /// ``` +/// +/// Test functions are automatically generated for each benchmark and are accessible to you when you +/// run `cargo test`. All tests are named `test_benchmark_`, expect you to pass them +/// the Runtime Trait, and run them in a test externalities environment. The test function runs your +/// benchmark just like a regular benchmark, but only testing at the lowest and highest values for +/// each component. The function will return `Ok(())` if the benchmarks return no errors. +/// +/// You can optionally add a `verify` code block at the end of a benchmark to test any final state +/// of your benchmark in a unit test. For example: +/// +/// ```ignore +/// sort_vector { +/// let x in 1 .. 10000; +/// let mut m = Vec::::new(); +/// for i in (0..x).rev() { +/// m.push(i); +/// } +/// }: { +/// m.sort(); +/// } verify { +/// ensure!(m[0] == 0, "You forgot to sort!") +/// } +/// ``` +/// +/// These `verify` blocks will not execute when running your actual benchmarks! +/// +/// You can construct benchmark tests like so: +/// +/// ```ignore +/// #[test] +/// fn test_benchmarks() { +/// new_test_ext().execute_with(|| { +/// assert_ok!(test_benchmark_dummy::()); +/// assert_err!(test_benchmark_other_name::(), "Bad origin"); +/// assert_ok!(test_benchmark_sort_vector::()); +/// assert_err!(test_benchmark_broken_benchmark::(), "You forgot to sort!"); +/// }); +/// } +/// ``` #[macro_export] macro_rules! benchmarks { ( @@ -134,9 +178,12 @@ macro_rules! benchmarks { } $( $rest:tt )* ) => { - $crate::benchmarks_iter!(NO_INSTANCE { - $( { $common , $common_from , $common_to , $common_instancer } )* - } ( ) $( $rest )* ); + $crate::benchmarks_iter!( + NO_INSTANCE + { $( { $common , $common_from , $common_to , $common_instancer } )* } + ( ) + $( $rest )* + ); } } @@ -150,9 +197,12 @@ macro_rules! benchmarks_instance { } $( $rest:tt )* ) => { - $crate::benchmarks_iter!(INSTANCE { - $( { $common , $common_from , $common_to , $common_instancer } )* - } ( ) $( $rest )* ); + $crate::benchmarks_iter!( + INSTANCE + { $( { $common , $common_from , $common_to , $common_instancer } )* } + ( ) + $( $rest )* + ); } } @@ -165,10 +215,16 @@ macro_rules! benchmarks_iter { { $( $common:tt )* } ( $( $names:ident )* ) $name:ident { $( $code:tt )* }: _ ( $origin:expr $( , $arg:expr )* ) + verify $postcode:block $( $rest:tt )* ) => { $crate::benchmarks_iter! { - $instance { $( $common )* } ( $( $names )* ) $name { $( $code )* }: $name ( $origin $( , $arg )* ) $( $rest )* + $instance + { $( $common )* } + ( $( $names )* ) + $name { $( $code )* }: $name ( $origin $( , $arg )* ) + verify $postcode + $( $rest )* } }; // no instance mutation arm: @@ -177,13 +233,18 @@ macro_rules! benchmarks_iter { { $( $common:tt )* } ( $( $names:ident )* ) $name:ident { $( $code:tt )* }: $dispatch:ident ( $origin:expr $( , $arg:expr )* ) + verify $postcode:block $( $rest:tt )* ) => { $crate::benchmarks_iter! { NO_INSTANCE - { $( $common )* } ( $( $names )* ) $name { $( $code )* }: { + { $( $common )* } + ( $( $names )* ) + $name { $( $code )* }: { as $crate::Dispatchable>::dispatch(Call::::$dispatch($($arg),*), $origin.into())?; - } $( $rest )* + } + verify $postcode + $( $rest )* } }; // instance mutation arm: @@ -192,13 +253,18 @@ macro_rules! benchmarks_iter { { $( $common:tt )* } ( $( $names:ident )* ) $name:ident { $( $code:tt )* }: $dispatch:ident ( $origin:expr $( , $arg:expr )* ) + verify $postcode:block $( $rest:tt )* ) => { $crate::benchmarks_iter! { INSTANCE - { $( $common )* } ( $( $names )* ) $name { $( $code )* }: { + { $( $common )* } + ( $( $names )* ) + $name { $( $code )* }: { as $crate::Dispatchable>::dispatch(Call::::$dispatch($($arg),*), $origin.into())?; - } $( $rest )* + } + verify $postcode + $( $rest )* } }; // iteration arm: @@ -207,18 +273,83 @@ macro_rules! benchmarks_iter { { $( $common:tt )* } ( $( $names:ident )* ) $name:ident { $( $code:tt )* }: $eval:block + verify $postcode:block $( $rest:tt )* ) => { $crate::benchmark_backend! { - $instance $name { $( $common )* } { } { $eval } { $( $code )* } + $instance + $name + { $( $common )* } + { } + { $eval } + { $( $code )* } + $postcode } - $crate::benchmarks_iter!( $instance { $( $common )* } ( $( $names )* $name ) $( $rest )* ); + $crate::benchmarks_iter!( + $instance + { $( $common )* } + ( $( $names )* $name ) + $( $rest )* + ); }; // iteration-exit arm ( $instance:ident { $( $common:tt )* } ( $( $names:ident )* ) ) => { $crate::selected_benchmark!( $instance $( $names ),* ); $crate::impl_benchmark!( $instance $( $names ),* ); - } + #[cfg(test)] + $crate::impl_benchmark_tests!( $instance $( $names ),* ); + }; + // add verify block to _() format + ( + $instance:ident + { $( $common:tt )* } + ( $( $names:ident )* ) + $name:ident { $( $code:tt )* }: _ ( $origin:expr $( , $arg:expr )* ) + $( $rest:tt )* + ) => { + $crate::benchmarks_iter! { + $instance + { $( $common )* } + ( $( $names )* ) + $name { $( $code )* }: _ ( $origin $( , $arg )* ) + verify { } + $( $rest )* + } + }; + // add verify block to name() format + ( + $instance:ident + { $( $common:tt )* } + ( $( $names:ident )* ) + $name:ident { $( $code:tt )* }: $dispatch:ident ( $origin:expr $( , $arg:expr )* ) + $( $rest:tt )* + ) => { + $crate::benchmarks_iter! { + $instance + { $( $common )* } + ( $( $names )* ) + $name { $( $code )* }: $dispatch ( $origin $( , $arg )* ) + verify { } + $( $rest )* + } + }; + // add verify block to {} format + ( + $instance:ident + { $( $common:tt )* } + ( $( $names:ident )* ) + $name:ident { $( $code:tt )* }: $eval:block + $( $rest:tt )* + ) => { + $crate::benchmarks_iter!( + $instance + { $( $common )* } + ( $( $names )* ) + $name { $( $code )* }: $eval + verify { } + $( $rest )* + ); + }; } #[macro_export] @@ -232,12 +363,12 @@ macro_rules! benchmark_backend { } { $eval:block } { let $pre_id:tt : $pre_ty:ty = $pre_ex:expr; $( $rest:tt )* - } ) => { + } $postcode:block) => { $crate::benchmark_backend! { $instance $name { $( $common )* } { $( PRE { $( $pre_parsed )* } )* PRE { $pre_id , $pre_ty , $pre_ex } - } { $eval } { $( $rest )* } + } { $eval } { $( $rest )* } $postcode } }; ($instance:ident $name:ident { @@ -247,12 +378,12 @@ macro_rules! benchmark_backend { } { $eval:block } { let $param:ident in ( $param_from:expr ) .. $param_to:expr => $param_instancer:expr; $( $rest:tt )* - }) => { + } $postcode:block) => { $crate::benchmark_backend! { $instance $name { $( $common )* } { $( $parsed )* PARAM { $param , $param_from , $param_to , $param_instancer } - } { $eval } { $( $rest )* } + } { $eval } { $( $rest )* } $postcode } }; // mutation arm to look after defaulting to a common param @@ -263,7 +394,7 @@ macro_rules! benchmark_backend { } { $eval:block } { let $param:ident in ...; $( $rest:tt )* - }) => { + } $postcode:block) => { $crate::benchmark_backend! { $instance $name { $( { $common , $common_from , $common_to , $common_instancer } )* @@ -275,7 +406,7 @@ macro_rules! benchmark_backend { .. ({ $( let $common = $common_to; )* $param }) => ({ $( let $common = || -> Result<(), &'static str> { $common_instancer ; Ok(()) }; )* $param()? }); $( $rest )* - } + } $postcode } }; // mutation arm to look after defaulting only the range to common param @@ -286,7 +417,7 @@ macro_rules! benchmark_backend { } { $eval:block } { let $param:ident in _ .. _ => $param_instancer:expr ; $( $rest:tt )* - }) => { + } $postcode:block) => { $crate::benchmark_backend! { $instance $name { $( { $common , $common_from , $common_to , $common_instancer } )* @@ -298,7 +429,7 @@ macro_rules! benchmark_backend { .. ({ $( let $common = $common_to; )* $param }) => $param_instancer ; $( $rest )* - } + } $postcode } }; // mutation arm to look after a single tt for param_from. @@ -309,12 +440,12 @@ macro_rules! benchmark_backend { } { $eval:block } { let $param:ident in $param_from:tt .. $param_to:expr => $param_instancer:expr ; $( $rest:tt )* - }) => { + } $postcode:block) => { $crate::benchmark_backend! { $instance $name { $( $common )* } { $( $parsed )* } { $eval } { let $param in ( $param_from ) .. $param_to => $param_instancer; $( $rest )* - } + } $postcode } }; // mutation arm to look after the default tail of `=> ()` @@ -325,12 +456,12 @@ macro_rules! benchmark_backend { } { $eval:block } { let $param:ident in $param_from:tt .. $param_to:expr; $( $rest:tt )* - }) => { + } $postcode:block) => { $crate::benchmark_backend! { $instance $name { $( $common )* } { $( $parsed )* } { $eval } { let $param in $param_from .. $param_to => (); $( $rest )* - } + } $postcode } }; // mutation arm to look after `let _ =` @@ -341,12 +472,12 @@ macro_rules! benchmark_backend { } { $eval:block } { let $pre_id:tt = $pre_ex:expr; $( $rest:tt )* - }) => { + } $postcode:block) => { $crate::benchmark_backend! { $instance $name { $( $common )* } { $( $parsed )* } { $eval } { let $pre_id : _ = $pre_ex; $( $rest )* - } + } $postcode } }; // no instance actioning arm @@ -355,7 +486,7 @@ macro_rules! benchmark_backend { } { $( PRE { $pre_id:tt , $pre_ty:ty , $pre_ex:expr } )* $( PARAM { $param:ident , $param_from:expr , $param_to:expr , $param_instancer:expr } )* - } { $eval:block } { $( $post:tt )* } ) => { + } { $eval:block } { $( $post:tt )* } $postcode:block) => { #[allow(non_camel_case_types)] struct $name; #[allow(unused_variables)] @@ -386,6 +517,25 @@ macro_rules! benchmark_backend { Ok(Box::new(move || -> Result<(), &'static str> { $eval; Ok(()) })) } + + fn verify(&self, components: &[($crate::BenchmarkParameter, u32)]) + -> Result Result<(), &'static str>>, &'static str> + { + $( + let $common = $common_from; + )* + $( + // Prepare instance + let $param = components.iter().find(|&c| c.0 == $crate::BenchmarkParameter::$param).unwrap().1; + )* + $( + let $pre_id : $pre_ty = $pre_ex; + )* + $( $param_instancer ; )* + $( $post )* + + Ok(Box::new(move || -> Result<(), &'static str> { $eval; $postcode; Ok(()) })) + } } }; // instance actioning arm @@ -394,7 +544,7 @@ macro_rules! benchmark_backend { } { $( PRE { $pre_id:tt , $pre_ty:ty , $pre_ex:expr } )* $( PARAM { $param:ident , $param_from:expr , $param_to:expr , $param_instancer:expr } )* - } { $eval:block } { $( $post:tt )* } ) => { + } { $eval:block } { $( $post:tt )* } $postcode:block) => { #[allow(non_camel_case_types)] struct $name; #[allow(unused_variables)] @@ -425,6 +575,25 @@ macro_rules! benchmark_backend { Ok(Box::new(move || -> Result<(), &'static str> { $eval; Ok(()) })) } + + fn verify(&self, components: &[($crate::BenchmarkParameter, u32)]) + -> Result Result<(), &'static str>>, &'static str> + { + $( + let $common = $common_from; + )* + $( + // Prepare instance + let $param = components.iter().find(|&c| c.0 == $crate::BenchmarkParameter::$param).unwrap().1; + )* + $( + let $pre_id : $pre_ty = $pre_ex; + )* + $( $param_instancer ; )* + $( $post )* + + Ok(Box::new(move || -> Result<(), &'static str> { $eval; $postcode; Ok(()) })) + } } } } @@ -469,6 +638,14 @@ macro_rules! selected_benchmark { $( Self::$bench => <$bench as $crate::BenchmarkingSetup>::instance(&$bench, components), )* } } + + fn verify(&self, components: &[($crate::BenchmarkParameter, u32)]) + -> Result Result<(), &'static str>>, &'static str> + { + match self { + $( Self::$bench => <$bench as $crate::BenchmarkingSetup>::verify(&$bench, components), )* + } + } } }; ( @@ -495,6 +672,14 @@ macro_rules! selected_benchmark { $( Self::$bench => <$bench as $crate::BenchmarkingSetupInstance>::instance(&$bench, components), )* } } + + fn verify(&self, components: &[($crate::BenchmarkParameter, u32)]) + -> Result Result<(), &'static str>>, &'static str> + { + match self { + $( Self::$bench => <$bench as $crate::BenchmarkingSetupInstance>::verify(&$bench, components), )* + } + } } } } @@ -542,6 +727,9 @@ macro_rules! impl_benchmark { let steps = steps.get(idx).cloned().unwrap_or(prev_steps); prev_steps = steps; + // Skip this loop if steps is zero + if steps == 0 { continue } + let lowest = lowest_range_values.get(idx).cloned().unwrap_or(*low); let highest = highest_range_values.get(idx).cloned().unwrap_or(*high); @@ -569,20 +757,25 @@ macro_rules! impl_benchmark { // Run the benchmark `repeat` times. for _ in 0..repeat { - // Set the block number to 1 so events are deposited. - frame_system::Module::::set_block_number(1.into()); // Set up the externalities environment for the setup we want to benchmark. let closure_to_benchmark = >::instance(&selected_benchmark, &c)?; + // Set the block number to at least 1 so events are deposited. + if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { + frame_system::Module::::set_block_number(1.into()); + } + // Commit the externalities to the database, flushing the DB cache. // This will enable worst case scenario for reading from the database. $crate::benchmarking::commit_db(); // Time the extrinsic logic. + frame_support::debug::trace!(target: "benchmark", "Start Benchmark: {:?} {:?}", name, component_value); let start_extrinsic = $crate::benchmarking::current_time(); closure_to_benchmark()?; let finish_extrinsic = $crate::benchmarking::current_time(); let elapsed_extrinsic = finish_extrinsic - start_extrinsic; + frame_support::debug::trace!(target: "benchmark", "End Benchmark: {} ns", elapsed_extrinsic); // Time the storage root recalculation. let start_storage_root = $crate::benchmarking::current_time(); @@ -642,6 +835,9 @@ macro_rules! impl_benchmark { let steps = steps.get(idx).cloned().unwrap_or(prev_steps); prev_steps = steps; + // Skip this loop if steps is zero + if steps == 0 { continue } + let lowest = lowest_range_values.get(idx).cloned().unwrap_or(*low); let highest = highest_range_values.get(idx).cloned().unwrap_or(*high); @@ -669,20 +865,25 @@ macro_rules! impl_benchmark { // Run the benchmark `repeat` times. for _ in 0..repeat { - // Set the block number to 1 so events are deposited. - frame_system::Module::::set_block_number(1.into()); // Set up the externalities environment for the setup we want to benchmark. let closure_to_benchmark = >::instance(&selected_benchmark, &c)?; + // Set the block number to at least 1 so events are deposited. + if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { + frame_system::Module::::set_block_number(1.into()); + } + // Commit the externalities to the database, flushing the DB cache. // This will enable worst case scenario for reading from the database. $crate::benchmarking::commit_db(); // Time the extrinsic logic. + frame_support::debug::trace!(target: "benchmark", "Start Benchmark: {:?} {:?}", name, component_value); let start_extrinsic = $crate::benchmarking::current_time(); closure_to_benchmark()?; let finish_extrinsic = $crate::benchmarking::current_time(); let elapsed_extrinsic = finish_extrinsic - start_extrinsic; + frame_support::debug::trace!(target: "benchmark", "End Benchmark: {} ns", elapsed_extrinsic); // Time the storage root recalculation. let start_storage_root = $crate::benchmarking::current_time(); @@ -703,6 +904,107 @@ macro_rules! impl_benchmark { } } +// This creates unit tests from the main benchmark macro. +// They run the benchmark using the `high` and `low` value for each component +// and ensure that everything completes successfully. +#[macro_export] +macro_rules! impl_benchmark_tests { + ( + NO_INSTANCE + $( $name:ident ),* + ) => { + $( + $crate::paste::item! { + fn [] () -> Result<(), &'static str> + where T: frame_system::Trait + { + let selected_benchmark = SelectedBenchmark::$name; + let components = >::components(&selected_benchmark); + + for (_, (name, low, high)) in components.iter().enumerate() { + // Test only the low and high value, assuming values in the middle won't break + for component_value in vec![low, high] { + // Select the max value for all the other components. + let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() + .enumerate() + .map(|(_, (n, _, h))| + if n == name { + (*n, *component_value) + } else { + (*n, *h) + } + ) + .collect(); + + // Set up the verification state + let closure_to_verify = >::verify(&selected_benchmark, &c)?; + + // Set the block number to at least 1 so events are deposited. + if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { + frame_system::Module::::set_block_number(1.into()); + } + + // Run verification + closure_to_verify()?; + + // Reset the state + $crate::benchmarking::wipe_db(); + } + } + Ok(()) + } + } + )* + }; + ( + INSTANCE + $( $name:ident ),* + ) => { + $( + $crate::paste::item! { + fn [] () -> Result<(), &'static str> + where T: frame_system::Trait + { + let selected_benchmark = SelectedBenchmark::$name; + let components = >::components(&selected_benchmark); + + for (_, (name, low, high)) in components.iter().enumerate() { + // Test only the low and high value, assuming values in the middle won't break + for component_value in vec![low, high] { + // Select the max value for all the other components. + let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter() + .enumerate() + .map(|(_, (n, _, h))| + if n == name { + (*n, *component_value) + } else { + (*n, *h) + } + ) + .collect(); + + // Set up the verification state + let closure_to_verify = >::verify(&selected_benchmark, &c)?; + + // Set the block number to at least 1 so events are deposited. + if $crate::Zero::is_zero(&frame_system::Module::::block_number()) { + frame_system::Module::::set_block_number(1.into()); + } + + // Run verification + closure_to_verify()?; + + // Reset the state + $crate::benchmarking::wipe_db(); + } + } + Ok(()) + } + } + )* + }; +} + /// This macro adds pallet benchmarks to a `Vec` object. /// @@ -744,7 +1046,7 @@ macro_rules! add_benchmark { &steps[..], repeat, )?, - pallet: pallet.to_vec(), + pallet: $name.to_vec(), benchmark: benchmark.to_vec(), }); } @@ -757,7 +1059,7 @@ macro_rules! add_benchmark { &steps[..], repeat, )?, - pallet: pallet.to_vec(), + pallet: $name.to_vec(), benchmark: benchmark.clone(), }); } diff --git a/frame/benchmarking/src/tests.rs b/frame/benchmarking/src/tests.rs index b3537617c7282da135db2c9ea59c6ff6506b3487..67ad9b4d22025dd8b4132ee1b023049132cf6da8 100644 --- a/frame/benchmarking/src/tests.rs +++ b/frame/benchmarking/src/tests.rs @@ -22,19 +22,29 @@ use super::*; use codec::Decode; use sp_std::prelude::*; use sp_runtime::{traits::{BlakeTwo256, IdentityLookup}, testing::{H256, Header}}; -use frame_support::{dispatch::DispatchResult, decl_module, impl_outer_origin}; +use frame_support::{ + dispatch::DispatchResult, + decl_module, decl_storage, impl_outer_origin, assert_ok, assert_err, ensure +}; use frame_system::{RawOrigin, ensure_signed, ensure_none}; +decl_storage! { + trait Store for Module as Test { + Value get(fn value): Option; + } +} + decl_module! { pub struct Module for enum Call where origin: T::Origin { - #[weight = frame_support::weights::SimpleDispatchInfo::default()] - fn dummy(origin, _n: u32) -> DispatchResult { + #[weight = 0] + fn set_value(origin, n: u32) -> DispatchResult { let _sender = ensure_signed(origin)?; + Value::put(n); Ok(()) } - #[weight = frame_support::weights::SimpleDispatchInfo::default()] - fn other_dummy(origin, _n: u32) -> DispatchResult { + #[weight = 0] + fn dummy(origin, _n: u32) -> DispatchResult { let _sender = ensure_none(origin)?; Ok(()) } @@ -68,6 +78,9 @@ impl frame_system::Trait for Test { type Event = (); type BlockHashCount = (); type MaximumBlockWeight = (); + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = (); type AvailableBlockRatio = (); type Version = (); @@ -96,31 +109,51 @@ benchmarks!{ let b in 1 .. 1000 => (); } - dummy { + set_value { let b in ...; - let caller = account("caller", 0, 0); + let caller = account::("caller", 0, 0); }: _ (RawOrigin::Signed(caller), b.into()) + verify { + assert_eq!(Value::get(), Some(b)); + } other_name { let b in ...; - let caller = account("caller", 0, 0); - }: other_dummy (RawOrigin::Signed(caller), b.into()) + }: dummy (RawOrigin::None, b.into()) sort_vector { - let x in 0 .. 10000; + let x in 1 .. 10000; let mut m = Vec::::new(); - for i in 0..x { + for i in (0..x).rev() { m.push(i); } }: { m.sort(); + } verify { + ensure!(m[0] == 0, "You forgot to sort!") + } + + bad_origin { + let b in ...; + let caller = account::("caller", 0, 0); + }: dummy (RawOrigin::Signed(caller), b.into()) + + bad_verify { + let x in 1 .. 10000; + let mut m = Vec::::new(); + for i in (0..x).rev() { + m.push(i); + } + }: { } + verify { + ensure!(m[0] == 0, "You forgot to sort!") } } #[test] fn benchmarks_macro_works() { - // Check benchmark creation for `dummy`. - let selected_benchmark = SelectedBenchmark::dummy; + // Check benchmark creation for `set_value`. + let selected_benchmark = SelectedBenchmark::set_value; let components = >::components(&selected_benchmark); assert_eq!(components, vec![(BenchmarkParameter::b, 1, 1000)]); @@ -148,7 +181,7 @@ fn benchmarks_macro_rename_works() { ).expect("failed to create closure"); new_test_ext().execute_with(|| { - assert_eq!(closure(), Err("Bad origin")); + assert_ok!(closure()); }); } @@ -157,7 +190,7 @@ fn benchmarks_macro_works_for_non_dispatchable() { let selected_benchmark = SelectedBenchmark::sort_vector; let components = >::components(&selected_benchmark); - assert_eq!(components, vec![(BenchmarkParameter::x, 0, 10000)]); + assert_eq!(components, vec![(BenchmarkParameter::x, 1, 10000)]); let closure = >::instance( &selected_benchmark, @@ -166,3 +199,29 @@ fn benchmarks_macro_works_for_non_dispatchable() { assert_eq!(closure(), Ok(())); } + +#[test] +fn benchmarks_macro_verify_works() { + // Check postcondition for benchmark `set_value` is valid. + let selected_benchmark = SelectedBenchmark::set_value; + + let closure = >::verify( + &selected_benchmark, + &[(BenchmarkParameter::b, 1)], + ).expect("failed to create closure"); + + new_test_ext().execute_with(|| { + assert_ok!(closure()); + }); +} + +#[test] +fn benchmarks_generate_unit_tests() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_set_value::()); + assert_ok!(test_benchmark_other_name::()); + assert_ok!(test_benchmark_sort_vector::()); + assert_err!(test_benchmark_bad_origin::(), "Bad origin"); + assert_err!(test_benchmark_bad_verify::(), "You forgot to sort!"); + }); +} diff --git a/frame/benchmarking/src/utils.rs b/frame/benchmarking/src/utils.rs index a6f262eab99ec87f9ecd9597fb411e0772587a1d..41b968fbfcad6be7be50a5d737399a9fcd41d0c1 100644 --- a/frame/benchmarking/src/utils.rs +++ b/frame/benchmarking/src/utils.rs @@ -113,8 +113,11 @@ pub trait BenchmarkingSetup { /// Return the components and their ranges which should be tested in this benchmark. fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)>; - /// Set up the storage, and prepare a closure to test in a single run of the benchmark. + /// Set up the storage, and prepare a closure to run the benchmark. fn instance(&self, components: &[(BenchmarkParameter, u32)]) -> Result Result<(), &'static str>>, &'static str>; + + /// Set up the storage, and prepare a closure to test and verify the benchmark + fn verify(&self, components: &[(BenchmarkParameter, u32)]) -> Result Result<(), &'static str>>, &'static str>; } /// The required setup for creating a benchmark. @@ -122,8 +125,11 @@ pub trait BenchmarkingSetupInstance { /// Return the components and their ranges which should be tested in this benchmark. fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)>; - /// Set up the storage, and prepare a closure to test in a single run of the benchmark. + /// Set up the storage, and prepare a closure to run the benchmark. fn instance(&self, components: &[(BenchmarkParameter, u32)]) -> Result Result<(), &'static str>>, &'static str>; + + /// Set up the storage, and prepare a closure to test and verify the benchmark + fn verify(&self, components: &[(BenchmarkParameter, u32)]) -> Result Result<(), &'static str>>, &'static str>; } /// Grab an account, seeded by a name and index. diff --git a/frame/collective/Cargo.toml b/frame/collective/Cargo.toml index 113705c2c8b90c95f068971bb0f810c7b401debc..2107bf0b2f5dbbd8834692890d376083b8e80fcc 100644 --- a/frame/collective/Cargo.toml +++ b/frame/collective/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-collective" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,20 +8,23 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Collective system: Members of a set of account IDs can make their collective feelings known through dispatched calls from one of two specialized origins." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../benchmarking", optional = true } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } [features] default = ["std"] @@ -40,6 +43,3 @@ runtime-benchmarks = [ "sp-runtime/runtime-benchmarks", "frame-system/runtime-benchmarks", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/collective/src/benchmarking.rs b/frame/collective/src/benchmarking.rs index 51db4ee109b06bc550759574432b3170f630e907..5c25051fd083716065cbb90a59b17041d7219724 100644 --- a/frame/collective/src/benchmarking.rs +++ b/frame/collective/src/benchmarking.rs @@ -19,167 +19,418 @@ use super::*; use frame_system::RawOrigin as SystemOrigin; +use frame_system::EventRecord; use frame_benchmarking::{benchmarks_instance, account}; +use sp_runtime::traits::Bounded; use frame_system::Module as System; use crate::Module as Collective; const SEED: u32 = 0; +const MAX_MEMBERS: u32 = 1000; +const MAX_PROPOSALS: u32 = 100; +const MAX_BYTES: u32 = 1_024; + +fn assert_last_event, I: Instance>(generic_event: >::Event) { + let events = System::::events(); + let system_event: ::Event = generic_event.into(); + // compare to the last event record + let EventRecord { event, .. } = &events[events.len() - 1]; + assert_eq!(event, &system_event); +} + benchmarks_instance! { - _{ - // User account seed. - let u in 1 .. 1000 => (); - // Old members. - let n in 1 .. 1000 => (); - // New members. - let m in 1 .. 1000 => (); - // Existing proposals. - let p in 1 .. 100 => (); - } + _{ } set_members { - let m in ...; - let n in ...; + let m in 1 .. MAX_MEMBERS; + let n in 1 .. MAX_MEMBERS; + + // Set old members. + // We compute the difference of old and new members, so it should influence timing. + let mut old_members = vec![]; + let mut last_old_member = T::AccountId::default(); + for i in 0 .. n { + last_old_member = account("old member", i, SEED); + old_members.push(last_old_member.clone()); + } + + Collective::::set_members(SystemOrigin::Root.into(), old_members, Some(last_old_member))?; // Construct `new_members`. // It should influence timing since it will sort this vector. let mut new_members = vec![]; + let mut last_member = T::AccountId::default(); for i in 0 .. m { - let member = account("member", i, SEED); - new_members.push(member); + last_member = account("member", i, SEED); + new_members.push(last_member.clone()); } - // Set old members. - // We compute the difference of old and new members, so it should influence timing. - let mut old_members = vec![]; - for i in 0 .. n { - let old_member = account("old member", i, SEED); - old_members.push(old_member); - } + }: _(SystemOrigin::Root, new_members.clone(), Some(last_member)) + verify { + new_members.sort(); + assert_eq!(Collective::::members(), new_members); + } - let prime = Some(account("prime", 0, SEED)); + execute { + let m in 1 .. MAX_MEMBERS; + let b in 1 .. MAX_BYTES; - Collective::::set_members(SystemOrigin::Root.into(), old_members, prime.clone())?; + // Construct `members`. + let mut members = vec![]; + for i in 0 .. m { + let member = account("member", i, SEED); + members.push(member); + } - }: _(SystemOrigin::Root, new_members, prime) + let caller: T::AccountId = account("caller", 0, SEED); + members.push(caller.clone()); - execute { - let u in ...; + Collective::::set_members(SystemOrigin::Root.into(), members, None)?; - let caller: T::AccountId = account("caller", u, SEED); - let proposal: T::Proposal = Call::::close(Default::default(), Default::default()).into(); + let proposal: T::Proposal = frame_system::Call::::remark(vec![1; b as usize]).into(); - Collective::::set_members(SystemOrigin::Root.into(), vec![caller.clone()], None)?; + }: _(SystemOrigin::Signed(caller), Box::new(proposal.clone())) + verify { + let proposal_hash = T::Hashing::hash_of(&proposal); + // Note that execution fails due to mis-matched origin + assert_last_event::(RawEvent::MemberExecuted(proposal_hash, false).into()); + } - }: _(SystemOrigin::Signed(caller), Box::new(proposal)) + // This tests when execution would happen immediately after proposal + propose_execute { + let m in 1 .. MAX_MEMBERS; + let b in 1 .. MAX_BYTES; - propose { - let u in ...; + // Construct `members`. + let mut members = vec![]; + for i in 0 .. m { + let member = account("member", i, SEED); + members.push(member); + } - let caller: T::AccountId = account("caller", u, SEED); - let proposal: T::Proposal = Call::::close(Default::default(), Default::default()).into(); + let caller: T::AccountId = account("caller", 0, SEED); + members.push(caller.clone()); - Collective::::set_members(SystemOrigin::Root.into(), vec![caller.clone()], None)?; + Collective::::set_members(SystemOrigin::Root.into(), members, None)?; - let member_count = 0; + let proposal: T::Proposal = frame_system::Call::::remark(vec![1; b as usize]).into(); + let threshold = 1; - }: _(SystemOrigin::Signed(caller), member_count, Box::new(proposal.into())) + }: propose(SystemOrigin::Signed(caller), threshold, Box::new(proposal.clone())) + verify { + let proposal_hash = T::Hashing::hash_of(&proposal); + // Note that execution fails due to mis-matched origin + assert_last_event::(RawEvent::Executed(proposal_hash, false).into()); + } - propose_else_branch { - let u in ...; - let p in ...; + // This tests when proposal is created and queued as "proposed" + propose_proposed { + let m in 1 .. MAX_MEMBERS; + let p in 0 .. MAX_PROPOSALS; + let b in 1 .. MAX_BYTES; - let caller: T::AccountId = account("caller", u, SEED); - let proposal: T::Proposal = Call::::close(Default::default(), Default::default()).into(); + // Construct `members`. + let mut members = vec![]; + for i in 0 .. m { + let member = account("member", i, SEED); + members.push(member); + } - Collective::::set_members(SystemOrigin::Root.into(), vec![caller.clone()], None)?; + let caller: T::AccountId = account("caller", 0, SEED); + members.push(caller.clone()); + Collective::::set_members(SystemOrigin::Root.into(), members, None)?; - let member_count = 3; + let threshold = m.max(2); // Add previous proposals. for i in 0 .. p { - let proposal: T::Proposal = Call::::close(Default::default(), (i + 1).into()).into(); - Collective::::propose(SystemOrigin::Signed(caller.clone()).into(), member_count.clone(), Box::new(proposal.into()))?; + // Proposals should be different so that different proposal hashes are generated + let proposal: T::Proposal = frame_system::Call::::remark(vec![i as u8; b as usize]).into(); + Collective::::propose(SystemOrigin::Signed(caller.clone()).into(), threshold, Box::new(proposal))?; } - }: propose(SystemOrigin::Signed(caller), member_count, Box::new(proposal.into())) - - vote { - let u in ...; + assert_eq!(Collective::::proposals().len(), p as usize); - let caller1: T::AccountId = account("caller1", u, SEED); - let caller2: T::AccountId = account("caller2", u, SEED); + let proposal: T::Proposal = frame_system::Call::::remark(vec![p as u8; b as usize]).into(); - let proposal: Box = Box::new(Call::::close(Default::default(), Default::default()).into()); + }: propose(SystemOrigin::Signed(caller.clone()), threshold, Box::new(proposal.clone())) + verify { + // New proposal is recorded + assert_eq!(Collective::::proposals().len(), (p + 1) as usize); let proposal_hash = T::Hashing::hash_of(&proposal); + assert_last_event::(RawEvent::Proposed(caller, p, proposal_hash, threshold).into()); + } - Collective::::set_members(SystemOrigin::Root.into(), vec![caller1.clone(), caller2.clone()], None)?; + vote_insert { + let m in 2 .. MAX_MEMBERS; + let p in 1 .. MAX_PROPOSALS; + let b in 1 .. MAX_BYTES; - let member_count = 3; - Collective::::propose(SystemOrigin::Signed(caller1.clone()).into(), member_count, proposal)?; + // Construct `members`. + let mut members = vec![]; + for i in 0 .. m { + let member = account("member", i, SEED); + members.push(member); + } - let index = 0; - let approve = true; + let caller: T::AccountId = account("caller", 0, SEED); + members.push(caller.clone()); + Collective::::set_members(SystemOrigin::Root.into(), members.clone(), None)?; - }: _(SystemOrigin::Signed(caller2), proposal_hash, index, approve) + // Threshold is 1 less than the number of members so that one person can vote nay + let threshold = m; - vote_not_approve { - let u in ...; + // Add previous proposals + let mut last_hash = T::Hash::default(); + for i in 0 .. p { + // Proposals should be different so that different proposal hashes are generated + let proposal: T::Proposal = frame_system::Call::::remark(vec![i as u8; b as usize]).into(); + Collective::::propose(SystemOrigin::Signed(caller.clone()).into(), threshold, Box::new(proposal.clone()))?; + last_hash = T::Hashing::hash_of(&proposal); + } - let caller1: T::AccountId = account("caller1", u, SEED); - let caller2: T::AccountId = account("caller2", u, SEED); + // Have everyone vote aye on last proposal, while keeping it from passing + for j in 2 .. m { + let voter = &members[j as usize]; + let approve = true; + Collective::::vote(SystemOrigin::Signed(voter.clone()).into(), last_hash.clone(), p - 1, approve)?; + } - let proposal: Box = Box::new(Call::::close(Default::default(), Default::default()).into()); - let proposal_hash = T::Hashing::hash_of(&proposal); + assert_eq!(Collective::::proposals().len(), p as usize); + + // Caller switches vote to nay, but does not kill the vote, just updates + inserts + let index = p - 1; + let approve = false; + + }: vote(SystemOrigin::Signed(caller), last_hash.clone(), index, approve) + verify { + // All proposals exist and the last proposal has just been updated. + assert_eq!(Collective::::proposals().len(), p as usize); + let voting = Collective::::voting(&last_hash).ok_or(Error::::ProposalMissing)?; + assert_eq!(voting.ayes.len(), (m - 2) as usize); + assert_eq!(voting.nays.len(), 1); + } - Collective::::set_members(SystemOrigin::Root.into(), vec![caller1.clone(), caller2.clone()], None)?; + vote_disapproved { + let m in 2 .. MAX_MEMBERS; + let p in 1 .. MAX_PROPOSALS; + let b in 1 .. MAX_BYTES; - let member_count = 3; - Collective::::propose(SystemOrigin::Signed(caller1.clone()).into(), member_count, proposal)?; + // Construct `members`. + let mut members = vec![]; + for i in 0 .. m { + let member = account("member", i, SEED); + members.push(member); + } - let index = 0; + let caller: T::AccountId = account("caller", 0, SEED); + members.push(caller.clone()); + Collective::::set_members(SystemOrigin::Root.into(), members.clone(), None)?; + + // Threshold is total members so that one nay will disapprove the vote + let threshold = m + 1; + + // Add previous proposals + let mut last_hash = T::Hash::default(); + for i in 0 .. p { + // Proposals should be different so that different proposal hashes are generated + let proposal: T::Proposal = frame_system::Call::::remark(vec![i as u8; b as usize]).into(); + Collective::::propose(SystemOrigin::Signed(caller.clone()).into(), threshold, Box::new(proposal.clone()))?; + last_hash = T::Hashing::hash_of(&proposal); + } + + // Have everyone vote aye on last proposal, while keeping it from passing + for j in 1 .. m { + let voter = &members[j as usize]; + let approve = true; + Collective::::vote(SystemOrigin::Signed(voter.clone()).into(), last_hash.clone(), p - 1, approve)?; + } + + assert_eq!(Collective::::proposals().len(), p as usize); + + // Caller switches vote to nay, which kills the vote + let index = p - 1; let approve = false; - }: vote(SystemOrigin::Signed(caller2), proposal_hash, index, approve) + }: vote(SystemOrigin::Signed(caller), last_hash.clone(), index, approve) + verify { + // The last proposal is removed. + assert_eq!(Collective::::proposals().len(), (p - 1) as usize); + assert_last_event::(RawEvent::Disapproved(last_hash).into()); + } vote_approved { - let u in ...; + let m in 2 .. MAX_MEMBERS; + let p in 1 .. MAX_PROPOSALS; + let b in 1 .. MAX_BYTES; - let caller1: T::AccountId = account("caller1", u, SEED); - let caller2: T::AccountId = account("caller2", u, SEED); - - let proposal: Box = Box::new(Call::::close(Default::default(), Default::default()).into()); - let proposal_hash = T::Hashing::hash_of(&proposal); + // Construct `members`. + let mut members = vec![]; + for i in 0 .. m { + let member = account("member", i, SEED); + members.push(member); + } - Collective::::set_members(SystemOrigin::Root.into(), vec![caller1.clone(), caller2.clone()], None)?; + let caller: T::AccountId = account("caller", 0, SEED); + members.push(caller.clone()); + Collective::::set_members(SystemOrigin::Root.into(), members.clone(), None)?; - let member_count = 2; - Collective::::propose(SystemOrigin::Signed(caller1.clone()).into(), member_count, proposal)?; + // Threshold is 2 so any two ayes will approve the vote + let threshold = 2; - let index = 0; - let approve = true; + // Add previous proposals + let mut last_hash = T::Hash::default(); + for i in 0 .. p { + // Proposals should be different so that different proposal hashes are generated + let proposal: T::Proposal = frame_system::Call::::remark(vec![i as u8; b as usize]).into(); + Collective::::propose(SystemOrigin::Signed(caller.clone()).into(), threshold, Box::new(proposal.clone()))?; + last_hash = T::Hashing::hash_of(&proposal); + } - }: vote(SystemOrigin::Signed(caller2), proposal_hash, index, approve) + // Caller switches vote to nay on their own proposal, allowing them to be the deciding approval vote + Collective::::vote(SystemOrigin::Signed(caller.clone()).into(), last_hash.clone(), p - 1, false)?; - close { - let u in ...; + // Have everyone vote nay on last proposal, while keeping it from failing + for j in 2 .. m { + let voter = &members[j as usize]; + let approve = false; + Collective::::vote(SystemOrigin::Signed(voter.clone()).into(), last_hash.clone(), p - 1, approve)?; + } - let caller1: T::AccountId = account("caller1", u, SEED); - let caller2: T::AccountId = account("caller2", u, SEED); - - let proposal: Box = Box::new(Call::::close(Default::default(), Default::default()).into()); - let proposal_hash = T::Hashing::hash_of(&proposal); + // Member zero is the first aye + Collective::::vote(SystemOrigin::Signed(members[0].clone()).into(), last_hash.clone(), p - 1, true)?; - Collective::::set_members(SystemOrigin::Root.into(), vec![caller1.clone(), caller2.clone()], None)?; - let member_count = 2; - Collective::::propose(SystemOrigin::Signed(caller1.clone()).into(), member_count, proposal)?; + assert_eq!(Collective::::proposals().len(), p as usize); - let index = 0; + // Caller switches vote to aye, which passes the vote + let index = p - 1; let approve = true; - let vote_end = T::MotionDuration::get() + 1u32.into(); - System::::set_block_number(vote_end); + }: vote(SystemOrigin::Signed(caller), last_hash.clone(), index, approve) + verify { + // The last proposal is removed. + assert_eq!(Collective::::proposals().len(), (p - 1) as usize); + assert_last_event::(RawEvent::Executed(last_hash, false).into()); + } + + close_disapproved { + let m in 2 .. MAX_MEMBERS; + let p in 1 .. MAX_PROPOSALS; + let b in 1 .. MAX_BYTES; + + // Construct `members`. + let mut members = vec![]; + for i in 0 .. m { + let member = account("member", i, SEED); + members.push(member); + } + let caller: T::AccountId = account("caller", 0, SEED); + members.push(caller.clone()); + + Collective::::set_members(SystemOrigin::Root.into(), members.clone(), Some(caller.clone()))?; + + // Threshold is one less than total members so that two nays will disapprove the vote + let threshold = m; + + // Add proposals + let mut last_hash = T::Hash::default(); + for i in 0 .. p { + // Proposals should be different so that different proposal hashes are generated + let proposal: T::Proposal = frame_system::Call::::remark(vec![i as u8; b as usize]).into(); + Collective::::propose(SystemOrigin::Signed(caller.clone()).into(), threshold, Box::new(proposal.clone()))?; + last_hash = T::Hashing::hash_of(&proposal); + } + + // Have everyone vote aye on last proposal, while keeping it from passing + // A few abstainers will be the nay votes needed to fail the vote + for j in 2 .. m { + let voter = &members[j as usize]; + let approve = true; + Collective::::vote(SystemOrigin::Signed(voter.clone()).into(), last_hash.clone(), p - 1, approve)?; + } + + // caller is prime, prime votes nay + Collective::::vote(SystemOrigin::Signed(caller.clone()).into(), last_hash.clone(), p - 1, false)?; + + System::::set_block_number(T::BlockNumber::max_value()); + assert_eq!(Collective::::proposals().len(), p as usize); + + // Prime nay will close it as disapproved + }: close(SystemOrigin::Signed(caller), last_hash, p - 1) + verify { + assert_eq!(Collective::::proposals().len(), (p - 1) as usize); + assert_last_event::(RawEvent::Disapproved(last_hash).into()); + } + + + close_approved { + let m in 2 .. MAX_MEMBERS; + let p in 1 .. MAX_PROPOSALS; + let b in 1 .. MAX_BYTES; + + // Construct `members`. + let mut members = vec![]; + for i in 0 .. m { + let member = account("member", i, SEED); + members.push(member); + } + let caller: T::AccountId = account("caller", 0, SEED); + members.push(caller.clone()); - }: _(SystemOrigin::Signed(caller2), proposal_hash, index) + Collective::::set_members(SystemOrigin::Root.into(), members.clone(), Some(caller.clone()))?; + + // Threshold is two, so any two ayes will pass the vote + let threshold = 2; + + // Add proposals + let mut last_hash = T::Hash::default(); + for i in 0 .. p { + // Proposals should be different so that different proposal hashes are generated + let proposal: T::Proposal = frame_system::Call::::remark(vec![i as u8; b as usize]).into(); + Collective::::propose(SystemOrigin::Signed(caller.clone()).into(), threshold, Box::new(proposal.clone()))?; + last_hash = T::Hashing::hash_of(&proposal); + } + + // Have everyone vote nay on last proposal, while keeping it from failing + // A few abstainers will be the aye votes needed to pass the vote + for j in 2 .. m { + let voter = &members[j as usize]; + let approve = false; + Collective::::vote(SystemOrigin::Signed(voter.clone()).into(), last_hash.clone(), p - 1, approve)?; + } + + // caller is prime, prime already votes aye by creating the proposal + System::::set_block_number(T::BlockNumber::max_value()); + assert_eq!(Collective::::proposals().len(), p as usize); + + // Prime aye will close it as approved + }: close(SystemOrigin::Signed(caller), last_hash, p - 1) + verify { + assert_eq!(Collective::::proposals().len(), (p - 1) as usize); + assert_last_event::(RawEvent::Executed(last_hash, false).into()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_set_members::()); + assert_ok!(test_benchmark_execute::()); + assert_ok!(test_benchmark_propose_execute::()); + assert_ok!(test_benchmark_propose_proposed::()); + assert_ok!(test_benchmark_vote_insert::()); + assert_ok!(test_benchmark_vote_disapproved::()); + assert_ok!(test_benchmark_vote_approved::()); + assert_ok!(test_benchmark_close_disapproved::()); + assert_ok!(test_benchmark_close_approved::()); + }); + } } diff --git a/frame/collective/src/lib.rs b/frame/collective/src/lib.rs index 49c1bd38914930a6e1cfb9d229b7f1e9aef0dac7..662465616ed1bec6c55ecff56e5e0a664ac519c2 100644 --- a/frame/collective/src/lib.rs +++ b/frame/collective/src/lib.rs @@ -40,11 +40,11 @@ use sp_std::{prelude::*, result}; use sp_core::u32_trait::Value as U32; use sp_runtime::RuntimeDebug; use sp_runtime::traits::Hash; -use frame_support::weights::SimpleDispatchInfo; use frame_support::{ dispatch::{Dispatchable, Parameter}, codec::{Encode, Decode}, traits::{Get, ChangeMembers, InitializeMembers, EnsureOrigin}, decl_module, decl_event, decl_storage, decl_error, ensure, + weights::DispatchClass, }; use frame_system::{self as system, ensure_signed, ensure_root}; @@ -65,7 +65,7 @@ pub trait Trait: frame_system::Trait { type Origin: From>; /// The outer call dispatch type. - type Proposal: Parameter + Dispatchable>::Origin> + From>; + type Proposal: Parameter + Dispatchable>::Origin> + From>; /// The outer event type. type Event: From> + Into<::Event>; @@ -187,7 +187,7 @@ decl_module! { /// - `prime`: The prime member whose vote sets the default. /// /// Requires root origin. - #[weight = SimpleDispatchInfo::FixedOperational(100_000)] + #[weight = (100_000_000, DispatchClass::Operational)] fn set_members(origin, new_members: Vec, prime: Option) { ensure_root(origin)?; let mut new_members = new_members; @@ -200,7 +200,7 @@ decl_module! { /// Dispatch a proposal from a member using the `Member` origin. /// /// Origin must be a member of the collective. - #[weight = SimpleDispatchInfo::FixedOperational(100_000)] + #[weight = (100_000_000, DispatchClass::Operational)] fn execute(origin, proposal: Box<>::Proposal>) { let who = ensure_signed(origin)?; ensure!(Self::is_member(&who), Error::::NotMember); @@ -214,7 +214,7 @@ decl_module! { /// - Bounded storage reads and writes. /// - Argument `threshold` has bearing on weight. /// # - #[weight = SimpleDispatchInfo::FixedOperational(5_000_000)] + #[weight = (5_000_000_000, DispatchClass::Operational)] fn propose(origin, #[compact] threshold: MemberCount, proposal: Box<>::Proposal>) { let who = ensure_signed(origin)?; ensure!(Self::is_member(&who), Error::::NotMember); @@ -244,7 +244,7 @@ decl_module! { /// - Bounded storage read and writes. /// - Will be slightly heavier if the proposal is approved / disapproved after the vote. /// # - #[weight = SimpleDispatchInfo::FixedOperational(200_000)] + #[weight = (200_000_000, DispatchClass::Operational)] fn vote(origin, proposal: T::Hash, #[compact] index: ProposalIndex, approve: bool) { let who = ensure_signed(origin)?; ensure!(Self::is_member(&who), Error::::NotMember); @@ -303,7 +303,7 @@ decl_module! { /// - `M` is number of members, /// - `P` is number of active proposals, /// - `L` is the encoded length of `proposal` preimage. - #[weight = SimpleDispatchInfo::FixedOperational(200_000)] + #[weight = (200_000_000, DispatchClass::Operational)] fn close(origin, proposal: T::Hash, #[compact] index: ProposalIndex) { let _ = ensure_signed(origin)?; @@ -553,6 +553,9 @@ mod tests { type Event = Event; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -589,7 +592,7 @@ mod tests { } ); - fn make_ext() -> sp_io::TestExternalities { + pub fn new_test_ext() -> sp_io::TestExternalities { let mut ext: sp_io::TestExternalities = GenesisConfig { collective_Instance1: Some(collective::GenesisConfig { members: vec![1, 2, 3], @@ -603,7 +606,7 @@ mod tests { #[test] fn motions_basic_environment_works() { - make_ext().execute_with(|| { + new_test_ext().execute_with(|| { assert_eq!(Collective::members(), vec![1, 2, 3]); assert_eq!(Collective::proposals(), Vec::::new()); }); @@ -615,7 +618,7 @@ mod tests { #[test] fn close_works() { - make_ext().execute_with(|| { + new_test_ext().execute_with(|| { let proposal = make_proposal(42); let hash = BlakeTwo256::hash_of(&proposal); @@ -643,7 +646,7 @@ mod tests { #[test] fn close_with_prime_works() { - make_ext().execute_with(|| { + new_test_ext().execute_with(|| { let proposal = make_proposal(42); let hash = BlakeTwo256::hash_of(&proposal); assert_ok!(Collective::set_members(Origin::ROOT, vec![1, 2, 3], Some(3))); @@ -666,7 +669,7 @@ mod tests { #[test] fn close_with_voting_prime_works() { - make_ext().execute_with(|| { + new_test_ext().execute_with(|| { let proposal = make_proposal(42); let hash = BlakeTwo256::hash_of(&proposal); assert_ok!(Collective::set_members(Origin::ROOT, vec![1, 2, 3], Some(1))); @@ -690,7 +693,7 @@ mod tests { #[test] fn removal_of_old_voters_votes_works() { - make_ext().execute_with(|| { + new_test_ext().execute_with(|| { let proposal = make_proposal(42); let hash = BlakeTwo256::hash_of(&proposal); let end = 4; @@ -724,7 +727,7 @@ mod tests { #[test] fn removal_of_old_voters_votes_works_with_set_members() { - make_ext().execute_with(|| { + new_test_ext().execute_with(|| { let proposal = make_proposal(42); let hash = BlakeTwo256::hash_of(&proposal); let end = 4; @@ -758,7 +761,7 @@ mod tests { #[test] fn propose_works() { - make_ext().execute_with(|| { + new_test_ext().execute_with(|| { let proposal = make_proposal(42); let hash = proposal.blake2_256().into(); let end = 4; @@ -787,7 +790,7 @@ mod tests { #[test] fn motions_ignoring_non_collective_proposals_works() { - make_ext().execute_with(|| { + new_test_ext().execute_with(|| { let proposal = make_proposal(42); assert_noop!( Collective::propose(Origin::signed(42), 3, Box::new(proposal.clone())), @@ -798,7 +801,7 @@ mod tests { #[test] fn motions_ignoring_non_collective_votes_works() { - make_ext().execute_with(|| { + new_test_ext().execute_with(|| { let proposal = make_proposal(42); let hash: H256 = proposal.blake2_256().into(); assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); @@ -811,7 +814,7 @@ mod tests { #[test] fn motions_ignoring_bad_index_collective_vote_works() { - make_ext().execute_with(|| { + new_test_ext().execute_with(|| { System::set_block_number(3); let proposal = make_proposal(42); let hash: H256 = proposal.blake2_256().into(); @@ -825,7 +828,7 @@ mod tests { #[test] fn motions_revoting_works() { - make_ext().execute_with(|| { + new_test_ext().execute_with(|| { let proposal = make_proposal(42); let hash: H256 = proposal.blake2_256().into(); let end = 4; @@ -876,7 +879,7 @@ mod tests { #[test] fn motions_reproposing_disapproved_works() { - make_ext().execute_with(|| { + new_test_ext().execute_with(|| { let proposal = make_proposal(42); let hash: H256 = proposal.blake2_256().into(); assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); @@ -889,7 +892,7 @@ mod tests { #[test] fn motions_disapproval_works() { - make_ext().execute_with(|| { + new_test_ext().execute_with(|| { let proposal = make_proposal(42); let hash: H256 = proposal.blake2_256().into(); assert_ok!(Collective::propose(Origin::signed(1), 3, Box::new(proposal.clone()))); @@ -931,7 +934,7 @@ mod tests { #[test] fn motions_approval_works() { - make_ext().execute_with(|| { + new_test_ext().execute_with(|| { let proposal = make_proposal(42); let hash: H256 = proposal.blake2_256().into(); assert_ok!(Collective::propose(Origin::signed(1), 2, Box::new(proposal.clone()))); diff --git a/frame/contracts/Cargo.toml b/frame/contracts/Cargo.toml index a9318002cee2a71d290b56687f7123d5a1f994d7..4e1128000e33894f218f3f94a42442c255bcada5 100644 --- a/frame/contracts/Cargo.toml +++ b/frame/contracts/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-contracts" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,28 +8,32 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for WASM contracts" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } pwasm-utils = { version = "0.12.0", default-features = false } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } parity-wasm = { version = "0.41.0", default-features = false } wasmi-validation = { version = "0.3.0", default-features = false } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-sandbox = { version = "0.8.0-alpha.5", default-features = false, path = "../../primitives/sandbox" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -pallet-contracts-primitives = { version = "2.0.0-alpha.5", default-features = false, path = "common" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-sandbox = { version = "0.8.0-dev", default-features = false, path = "../../primitives/sandbox" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +pallet-contracts-primitives = { version = "2.0.0-dev", default-features = false, path = "common" } +pallet-transaction-payment = { version = "2.0.0-dev", default-features = false, path = "../transaction-payment" } [dev-dependencies] wabt = "0.9.2" assert_matches = "1.3.0" hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } -pallet-timestamp = { version = "2.0.0-alpha.5", path = "../timestamp" } -pallet-randomness-collective-flip = { version = "2.0.0-alpha.5", path = "../randomness-collective-flip" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } +pallet-timestamp = { version = "2.0.0-dev", path = "../timestamp" } +pallet-randomness-collective-flip = { version = "2.0.0-dev", path = "../randomness-collective-flip" } [features] default = ["std"] @@ -47,7 +51,5 @@ std = [ "pwasm-utils/std", "wasmi-validation/std", "pallet-contracts-primitives/std", + "pallet-transaction-payment/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/contracts/common/Cargo.toml b/frame/contracts/common/Cargo.toml index d181896bd2a6c2bf4d25524e7fec277823f3cd02..edf1867be0a7109b968f2645b2b034749fe8a5a3 100644 --- a/frame/contracts/common/Cargo.toml +++ b/frame/contracts/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-contracts-primitives" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,11 +8,14 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "A crate that hosts a common definitions that are relevant for the pallet-contracts." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] # This crate should not rely on any of the frame primitives. codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/runtime" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/runtime" } [features] default = ["std"] @@ -21,6 +24,3 @@ std = [ "sp-runtime/std", "sp-std/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/contracts/rpc/Cargo.toml b/frame/contracts/rpc/Cargo.toml index fa13d8ec3fe4a3fbca2f89750772a3f3a8f91251..66d75759f1a25cdb2ff9b6757f44fa6ec79919e3 100644 --- a/frame/contracts/rpc/Cargo.toml +++ b/frame/contracts/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-contracts-rpc" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,22 +8,22 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Node-specific RPC methods for interaction with contracts." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0" } jsonrpc-core = "14.0.3" jsonrpc-core-client = "14.0.5" jsonrpc-derive = "14.0.3" -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-rpc = { version = "2.0.0-alpha.5", path = "../../../primitives/rpc" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-rpc = { version = "2.0.0-dev", path = "../../../primitives/rpc" } serde = { version = "1.0.101", features = ["derive"] } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.5", path = "../../../primitives/api" } -pallet-contracts-primitives = { version = "2.0.0-alpha.5", path = "../common" } -pallet-contracts-rpc-runtime-api = { version = "0.8.0-alpha.5", path = "./runtime-api" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-dev", path = "../../../primitives/api" } +pallet-contracts-primitives = { version = "2.0.0-dev", path = "../common" } +pallet-contracts-rpc-runtime-api = { version = "0.8.0-dev", path = "./runtime-api" } [dev-dependencies] serde_json = "1.0.41" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/contracts/rpc/runtime-api/Cargo.toml b/frame/contracts/rpc/runtime-api/Cargo.toml index 692bd3f25ef85b3c695240897cc55bb0546eddcd..81c6ce3760129a58a753d0bd47519ff117fda388 100644 --- a/frame/contracts/rpc/runtime-api/Cargo.toml +++ b/frame/contracts/rpc/runtime-api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-contracts-rpc-runtime-api" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,12 +8,15 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Runtime API definition required by Contracts RPC extensions." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../../../primitives/api" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../../../../primitives/api" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../../../primitives/runtime" } -pallet-contracts-primitives = { version = "2.0.0-alpha.5", default-features = false, path = "../../common" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../../../primitives/std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../../../primitives/runtime" } +pallet-contracts-primitives = { version = "2.0.0-dev", default-features = false, path = "../../common" } [features] default = ["std"] @@ -24,6 +27,3 @@ std = [ "sp-runtime/std", "pallet-contracts-primitives/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/contracts/rpc/src/lib.rs b/frame/contracts/rpc/src/lib.rs index 52dddb177bbc74538f869419d20beebd1cc2b916..53e8d938703c866de09d1234e2a34ebb51122e09 100644 --- a/frame/contracts/rpc/src/lib.rs +++ b/frame/contracts/rpc/src/lib.rs @@ -46,9 +46,10 @@ const CONTRACT_IS_A_TOMBSTONE: i64 = 3; /// This value is used to set the upper bound for maximal contract calls to /// prevent blocking the RPC for too long. /// -/// Based on W3F research spreadsheet: -/// https://docs.google.com/spreadsheets/d/1h0RqncdqiWI4KgxO0z9JIpZEJESXjX_ZCK6LFX6veDo/view -const GAS_PER_SECOND: u64 = 1_000_000_000; +/// As 1 gas is equal to 1 weight we base this on the conducted benchmarks which +/// determined runtime weights: +/// https://github.com/paritytech/substrate/pull/5446 +const GAS_PER_SECOND: u64 = 1_000_000_000_000; /// A private newtype for converting `ContractAccessError` into an RPC error. struct ContractAccessError(pallet_contracts_primitives::ContractAccessError); diff --git a/frame/contracts/src/account_db.rs b/frame/contracts/src/account_db.rs index 165581e67646df41afa60945e5baefd6e62feb30..aae853d2ff9965c146eca1f816d6c59ed58c459d 100644 --- a/frame/contracts/src/account_db.rs +++ b/frame/contracts/src/account_db.rs @@ -128,7 +128,7 @@ impl AccountDb for DirectAccountDb { trie_id: Option<&TrieId>, location: &StorageKey ) -> Option> { - trie_id.and_then(|id| child::get_raw(id, crate::trie_unique_id(&id[..]), &blake2_256(location))) + trie_id.and_then(|id| child::get_raw(&crate::child_trie_info(&id[..]), &blake2_256(location))) } fn get_code_hash(&self, account: &T::AccountId) -> Option> { >::get(account).and_then(|i| i.as_alive().map(|i| i.code_hash)) @@ -167,13 +167,13 @@ impl AccountDb for DirectAccountDb { (false, Some(info), _) => info, // Existing contract is being removed. (true, Some(info), None) => { - child::kill_storage(&info.trie_id, info.child_trie_unique_id()); + child::kill_storage(&info.child_trie_info()); >::remove(&address); continue; } // Existing contract is being replaced by a new one. (true, Some(info), Some(code_hash)) => { - child::kill_storage(&info.trie_id, info.child_trie_unique_id()); + child::kill_storage(&info.child_trie_info()); AliveContractInfo:: { code_hash, storage_size: T::StorageSizeOffset::get(), @@ -212,17 +212,16 @@ impl AccountDb for DirectAccountDb { for (k, v) in changed.storage.into_iter() { if let Some(value) = child::get_raw( - &new_info.trie_id[..], - new_info.child_trie_unique_id(), + &new_info.child_trie_info(), &blake2_256(&k), ) { new_info.storage_size -= value.len() as u32; } if let Some(value) = v { new_info.storage_size += value.len() as u32; - child::put_raw(&new_info.trie_id[..], new_info.child_trie_unique_id(), &blake2_256(&k), &value[..]); + child::put_raw(&new_info.child_trie_info(), &blake2_256(&k), &value[..]); } else { - child::kill(&new_info.trie_id[..], new_info.child_trie_unique_id(), &blake2_256(&k)); + child::kill(&new_info.child_trie_info(), &blake2_256(&k)); } } diff --git a/frame/contracts/src/exec.rs b/frame/contracts/src/exec.rs index 402622331d0ecc33ae66a8920b0568aaad099dd2..9cc1c50260db9fed6581217e08107a61d4aafe6c 100644 --- a/frame/contracts/src/exec.rs +++ b/frame/contracts/src/exec.rs @@ -17,7 +17,7 @@ use super::{CodeHash, Config, ContractAddressFor, Event, RawEvent, Trait, TrieId, BalanceOf, ContractInfo}; use crate::account_db::{AccountDb, DirectAccountDb, OverlayAccountDb}; -use crate::gas::{Gas, GasMeter, Token, approx_gas_for_balance}; +use crate::gas::{Gas, GasMeter, Token}; use crate::rent; use sp_std::prelude::*; @@ -203,6 +203,9 @@ pub trait Ext { /// /// Returns `None` if the value doesn't exist. fn get_runtime_storage(&self, key: &[u8]) -> Option>; + + /// Returns the price of one weight unit. + fn get_weight_price(&self) -> BalanceOf; } /// Loader is a companion of the `Vm` trait. It loads an appropriate abstract @@ -605,21 +608,19 @@ pub enum TransferFeeKind { #[cfg_attr(test, derive(Debug, PartialEq, Eq))] #[derive(Copy, Clone)] -pub struct TransferFeeToken { +pub struct TransferFeeToken { kind: TransferFeeKind, - gas_price: Balance, } -impl Token for TransferFeeToken> { +impl Token for TransferFeeToken { type Metadata = Config; #[inline] fn calculate_amount(&self, metadata: &Config) -> Gas { - let balance_fee = match self.kind { - TransferFeeKind::ContractInstantiate => metadata.contract_account_instantiate_fee, - TransferFeeKind::Transfer => return metadata.schedule.transfer_cost, - }; - approx_gas_for_balance(self.gas_price, balance_fee) + match self.kind { + TransferFeeKind::ContractInstantiate => metadata.schedule.instantiate_cost, + TransferFeeKind::Transfer => metadata.schedule.transfer_cost, + } } } @@ -668,7 +669,6 @@ fn transfer<'a, T: Trait, V: Vm, L: Loader>( }; TransferFeeToken { kind, - gas_price: gas_meter.gas_price(), } }; @@ -868,6 +868,13 @@ where fn get_runtime_storage(&self, key: &[u8]) -> Option> { unhashed::get_raw(&key) } + + fn get_weight_price(&self) -> BalanceOf { + use pallet_transaction_payment::Module as Payment; + use sp_runtime::SaturatedConversion; + let price = Payment::::weight_to_fee_with_adjustment::(1); + price.saturated_into() + } } /// These tests exercise the executive layer. @@ -889,6 +896,7 @@ mod tests { use crate::{ account_db::AccountDb, gas::GasMeter, tests::{ExtBuilder, Test}, exec::{ExecReturnValue, ExecError, STATUS_SUCCESS}, CodeHash, Config, + gas::Gas, }; use std::{cell::RefCell, rc::Rc, collections::HashMap, marker::PhantomData}; use assert_matches::assert_matches; @@ -898,6 +906,8 @@ mod tests { const BOB: u64 = 2; const CHARLIE: u64 = 3; + const GAS_LIMIT: Gas = 10_000_000_000; + impl<'a, T, V, L> ExecutionContext<'a, T, V, L> where T: crate::Trait { @@ -1003,7 +1013,7 @@ mod tests { #[test] fn it_works() { let value = Default::default(); - let mut gas_meter = GasMeter::::with_limit(10000, 1); + let mut gas_meter = GasMeter::::new(GAS_LIMIT); let data = vec![]; let vm = MockVm::new(); @@ -1044,7 +1054,7 @@ mod tests { ctx.overlay.set_balance(&origin, 100); ctx.overlay.set_balance(&dest, 0); - let mut gas_meter = GasMeter::::with_limit(1000, 1); + let mut gas_meter = GasMeter::::new(GAS_LIMIT); let result = ctx.call(dest, 0, &mut gas_meter, vec![]); assert_matches!(result, Ok(_)); @@ -1064,7 +1074,7 @@ mod tests { ctx.overlay.set_balance(&origin, 100); - let mut gas_meter = GasMeter::::with_limit(1000, 1); + let mut gas_meter = GasMeter::::new(GAS_LIMIT); let result = ctx.instantiate(1, &mut gas_meter, &code, vec![]); assert_matches!(result, Ok(_)); @@ -1093,7 +1103,7 @@ mod tests { let output = ctx.call( dest, 55, - &mut GasMeter::::with_limit(1000, 1), + &mut GasMeter::::new(GAS_LIMIT), vec![], ).unwrap(); @@ -1126,7 +1136,7 @@ mod tests { let output = ctx.call( dest, 55, - &mut GasMeter::::with_limit(1000, 1), + &mut GasMeter::::new(GAS_LIMIT), vec![], ).unwrap(); @@ -1152,7 +1162,7 @@ mod tests { ctx.overlay.set_balance(&origin, 100); ctx.overlay.set_balance(&dest, 0); - let mut gas_meter = GasMeter::::with_limit(1000, 1); + let mut gas_meter = GasMeter::::new(GAS_LIMIT); let result = ctx.call(dest, 50, &mut gas_meter, vec![]); assert_matches!(result, Ok(_)); @@ -1163,7 +1173,6 @@ mod tests { ExecFeeToken::Call, TransferFeeToken { kind: TransferFeeKind::Transfer, - gas_price: 1u64 }, ); }); @@ -1178,7 +1187,7 @@ mod tests { ctx.overlay.set_balance(&origin, 100); ctx.overlay.set_balance(&dest, 15); - let mut gas_meter = GasMeter::::with_limit(1000, 1); + let mut gas_meter = GasMeter::::new(GAS_LIMIT); let result = ctx.call(dest, 50, &mut gas_meter, vec![]); assert_matches!(result, Ok(_)); @@ -1189,7 +1198,6 @@ mod tests { ExecFeeToken::Call, TransferFeeToken { kind: TransferFeeKind::Transfer, - gas_price: 1u64 }, ); }); @@ -1207,7 +1215,7 @@ mod tests { ctx.overlay.set_balance(&origin, 100); ctx.overlay.set_balance(&dest, 15); - let mut gas_meter = GasMeter::::with_limit(1000, 1); + let mut gas_meter = GasMeter::::new(GAS_LIMIT); let result = ctx.instantiate(50, &mut gas_meter, &code, vec![]); assert_matches!(result, Ok(_)); @@ -1218,7 +1226,6 @@ mod tests { ExecFeeToken::Instantiate, TransferFeeToken { kind: TransferFeeKind::ContractInstantiate, - gas_price: 1u64 }, ); }); @@ -1242,7 +1249,7 @@ mod tests { let result = ctx.call( dest, 100, - &mut GasMeter::::with_limit(1000, 1), + &mut GasMeter::::new(GAS_LIMIT), vec![], ); @@ -1279,7 +1286,7 @@ mod tests { let result = ctx.call( dest, 0, - &mut GasMeter::::with_limit(1000, 1), + &mut GasMeter::::new(GAS_LIMIT), vec![], ); @@ -1310,7 +1317,7 @@ mod tests { let result = ctx.call( dest, 0, - &mut GasMeter::::with_limit(1000, 1), + &mut GasMeter::::new(GAS_LIMIT), vec![], ); @@ -1338,7 +1345,7 @@ mod tests { let result = ctx.call( BOB, 0, - &mut GasMeter::::with_limit(10000, 1), + &mut GasMeter::::new(GAS_LIMIT), vec![1, 2, 3, 4], ); assert_matches!(result, Ok(_)); @@ -1363,7 +1370,7 @@ mod tests { let result = ctx.instantiate( 1, - &mut GasMeter::::with_limit(10000, 1), + &mut GasMeter::::new(GAS_LIMIT), &input_data_ch, vec![1, 2, 3, 4], ); @@ -1413,7 +1420,7 @@ mod tests { let result = ctx.call( BOB, value, - &mut GasMeter::::with_limit(100000, 1), + &mut GasMeter::::new(GAS_LIMIT), vec![], ); @@ -1459,7 +1466,7 @@ mod tests { let result = ctx.call( dest, 0, - &mut GasMeter::::with_limit(10000, 1), + &mut GasMeter::::new(GAS_LIMIT), vec![], ); @@ -1500,7 +1507,7 @@ mod tests { let result = ctx.call( BOB, 0, - &mut GasMeter::::with_limit(10000, 1), + &mut GasMeter::::new(GAS_LIMIT), vec![], ); @@ -1522,7 +1529,7 @@ mod tests { assert_matches!( ctx.instantiate( 0, // <- zero endowment - &mut GasMeter::::with_limit(10000, 1), + &mut GasMeter::::new(GAS_LIMIT), &dummy_ch, vec![], ), @@ -1548,7 +1555,7 @@ mod tests { let instantiated_contract_address = assert_matches!( ctx.instantiate( 100, - &mut GasMeter::::with_limit(10000, 1), + &mut GasMeter::::new(GAS_LIMIT), &dummy_ch, vec![], ), @@ -1588,7 +1595,7 @@ mod tests { let instantiated_contract_address = assert_matches!( ctx.instantiate( 100, - &mut GasMeter::::with_limit(10000, 1), + &mut GasMeter::::new(GAS_LIMIT), &dummy_ch, vec![], ), @@ -1633,7 +1640,7 @@ mod tests { ctx.overlay.instantiate_contract(&BOB, instantiator_ch).unwrap(); assert_matches!( - ctx.call(BOB, 20, &mut GasMeter::::with_limit(1000, 1), vec![]), + ctx.call(BOB, 20, &mut GasMeter::::new(GAS_LIMIT), vec![]), Ok(_) ); @@ -1693,7 +1700,7 @@ mod tests { ctx.overlay.instantiate_contract(&BOB, instantiator_ch).unwrap(); assert_matches!( - ctx.call(BOB, 20, &mut GasMeter::::with_limit(1000, 1), vec![]), + ctx.call(BOB, 20, &mut GasMeter::::new(GAS_LIMIT), vec![]), Ok(_) ); @@ -1730,7 +1737,7 @@ mod tests { assert_matches!( ctx.instantiate( 100, - &mut GasMeter::::with_limit(10000, 1), + &mut GasMeter::::new(GAS_LIMIT), &terminate_ch, vec![], ), @@ -1766,7 +1773,7 @@ mod tests { let result = ctx.instantiate( 1, - &mut GasMeter::::with_limit(10000, 1), + &mut GasMeter::::new(GAS_LIMIT), &rent_allowance_ch, vec![], ); diff --git a/frame/contracts/src/gas.rs b/frame/contracts/src/gas.rs index 362f15f3aae795ef44b346285de79cd557291a23..38f231c008f8b635c10d67102ecd0c2bf3e703c4 100644 --- a/frame/contracts/src/gas.rs +++ b/frame/contracts/src/gas.rs @@ -14,22 +14,18 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use crate::{GasSpent, Module, Trait, BalanceOf, NegativeImbalanceOf}; -use sp_std::convert::TryFrom; -use sp_runtime::traits::{ - CheckedMul, Zero, SaturatedConversion, AtLeast32Bit, UniqueSaturatedInto, -}; -use frame_support::{ - traits::{Currency, ExistenceRequirement, Imbalance, OnUnbalanced, WithdrawReason}, StorageValue, - dispatch::DispatchError, +use crate::Trait; +use sp_std::marker::PhantomData; +use sp_runtime::traits::Zero; +use frame_support::dispatch::{ + DispatchError, DispatchResultWithPostInfo, PostDispatchInfo, DispatchErrorWithPostInfo, }; #[cfg(test)] use std::{any::Any, fmt::Debug}; -// Gas units are chosen to be represented by u64 so that gas metering instructions can operate on -// them efficiently. -pub type Gas = u64; +// Gas is essentially the same as weight. It is a 1 to 1 correspondence. +pub type Gas = frame_support::weights::Weight; #[must_use] #[derive(Debug, PartialEq, Eq)] @@ -88,20 +84,19 @@ pub struct ErasedToken { } pub struct GasMeter { - limit: Gas, + gas_limit: Gas, /// Amount of gas left from initial gas limit. Can reach zero. gas_left: Gas, - gas_price: BalanceOf, - + _phantom: PhantomData, #[cfg(test)] tokens: Vec, } impl GasMeter { - pub fn with_limit(gas_limit: Gas, gas_price: BalanceOf) -> GasMeter { + pub fn new(gas_limit: Gas) -> Self { GasMeter { - limit: gas_limit, + gas_limit, gas_left: gas_limit, - gas_price, + _phantom: PhantomData, #[cfg(test)] tokens: Vec::new(), } @@ -147,6 +142,14 @@ impl GasMeter { } } + // Account for not fully used gas. + // + // This can be used after dispatching a runtime call to refund gas that was not + // used by the dispatchable. + pub fn refund(&mut self, gas: Gas) { + self.gas_left = self.gas_left.saturating_add(gas).max(self.gas_limit); + } + /// Allocate some amount of gas and perform some work with /// a newly created nested gas meter. /// @@ -165,7 +168,7 @@ impl GasMeter { f(None) } else { self.gas_left = self.gas_left - amount; - let mut nested = GasMeter::with_limit(amount, self.gas_price); + let mut nested = GasMeter::new(amount); let r = f(Some(&mut nested)); @@ -175,8 +178,9 @@ impl GasMeter { } } - pub fn gas_price(&self) -> BalanceOf { - self.gas_price + /// Returns how much gas left from the initial budget. + fn gas_spent(&self) -> Gas { + self.gas_limit - self.gas_left } /// Returns how much gas left from the initial budget. @@ -184,9 +188,17 @@ impl GasMeter { self.gas_left } - /// Returns how much gas was spent. - fn spent(&self) -> Gas { - self.limit - self.gas_left + /// Turn this GasMeter into a DispatchResult that contains the actually used gas. + pub fn into_dispatch_result(self, result: Result) -> DispatchResultWithPostInfo where + E: Into, + { + let post_info = PostDispatchInfo { + actual_weight: Some(self.gas_spent()), + }; + + result + .map(|_| post_info) + .map_err(|e| DispatchErrorWithPostInfo { post_info, error: e.into() }) } #[cfg(test)] @@ -195,68 +207,6 @@ impl GasMeter { } } -/// Buy the given amount of gas. -/// -/// Cost is calculated by multiplying the gas cost (taken from the storage) by the `gas_limit`. -/// The funds are deducted from `transactor`. -pub fn buy_gas( - transactor: &T::AccountId, - gas_limit: Gas, -) -> Result<(GasMeter, NegativeImbalanceOf), DispatchError> { - // Buy the specified amount of gas. - let gas_price = >::gas_price(); - let cost = if gas_price.is_zero() { - >::zero() - } else { - as TryFrom>::try_from(gas_limit).ok() - .and_then(|gas_limit| gas_price.checked_mul(&gas_limit)) - .ok_or("overflow multiplying gas limit by price")? - }; - - let imbalance = T::Currency::withdraw( - transactor, - cost, - WithdrawReason::Fee.into(), - ExistenceRequirement::KeepAlive - )?; - - Ok((GasMeter::with_limit(gas_limit, gas_price), imbalance)) -} - -/// Refund the unused gas. -pub fn refund_unused_gas( - transactor: &T::AccountId, - gas_meter: GasMeter, - imbalance: NegativeImbalanceOf, -) { - let gas_spent = gas_meter.spent(); - let gas_left = gas_meter.gas_left(); - - // Increase total spent gas. - // This cannot overflow, since `gas_spent` is never greater than `block_gas_limit`, which - // also has Gas type. - GasSpent::mutate(|block_gas_spent| *block_gas_spent += gas_spent); - - // Refund gas left by the price it was bought at. - let refund = gas_meter.gas_price * gas_left.unique_saturated_into(); - let refund_imbalance = T::Currency::deposit_creating(transactor, refund); - if let Ok(imbalance) = imbalance.offset(refund_imbalance) { - T::GasPayment::on_unbalanced(imbalance); - } -} - -/// A little handy utility for converting a value in balance units into approximate value in gas units -/// at the given gas price. -pub fn approx_gas_for_balance(gas_price: Balance, balance: Balance) -> Gas - where Balance: AtLeast32Bit -{ - if gas_price.is_zero() { - Zero::zero() - } else { - (balance / gas_price).saturated_into::() - } -} - /// A simple utility macro that helps to match against a /// list of tokens. #[macro_export] @@ -298,7 +248,7 @@ macro_rules! match_tokens { #[cfg(test)] mod tests { use super::{GasMeter, Token}; - use crate::{tests::Test, gas::approx_gas_for_balance}; + use crate::tests::Test; /// A trivial token that charges the specified number of gas units. #[derive(Copy, Clone, PartialEq, Eq, Debug)] @@ -326,26 +276,24 @@ mod tests { #[test] fn it_works() { - let gas_meter = GasMeter::::with_limit(50000, 10); + let gas_meter = GasMeter::::new(50000); assert_eq!(gas_meter.gas_left(), 50000); } #[test] fn simple() { - let mut gas_meter = GasMeter::::with_limit(50000, 10); + let mut gas_meter = GasMeter::::new(50000); let result = gas_meter .charge(&MultiplierTokenMetadata { multiplier: 3 }, MultiplierToken(10)); assert!(!result.is_out_of_gas()); assert_eq!(gas_meter.gas_left(), 49_970); - assert_eq!(gas_meter.spent(), 30); - assert_eq!(gas_meter.gas_price(), 10); } #[test] fn tracing() { - let mut gas_meter = GasMeter::::with_limit(50000, 10); + let mut gas_meter = GasMeter::::new(50000); assert!(!gas_meter.charge(&(), SimpleToken(1)).is_out_of_gas()); assert!(!gas_meter .charge(&MultiplierTokenMetadata { multiplier: 3 }, MultiplierToken(10)) @@ -358,7 +306,7 @@ mod tests { // This test makes sure that nothing can be executed if there is no gas. #[test] fn refuse_to_execute_anything_if_zero() { - let mut gas_meter = GasMeter::::with_limit(0, 10); + let mut gas_meter = GasMeter::::new(0); assert!(gas_meter.charge(&(), SimpleToken(1)).is_out_of_gas()); } @@ -369,7 +317,7 @@ mod tests { // if the gas meter runs out of gas. However, this is just a nice property to have. #[test] fn overcharge_is_unrecoverable() { - let mut gas_meter = GasMeter::::with_limit(200, 10); + let mut gas_meter = GasMeter::::new(200); // The first charge is should lead to OOG. assert!(gas_meter.charge(&(), SimpleToken(300)).is_out_of_gas()); @@ -383,25 +331,7 @@ mod tests { // possible. #[test] fn charge_exact_amount() { - let mut gas_meter = GasMeter::::with_limit(25, 10); + let mut gas_meter = GasMeter::::new(25); assert!(!gas_meter.charge(&(), SimpleToken(25)).is_out_of_gas()); } - - // A unit test for `fn approx_gas_for_balance()`, and makes - // sure setting gas_price 0 does not cause `div by zero` error. - #[test] - fn approx_gas_for_balance_works() { - let tests = vec![ - (approx_gas_for_balance(0_u64, 123), 0), - (approx_gas_for_balance(0_u64, 456), 0), - (approx_gas_for_balance(1_u64, 123), 123), - (approx_gas_for_balance(1_u64, 456), 456), - (approx_gas_for_balance(100_u64, 900), 9), - (approx_gas_for_balance(123_u64, 900), 7), - ]; - - for (lhs, rhs) in tests { - assert_eq!(lhs, rhs); - } - } } diff --git a/frame/contracts/src/lib.rs b/frame/contracts/src/lib.rs index 090b345099e916a827382adbd5903bb4cecfeb6c..df53cf0a0eadd3f0bc9c30520a35a01591808653 100644 --- a/frame/contracts/src/lib.rs +++ b/frame/contracts/src/lib.rs @@ -64,15 +64,6 @@ //! initialize the contract. //! * `call` - Makes a call to an account, optionally transferring some balance. //! -//! ### Signed Extensions -//! -//! The contracts module defines the following extension: -//! -//! - [`CheckBlockGasLimit`]: Ensures that the transaction does not exceeds the block gas limit. -//! -//! The signed extension needs to be added as signed extra to the transaction type to be used in the -//! runtime. -//! //! ## Usage //! //! The Contract module is a work in progress. The following examples show how this Contract module @@ -113,21 +104,21 @@ use sp_std::{prelude::*, marker::PhantomData, fmt::Debug}; use codec::{Codec, Encode, Decode}; use sp_io::hashing::blake2_256; use sp_runtime::{ - traits::{Hash, StaticLookup, Zero, MaybeSerializeDeserialize, Member, SignedExtension}, - transaction_validity::{ - ValidTransaction, InvalidTransaction, TransactionValidity, TransactionValidityError, + traits::{ + Hash, StaticLookup, Zero, MaybeSerializeDeserialize, Member, }, RuntimeDebug, }; -use frame_support::dispatch::{DispatchResult, Dispatchable}; +use frame_support::dispatch::{ + PostDispatchInfo, DispatchResult, Dispatchable, DispatchResultWithPostInfo +}; use frame_support::{ - Parameter, decl_module, decl_event, decl_storage, decl_error, storage::child, - parameter_types, IsSubType, - weights::DispatchInfo, + Parameter, decl_module, decl_event, decl_storage, decl_error, + parameter_types, IsSubType, storage::child::{self, ChildInfo}, }; use frame_support::traits::{OnUnbalanced, Currency, Get, Time, Randomness}; +use frame_support::weights::{FunctionOf, DispatchClass, Weight, GetDispatchInfo, Pays}; use frame_system::{self as system, ensure_signed, RawOrigin, ensure_root}; -use sp_core::storage::well_known_keys::CHILD_STORAGE_KEY_PREFIX; use pallet_contracts_primitives::{RentProjection, ContractAccessError}; pub type CodeHash = ::Hash; @@ -226,15 +217,14 @@ pub struct RawAliveContractInfo { impl RawAliveContractInfo { /// Associated child trie unique id is built from the hash part of the trie id. - pub fn child_trie_unique_id(&self) -> child::ChildInfo { - trie_unique_id(&self.trie_id[..]) + pub fn child_trie_info(&self) -> ChildInfo { + child_trie_info(&self.trie_id[..]) } } /// Associated child trie unique id is built from the hash part of the trie id. -pub(crate) fn trie_unique_id(trie_id: &[u8]) -> child::ChildInfo { - let start = CHILD_STORAGE_KEY_PREFIX.len() + b"default:".len(); - child::ChildInfo::new_default(&trie_id[start ..]) +pub(crate) fn child_trie_info(trie_id: &[u8]) -> ChildInfo { + ChildInfo::new_default(trie_id) } pub type TombstoneContractInfo = @@ -267,10 +257,6 @@ pub trait TrieIdGenerator { /// /// The implementation must ensure every new trie id is unique: two consecutive calls with the /// same parameter needs to return different trie id values. - /// - /// Also, the implementation is responsible for ensuring that `TrieId` starts with - /// `:child_storage:`. - /// TODO: We want to change this, see https://github.com/paritytech/substrate/issues/2325 fn trie_id(account_id: &AccountId) -> TrieId; } @@ -294,19 +280,13 @@ where let mut buf = Vec::new(); buf.extend_from_slice(account_id.as_ref()); buf.extend_from_slice(&new_seed.to_le_bytes()[..]); - - // TODO: see https://github.com/paritytech/substrate/issues/2325 - CHILD_STORAGE_KEY_PREFIX.iter() - .chain(b"default:") - .chain(T::Hashing::hash(&buf[..]).as_ref().iter()) - .cloned() - .collect() + T::Hashing::hash(&buf[..]).as_ref().into() } } -pub type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; +pub type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; pub type NegativeImbalanceOf = - <::Currency as Currency<::AccountId>>::NegativeImbalance; + <::Currency as Currency<::AccountId>>::NegativeImbalance; parameter_types! { /// A reasonable default value for [`Trait::SignedClaimedHandicap`]. @@ -321,35 +301,21 @@ parameter_types! { pub const DefaultRentDepositOffset: u32 = 1000; /// A reasonable default value for [`Trait::SurchargeReward`]. pub const DefaultSurchargeReward: u32 = 150; - /// A reasonable default value for [`Trait::TransferFee`]. - pub const DefaultTransferFee: u32 = 0; - /// A reasonable default value for [`Trait::InstantiationFee`]. - pub const DefaultInstantiationFee: u32 = 0; - /// A reasonable default value for [`Trait::TransactionBaseFee`]. - pub const DefaultTransactionBaseFee: u32 = 0; - /// A reasonable default value for [`Trait::TransactionByteFee`]. - pub const DefaultTransactionByteFee: u32 = 0; - /// A reasonable default value for [`Trait::ContractFee`]. - pub const DefaultContractFee: u32 = 21; - /// A reasonable default value for [`Trait::CallBaseFee`]. - pub const DefaultCallBaseFee: u32 = 1000; - /// A reasonable default value for [`Trait::InstantiateBaseFee`]. - pub const DefaultInstantiateBaseFee: u32 = 1000; /// A reasonable default value for [`Trait::MaxDepth`]. pub const DefaultMaxDepth: u32 = 32; /// A reasonable default value for [`Trait::MaxValueSize`]. pub const DefaultMaxValueSize: u32 = 16_384; - /// A reasonable default value for [`Trait::BlockGasLimit`]. - pub const DefaultBlockGasLimit: u32 = 10_000_000; } -pub trait Trait: frame_system::Trait { - type Currency: Currency; +pub trait Trait: frame_system::Trait + pallet_transaction_payment::Trait { type Time: Time; type Randomness: Randomness; /// The outer call dispatch type. - type Call: Parameter + Dispatchable::Origin> + IsSubType, Self>; + type Call: + Parameter + + Dispatchable::Origin> + + IsSubType, Self> + GetDispatchInfo; /// The overarching event type. type Event: From> + Into<::Event>; @@ -357,18 +323,9 @@ pub trait Trait: frame_system::Trait { /// A function type to get the contract address given the instantiator. type DetermineContractAddress: ContractAddressFor, Self::AccountId>; - /// A function type that computes the fee for dispatching the given `Call`. - /// - /// It is recommended (though not required) for this function to return a fee that would be - /// taken by the Executive module for regular dispatch. - type ComputeDispatchFee: ComputeDispatchFee<::Call, BalanceOf>; - /// trie id generator type TrieIdGenerator: TrieIdGenerator; - /// Handler for the unbalanced reduction when making a gas payment. - type GasPayment: OnUnbalanced>; - /// Handler for rent payments. type RentPayment: OnUnbalanced>; @@ -401,29 +358,11 @@ pub trait Trait: frame_system::Trait { /// to removal of a contract. type SurchargeReward: Get>; - /// The fee to be paid for making a transaction; the base. - type TransactionBaseFee: Get>; - - /// The fee to be paid for making a transaction; the per-byte portion. - type TransactionByteFee: Get>; - - /// The fee required to instantiate a contract instance. - type ContractFee: Get>; - - /// The base fee charged for calling into a contract. - type CallBaseFee: Get; - - /// The base fee charged for instantiating a contract. - type InstantiateBaseFee: Get; - /// The maximum nesting level of a call/instantiate stack. type MaxDepth: Get; /// The maximum size of a storage value in bytes. type MaxValueSize: Get; - - /// The maximum amount of gas that could be expended per block. - type BlockGasLimit: Get; } /// Simple contract address determiner. @@ -449,19 +388,6 @@ where } } -/// The default dispatch fee computor computes the fee in the same way that -/// the implementation of `ChargeTransactionPayment` for the Balances module does. Note that this only takes a fixed -/// fee based on size. Unlike the balances module, weight-fee is applied. -pub struct DefaultDispatchFeeComputor(PhantomData); -impl ComputeDispatchFee<::Call, BalanceOf> for DefaultDispatchFeeComputor { - fn compute_dispatch_fee(call: &::Call) -> BalanceOf { - let encoded_len = call.using_encoded(|encoded| encoded.len() as u32); - let base_fee = T::TransactionBaseFee::get(); - let byte_fee = T::TransactionByteFee::get(); - base_fee + byte_fee * encoded_len.into() - } -} - decl_error! { /// Error for the contracts module. pub enum Error for Module { @@ -514,24 +440,6 @@ decl_module! { /// to removal of a contract. const SurchargeReward: BalanceOf = T::SurchargeReward::get(); - /// The fee to be paid for making a transaction; the base. - const TransactionBaseFee: BalanceOf = T::TransactionBaseFee::get(); - - /// The fee to be paid for making a transaction; the per-byte portion. - const TransactionByteFee: BalanceOf = T::TransactionByteFee::get(); - - /// The fee required to instantiate a contract instance. A reasonable default value - /// is 21. - const ContractFee: BalanceOf = T::ContractFee::get(); - - /// The base fee charged for calling into a contract. A reasonable default - /// value is 135. - const CallBaseFee: Gas = T::CallBaseFee::get(); - - /// The base fee charged for instantiating a contract. A reasonable default value - /// is 175. - const InstantiateBaseFee: Gas = T::InstantiateBaseFee::get(); - /// The maximum nesting level of a call/instantiate stack. A reasonable default /// value is 100. const MaxDepth: u32 = T::MaxDepth::get(); @@ -539,16 +447,12 @@ decl_module! { /// The maximum size of a storage value in bytes. A reasonable default is 16 KiB. const MaxValueSize: u32 = T::MaxValueSize::get(); - /// The maximum amount of gas that could be expended per block. A reasonable - /// default value is 10_000_000. - const BlockGasLimit: Gas = T::BlockGasLimit::get(); - fn deposit_event() = default; /// Updates the schedule for metering contracts. /// /// The schedule must have a greater version than the stored schedule. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn update_schedule(origin, schedule: Schedule) -> DispatchResult { ensure_root(origin)?; if >::current_schedule().version >= schedule.version { @@ -563,24 +467,21 @@ decl_module! { /// Stores the given binary Wasm code into the chain's storage and returns its `codehash`. /// You can instantiate contracts only with stored code. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = FunctionOf( + |args: (&Vec,)| Module::::calc_code_put_costs(args.0), + DispatchClass::Normal, + Pays::Yes + )] pub fn put_code( origin, - #[compact] gas_limit: Gas, code: Vec ) -> DispatchResult { - let origin = ensure_signed(origin)?; - - let (mut gas_meter, imbalance) = gas::buy_gas::(&origin, gas_limit)?; - + ensure_signed(origin)?; let schedule = >::current_schedule(); - let result = wasm::save_code::(code, &mut gas_meter, &schedule); + let result = wasm::save_code::(code, &schedule); if let Ok(code_hash) = result { Self::deposit_event(RawEvent::CodeStored(code_hash)); } - - gas::refund_unused_gas::(&origin, gas_meter, imbalance); - result.map(|_| ()).map_err(Into::into) } @@ -591,20 +492,26 @@ decl_module! { /// * If the account is a regular account, any value will be transferred. /// * If no account exists and the call value is not less than `existential_deposit`, /// a regular account will be created and any value will be transferred. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = FunctionOf( + |args: (&::Source, &BalanceOf, &Weight, &Vec)| *args.2, + DispatchClass::Normal, + Pays::Yes + )] pub fn call( origin, dest: ::Source, #[compact] value: BalanceOf, #[compact] gas_limit: Gas, data: Vec - ) -> DispatchResult { + ) -> DispatchResultWithPostInfo { let origin = ensure_signed(origin)?; let dest = T::Lookup::lookup(dest)?; + let mut gas_meter = GasMeter::new(gas_limit); - Self::bare_call(origin, dest, value, gas_limit, data) - .map(|_| ()) - .map_err(|e| e.reason.into()) + let result = Self::execute_wasm(origin, &mut gas_meter, |ctx, gas_meter| { + ctx.call(dest, value, gas_meter, data) + }); + gas_meter.into_dispatch_result(result.map_err(|e| e.reason)) } /// Instantiates a new contract from the `codehash` generated by `put_code`, optionally transferring some balance. @@ -617,22 +524,26 @@ decl_module! { /// after the execution is saved as the `code` of the account. That code will be invoked /// upon any call received by this account. /// - The contract is initialized. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = FunctionOf( + |args: (&BalanceOf, &Weight, &CodeHash, &Vec)| *args.1, + DispatchClass::Normal, + Pays::Yes + )] pub fn instantiate( origin, #[compact] endowment: BalanceOf, #[compact] gas_limit: Gas, code_hash: CodeHash, data: Vec - ) -> DispatchResult { + ) -> DispatchResultWithPostInfo { let origin = ensure_signed(origin)?; + let mut gas_meter = GasMeter::new(gas_limit); - Self::execute_wasm(origin, gas_limit, |ctx, gas_meter| { + let result = Self::execute_wasm(origin, &mut gas_meter, |ctx, gas_meter| { ctx.instantiate(endowment, gas_meter, &code_hash, data) .map(|(_address, output)| output) - }) - .map(|_| ()) - .map_err(|e| e.reason.into()) + }); + gas_meter.into_dispatch_result(result.map_err(|e| e.reason)) } /// Allows block producers to claim a small reward for evicting a contract. If a block producer @@ -640,7 +551,7 @@ decl_module! { /// /// If contract is not evicted as a result of this call, no actions are taken and /// the sender is not eligible for the reward. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn claim_surcharge(origin, dest: T::AccountId, aux_sender: Option) { let origin = origin.into(); let (signed, rewarded) = match (origin, aux_sender) { @@ -667,10 +578,6 @@ decl_module! { T::Currency::deposit_into_existing(&rewarded, T::SurchargeReward::get())?; } } - - fn on_finalize() { - GasSpent::kill(); - } } } @@ -687,7 +594,8 @@ impl Module { gas_limit: Gas, input_data: Vec, ) -> ExecResult { - Self::execute_wasm(origin, gas_limit, |ctx, gas_meter| { + let mut gas_meter = GasMeter::new(gas_limit); + Self::execute_wasm(origin, &mut gas_meter, |ctx, gas_meter| { ctx.call(dest, value, gas_meter, input_data) }) } @@ -719,40 +627,27 @@ impl Module { } impl Module { + fn calc_code_put_costs(code: &Vec) -> Gas { + >::current_schedule().put_code_per_byte_cost.saturating_mul(code.len() as Gas) + } + fn execute_wasm( origin: T::AccountId, - gas_limit: Gas, + gas_meter: &mut GasMeter, func: impl FnOnce(&mut ExecutionContext, &mut GasMeter) -> ExecResult ) -> ExecResult { - // Pay for the gas upfront. - // - // NOTE: it is very important to avoid any state changes before - // paying for the gas. - let (mut gas_meter, imbalance) = - try_or_exec_error!( - gas::buy_gas::(&origin, gas_limit), - // We don't have a spare buffer here in the first place, so create a new empty one. - Vec::new() - ); - let cfg = Config::preload(); let vm = WasmVm::new(&cfg.schedule); let loader = WasmLoader::new(&cfg.schedule); let mut ctx = ExecutionContext::top_level(origin.clone(), &cfg, &vm, &loader); - let result = func(&mut ctx, &mut gas_meter); + let result = func(&mut ctx, gas_meter); if result.as_ref().map(|output| output.is_success()).unwrap_or(false) { // Commit all changes that made it thus far into the persistent storage. DirectAccountDb.commit(ctx.overlay.into_change_set()); } - // Refund cost of the unused gas. - // - // NOTE: This should go after the commit to the storage, since the storage changes - // can alter the balance of the caller. - gas::refund_unused_gas::(&origin, gas_meter, imbalance); - // Execute deferred actions. ctx.deferred.into_iter().for_each(|deferred| { use self::exec::DeferredAction::*; @@ -768,7 +663,13 @@ impl Module { origin: who, call, } => { + let info = call.get_dispatch_info(); let result = call.dispatch(RawOrigin::Signed(who.clone()).into()); + let post_info = match result { + Ok(post_info) => post_info, + Err(err) => err.post_info, + }; + gas_meter.refund(post_info.calc_unspent(&info)); Self::deposit_event(RawEvent::Dispatched(who, result.is_ok())); } RestoreTo { @@ -821,13 +722,11 @@ impl Module { let key_values_taken = delta.iter() .filter_map(|key| { child::get_raw( - &origin_contract.trie_id, - origin_contract.child_trie_unique_id(), + &origin_contract.child_trie_info(), &blake2_256(key), ).map(|value| { child::kill( - &origin_contract.trie_id, - origin_contract.child_trie_unique_id(), + &origin_contract.child_trie_info(), &blake2_256(key), ); @@ -839,8 +738,8 @@ impl Module { let tombstone = >::new( // This operation is cheap enough because last_write (delta not included) // is not this block as it has been checked earlier. - &child::child_root( - &origin_contract.trie_id, + &child::root( + &origin_contract.child_trie_info(), )[..], code_hash, ); @@ -848,8 +747,7 @@ impl Module { if tombstone != dest_tombstone { for (key, value) in key_values_taken { child::put_raw( - &origin_contract.trie_id, - origin_contract.child_trie_unique_id(), + &origin_contract.child_trie_info(), &blake2_256(key), &value, ); @@ -929,8 +827,6 @@ decl_event! { decl_storage! { trait Store for Module as Contracts { - /// Gas spent so far in this block. - GasSpent get(fn gas_spent): Gas; /// Current cost schedule for contracts. CurrentSchedule get(fn current_schedule) config(): Schedule = Schedule::default(); /// A mapping from an original code hash to the original code, untouched by instrumentation. @@ -941,8 +837,6 @@ decl_storage! { pub AccountCounter: u64 = 0; /// The code associated with a given account. pub ContractInfoOf: map hasher(twox_64_concat) T::AccountId => Option>; - /// The price of one unit of gas. - GasPrice get(fn gas_price) config(): BalanceOf = 1.into(); } } @@ -956,7 +850,6 @@ pub struct Config { pub tombstone_deposit: BalanceOf, pub max_depth: u32, pub max_value_size: u32, - pub contract_account_instantiate_fee: BalanceOf, } impl Config { @@ -967,7 +860,6 @@ impl Config { tombstone_deposit: T::TombstoneDeposit::get(), max_depth: T::MaxDepth::get(), max_value_size: T::MaxValueSize::get(), - contract_account_instantiate_fee: T::ContractFee::get(), } } } @@ -1006,6 +898,9 @@ pub struct Schedule { /// Base gas cost to instantiate a contract. pub instantiate_base_cost: Gas, + /// Base gas cost to dispatch a runtime call. + pub dispatch_base_cost: Gas, + /// Gas cost per one byte read from the sandbox memory. pub sandbox_data_read_cost: Gas, @@ -1015,6 +910,9 @@ pub struct Schedule { /// Cost for a simple balance transfer. pub transfer_cost: Gas, + /// Cost for instantiating a new contract. + pub instantiate_cost: Gas, + /// The maximum number of topics supported by an event. pub max_event_topics: u32, @@ -1038,22 +936,29 @@ pub struct Schedule { pub max_subject_len: u32, } +// 500 (2 instructions per nano second on 2GHZ) * 1000x slowdown through wasmi +// This is a wild guess and should be viewed as a rough estimation. +// Proper benchmarks are needed before this value and its derivatives can be used in production. +const WASM_INSTRUCTION_COST: Gas = 500_000; + impl Default for Schedule { fn default() -> Schedule { Schedule { version: 0, - put_code_per_byte_cost: 1, - grow_mem_cost: 1, - regular_op_cost: 1, - return_data_per_byte_cost: 1, - event_data_per_byte_cost: 1, - event_per_topic_cost: 1, - event_base_cost: 1, - call_base_cost: 135, - instantiate_base_cost: 175, - sandbox_data_read_cost: 1, - sandbox_data_write_cost: 1, - transfer_cost: 100, + put_code_per_byte_cost: WASM_INSTRUCTION_COST, + grow_mem_cost: WASM_INSTRUCTION_COST, + regular_op_cost: WASM_INSTRUCTION_COST, + return_data_per_byte_cost: WASM_INSTRUCTION_COST, + event_data_per_byte_cost: WASM_INSTRUCTION_COST, + event_per_topic_cost: WASM_INSTRUCTION_COST, + event_base_cost: WASM_INSTRUCTION_COST, + call_base_cost: 135 * WASM_INSTRUCTION_COST, + dispatch_base_cost: 135 * WASM_INSTRUCTION_COST, + instantiate_base_cost: 175 * WASM_INSTRUCTION_COST, + sandbox_data_read_cost: WASM_INSTRUCTION_COST, + sandbox_data_write_cost: WASM_INSTRUCTION_COST, + transfer_cost: 100 * WASM_INSTRUCTION_COST, + instantiate_cost: 200 * WASM_INSTRUCTION_COST, max_event_topics: 4, max_stack_height: 64 * 1024, max_memory_pages: 16, @@ -1063,69 +968,3 @@ impl Default for Schedule { } } } - -/// `SignedExtension` that checks if a transaction would exhausts the block gas limit. -#[derive(Encode, Decode, Clone, Eq, PartialEq)] -pub struct CheckBlockGasLimit(PhantomData); - -impl Default for CheckBlockGasLimit { - fn default() -> Self { - Self(PhantomData) - } -} - -impl sp_std::fmt::Debug for CheckBlockGasLimit { - #[cfg(feature = "std")] - fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - write!(f, "CheckBlockGasLimit") - } - - #[cfg(not(feature = "std"))] - fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - Ok(()) - } -} - -impl SignedExtension for CheckBlockGasLimit { - const IDENTIFIER: &'static str = "CheckBlockGasLimit"; - type AccountId = T::AccountId; - type Call = ::Call; - type AdditionalSigned = (); - type DispatchInfo = DispatchInfo; - type Pre = (); - - fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } - - fn validate( - &self, - _: &Self::AccountId, - call: &Self::Call, - _: Self::DispatchInfo, - _: usize, - ) -> TransactionValidity { - let call = match call.is_sub_type() { - Some(call) => call, - None => return Ok(ValidTransaction::default()), - }; - - match call { - Call::claim_surcharge(_, _) | Call::update_schedule(_) => - Ok(ValidTransaction::default()), - Call::put_code(gas_limit, _) - | Call::call(_, _, gas_limit, _) - | Call::instantiate(_, gas_limit, _, _) - => { - // Check if the specified amount of gas is available in the current block. - // This cannot underflow since `gas_spent` is never greater than `T::BlockGasLimit`. - let gas_available = T::BlockGasLimit::get() - >::gas_spent(); - if *gas_limit > gas_available { - // gas limit reached, revert the transaction and retry again in the future - InvalidTransaction::ExhaustsResources.into() - } else { - Ok(ValidTransaction::default()) - } - }, - Call::__PhantomItem(_, _) => unreachable!("Variant is never constructed"), - } - } -} diff --git a/frame/contracts/src/rent.rs b/frame/contracts/src/rent.rs index 8b6825419c81e570033f695f52cf03c1f20847b6..1aa52fff314356e4758947e128139e92a4961c73 100644 --- a/frame/contracts/src/rent.rs +++ b/frame/contracts/src/rent.rs @@ -223,8 +223,7 @@ fn enact_verdict( Verdict::Kill => { >::remove(account); child::kill_storage( - &alive_contract_info.trie_id, - alive_contract_info.child_trie_unique_id(), + &alive_contract_info.child_trie_info(), ); >::deposit_event(RawEvent::Evicted(account.clone(), false)); None @@ -235,7 +234,9 @@ fn enact_verdict( } // Note: this operation is heavy. - let child_storage_root = child::child_root(&alive_contract_info.trie_id); + let child_storage_root = child::root( + &alive_contract_info.child_trie_info(), + ); let tombstone = >::new( &child_storage_root[..], @@ -245,8 +246,7 @@ fn enact_verdict( >::insert(account, &tombstone_info); child::kill_storage( - &alive_contract_info.trie_id, - alive_contract_info.child_trie_unique_id(), + &alive_contract_info.child_trie_info(), ); >::deposit_event(RawEvent::Evicted(account.clone(), true)); diff --git a/frame/contracts/src/tests.rs b/frame/contracts/src/tests.rs index 04b9b1ee4ca7775ebb53d434bbcc64cd465cef49..218a5c99372d40d78ba6c68a099d89faf3f79200 100644 --- a/frame/contracts/src/tests.rs +++ b/frame/contracts/src/tests.rs @@ -22,20 +22,22 @@ use crate::{ BalanceOf, ComputeDispatchFee, ContractAddressFor, ContractInfo, ContractInfoOf, GenesisConfig, Module, RawAliveContractInfo, RawEvent, Trait, TrieId, TrieIdFromParentCounter, Schedule, - TrieIdGenerator, CheckBlockGasLimit, account_db::{AccountDb, DirectAccountDb, OverlayAccountDb}, + TrieIdGenerator, account_db::{AccountDb, DirectAccountDb, OverlayAccountDb}, + gas::Gas, }; use assert_matches::assert_matches; use hex_literal::*; use codec::{Decode, Encode, KeyedVec}; use sp_runtime::{ Perbill, BuildStorage, transaction_validity::{InvalidTransaction, ValidTransaction}, - traits::{BlakeTwo256, Hash, IdentityLookup, SignedExtension}, + traits::{BlakeTwo256, Hash, IdentityLookup, SignedExtension, Convert}, testing::{Digest, DigestItem, Header, UintAuthorityId, H256}, }; use frame_support::{ - assert_ok, assert_err, impl_outer_dispatch, impl_outer_event, impl_outer_origin, parameter_types, + assert_ok, assert_err, assert_err_ignore_postinfo, impl_outer_dispatch, impl_outer_event, + impl_outer_origin, parameter_types, storage::child, StorageMap, StorageValue, traits::{Currency, Get}, - weights::{DispatchInfo, DispatchClass, Weight}, + weights::{DispatchInfo, DispatchClass, Weight, PostDispatchInfo, Pays}, }; use std::{cell::RefCell, sync::atomic::{AtomicUsize, Ordering}}; use sp_core::storage::well_known_keys; @@ -70,9 +72,6 @@ impl_outer_dispatch! { thread_local! { static EXISTENTIAL_DEPOSIT: RefCell = RefCell::new(0); - static TRANSFER_FEE: RefCell = RefCell::new(0); - static INSTANTIATION_FEE: RefCell = RefCell::new(0); - static BLOCK_GAS_LIMIT: RefCell = RefCell::new(0); } pub struct ExistentialDeposit; @@ -80,16 +79,6 @@ impl Get for ExistentialDeposit { fn get() -> u64 { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow()) } } -pub struct TransferFee; -impl Get for TransferFee { - fn get() -> u64 { TRANSFER_FEE.with(|v| *v.borrow()) } -} - -pub struct BlockGasLimit; -impl Get for BlockGasLimit { - fn get() -> u64 { BLOCK_GAS_LIMIT.with(|v| *v.borrow()) } -} - #[derive(Clone, Eq, PartialEq, Debug)] pub struct Test; parameter_types! { @@ -111,6 +100,9 @@ impl frame_system::Trait for Test { type Event = MetaEvent; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); @@ -141,24 +133,35 @@ parameter_types! { pub const RentByteFee: u64 = 4; pub const RentDepositOffset: u64 = 10_000; pub const SurchargeReward: u64 = 150; - pub const TransactionBaseFee: u64 = 2; - pub const TransactionByteFee: u64 = 6; - pub const ContractFee: u64 = 21; - pub const CallBaseFee: u64 = 135; - pub const InstantiateBaseFee: u64 = 175; pub const MaxDepth: u32 = 100; pub const MaxValueSize: u32 = 16_384; } -impl Trait for Test { + +parameter_types! { + pub const TransactionByteFee: u64 = 0; +} + +impl Convert> for Test { + fn convert(w: Weight) -> BalanceOf { + w + } +} + +impl pallet_transaction_payment::Trait for Test { type Currency = Balances; + type OnTransactionPayment = (); + type TransactionByteFee = TransactionByteFee; + type WeightToFee = Test; + type FeeMultiplierUpdate = (); +} + +impl Trait for Test { type Time = Timestamp; type Randomness = Randomness; type Call = Call; type DetermineContractAddress = DummyContractAddressFor; type Event = MetaEvent; - type ComputeDispatchFee = DummyComputeDispatchFee; type TrieIdGenerator = DummyTrieIdGenerator; - type GasPayment = (); type RentPayment = (); type SignedClaimHandicap = SignedClaimHandicap; type TombstoneDeposit = TombstoneDeposit; @@ -166,14 +169,8 @@ impl Trait for Test { type RentByteFee = RentByteFee; type RentDepositOffset = RentDepositOffset; type SurchargeReward = SurchargeReward; - type TransactionBaseFee = TransactionBaseFee; - type TransactionByteFee = TransactionByteFee; - type ContractFee = ContractFee; - type CallBaseFee = CallBaseFee; - type InstantiateBaseFee = InstantiateBaseFee; type MaxDepth = MaxDepth; type MaxValueSize = MaxValueSize; - type BlockGasLimit = BlockGasLimit; } type Balances = pallet_balances::Module; @@ -199,10 +196,7 @@ impl TrieIdGenerator for DummyTrieIdGenerator { *v }); - // TODO: see https://github.com/paritytech/substrate/issues/2325 let mut res = vec![]; - res.extend_from_slice(well_known_keys::CHILD_STORAGE_KEY_PREFIX); - res.extend_from_slice(b"default:"); res.extend_from_slice(&new_seed.to_le_bytes()); res.extend_from_slice(&account_id.to_le_bytes()); res @@ -221,21 +215,15 @@ const BOB: u64 = 2; const CHARLIE: u64 = 3; const DJANGO: u64 = 4; +const GAS_LIMIT: Gas = 10_000_000_000; + pub struct ExtBuilder { existential_deposit: u64, - gas_price: u64, - block_gas_limit: u64, - transfer_fee: u64, - instantiation_fee: u64, } impl Default for ExtBuilder { fn default() -> Self { Self { existential_deposit: 1, - gas_price: 2, - block_gas_limit: 100_000_000, - transfer_fee: 0, - instantiation_fee: 0, } } } @@ -244,27 +232,8 @@ impl ExtBuilder { self.existential_deposit = existential_deposit; self } - pub fn gas_price(mut self, gas_price: u64) -> Self { - self.gas_price = gas_price; - self - } - pub fn block_gas_limit(mut self, block_gas_limit: u64) -> Self { - self.block_gas_limit = block_gas_limit; - self - } - pub fn transfer_fee(mut self, transfer_fee: u64) -> Self { - self.transfer_fee = transfer_fee; - self - } - pub fn instantiation_fee(mut self, instantiation_fee: u64) -> Self { - self.instantiation_fee = instantiation_fee; - self - } pub fn set_associated_consts(&self) { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); - TRANSFER_FEE.with(|v| *v.borrow_mut() = self.transfer_fee); - INSTANTIATION_FEE.with(|v| *v.borrow_mut() = self.instantiation_fee); - BLOCK_GAS_LIMIT.with(|v| *v.borrow_mut() = self.block_gas_limit); } pub fn build(self) -> sp_io::TestExternalities { self.set_associated_consts(); @@ -272,12 +241,11 @@ impl ExtBuilder { pallet_balances::GenesisConfig:: { balances: vec![], }.assimilate_storage(&mut t).unwrap(); - GenesisConfig:: { + GenesisConfig { current_schedule: Schedule { enable_println: true, ..Default::default() }, - gas_price: self.gas_price, }.assimilate_storage(&mut t).unwrap(); let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); @@ -295,17 +263,21 @@ fn compile_module(wabt_module: &str) Ok((wasm, code_hash)) } -// Perform a simple transfer to a non-existent account supplying way more gas than needed. -// Then we check that the all unused gas is refunded. +// Perform a simple transfer to a non-existent account. +// Then we check that only the base costs are returned as actual costs. #[test] -fn refunds_unused_gas() { - ExtBuilder::default().gas_price(2).build().execute_with(|| { +fn returns_base_call_cost() { + ExtBuilder::default().build().execute_with(|| { Balances::deposit_creating(&ALICE, 100_000_000); - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, Vec::new())); - - // 2 * 135 - gas price multiplied by the call base fee. - assert_eq!(Balances::free_balance(ALICE), 100_000_000 - (2 * 135)); + assert_eq!( + Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, Vec::new()), + Ok( + PostDispatchInfo { + actual_weight: Some(67500000), + } + ) + ); }); } @@ -382,50 +354,21 @@ fn account_removal_does_not_remove_storage() { }); } -const CODE_RETURN_FROM_START_FN: &str = r#" -(module - (import "env" "ext_return" (func $ext_return (param i32 i32))) - (import "env" "ext_deposit_event" (func $ext_deposit_event (param i32 i32 i32 i32))) - (import "env" "memory" (memory 1 1)) - - (start $start) - (func $start - (call $ext_deposit_event - (i32.const 0) ;; The topics buffer - (i32.const 0) ;; The topics buffer's length - (i32.const 8) ;; The data buffer - (i32.const 4) ;; The data buffer's length - ) - (call $ext_return - (i32.const 8) - (i32.const 4) - ) - (unreachable) - ) - - (func (export "call") - (unreachable) - ) - (func (export "deploy")) - - (data (i32.const 8) "\01\02\03\04") -) -"#; - #[test] fn instantiate_and_call_and_deposit_event() { - let (wasm, code_hash) = compile_module::(CODE_RETURN_FROM_START_FN).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("return_from_start_fn.wat")) + .unwrap(); ExtBuilder::default().existential_deposit(100).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); // Check at the end to get hash on error easily let creation = Contracts::instantiate( Origin::signed(ALICE), 100, - 100_000, + GAS_LIMIT, code_hash.into(), vec![], ); @@ -480,23 +423,6 @@ fn instantiate_and_call_and_deposit_event() { }); } -const CODE_DISPATCH_CALL: &str = r#" -(module - (import "env" "ext_dispatch_call" (func $ext_dispatch_call (param i32 i32))) - (import "env" "memory" (memory 1 1)) - - (func (export "call") - (call $ext_dispatch_call - (i32.const 8) ;; Pointer to the start of encoded call buffer - (i32.const 11) ;; Length of the buffer - ) - ) - (func (export "deploy")) - - (data (i32.const 8) "\00\00\03\00\00\00\00\00\00\00\C8") -) -"#; - #[test] fn dispatch_call() { // This test can fail due to the encoding changes. In case it becomes too annoying @@ -504,12 +430,13 @@ fn dispatch_call() { let encoded = Encode::encode(&Call::Balances(pallet_balances::Call::transfer(CHARLIE, 50))); assert_eq!(&encoded[..], &hex!("00000300000000000000C8")[..]); - let (wasm, code_hash) = compile_module::(CODE_DISPATCH_CALL).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("dispatch_call.wat")) + .unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); // Let's keep this assert even though it's redundant. If you ever need to update the // wasm source this test will fail and will show you the actual hash. @@ -534,7 +461,7 @@ fn dispatch_call() { assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100, - 100_000, + GAS_LIMIT, code_hash.into(), vec![], )); @@ -543,7 +470,7 @@ fn dispatch_call() { Origin::signed(ALICE), BOB, // newly created account 0, - 100_000, + GAS_LIMIT, vec![], )); @@ -617,24 +544,6 @@ fn dispatch_call() { }); } -const CODE_DISPATCH_CALL_THEN_TRAP: &str = r#" -(module - (import "env" "ext_dispatch_call" (func $ext_dispatch_call (param i32 i32))) - (import "env" "memory" (memory 1 1)) - - (func (export "call") - (call $ext_dispatch_call - (i32.const 8) ;; Pointer to the start of encoded call buffer - (i32.const 11) ;; Length of the buffer - ) - (unreachable) ;; trap so that the top level transaction fails - ) - (func (export "deploy")) - - (data (i32.const 8) "\00\00\03\00\00\00\00\00\00\00\C8") -) -"#; - #[test] fn dispatch_call_not_dispatched_after_top_level_transaction_failure() { // This test can fail due to the encoding changes. In case it becomes too annoying @@ -642,12 +551,13 @@ fn dispatch_call_not_dispatched_after_top_level_transaction_failure() { let encoded = Encode::encode(&Call::Balances(pallet_balances::Call::transfer(CHARLIE, 50))); assert_eq!(&encoded[..], &hex!("00000300000000000000C8")[..]); - let (wasm, code_hash) = compile_module::(CODE_DISPATCH_CALL_THEN_TRAP).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("dispatch_call_then_trap.wat")) + .unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); // Let's keep this assert even though it's redundant. If you ever need to update the // wasm source this test will fail and will show you the actual hash. @@ -672,19 +582,19 @@ fn dispatch_call_not_dispatched_after_top_level_transaction_failure() { assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100, - 100_000, + GAS_LIMIT, code_hash.into(), vec![], )); // Call the newly instantiated contract. The contract is expected to dispatch a call // and then trap. - assert_err!( + assert_err_ignore_postinfo!( Contracts::call( Origin::signed(ALICE), BOB, // newly created account 0, - 100_000, + GAS_LIMIT, vec![], ), "contract trapped during execution" @@ -732,19 +642,10 @@ fn dispatch_call_not_dispatched_after_top_level_transaction_failure() { }); } -const CODE_RUN_OUT_OF_GAS: &str = r#" -(module - (func (export "call") - (loop $inf (br $inf)) ;; just run out of gas - (unreachable) - ) - (func (export "deploy")) -) -"#; - #[test] fn run_out_of_gas() { - let (wasm, code_hash) = compile_module::(CODE_RUN_OUT_OF_GAS).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("run_out_of_gas.wat")) + .unwrap(); ExtBuilder::default() .existential_deposit(50) @@ -752,24 +653,24 @@ fn run_out_of_gas() { .execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100, - 100_000, + GAS_LIMIT, code_hash.into(), vec![], )); // Call the contract with a fixed gas limit. It must run out of gas because it just // loops forever. - assert_err!( + assert_err_ignore_postinfo!( Contracts::call( Origin::signed(ALICE), BOB, // newly created account 0, - 1000, + 67_500_000, vec![], ), "ran out of gas during contract execution" @@ -777,110 +678,6 @@ fn run_out_of_gas() { }); } -const CODE_SET_RENT: &str = r#" -(module - (import "env" "ext_dispatch_call" (func $ext_dispatch_call (param i32 i32))) - (import "env" "ext_set_storage" (func $ext_set_storage (param i32 i32 i32))) - (import "env" "ext_clear_storage" (func $ext_clear_storage (param i32))) - (import "env" "ext_set_rent_allowance" (func $ext_set_rent_allowance (param i32 i32))) - (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) - (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) - (import "env" "memory" (memory 1 1)) - - ;; insert a value of 4 bytes into storage - (func $call_0 - (call $ext_set_storage - (i32.const 1) - (i32.const 0) - (i32.const 4) - ) - ) - - ;; remove the value inserted by call_1 - (func $call_1 - (call $ext_clear_storage - (i32.const 1) - ) - ) - - ;; transfer 50 to ALICE - (func $call_2 - (call $ext_dispatch_call - (i32.const 68) - (i32.const 11) - ) - ) - - ;; do nothing - (func $call_else) - - (func $assert (param i32) - (block $ok - (br_if $ok - (get_local 0) - ) - (unreachable) - ) - ) - - ;; Dispatch the call according to input size - (func (export "call") - (local $input_size i32) - (set_local $input_size - (call $ext_scratch_size) - ) - (block $IF_ELSE - (block $IF_2 - (block $IF_1 - (block $IF_0 - (br_table $IF_0 $IF_1 $IF_2 $IF_ELSE - (get_local $input_size) - ) - (unreachable) - ) - (call $call_0) - return - ) - (call $call_1) - return - ) - (call $call_2) - return - ) - (call $call_else) - ) - - ;; Set into storage a 4 bytes value - ;; Set call set_rent_allowance with input - (func (export "deploy") - (local $input_size i32) - (set_local $input_size - (call $ext_scratch_size) - ) - (call $ext_set_storage - (i32.const 0) - (i32.const 0) - (i32.const 4) - ) - (call $ext_scratch_read - (i32.const 0) - (i32.const 0) - (get_local $input_size) - ) - (call $ext_set_rent_allowance - (i32.const 0) - (get_local $input_size) - ) - ) - - ;; Encoding of 10 in balance - (data (i32.const 0) "\28") - - ;; Encoding of call transfer 50 to CHARLIE - (data (i32.const 68) "\00\00\03\00\00\00\00\00\00\00\C8") -) -"#; - /// Input data for each call in set_rent code mod call { pub fn set_storage_4_byte() -> Vec { vec![] } @@ -898,11 +695,11 @@ fn test_set_rent_code_and_hash() { let encoded = Encode::encode(&Call::Balances(pallet_balances::Call::transfer(CHARLIE, 50))); assert_eq!(&encoded[..], &hex!("00000300000000000000C8")[..]); - let (wasm, code_hash) = compile_module::(CODE_SET_RENT).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("set_rent.wat")).unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); // If you ever need to update the wasm source this test will fail // and will show you the actual hash. @@ -928,27 +725,27 @@ fn test_set_rent_code_and_hash() { #[test] fn storage_size() { - let (wasm, code_hash) = compile_module::(CODE_SET_RENT).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("set_rent.wat")).unwrap(); // Storage size ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 30_000, - 100_000, code_hash.into(), + GAS_LIMIT, code_hash.into(), ::Balance::from(1_000u32).encode() // rent allowance )); let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); assert_eq!(bob_contract.storage_size, ::StorageSizeOffset::get() + 4); - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::set_storage_4_byte())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::set_storage_4_byte())); let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); assert_eq!(bob_contract.storage_size, ::StorageSizeOffset::get() + 4 + 4); - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::remove_storage_4_byte())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::remove_storage_4_byte())); let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); assert_eq!(bob_contract.storage_size, ::StorageSizeOffset::get() + 4); }); @@ -966,16 +763,16 @@ fn initialize_block(number: u64) { #[test] fn deduct_blocks() { - let (wasm, code_hash) = compile_module::(CODE_SET_RENT).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("set_rent.wat")).unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 30_000, - 100_000, code_hash.into(), + GAS_LIMIT, code_hash.into(), ::Balance::from(1_000u32).encode() // rent allowance )); @@ -987,7 +784,7 @@ fn deduct_blocks() { initialize_block(5); // Trigger rent through call - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null())); // Check result let rent = (8 + 4 - 3) // storage size = size_offset + deploy_set_storage - deposit_offset @@ -1002,7 +799,7 @@ fn deduct_blocks() { initialize_block(12); // Trigger rent through call - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null())); // Check result let rent_2 = (8 + 4 - 2) // storage size = size_offset + deploy_set_storage - deposit_offset @@ -1014,7 +811,7 @@ fn deduct_blocks() { assert_eq!(Balances::free_balance(BOB), 30_000 - rent - rent_2); // Second call on same block should have no effect on rent - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null())); let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap(); assert_eq!(bob_contract.rent_allowance, 1_000 - rent - rent_2); @@ -1027,7 +824,7 @@ fn deduct_blocks() { fn call_contract_removals() { removals(|| { // Call on already-removed account might fail, and this is fine. - Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null()); + Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null()); true }); } @@ -1060,16 +857,16 @@ fn claim_surcharge_malus() { /// Claim surcharge with the given trigger_call at the given blocks. /// If `removes` is true then assert that the contract is a tombstone. fn claim_surcharge(blocks: u64, trigger_call: impl Fn() -> bool, removes: bool) { - let (wasm, code_hash) = compile_module::(CODE_SET_RENT).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("set_rent.wat")).unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100, - 100_000, code_hash.into(), + GAS_LIMIT, code_hash.into(), ::Balance::from(1_000u32).encode() // rent allowance )); @@ -1092,17 +889,17 @@ fn claim_surcharge(blocks: u64, trigger_call: impl Fn() -> bool, removes: bool) /// * if allowance is exceeded /// * if balance is reached and balance < subsistence threshold fn removals(trigger_call: impl Fn() -> bool) { - let (wasm, code_hash) = compile_module::(CODE_SET_RENT).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("set_rent.wat")).unwrap(); // Balance reached and superior to subsistence threshold ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm.clone())); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm.clone())); assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100, - 100_000, code_hash.into(), + GAS_LIMIT, code_hash.into(), ::Balance::from(1_000u32).encode() // rent allowance )); @@ -1134,11 +931,11 @@ fn removals(trigger_call: impl Fn() -> bool) { ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm.clone())); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm.clone())); assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 1_000, - 100_000, code_hash.into(), + GAS_LIMIT, code_hash.into(), ::Balance::from(100u32).encode() // rent allowance )); @@ -1169,11 +966,11 @@ fn removals(trigger_call: impl Fn() -> bool) { ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm.clone())); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm.clone())); assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 50+Balances::minimum_balance(), - 100_000, code_hash.into(), + GAS_LIMIT, code_hash.into(), ::Balance::from(1_000u32).encode() // rent allowance )); @@ -1183,7 +980,7 @@ fn removals(trigger_call: impl Fn() -> bool) { assert_eq!(Balances::free_balance(BOB), 50 + Balances::minimum_balance()); // Transfer funds - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::transfer())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::transfer())); assert_eq!(ContractInfoOf::::get(BOB).unwrap().get_alive().unwrap().rent_allowance, 1_000); assert_eq!(Balances::free_balance(BOB), Balances::minimum_balance()); @@ -1207,29 +1004,29 @@ fn removals(trigger_call: impl Fn() -> bool) { #[test] fn call_removed_contract() { - let (wasm, code_hash) = compile_module::(CODE_SET_RENT).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("set_rent.wat")).unwrap(); // Balance reached and superior to subsistence threshold ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm.clone())); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm.clone())); assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100, - 100_000, code_hash.into(), + GAS_LIMIT, code_hash.into(), ::Balance::from(1_000u32).encode() // rent allowance )); // Calling contract should succeed. - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null())); // Advance blocks initialize_block(10); // Calling contract should remove contract and fail. - assert_err!( - Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null()), + assert_err_ignore_postinfo!( + Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null()), "contract has been evicted" ); // Calling a contract that is about to evict shall emit an event. @@ -1242,75 +1039,26 @@ fn call_removed_contract() { ]); // Subsequent contract calls should also fail. - assert_err!( - Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null()), + assert_err_ignore_postinfo!( + Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null()), "contract has been evicted" ); }) } -const CODE_CHECK_DEFAULT_RENT_ALLOWANCE: &str = r#" -(module - (import "env" "ext_rent_allowance" (func $ext_rent_allowance)) - (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) - (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) - (import "env" "memory" (memory 1 1)) - - (func $assert (param i32) - (block $ok - (br_if $ok - (get_local 0) - ) - (unreachable) - ) - ) - - (func (export "call")) - - (func (export "deploy") - ;; fill the scratch buffer with the rent allowance. - (call $ext_rent_allowance) - - ;; assert $ext_scratch_size == 8 - (call $assert - (i32.eq - (call $ext_scratch_size) - (i32.const 8) - ) - ) - - ;; copy contents of the scratch buffer into the contract's memory. - (call $ext_scratch_read - (i32.const 8) ;; Pointer in memory to the place where to copy. - (i32.const 0) ;; Offset from the start of the scratch buffer. - (i32.const 8) ;; Count of bytes to copy. - ) - - ;; assert that contents of the buffer is equal to >::max_value(). - (call $assert - (i64.eq - (i64.load - (i32.const 8) - ) - (i64.const 0xFFFFFFFFFFFFFFFF) - ) - ) - ) -) -"#; - #[test] fn default_rent_allowance_on_instantiate() { - let (wasm, code_hash) = compile_module::(CODE_CHECK_DEFAULT_RENT_ALLOWANCE).unwrap(); + let (wasm, code_hash) = compile_module::( + &load_wasm("check_default_rent_allowance.wat")).unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 30_000, - 100_000, + GAS_LIMIT, code_hash.into(), vec![], )); @@ -1323,7 +1071,7 @@ fn default_rent_allowance_on_instantiate() { initialize_block(5); // Trigger rent through call - assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null())); + assert_ok!(Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null())); // Check contract is still alive let bob_contract = ContractInfoOf::::get(BOB).unwrap().get_alive(); @@ -1331,65 +1079,6 @@ fn default_rent_allowance_on_instantiate() { }); } -const CODE_RESTORATION: &str = r#" -(module - (import "env" "ext_set_storage" (func $ext_set_storage (param i32 i32 i32))) - (import "env" "ext_restore_to" (func $ext_restore_to (param i32 i32 i32 i32 i32 i32 i32 i32))) - (import "env" "memory" (memory 1 1)) - - (func (export "call") - (call $ext_restore_to - ;; Pointer and length of the encoded dest buffer. - (i32.const 256) - (i32.const 8) - ;; Pointer and length of the encoded code hash buffer - (i32.const 264) - (i32.const 32) - ;; Pointer and length of the encoded rent_allowance buffer - (i32.const 296) - (i32.const 8) - ;; Pointer and number of items in the delta buffer. - ;; This buffer specifies multiple keys for removal before restoration. - (i32.const 100) - (i32.const 1) - ) - ) - (func (export "deploy") - ;; Data to restore - (call $ext_set_storage - (i32.const 0) - (i32.const 0) - (i32.const 4) - ) - - ;; ACL - (call $ext_set_storage - (i32.const 100) - (i32.const 0) - (i32.const 4) - ) - ) - - ;; Data to restore - (data (i32.const 0) "\28") - - ;; Buffer that has ACL storage keys. - (data (i32.const 100) "\01") - - ;; Address of bob - (data (i32.const 256) "\02\00\00\00\00\00\00\00") - - ;; Code hash of SET_RENT - (data (i32.const 264) - "\c2\1c\41\10\a5\22\d8\59\1c\4c\77\35\dd\2d\bf\a1" - "\13\0b\50\93\76\9b\92\31\97\b7\c5\74\26\aa\38\2a" - ) - - ;; Rent allowance - (data (i32.const 296) "\32\00\00\00\00\00\00\00") -) -"#; - #[test] fn restorations_dirty_storage_and_different_storage() { restoration(true, true); @@ -1411,14 +1100,15 @@ fn restoration_success() { } fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: bool) { - let (set_rent_wasm, set_rent_code_hash) = compile_module::(CODE_SET_RENT).unwrap(); + let (set_rent_wasm, set_rent_code_hash) = + compile_module::(&load_wasm("set_rent.wat")).unwrap(); let (restoration_wasm, restoration_code_hash) = - compile_module::(CODE_RESTORATION).unwrap(); + compile_module::(&load_wasm("restoration.wat")).unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, restoration_wasm)); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, set_rent_wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), restoration_wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), set_rent_wasm)); // If you ever need to update the wasm source this test will fail // and will show you the actual hash. @@ -1450,7 +1140,7 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 30_000, - 100_000, + GAS_LIMIT, set_rent_code_hash.into(), ::Balance::from(0u32).encode() )); @@ -1463,7 +1153,7 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: if test_different_storage { assert_ok!(Contracts::call( Origin::signed(ALICE), - BOB, 0, 100_000, + BOB, 0, GAS_LIMIT, call::set_storage_4_byte()) ); } @@ -1476,8 +1166,8 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: .get_alive().unwrap().code_hash; // Call `BOB`, which makes it pay rent. Since the rent allowance is set to 0 // we expect that it will get removed leaving tombstone. - assert_err!( - Contracts::call(Origin::signed(ALICE), BOB, 0, 100_000, call::null()), + assert_err_ignore_postinfo!( + Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, call::null()), "contract has been evicted" ); assert!(ContractInfoOf::::get(BOB).unwrap().get_tombstone().is_some()); @@ -1499,7 +1189,7 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: assert_ok!(Contracts::instantiate( Origin::signed(CHARLIE), 30_000, - 100_000, + GAS_LIMIT, restoration_code_hash.into(), ::Balance::from(0u32).encode() )); @@ -1519,7 +1209,7 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: Origin::signed(ALICE), DJANGO, 0, - 100_000, + GAS_LIMIT, vec![], )); @@ -1625,81 +1315,19 @@ fn restoration(test_different_storage: bool, test_restore_to_with_dirty_storage: }); } -const CODE_STORAGE_SIZE: &str = r#" -(module - (import "env" "ext_get_storage" (func $ext_get_storage (param i32) (result i32))) - (import "env" "ext_set_storage" (func $ext_set_storage (param i32 i32 i32))) - (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) - (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) - (import "env" "memory" (memory 16 16)) - - (func $assert (param i32) - (block $ok - (br_if $ok - (get_local 0) - ) - (unreachable) - ) - ) - - (func (export "call") - ;; assert $ext_scratch_size == 8 - (call $assert - (i32.eq - (call $ext_scratch_size) - (i32.const 4) - ) - ) - - ;; copy contents of the scratch buffer into the contract's memory. - (call $ext_scratch_read - (i32.const 32) ;; Pointer in memory to the place where to copy. - (i32.const 0) ;; Offset from the start of the scratch buffer. - (i32.const 4) ;; Count of bytes to copy. - ) - - ;; place a garbage value in storage, the size of which is specified by the call input. - (call $ext_set_storage - (i32.const 0) ;; Pointer to storage key - (i32.const 0) ;; Pointer to value - (i32.load (i32.const 32)) ;; Size of value - ) - - (call $assert - (i32.eq - (call $ext_get_storage - (i32.const 0) ;; Pointer to storage key - ) - (i32.const 0) - ) - ) - - (call $assert - (i32.eq - (call $ext_scratch_size) - (i32.load (i32.const 32)) - ) - ) - ) - - (func (export "deploy")) - - (data (i32.const 0) "\01") ;; Storage key (32 B) -) -"#; - #[test] fn storage_max_value_limit() { - let (wasm, code_hash) = compile_module::(CODE_STORAGE_SIZE).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("storage_size.wat")) + .unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 30_000, - 100_000, + GAS_LIMIT, code_hash.into(), vec![], )); @@ -1713,17 +1341,17 @@ fn storage_max_value_limit() { Origin::signed(ALICE), BOB, 0, - 100_000, + GAS_LIMIT, Encode::encode(&self::MaxValueSize::get()), )); // Call contract with too large a storage value. - assert_err!( + assert_err_ignore_postinfo!( Contracts::call( Origin::signed(ALICE), BOB, 0, - 100_000, + GAS_LIMIT, Encode::encode(&(self::MaxValueSize::get() + 1)), ), "contract trapped during execution" @@ -1731,341 +1359,23 @@ fn storage_max_value_limit() { }); } -const CODE_RETURN_WITH_DATA: &str = r#" -(module - (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) - (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) - (import "env" "ext_scratch_write" (func $ext_scratch_write (param i32 i32))) - (import "env" "memory" (memory 1 1)) - - ;; Deploy routine is the same as call. - (func (export "deploy") (result i32) - (call $call) - ) - - ;; Call reads the first 4 bytes (LE) as the exit status and returns the rest as output data. - (func $call (export "call") (result i32) - (local $buf_size i32) - (local $exit_status i32) - - ;; Find out the size of the scratch buffer - (set_local $buf_size (call $ext_scratch_size)) - - ;; Copy scratch buffer into this contract memory. - (call $ext_scratch_read - (i32.const 0) ;; The pointer where to store the scratch buffer contents, - (i32.const 0) ;; Offset from the start of the scratch buffer. - (get_local $buf_size) ;; Count of bytes to copy. - ) - - ;; Copy all but the first 4 bytes of the input data as the output data. - (call $ext_scratch_write - (i32.const 4) ;; Pointer to the data to return. - (i32.sub ;; Count of bytes to copy. - (get_local $buf_size) - (i32.const 4) - ) - ) - - ;; Return the first 4 bytes of the input data as the exit status. - (i32.load (i32.const 0)) - ) -) -"#; - -const CODE_CALLER_CONTRACT: &str = r#" -(module - (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) - (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) - (import "env" "ext_balance" (func $ext_balance)) - (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32) (result i32))) - (import "env" "ext_instantiate" (func $ext_instantiate (param i32 i32 i64 i32 i32 i32 i32) (result i32))) - (import "env" "ext_println" (func $ext_println (param i32 i32))) - (import "env" "memory" (memory 1 1)) - - (func $assert (param i32) - (block $ok - (br_if $ok - (get_local 0) - ) - (unreachable) - ) - ) - - (func $current_balance (param $sp i32) (result i64) - (call $ext_balance) - (call $assert - (i32.eq (call $ext_scratch_size) (i32.const 8)) - ) - (call $ext_scratch_read - (i32.sub (get_local $sp) (i32.const 8)) - (i32.const 0) - (i32.const 8) - ) - (i64.load (i32.sub (get_local $sp) (i32.const 8))) - ) - - (func (export "deploy")) - - (func (export "call") - (local $sp i32) - (local $exit_code i32) - (local $balance i64) - - ;; Input data is the code hash of the contract to be deployed. - (call $assert - (i32.eq - (call $ext_scratch_size) - (i32.const 32) - ) - ) - - ;; Copy code hash from scratch buffer into this contract's memory. - (call $ext_scratch_read - (i32.const 24) ;; The pointer where to store the scratch buffer contents, - (i32.const 0) ;; Offset from the start of the scratch buffer. - (i32.const 32) ;; Count of bytes to copy. - ) - - ;; Read current balance into local variable. - (set_local $sp (i32.const 1024)) - (set_local $balance - (call $current_balance (get_local $sp)) - ) - - ;; Fail to deploy the contract since it returns a non-zero exit status. - (set_local $exit_code - (call $ext_instantiate - (i32.const 24) ;; Pointer to the code hash. - (i32.const 32) ;; Length of the code hash. - (i64.const 0) ;; How much gas to devote for the execution. 0 = all. - (i32.const 0) ;; Pointer to the buffer with value to transfer - (i32.const 8) ;; Length of the buffer with value to transfer. - (i32.const 9) ;; Pointer to input data buffer address - (i32.const 7) ;; Length of input data buffer - ) - ) - - ;; Check non-zero exit status. - (call $assert - (i32.eq (get_local $exit_code) (i32.const 0x11)) - ) - - ;; Check that scratch buffer is empty since contract instantiation failed. - (call $assert - (i32.eq (call $ext_scratch_size) (i32.const 0)) - ) - - ;; Check that balance has not changed. - (call $assert - (i64.eq (get_local $balance) (call $current_balance (get_local $sp))) - ) - - ;; Fail to deploy the contract due to insufficient gas. - (set_local $exit_code - (call $ext_instantiate - (i32.const 24) ;; Pointer to the code hash. - (i32.const 32) ;; Length of the code hash. - (i64.const 200) ;; How much gas to devote for the execution. - (i32.const 0) ;; Pointer to the buffer with value to transfer - (i32.const 8) ;; Length of the buffer with value to transfer. - (i32.const 8) ;; Pointer to input data buffer address - (i32.const 8) ;; Length of input data buffer - ) - ) - - ;; Check for special trap exit status. - (call $assert - (i32.eq (get_local $exit_code) (i32.const 0x0100)) - ) - - ;; Check that scratch buffer is empty since contract instantiation failed. - (call $assert - (i32.eq (call $ext_scratch_size) (i32.const 0)) - ) - - ;; Check that balance has not changed. - (call $assert - (i64.eq (get_local $balance) (call $current_balance (get_local $sp))) - ) - - ;; Deploy the contract successfully. - (set_local $exit_code - (call $ext_instantiate - (i32.const 24) ;; Pointer to the code hash. - (i32.const 32) ;; Length of the code hash. - (i64.const 0) ;; How much gas to devote for the execution. 0 = all. - (i32.const 0) ;; Pointer to the buffer with value to transfer - (i32.const 8) ;; Length of the buffer with value to transfer. - (i32.const 8) ;; Pointer to input data buffer address - (i32.const 8) ;; Length of input data buffer - ) - ) - - ;; Check for success exit status. - (call $assert - (i32.eq (get_local $exit_code) (i32.const 0x00)) - ) - - ;; Check that scratch buffer contains the address of the new contract. - (call $assert - (i32.eq (call $ext_scratch_size) (i32.const 8)) - ) - - ;; Copy contract address from scratch buffer into this contract's memory. - (call $ext_scratch_read - (i32.const 16) ;; The pointer where to store the scratch buffer contents, - (i32.const 0) ;; Offset from the start of the scratch buffer. - (i32.const 8) ;; Count of bytes to copy. - ) - - ;; Check that balance has been deducted. - (set_local $balance - (i64.sub (get_local $balance) (i64.load (i32.const 0))) - ) - (call $assert - (i64.eq (get_local $balance) (call $current_balance (get_local $sp))) - ) - - ;; Call the new contract and expect it to return failing exit code. - (set_local $exit_code - (call $ext_call - (i32.const 16) ;; Pointer to "callee" address. - (i32.const 8) ;; Length of "callee" address. - (i64.const 0) ;; How much gas to devote for the execution. 0 = all. - (i32.const 0) ;; Pointer to the buffer with value to transfer - (i32.const 8) ;; Length of the buffer with value to transfer. - (i32.const 9) ;; Pointer to input data buffer address - (i32.const 7) ;; Length of input data buffer - ) - ) - - ;; Check non-zero exit status. - (call $assert - (i32.eq (get_local $exit_code) (i32.const 0x11)) - ) - - ;; Check that scratch buffer contains the expected return data. - (call $assert - (i32.eq (call $ext_scratch_size) (i32.const 3)) - ) - (i32.store - (i32.sub (get_local $sp) (i32.const 4)) - (i32.const 0) - ) - (call $ext_scratch_read - (i32.sub (get_local $sp) (i32.const 4)) - (i32.const 0) - (i32.const 3) - ) - (call $assert - (i32.eq - (i32.load (i32.sub (get_local $sp) (i32.const 4))) - (i32.const 0x00776655) - ) - ) - - ;; Check that balance has not changed. - (call $assert - (i64.eq (get_local $balance) (call $current_balance (get_local $sp))) - ) - - ;; Fail to call the contract due to insufficient gas. - (set_local $exit_code - (call $ext_call - (i32.const 16) ;; Pointer to "callee" address. - (i32.const 8) ;; Length of "callee" address. - (i64.const 100) ;; How much gas to devote for the execution. - (i32.const 0) ;; Pointer to the buffer with value to transfer - (i32.const 8) ;; Length of the buffer with value to transfer. - (i32.const 8) ;; Pointer to input data buffer address - (i32.const 8) ;; Length of input data buffer - ) - ) - - ;; Check for special trap exit status. - (call $assert - (i32.eq (get_local $exit_code) (i32.const 0x0100)) - ) - - ;; Check that scratch buffer is empty since call trapped. - (call $assert - (i32.eq (call $ext_scratch_size) (i32.const 0)) - ) - - ;; Check that balance has not changed. - (call $assert - (i64.eq (get_local $balance) (call $current_balance (get_local $sp))) - ) - - ;; Call the contract successfully. - (set_local $exit_code - (call $ext_call - (i32.const 16) ;; Pointer to "callee" address. - (i32.const 8) ;; Length of "callee" address. - (i64.const 0) ;; How much gas to devote for the execution. 0 = all. - (i32.const 0) ;; Pointer to the buffer with value to transfer - (i32.const 8) ;; Length of the buffer with value to transfer. - (i32.const 8) ;; Pointer to input data buffer address - (i32.const 8) ;; Length of input data buffer - ) - ) - - ;; Check for success exit status. - (call $assert - (i32.eq (get_local $exit_code) (i32.const 0x00)) - ) - - ;; Check that scratch buffer contains the expected return data. - (call $assert - (i32.eq (call $ext_scratch_size) (i32.const 4)) - ) - (i32.store - (i32.sub (get_local $sp) (i32.const 4)) - (i32.const 0) - ) - (call $ext_scratch_read - (i32.sub (get_local $sp) (i32.const 4)) - (i32.const 0) - (i32.const 4) - ) - (call $assert - (i32.eq - (i32.load (i32.sub (get_local $sp) (i32.const 4))) - (i32.const 0x77665544) - ) - ) - - ;; Check that balance has been deducted. - (set_local $balance - (i64.sub (get_local $balance) (i64.load (i32.const 0))) - ) - (call $assert - (i64.eq (get_local $balance) (call $current_balance (get_local $sp))) - ) - ) - - (data (i32.const 0) "\00\80") ;; The value to transfer on instantiation and calls. - ;; Chosen to be greater than existential deposit. - (data (i32.const 8) "\00\11\22\33\44\55\66\77") ;; The input data to instantiations and calls. -) -"#; - #[test] fn deploy_and_call_other_contract() { - let (callee_wasm, callee_code_hash) = compile_module::(CODE_RETURN_WITH_DATA).unwrap(); - let (caller_wasm, caller_code_hash) = compile_module::(CODE_CALLER_CONTRACT).unwrap(); + let (callee_wasm, callee_code_hash) = + compile_module::(&load_wasm("return_with_data.wat")).unwrap(); + let (caller_wasm, caller_code_hash) = + compile_module::(&load_wasm("caller_contract.wat")).unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, callee_wasm)); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, caller_wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), callee_wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), caller_wasm)); assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100_000, - 100_000, + GAS_LIMIT, caller_code_hash.into(), vec![], )); @@ -2076,97 +1386,24 @@ fn deploy_and_call_other_contract() { Origin::signed(ALICE), BOB, 0, - 200_000, + GAS_LIMIT, callee_code_hash.as_ref().to_vec(), )); }); } -#[test] -fn deploy_works_without_gas_price() { - let (wasm, code_hash) = compile_module::(CODE_GET_RUNTIME_STORAGE).unwrap(); - ExtBuilder::default().existential_deposit(50).gas_price(0).build().execute_with(|| { - Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); - assert_ok!(Contracts::instantiate( - Origin::signed(ALICE), - 100, - 100_000, - code_hash.into(), - vec![], - )); - }); -} - -const CODE_DRAIN: &str = r#" -(module - (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) - (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) - (import "env" "ext_balance" (func $ext_balance)) - (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32) (result i32))) - (import "env" "memory" (memory 1 1)) - - (func $assert (param i32) - (block $ok - (br_if $ok - (get_local 0) - ) - (unreachable) - ) - ) - - (func (export "deploy")) - - (func (export "call") - ;; Send entire remaining balance to the 0 address. - (call $ext_balance) - - ;; Balance should be encoded as a u64. - (call $assert - (i32.eq - (call $ext_scratch_size) - (i32.const 8) - ) - ) - - ;; Read balance into memory. - (call $ext_scratch_read - (i32.const 8) ;; Pointer to write balance to - (i32.const 0) ;; Offset into scratch buffer - (i32.const 8) ;; Length of encoded balance - ) - - ;; Self-destruct by sending full balance to the 0 address. - (call $assert - (i32.eq - (call $ext_call - (i32.const 0) ;; Pointer to destination address - (i32.const 8) ;; Length of destination address - (i64.const 0) ;; How much gas to devote for the execution. 0 = all. - (i32.const 8) ;; Pointer to the buffer with value to transfer - (i32.const 8) ;; Length of the buffer with value to transfer - (i32.const 0) ;; Pointer to input data buffer address - (i32.const 0) ;; Length of input data buffer - ) - (i32.const 0) - ) - ) - ) -) -"#; - #[test] fn cannot_self_destruct_through_draning() { - let (wasm, code_hash) = compile_module::(CODE_DRAIN).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("drain.wat")).unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); // Instantiate the BOB contract. assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100_000, - 100_000, + GAS_LIMIT, code_hash.into(), vec![], )); @@ -2179,12 +1416,12 @@ fn cannot_self_destruct_through_draning() { // Call BOB with no input data, forcing it to run until out-of-balance // and eventually trapping because below existential deposit. - assert_err!( + assert_err_ignore_postinfo!( Contracts::call( Origin::signed(ALICE), BOB, 0, - 100_000, + GAS_LIMIT, vec![], ), "contract trapped during execution" @@ -2192,93 +1429,19 @@ fn cannot_self_destruct_through_draning() { }); } -const CODE_SELF_DESTRUCT: &str = r#" -(module - (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) - (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) - (import "env" "ext_address" (func $ext_address)) - (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32) (result i32))) - (import "env" "ext_terminate" (func $ext_terminate (param i32 i32))) - (import "env" "memory" (memory 1 1)) - - (func $assert (param i32) - (block $ok - (br_if $ok - (get_local 0) - ) - (unreachable) - ) - ) - - (func (export "deploy")) - - (func (export "call") - ;; If the input data is not empty, then recursively call self with empty input data. - ;; This should trap instead of self-destructing since a contract cannot be removed live in - ;; the execution stack cannot be removed. If the recursive call traps, then trap here as - ;; well. - (if (call $ext_scratch_size) - (then - (call $ext_address) - - ;; Expect address to be 8 bytes. - (call $assert - (i32.eq - (call $ext_scratch_size) - (i32.const 8) - ) - ) - - ;; Read own address into memory. - (call $ext_scratch_read - (i32.const 16) ;; Pointer to write address to - (i32.const 0) ;; Offset into scratch buffer - (i32.const 8) ;; Length of encoded address - ) - - ;; Recursively call self with empty input data. - (call $assert - (i32.eq - (call $ext_call - (i32.const 16) ;; Pointer to own address - (i32.const 8) ;; Length of own address - (i64.const 0) ;; How much gas to devote for the execution. 0 = all. - (i32.const 8) ;; Pointer to the buffer with value to transfer - (i32.const 8) ;; Length of the buffer with value to transfer - (i32.const 0) ;; Pointer to input data buffer address - (i32.const 0) ;; Length of input data buffer - ) - (i32.const 0) - ) - ) - ) - (else - ;; Try to terminate and give balance to django. - (call $ext_terminate - (i32.const 32) ;; Pointer to beneficiary address - (i32.const 8) ;; Length of beneficiary address - ) - (unreachable) ;; ext_terminate never returns - ) - ) - ) - ;; Address of django - (data (i32.const 32) "\04\00\00\00\00\00\00\00") -) -"#; - #[test] fn cannot_self_destruct_while_live() { - let (wasm, code_hash) = compile_module::(CODE_SELF_DESTRUCT).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("self_destruct.wat")) + .unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); // Instantiate the BOB contract. assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100_000, - 100_000, + GAS_LIMIT, code_hash.into(), vec![], )); @@ -2291,12 +1454,12 @@ fn cannot_self_destruct_while_live() { // Call BOB with input data, forcing it make a recursive call to itself to // self-destruct, resulting in a trap. - assert_err!( + assert_err_ignore_postinfo!( Contracts::call( Origin::signed(ALICE), BOB, 0, - 100_000, + GAS_LIMIT, vec![0], ), "contract trapped during execution" @@ -2312,16 +1475,17 @@ fn cannot_self_destruct_while_live() { #[test] fn self_destruct_works() { - let (wasm, code_hash) = compile_module::(CODE_SELF_DESTRUCT).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("self_destruct.wat")) + .unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); // Instantiate the BOB contract. assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100_000, - 100_000, + GAS_LIMIT, code_hash.into(), vec![], )); @@ -2338,10 +1502,10 @@ fn self_destruct_works() { Origin::signed(ALICE), BOB, 0, - 100_000, + GAS_LIMIT, vec![], ), - Ok(()) + Ok(_) ); // Check that account is gone @@ -2352,176 +1516,27 @@ fn self_destruct_works() { }); } -const CODE_DESTROY_AND_TRANSFER: &str = r#" -(module - (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) - (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) - (import "env" "ext_get_storage" (func $ext_get_storage (param i32) (result i32))) - (import "env" "ext_set_storage" (func $ext_set_storage (param i32 i32 i32))) - (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32) (result i32))) - (import "env" "ext_instantiate" (func $ext_instantiate (param i32 i32 i64 i32 i32 i32 i32) (result i32))) - (import "env" "memory" (memory 1 1)) - - (func $assert (param i32) - (block $ok - (br_if $ok - (get_local 0) - ) - (unreachable) - ) - ) - - (func (export "deploy") - ;; Input data is the code hash of the contract to be deployed. - (call $assert - (i32.eq - (call $ext_scratch_size) - (i32.const 32) - ) - ) - - ;; Copy code hash from scratch buffer into this contract's memory. - (call $ext_scratch_read - (i32.const 48) ;; The pointer where to store the scratch buffer contents, - (i32.const 0) ;; Offset from the start of the scratch buffer. - (i32.const 32) ;; Count of bytes to copy. - ) - - ;; Deploy the contract with the provided code hash. - (call $assert - (i32.eq - (call $ext_instantiate - (i32.const 48) ;; Pointer to the code hash. - (i32.const 32) ;; Length of the code hash. - (i64.const 0) ;; How much gas to devote for the execution. 0 = all. - (i32.const 0) ;; Pointer to the buffer with value to transfer - (i32.const 8) ;; Length of the buffer with value to transfer. - (i32.const 0) ;; Pointer to input data buffer address - (i32.const 0) ;; Length of input data buffer - ) - (i32.const 0) - ) - ) - - ;; Read the address of the instantiated contract into memory. - (call $assert - (i32.eq - (call $ext_scratch_size) - (i32.const 8) - ) - ) - (call $ext_scratch_read - (i32.const 80) ;; The pointer where to store the scratch buffer contents, - (i32.const 0) ;; Offset from the start of the scratch buffer. - (i32.const 8) ;; Count of bytes to copy. - ) - - ;; Store the return address. - (call $ext_set_storage - (i32.const 16) ;; Pointer to the key - (i32.const 80) ;; Pointer to the value - (i32.const 8) ;; Length of the value - ) - ) - - (func (export "call") - ;; Read address of destination contract from storage. - (call $assert - (i32.eq - (call $ext_get_storage - (i32.const 16) ;; Pointer to the key - ) - (i32.const 0) - ) - ) - (call $assert - (i32.eq - (call $ext_scratch_size) - (i32.const 8) - ) - ) - (call $ext_scratch_read - (i32.const 80) ;; The pointer where to store the contract address. - (i32.const 0) ;; Offset from the start of the scratch buffer. - (i32.const 8) ;; Count of bytes to copy. - ) - - ;; Calling the destination contract with non-empty input data should fail. - (call $assert - (i32.eq - (call $ext_call - (i32.const 80) ;; Pointer to destination address - (i32.const 8) ;; Length of destination address - (i64.const 0) ;; How much gas to devote for the execution. 0 = all. - (i32.const 0) ;; Pointer to the buffer with value to transfer - (i32.const 8) ;; Length of the buffer with value to transfer - (i32.const 0) ;; Pointer to input data buffer address - (i32.const 1) ;; Length of input data buffer - ) - (i32.const 0x0100) - ) - ) - - ;; Call the destination contract regularly, forcing it to self-destruct. - (call $assert - (i32.eq - (call $ext_call - (i32.const 80) ;; Pointer to destination address - (i32.const 8) ;; Length of destination address - (i64.const 0) ;; How much gas to devote for the execution. 0 = all. - (i32.const 8) ;; Pointer to the buffer with value to transfer - (i32.const 8) ;; Length of the buffer with value to transfer - (i32.const 0) ;; Pointer to input data buffer address - (i32.const 0) ;; Length of input data buffer - ) - (i32.const 0) - ) - ) - - ;; Calling the destination address with non-empty input data should now work since the - ;; contract has been removed. Also transfer a balance to the address so we can ensure this - ;; does not keep the contract alive. - (call $assert - (i32.eq - (call $ext_call - (i32.const 80) ;; Pointer to destination address - (i32.const 8) ;; Length of destination address - (i64.const 0) ;; How much gas to devote for the execution. 0 = all. - (i32.const 0) ;; Pointer to the buffer with value to transfer - (i32.const 8) ;; Length of the buffer with value to transfer - (i32.const 0) ;; Pointer to input data buffer address - (i32.const 1) ;; Length of input data buffer - ) - (i32.const 0) - ) - ) - ) - - (data (i32.const 0) "\00\00\01") ;; Endowment to send when creating contract. - (data (i32.const 8) "") ;; Value to send when calling contract. - (data (i32.const 16) "") ;; The key to store the contract address under. -) -"#; - // This tests that one contract cannot prevent another from self-destructing by sending it // additional funds after it has been drained. #[test] fn destroy_contract_and_transfer_funds() { - let (callee_wasm, callee_code_hash) = compile_module::(CODE_SELF_DESTRUCT).unwrap(); - let (caller_wasm, caller_code_hash) = compile_module::(CODE_DESTROY_AND_TRANSFER).unwrap(); + let (callee_wasm, callee_code_hash) = + compile_module::(&load_wasm("self_destruct.wat")).unwrap(); + let (caller_wasm, caller_code_hash) = + compile_module::(&load_wasm("destroy_and_transfer.wat")).unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, callee_wasm)); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, caller_wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), callee_wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), caller_wasm)); // This deploys the BOB contract, which in turn deploys the CHARLIE contract during // construction. assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 200_000, - 100_000, + GAS_LIMIT, caller_code_hash.into(), callee_code_hash.as_ref().to_vec(), )); @@ -2537,7 +1552,7 @@ fn destroy_contract_and_transfer_funds() { Origin::signed(ALICE), BOB, 0, - 100_000, + GAS_LIMIT, CHARLIE.encode(), )); @@ -2546,77 +1561,21 @@ fn destroy_contract_and_transfer_funds() { }); } -const CODE_SELF_DESTRUCTING_CONSTRUCTOR: &str = r#" -(module - (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) - (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) - (import "env" "ext_balance" (func $ext_balance)) - (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32) (result i32))) - (import "env" "memory" (memory 1 1)) - - (func $assert (param i32) - (block $ok - (br_if $ok - (get_local 0) - ) - (unreachable) - ) - ) - - (func (export "deploy") - ;; Send entire remaining balance to the 0 address. - (call $ext_balance) - - ;; Balance should be encoded as a u64. - (call $assert - (i32.eq - (call $ext_scratch_size) - (i32.const 8) - ) - ) - - ;; Read balance into memory. - (call $ext_scratch_read - (i32.const 8) ;; Pointer to write balance to - (i32.const 0) ;; Offset into scratch buffer - (i32.const 8) ;; Length of encoded balance - ) - - ;; Self-destruct by sending full balance to the 0 address. - (call $assert - (i32.eq - (call $ext_call - (i32.const 0) ;; Pointer to destination address - (i32.const 8) ;; Length of destination address - (i64.const 0) ;; How much gas to devote for the execution. 0 = all. - (i32.const 8) ;; Pointer to the buffer with value to transfer - (i32.const 8) ;; Length of the buffer with value to transfer - (i32.const 0) ;; Pointer to input data buffer address - (i32.const 0) ;; Length of input data buffer - ) - (i32.const 0) - ) - ) - ) - - (func (export "call")) -) -"#; - #[test] fn cannot_self_destruct_in_constructor() { - let (wasm, code_hash) = compile_module::(CODE_SELF_DESTRUCTING_CONSTRUCTOR).unwrap(); + let (wasm, code_hash) = + compile_module::(&load_wasm("self_destructing_constructor.wat")).unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); // Fail to instantiate the BOB because the call that is issued in the deploy // function exhausts all balances which puts it below the existential deposit. - assert_err!( + assert_err_ignore_postinfo!( Contracts::instantiate( Origin::signed(ALICE), 100_000, - 100_000, + GAS_LIMIT, code_hash.into(), vec![], ), @@ -2625,102 +1584,9 @@ fn cannot_self_destruct_in_constructor() { }); } -#[test] -fn check_block_gas_limit_works() { - ExtBuilder::default().block_gas_limit(50).build().execute_with(|| { - let info = DispatchInfo { weight: 100, class: DispatchClass::Normal, pays_fee: true }; - let check = CheckBlockGasLimit::(Default::default()); - let call: Call = crate::Call::put_code(1000, vec![]).into(); - - assert_eq!( - check.validate(&0, &call, info, 0), InvalidTransaction::ExhaustsResources.into(), - ); - - let call: Call = crate::Call::update_schedule(Default::default()).into(); - assert_eq!(check.validate(&0, &call, info, 0), Ok(Default::default())); - }); -} - -const CODE_GET_RUNTIME_STORAGE: &str = r#" -(module - (import "env" "ext_get_runtime_storage" - (func $ext_get_runtime_storage (param i32 i32) (result i32)) - ) - (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) - (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) - (import "env" "ext_scratch_write" (func $ext_scratch_write (param i32 i32))) - (import "env" "memory" (memory 1 1)) - - (func (export "deploy")) - - (func $assert (param i32) - (block $ok - (br_if $ok - (get_local 0) - ) - (unreachable) - ) - ) - - (func $call (export "call") - ;; Load runtime storage for the first key and assert that it exists. - (call $assert - (i32.eq - (call $ext_get_runtime_storage - (i32.const 16) - (i32.const 4) - ) - (i32.const 0) - ) - ) - - ;; assert $ext_scratch_size == 4 - (call $assert - (i32.eq - (call $ext_scratch_size) - (i32.const 4) - ) - ) - - ;; copy contents of the scratch buffer into the contract's memory. - (call $ext_scratch_read - (i32.const 4) ;; Pointer in memory to the place where to copy. - (i32.const 0) ;; Offset from the start of the scratch buffer. - (i32.const 4) ;; Count of bytes to copy. - ) - - ;; assert that contents of the buffer is equal to the i32 value of 0x14144020. - (call $assert - (i32.eq - (i32.load - (i32.const 4) - ) - (i32.const 0x14144020) - ) - ) - - ;; Load the second key and assert that it doesn't exist. - (call $assert - (i32.eq - (call $ext_get_runtime_storage - (i32.const 20) - (i32.const 4) - ) - (i32.const 1) - ) - ) - ) - - ;; The first key, 4 bytes long. - (data (i32.const 16) "\01\02\03\04") - ;; The second key, 4 bytes long. - (data (i32.const 20) "\02\03\04\05") -) -"#; - -#[test] fn get_runtime_storage() { - let (wasm, code_hash) = compile_module::(CODE_GET_RUNTIME_STORAGE).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("get_runtime_storage.wat")) + .unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); @@ -2729,11 +1595,11 @@ fn get_runtime_storage() { 0x14144020u32.to_le_bytes().to_vec().as_ref() ); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100, - 100_000, + GAS_LIMIT, code_hash.into(), vec![], )); @@ -2741,108 +1607,25 @@ fn get_runtime_storage() { Origin::signed(ALICE), BOB, 0, - 100_000, + GAS_LIMIT, vec![], )); }); } -const CODE_CRYPTO_HASHES: &str = r#" -(module - (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) - (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) - (import "env" "ext_scratch_write" (func $ext_scratch_write (param i32 i32))) - - (import "env" "ext_hash_sha2_256" (func $ext_hash_sha2_256 (param i32 i32 i32))) - (import "env" "ext_hash_keccak_256" (func $ext_hash_keccak_256 (param i32 i32 i32))) - (import "env" "ext_hash_blake2_256" (func $ext_hash_blake2_256 (param i32 i32 i32))) - (import "env" "ext_hash_blake2_128" (func $ext_hash_blake2_128 (param i32 i32 i32))) - - (import "env" "memory" (memory 1 1)) - - (type $hash_fn_sig (func (param i32 i32 i32))) - (table 8 funcref) - (elem (i32.const 1) - $ext_hash_sha2_256 - $ext_hash_keccak_256 - $ext_hash_blake2_256 - $ext_hash_blake2_128 - ) - (data (i32.const 1) "20202010201008") ;; Output sizes of the hashes in order in hex. - - ;; Not in use by the tests besides instantiating the contract. - (func (export "deploy")) - - ;; Called by the tests. - ;; - ;; The `call` function expects data in a certain format in the scratch - ;; buffer. - ;; - ;; 1. The first byte encodes an identifier for the crypto hash function - ;; under test. (*) - ;; 2. The rest encodes the input data that is directly fed into the - ;; crypto hash function chosen in 1. - ;; - ;; The `deploy` function then computes the chosen crypto hash function - ;; given the input and puts the result back into the scratch buffer. - ;; After contract execution the test driver then asserts that the returned - ;; values are equal to the expected bytes for the input and chosen hash - ;; function. - ;; - ;; (*) The possible value for the crypto hash identifiers can be found below: - ;; - ;; | value | Algorithm | Bit Width | - ;; |-------|-----------|-----------| - ;; | 0 | SHA2 | 256 | - ;; | 1 | KECCAK | 256 | - ;; | 2 | BLAKE2 | 256 | - ;; | 3 | BLAKE2 | 128 | - ;; --------------------------------- - (func (export "call") (result i32) - (local $chosen_hash_fn i32) - (local $input_ptr i32) - (local $input_len i32) - (local $output_ptr i32) - (local $output_len i32) - (local.set $input_ptr (i32.const 10)) - (call $ext_scratch_read (local.get $input_ptr) (i32.const 0) (call $ext_scratch_size)) - (local.set $chosen_hash_fn (i32.load8_u (local.get $input_ptr))) - (if (i32.gt_u (local.get $chosen_hash_fn) (i32.const 7)) - ;; We check that the chosen hash fn identifier is within bounds: [0,7] - (unreachable) - ) - (local.set $input_ptr (i32.add (local.get $input_ptr) (i32.const 1))) - (local.set $input_len (i32.sub (call $ext_scratch_size) (i32.const 1))) - (local.set $output_ptr (i32.const 100)) - (local.set $output_len (i32.load8_u (local.get $chosen_hash_fn))) - (call_indirect (type $hash_fn_sig) - (local.get $input_ptr) - (local.get $input_len) - (local.get $output_ptr) - (local.get $chosen_hash_fn) ;; Which crypto hash function to execute. - ) - (call $ext_scratch_write - (local.get $output_ptr) ;; Linear memory location of the output buffer. - (local.get $output_len) ;; Number of output buffer bytes. - ) - (i32.const 0) - ) -) -"#; - #[test] fn crypto_hashes() { - let (wasm, code_hash) = compile_module::(&CODE_CRYPTO_HASHES).unwrap(); + let (wasm, code_hash) = compile_module::(&load_wasm("crypto_hashes.wat")).unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { Balances::deposit_creating(&ALICE, 1_000_000); - assert_ok!(Contracts::put_code(Origin::signed(ALICE), 100_000, wasm)); + assert_ok!(Contracts::put_code(Origin::signed(ALICE), wasm)); // Instantiate the CRYPTO_HASHES contract. assert_ok!(Contracts::instantiate( Origin::signed(ALICE), 100_000, - 100_000, + GAS_LIMIT, code_hash.into(), vec![], )); @@ -2871,7 +1654,7 @@ fn crypto_hashes() { ALICE, BOB, 0, - 100_000, + GAS_LIMIT, params, ).unwrap(); assert_eq!(result.status, 0); @@ -2880,3 +1663,8 @@ fn crypto_hashes() { } }) } + +fn load_wasm(file_name: &str) -> String { + let path = ["tests/", file_name].concat(); + std::fs::read_to_string(&path).expect(&format!("Unable to read {} file", path)) +} diff --git a/frame/contracts/src/wasm/code_cache.rs b/frame/contracts/src/wasm/code_cache.rs index cb942a25892cd1553b381ab8f856329c395e188a..ba7a02356d2826c649afd8ae6dcd5b8bfb62ab27 100644 --- a/frame/contracts/src/wasm/code_cache.rs +++ b/frame/contracts/src/wasm/code_cache.rs @@ -26,49 +26,20 @@ //! this guarantees that every instrumented contract code in cache cannot have the version equal to the current one. //! Thus, before executing a contract it should be reinstrument with new schedule. -use crate::gas::{Gas, GasMeter, Token}; use crate::wasm::{prepare, runtime::Env, PrefabWasmModule}; use crate::{CodeHash, CodeStorage, PristineCode, Schedule, Trait}; use sp_std::prelude::*; -use sp_runtime::traits::{Hash, Bounded}; +use sp_runtime::traits::Hash; use frame_support::StorageMap; -/// Gas metering token that used for charging storing code into the code storage. -/// -/// Specifies the code length in bytes. -#[cfg_attr(test, derive(Debug, PartialEq, Eq))] -#[derive(Copy, Clone)] -pub struct PutCodeToken(u32); - -impl Token for PutCodeToken { - type Metadata = Schedule; - - fn calculate_amount(&self, metadata: &Schedule) -> Gas { - metadata - .put_code_per_byte_cost - .checked_mul(self.0.into()) - .unwrap_or_else(|| Bounded::max_value()) - } -} - /// Put code in the storage. The hash of code is used as a key and is returned /// as a result of this function. /// /// This function instruments the given code and caches it in the storage. pub fn save( original_code: Vec, - gas_meter: &mut GasMeter, schedule: &Schedule, ) -> Result, &'static str> { - // The first time instrumentation is on the user. However, consequent reinstrumentation - // due to the schedule changes is on governance system. - if gas_meter - .charge(schedule, PutCodeToken(original_code.len() as u32)) - .is_out_of_gas() - { - return Err("there is not enough gas for storing the code"); - } - let prefab_module = prepare::prepare_contract::(&original_code, schedule)?; let code_hash = T::Hashing::hash(&original_code); diff --git a/frame/contracts/src/wasm/mod.rs b/frame/contracts/src/wasm/mod.rs index 8911fb72b6130c66fa1cf7d88d6fcc86b062b5dd..cb69cd689b2657fa159eec447f1ed569cfc4a434 100644 --- a/frame/contracts/src/wasm/mod.rs +++ b/frame/contracts/src/wasm/mod.rs @@ -157,12 +157,14 @@ mod tests { use crate::gas::{Gas, GasMeter}; use crate::tests::{Test, Call}; use crate::wasm::prepare::prepare_contract; - use crate::CodeHash; + use crate::{CodeHash, BalanceOf}; use wabt; use hex_literal::hex; use assert_matches::assert_matches; use sp_runtime::DispatchError; + const GAS_LIMIT: Gas = 10_000_000_000; + #[derive(Debug, PartialEq, Eq)] struct DispatchEntry(Call); @@ -373,6 +375,9 @@ mod tests { ) ) } + fn get_weight_price(&self) -> BalanceOf { + 1312_u32.into() + } } impl Ext for &mut MockExt { @@ -478,6 +483,9 @@ mod tests { fn get_runtime_storage(&self, key: &[u8]) -> Option> { (**self).get_runtime_storage(key) } + fn get_weight_price(&self) -> BalanceOf { + (**self).get_weight_price() + } } fn execute( @@ -544,7 +552,7 @@ mod tests { CODE_TRANSFER, vec![], &mut mock_ext, - &mut GasMeter::with_limit(50_000, 1), + &mut GasMeter::new(GAS_LIMIT), ).unwrap(); assert_eq!( @@ -553,7 +561,7 @@ mod tests { to: 7, value: 153, data: Vec::new(), - gas_left: 49978, + gas_left: 9989000000, }] ); } @@ -604,7 +612,7 @@ mod tests { CODE_CALL, vec![], &mut mock_ext, - &mut GasMeter::with_limit(50_000, 1), + &mut GasMeter::new(GAS_LIMIT), ).unwrap(); assert_eq!( @@ -613,7 +621,7 @@ mod tests { to: 9, value: 6, data: vec![1, 2, 3, 4], - gas_left: 49971, + gas_left: 9985500000, }] ); } @@ -666,7 +674,7 @@ mod tests { CODE_INSTANTIATE, vec![], &mut mock_ext, - &mut GasMeter::with_limit(50_000, 1), + &mut GasMeter::new(GAS_LIMIT), ).unwrap(); assert_eq!( @@ -675,7 +683,7 @@ mod tests { code_hash: [0x11; 32].into(), endowment: 3, data: vec![1, 2, 3, 4], - gas_left: 49947, + gas_left: 9973500000, }] ); } @@ -709,14 +717,14 @@ mod tests { CODE_TERMINATE, vec![], &mut mock_ext, - &mut GasMeter::with_limit(50_000, 1), + &mut GasMeter::new(GAS_LIMIT), ).unwrap(); assert_eq!( &mock_ext.terminations, &[TerminationEntry { beneficiary: 0x09, - gas_left: 49989, + gas_left: 9994500000, }] ); } @@ -767,7 +775,7 @@ mod tests { &CODE_TRANSFER_LIMITED_GAS, vec![], &mut mock_ext, - &mut GasMeter::with_limit(50_000, 1), + &mut GasMeter::new(GAS_LIMIT), ).unwrap(); assert_eq!( @@ -860,7 +868,7 @@ mod tests { CODE_GET_STORAGE, vec![], mock_ext, - &mut GasMeter::with_limit(50_000, 1), + &mut GasMeter::new(GAS_LIMIT), ).unwrap(); assert_eq!(output, ExecReturnValue { status: STATUS_SUCCESS, data: [0x22; 32].to_vec() }); @@ -924,7 +932,7 @@ mod tests { CODE_CALLER, vec![], MockExt::default(), - &mut GasMeter::with_limit(50_000, 1), + &mut GasMeter::new(GAS_LIMIT), ).unwrap(); } @@ -986,7 +994,7 @@ mod tests { CODE_ADDRESS, vec![], MockExt::default(), - &mut GasMeter::with_limit(50_000, 1), + &mut GasMeter::new(GAS_LIMIT), ).unwrap(); } @@ -1041,7 +1049,7 @@ mod tests { #[test] fn balance() { - let mut gas_meter = GasMeter::with_limit(50_000, 1); + let mut gas_meter = GasMeter::new(GAS_LIMIT); let _ = execute( CODE_BALANCE, vec![], @@ -1101,7 +1109,7 @@ mod tests { #[test] fn gas_price() { - let mut gas_meter = GasMeter::with_limit(50_000, 1312); + let mut gas_meter = GasMeter::new(GAS_LIMIT); let _ = execute( CODE_GAS_PRICE, vec![], @@ -1159,7 +1167,7 @@ mod tests { #[test] fn gas_left() { - let mut gas_meter = GasMeter::with_limit(50_000, 1312); + let mut gas_meter = GasMeter::new(GAS_LIMIT); let output = execute( CODE_GAS_LEFT, @@ -1169,7 +1177,7 @@ mod tests { ).unwrap(); let gas_left = Gas::decode(&mut output.data.as_slice()).unwrap(); - assert!(gas_left < 50_000, "gas_left must be less than initial"); + assert!(gas_left < GAS_LIMIT, "gas_left must be less than initial"); assert!(gas_left > gas_meter.gas_left(), "gas_left must be greater than final"); } @@ -1224,7 +1232,7 @@ mod tests { #[test] fn value_transferred() { - let mut gas_meter = GasMeter::with_limit(50_000, 1); + let mut gas_meter = GasMeter::new(GAS_LIMIT); let _ = execute( CODE_VALUE_TRANSFERRED, vec![], @@ -1260,7 +1268,7 @@ mod tests { CODE_DISPATCH_CALL, vec![], &mut mock_ext, - &mut GasMeter::with_limit(50_000, 1), + &mut GasMeter::new(GAS_LIMIT), ).unwrap(); assert_eq!( @@ -1300,7 +1308,7 @@ mod tests { CODE_RETURN_FROM_START_FN, vec![], MockExt::default(), - &mut GasMeter::with_limit(50_000, 1), + &mut GasMeter::new(GAS_LIMIT), ).unwrap(); assert_eq!(output, ExecReturnValue { status: STATUS_SUCCESS, data: vec![1, 2, 3, 4] }); @@ -1357,7 +1365,7 @@ mod tests { #[test] fn now() { - let mut gas_meter = GasMeter::with_limit(50_000, 1); + let mut gas_meter = GasMeter::new(GAS_LIMIT); let _ = execute( CODE_TIMESTAMP_NOW, vec![], @@ -1416,7 +1424,7 @@ mod tests { #[test] fn minimum_balance() { - let mut gas_meter = GasMeter::with_limit(50_000, 1); + let mut gas_meter = GasMeter::new(GAS_LIMIT); let _ = execute( CODE_MINIMUM_BALANCE, vec![], @@ -1475,7 +1483,7 @@ mod tests { #[test] fn tombstone_deposit() { - let mut gas_meter = GasMeter::with_limit(50_000, 1); + let mut gas_meter = GasMeter::new(GAS_LIMIT); let _ = execute( CODE_TOMBSTONE_DEPOSIT, vec![], @@ -1543,7 +1551,7 @@ mod tests { #[test] fn random() { - let mut gas_meter = GasMeter::with_limit(50_000, 1); + let mut gas_meter = GasMeter::new(GAS_LIMIT); let output = execute( CODE_RANDOM, @@ -1588,7 +1596,7 @@ mod tests { #[test] fn deposit_event() { let mut mock_ext = MockExt::default(); - let mut gas_meter = GasMeter::with_limit(50_000, 1); + let mut gas_meter = GasMeter::new(GAS_LIMIT); let _ = execute( CODE_DEPOSIT_EVENT, vec![], @@ -1601,7 +1609,7 @@ mod tests { vec![0x00, 0x01, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe5, 0x14, 0x00]) ]); - assert_eq!(gas_meter.gas_left(), 49934); + assert_eq!(gas_meter.gas_left(), 9967000000); } const CODE_DEPOSIT_EVENT_MAX_TOPICS: &str = r#" @@ -1634,7 +1642,7 @@ mod tests { #[test] fn deposit_event_max_topics() { // Checks that the runtime traps if there are more than `max_topic_events` topics. - let mut gas_meter = GasMeter::with_limit(50_000, 1); + let mut gas_meter = GasMeter::new(GAS_LIMIT); assert_matches!( execute( @@ -1678,7 +1686,7 @@ mod tests { #[test] fn deposit_event_duplicates() { // Checks that the runtime traps if there are duplicates. - let mut gas_meter = GasMeter::with_limit(50_000, 1); + let mut gas_meter = GasMeter::new(GAS_LIMIT); assert_matches!( execute( @@ -1749,7 +1757,7 @@ mod tests { CODE_BLOCK_NUMBER, vec![], MockExt::default(), - &mut GasMeter::with_limit(50_000, 1), + &mut GasMeter::new(GAS_LIMIT), ).unwrap(); } @@ -1789,7 +1797,7 @@ mod tests { CODE_SIMPLE_ASSERT, input_data, MockExt::default(), - &mut GasMeter::with_limit(50_000, 1), + &mut GasMeter::new(GAS_LIMIT), ).unwrap(); assert_eq!(output.data.len(), 0); @@ -1805,7 +1813,7 @@ mod tests { CODE_SIMPLE_ASSERT, input_data, MockExt::default(), - &mut GasMeter::with_limit(50_000, 1), + &mut GasMeter::new(GAS_LIMIT), ).err().unwrap(); assert_eq!(error.buffer.capacity(), 1_234); @@ -1859,7 +1867,7 @@ mod tests { CODE_RETURN_WITH_DATA, hex!("00112233445566778899").to_vec(), MockExt::default(), - &mut GasMeter::with_limit(50_000, 1), + &mut GasMeter::new(GAS_LIMIT), ).unwrap(); assert_eq!(output, ExecReturnValue { status: 0, data: hex!("445566778899").to_vec() }); @@ -1872,7 +1880,7 @@ mod tests { CODE_RETURN_WITH_DATA, hex!("112233445566778899").to_vec(), MockExt::default(), - &mut GasMeter::with_limit(50_000, 1), + &mut GasMeter::new(GAS_LIMIT), ).unwrap(); assert_eq!(output, ExecReturnValue { status: 17, data: hex!("5566778899").to_vec() }); @@ -1958,7 +1966,7 @@ mod tests { #[test] fn get_runtime_storage() { - let mut gas_meter = GasMeter::with_limit(50_000, 1); + let mut gas_meter = GasMeter::new(GAS_LIMIT); let mock_ext = MockExt::default(); // "\01\02\03\04" - Some(0x14144020) diff --git a/frame/contracts/src/wasm/runtime.rs b/frame/contracts/src/wasm/runtime.rs index 7cede5542fc6f8bde389f6b99fc960b7b3512e24..f87f5d1ef53cc962ee88cad097b6e524259d6f54 100644 --- a/frame/contracts/src/wasm/runtime.rs +++ b/frame/contracts/src/wasm/runtime.rs @@ -16,11 +16,11 @@ //! Environment definition of the wasm smart-contract runtime. -use crate::{Schedule, Trait, CodeHash, ComputeDispatchFee, BalanceOf}; +use crate::{Schedule, Trait, CodeHash, BalanceOf}; use crate::exec::{ Ext, ExecResult, ExecError, ExecReturnValue, StorageKey, TopicOf, STATUS_SUCCESS, }; -use crate::gas::{Gas, GasMeter, Token, GasMeterResult, approx_gas_for_balance}; +use crate::gas::{Gas, GasMeter, Token, GasMeterResult}; use sp_sandbox; use frame_system; use sp_std::{prelude::*, mem, convert::TryInto}; @@ -32,6 +32,7 @@ use sp_io::hashing::{ blake2_128, sha2_256, }; +use frame_support::weights::GetDispatchInfo; /// The value returned from ext_call and ext_instantiate contract external functions if the call or /// instantiation traps. This value is chosen as if the execution does not trap, the return value @@ -153,8 +154,8 @@ pub enum RuntimeToken { /// The given number of bytes is read from the sandbox memory and /// is returned as the return data buffer of the call. ReturnData(u32), - /// Dispatch fee calculated by `T::ComputeDispatchFee`. - ComputedDispatchFee(Gas), + /// Dispatched a call with the given weight. + DispatchWithWeight(Gas), /// (topic_count, data_bytes): A buffer of the given size is posted as an event indexed with the /// given number of topics. DepositEvent(u32, u32), @@ -195,7 +196,7 @@ impl Token for RuntimeToken { data_and_topics_cost.checked_add(metadata.event_base_cost) ) }, - ComputedDispatchFee(gas) => Some(gas), + DispatchWithWeight(gas) => gas.checked_add(metadata.dispatch_base_cost), }; value.unwrap_or_else(|| Bounded::max_value()) @@ -692,7 +693,7 @@ define_env!(Env, , // The data is encoded as T::Balance. The current contents of the scratch buffer are overwritten. ext_gas_price(ctx) => { ctx.scratch_buf.clear(); - ctx.gas_meter.gas_price().encode_to(&mut ctx.scratch_buf); + ctx.ext.get_weight_price().encode_to(&mut ctx.scratch_buf); Ok(()) }, @@ -783,16 +784,14 @@ define_env!(Env, , let call: <::T as Trait>::Call = read_sandbox_memory_as(ctx, call_ptr, call_len)?; - // Charge gas for dispatching this call. - let fee = { - let balance_fee = <::T as Trait>::ComputeDispatchFee::compute_dispatch_fee(&call); - approx_gas_for_balance(ctx.gas_meter.gas_price(), balance_fee) - }; + // We already deducted the len costs when reading from the sandbox. + // Bill on the actual weight of the dispatched call. + let info = call.get_dispatch_info(); charge_gas( &mut ctx.gas_meter, ctx.schedule, &mut ctx.special_trap, - RuntimeToken::ComputedDispatchFee(fee) + RuntimeToken::DispatchWithWeight(info.weight) )?; ctx.ext.note_dispatch_call(call); diff --git a/frame/contracts/tests/caller_contract.wat b/frame/contracts/tests/caller_contract.wat new file mode 100644 index 0000000000000000000000000000000000000000..4bc122c0b1863002ffc9cb0b4d8042231c2dd576 --- /dev/null +++ b/frame/contracts/tests/caller_contract.wat @@ -0,0 +1,275 @@ +(module + (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) + (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) + (import "env" "ext_balance" (func $ext_balance)) + (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32) (result i32))) + (import "env" "ext_instantiate" (func $ext_instantiate (param i32 i32 i64 i32 i32 i32 i32) (result i32))) + (import "env" "ext_println" (func $ext_println (param i32 i32))) + (import "env" "memory" (memory 1 1)) + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + (func $current_balance (param $sp i32) (result i64) + (call $ext_balance) + (call $assert + (i32.eq (call $ext_scratch_size) (i32.const 8)) + ) + (call $ext_scratch_read + (i32.sub (get_local $sp) (i32.const 8)) + (i32.const 0) + (i32.const 8) + ) + (i64.load (i32.sub (get_local $sp) (i32.const 8))) + ) + + (func (export "deploy")) + + (func (export "call") + (local $sp i32) + (local $exit_code i32) + (local $balance i64) + + ;; Input data is the code hash of the contract to be deployed. + (call $assert + (i32.eq + (call $ext_scratch_size) + (i32.const 32) + ) + ) + + ;; Copy code hash from scratch buffer into this contract's memory. + (call $ext_scratch_read + (i32.const 24) ;; The pointer where to store the scratch buffer contents, + (i32.const 0) ;; Offset from the start of the scratch buffer. + (i32.const 32) ;; Count of bytes to copy. + ) + + ;; Read current balance into local variable. + (set_local $sp (i32.const 1024)) + (set_local $balance + (call $current_balance (get_local $sp)) + ) + + ;; Fail to deploy the contract since it returns a non-zero exit status. + (set_local $exit_code + (call $ext_instantiate + (i32.const 24) ;; Pointer to the code hash. + (i32.const 32) ;; Length of the code hash. + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 0) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer. + (i32.const 9) ;; Pointer to input data buffer address + (i32.const 7) ;; Length of input data buffer + ) + ) + + ;; Check non-zero exit status. + (call $assert + (i32.eq (get_local $exit_code) (i32.const 0x11)) + ) + + ;; Check that scratch buffer is empty since contract instantiation failed. + (call $assert + (i32.eq (call $ext_scratch_size) (i32.const 0)) + ) + + ;; Check that balance has not changed. + (call $assert + (i64.eq (get_local $balance) (call $current_balance (get_local $sp))) + ) + + ;; Fail to deploy the contract due to insufficient gas. + (set_local $exit_code + (call $ext_instantiate + (i32.const 24) ;; Pointer to the code hash. + (i32.const 32) ;; Length of the code hash. + (i64.const 200) ;; How much gas to devote for the execution. + (i32.const 0) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer. + (i32.const 8) ;; Pointer to input data buffer address + (i32.const 8) ;; Length of input data buffer + ) + ) + + ;; Check for special trap exit status. + (call $assert + (i32.eq (get_local $exit_code) (i32.const 0x0100)) + ) + + ;; Check that scratch buffer is empty since contract instantiation failed. + (call $assert + (i32.eq (call $ext_scratch_size) (i32.const 0)) + ) + + ;; Check that balance has not changed. + (call $assert + (i64.eq (get_local $balance) (call $current_balance (get_local $sp))) + ) + + ;; Deploy the contract successfully. + (set_local $exit_code + (call $ext_instantiate + (i32.const 24) ;; Pointer to the code hash. + (i32.const 32) ;; Length of the code hash. + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 0) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer. + (i32.const 8) ;; Pointer to input data buffer address + (i32.const 8) ;; Length of input data buffer + ) + ) + + ;; Check for success exit status. + (call $assert + (i32.eq (get_local $exit_code) (i32.const 0x00)) + ) + + ;; Check that scratch buffer contains the address of the new contract. + (call $assert + (i32.eq (call $ext_scratch_size) (i32.const 8)) + ) + + ;; Copy contract address from scratch buffer into this contract's memory. + (call $ext_scratch_read + (i32.const 16) ;; The pointer where to store the scratch buffer contents, + (i32.const 0) ;; Offset from the start of the scratch buffer. + (i32.const 8) ;; Count of bytes to copy. + ) + + ;; Check that balance has been deducted. + (set_local $balance + (i64.sub (get_local $balance) (i64.load (i32.const 0))) + ) + (call $assert + (i64.eq (get_local $balance) (call $current_balance (get_local $sp))) + ) + + ;; Call the new contract and expect it to return failing exit code. + (set_local $exit_code + (call $ext_call + (i32.const 16) ;; Pointer to "callee" address. + (i32.const 8) ;; Length of "callee" address. + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 0) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer. + (i32.const 9) ;; Pointer to input data buffer address + (i32.const 7) ;; Length of input data buffer + ) + ) + + ;; Check non-zero exit status. + (call $assert + (i32.eq (get_local $exit_code) (i32.const 0x11)) + ) + + ;; Check that scratch buffer contains the expected return data. + (call $assert + (i32.eq (call $ext_scratch_size) (i32.const 3)) + ) + (i32.store + (i32.sub (get_local $sp) (i32.const 4)) + (i32.const 0) + ) + (call $ext_scratch_read + (i32.sub (get_local $sp) (i32.const 4)) + (i32.const 0) + (i32.const 3) + ) + (call $assert + (i32.eq + (i32.load (i32.sub (get_local $sp) (i32.const 4))) + (i32.const 0x00776655) + ) + ) + + ;; Check that balance has not changed. + (call $assert + (i64.eq (get_local $balance) (call $current_balance (get_local $sp))) + ) + + ;; Fail to call the contract due to insufficient gas. + (set_local $exit_code + (call $ext_call + (i32.const 16) ;; Pointer to "callee" address. + (i32.const 8) ;; Length of "callee" address. + (i64.const 100) ;; How much gas to devote for the execution. + (i32.const 0) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer. + (i32.const 8) ;; Pointer to input data buffer address + (i32.const 8) ;; Length of input data buffer + ) + ) + + ;; Check for special trap exit status. + (call $assert + (i32.eq (get_local $exit_code) (i32.const 0x0100)) + ) + + ;; Check that scratch buffer is empty since call trapped. + (call $assert + (i32.eq (call $ext_scratch_size) (i32.const 0)) + ) + + ;; Check that balance has not changed. + (call $assert + (i64.eq (get_local $balance) (call $current_balance (get_local $sp))) + ) + + ;; Call the contract successfully. + (set_local $exit_code + (call $ext_call + (i32.const 16) ;; Pointer to "callee" address. + (i32.const 8) ;; Length of "callee" address. + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 0) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer. + (i32.const 8) ;; Pointer to input data buffer address + (i32.const 8) ;; Length of input data buffer + ) + ) + + ;; Check for success exit status. + (call $assert + (i32.eq (get_local $exit_code) (i32.const 0x00)) + ) + + ;; Check that scratch buffer contains the expected return data. + (call $assert + (i32.eq (call $ext_scratch_size) (i32.const 4)) + ) + (i32.store + (i32.sub (get_local $sp) (i32.const 4)) + (i32.const 0) + ) + (call $ext_scratch_read + (i32.sub (get_local $sp) (i32.const 4)) + (i32.const 0) + (i32.const 4) + ) + (call $assert + (i32.eq + (i32.load (i32.sub (get_local $sp) (i32.const 4))) + (i32.const 0x77665544) + ) + ) + + ;; Check that balance has been deducted. + (set_local $balance + (i64.sub (get_local $balance) (i64.load (i32.const 0))) + ) + (call $assert + (i64.eq (get_local $balance) (call $current_balance (get_local $sp))) + ) + ) + + (data (i32.const 0) "\00\80") ;; The value to transfer on instantiation and calls. + ;; Chosen to be greater than existential deposit. + (data (i32.const 8) "\00\11\22\33\44\55\66\77") ;; The input data to instantiations and calls. +) diff --git a/frame/contracts/tests/check_default_rent_allowance.wat b/frame/contracts/tests/check_default_rent_allowance.wat new file mode 100644 index 0000000000000000000000000000000000000000..12b3004adf7dea2c7db532372b0a7f9c867c6d3f --- /dev/null +++ b/frame/contracts/tests/check_default_rent_allowance.wat @@ -0,0 +1,47 @@ +(module + (import "env" "ext_rent_allowance" (func $ext_rent_allowance)) + (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) + (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) + (import "env" "memory" (memory 1 1)) + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + (func (export "call")) + + (func (export "deploy") + ;; fill the scratch buffer with the rent allowance. + (call $ext_rent_allowance) + + ;; assert $ext_scratch_size == 8 + (call $assert + (i32.eq + (call $ext_scratch_size) + (i32.const 8) + ) + ) + + ;; copy contents of the scratch buffer into the contract's memory. + (call $ext_scratch_read + (i32.const 8) ;; Pointer in memory to the place where to copy. + (i32.const 0) ;; Offset from the start of the scratch buffer. + (i32.const 8) ;; Count of bytes to copy. + ) + + ;; assert that contents of the buffer is equal to >::max_value(). + (call $assert + (i64.eq + (i64.load + (i32.const 8) + ) + (i64.const 0xFFFFFFFFFFFFFFFF) + ) + ) + ) +) diff --git a/frame/contracts/tests/crypto_hashes.wat b/frame/contracts/tests/crypto_hashes.wat new file mode 100644 index 0000000000000000000000000000000000000000..6dbca33928cb791bb32c2c6c857d687e74271b1e --- /dev/null +++ b/frame/contracts/tests/crypto_hashes.wat @@ -0,0 +1,80 @@ +(module + (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) + (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) + (import "env" "ext_scratch_write" (func $ext_scratch_write (param i32 i32))) + + (import "env" "ext_hash_sha2_256" (func $ext_hash_sha2_256 (param i32 i32 i32))) + (import "env" "ext_hash_keccak_256" (func $ext_hash_keccak_256 (param i32 i32 i32))) + (import "env" "ext_hash_blake2_256" (func $ext_hash_blake2_256 (param i32 i32 i32))) + (import "env" "ext_hash_blake2_128" (func $ext_hash_blake2_128 (param i32 i32 i32))) + + (import "env" "memory" (memory 1 1)) + + (type $hash_fn_sig (func (param i32 i32 i32))) + (table 8 funcref) + (elem (i32.const 1) + $ext_hash_sha2_256 + $ext_hash_keccak_256 + $ext_hash_blake2_256 + $ext_hash_blake2_128 + ) + (data (i32.const 1) "20202010201008") ;; Output sizes of the hashes in order in hex. + + ;; Not in use by the tests besides instantiating the contract. + (func (export "deploy")) + + ;; Called by the tests. + ;; + ;; The `call` function expects data in a certain format in the scratch + ;; buffer. + ;; + ;; 1. The first byte encodes an identifier for the crypto hash function + ;; under test. (*) + ;; 2. The rest encodes the input data that is directly fed into the + ;; crypto hash function chosen in 1. + ;; + ;; The `deploy` function then computes the chosen crypto hash function + ;; given the input and puts the result back into the scratch buffer. + ;; After contract execution the test driver then asserts that the returned + ;; values are equal to the expected bytes for the input and chosen hash + ;; function. + ;; + ;; (*) The possible value for the crypto hash identifiers can be found below: + ;; + ;; | value | Algorithm | Bit Width | + ;; |-------|-----------|-----------| + ;; | 0 | SHA2 | 256 | + ;; | 1 | KECCAK | 256 | + ;; | 2 | BLAKE2 | 256 | + ;; | 3 | BLAKE2 | 128 | + ;; --------------------------------- + (func (export "call") (result i32) + (local $chosen_hash_fn i32) + (local $input_ptr i32) + (local $input_len i32) + (local $output_ptr i32) + (local $output_len i32) + (local.set $input_ptr (i32.const 10)) + (call $ext_scratch_read (local.get $input_ptr) (i32.const 0) (call $ext_scratch_size)) + (local.set $chosen_hash_fn (i32.load8_u (local.get $input_ptr))) + (if (i32.gt_u (local.get $chosen_hash_fn) (i32.const 7)) + ;; We check that the chosen hash fn identifier is within bounds: [0,7] + (unreachable) + ) + (local.set $input_ptr (i32.add (local.get $input_ptr) (i32.const 1))) + (local.set $input_len (i32.sub (call $ext_scratch_size) (i32.const 1))) + (local.set $output_ptr (i32.const 100)) + (local.set $output_len (i32.load8_u (local.get $chosen_hash_fn))) + (call_indirect (type $hash_fn_sig) + (local.get $input_ptr) + (local.get $input_len) + (local.get $output_ptr) + (local.get $chosen_hash_fn) ;; Which crypto hash function to execute. + ) + (call $ext_scratch_write + (local.get $output_ptr) ;; Linear memory location of the output buffer. + (local.get $output_len) ;; Number of output buffer bytes. + ) + (i32.const 0) + ) +) diff --git a/frame/contracts/tests/destroy_and_transfer.wat b/frame/contracts/tests/destroy_and_transfer.wat new file mode 100644 index 0000000000000000000000000000000000000000..c8cf7271d74193bccf3d8720c34c12b43471bd45 --- /dev/null +++ b/frame/contracts/tests/destroy_and_transfer.wat @@ -0,0 +1,148 @@ +(module + (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) + (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) + (import "env" "ext_get_storage" (func $ext_get_storage (param i32) (result i32))) + (import "env" "ext_set_storage" (func $ext_set_storage (param i32 i32 i32))) + (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32) (result i32))) + (import "env" "ext_instantiate" (func $ext_instantiate (param i32 i32 i64 i32 i32 i32 i32) (result i32))) + (import "env" "memory" (memory 1 1)) + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + (func (export "deploy") + ;; Input data is the code hash of the contract to be deployed. + (call $assert + (i32.eq + (call $ext_scratch_size) + (i32.const 32) + ) + ) + + ;; Copy code hash from scratch buffer into this contract's memory. + (call $ext_scratch_read + (i32.const 48) ;; The pointer where to store the scratch buffer contents, + (i32.const 0) ;; Offset from the start of the scratch buffer. + (i32.const 32) ;; Count of bytes to copy. + ) + + ;; Deploy the contract with the provided code hash. + (call $assert + (i32.eq + (call $ext_instantiate + (i32.const 48) ;; Pointer to the code hash. + (i32.const 32) ;; Length of the code hash. + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 0) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer. + (i32.const 0) ;; Pointer to input data buffer address + (i32.const 0) ;; Length of input data buffer + ) + (i32.const 0) + ) + ) + + ;; Read the address of the instantiated contract into memory. + (call $assert + (i32.eq + (call $ext_scratch_size) + (i32.const 8) + ) + ) + (call $ext_scratch_read + (i32.const 80) ;; The pointer where to store the scratch buffer contents, + (i32.const 0) ;; Offset from the start of the scratch buffer. + (i32.const 8) ;; Count of bytes to copy. + ) + + ;; Store the return address. + (call $ext_set_storage + (i32.const 16) ;; Pointer to the key + (i32.const 80) ;; Pointer to the value + (i32.const 8) ;; Length of the value + ) + ) + + (func (export "call") + ;; Read address of destination contract from storage. + (call $assert + (i32.eq + (call $ext_get_storage + (i32.const 16) ;; Pointer to the key + ) + (i32.const 0) + ) + ) + (call $assert + (i32.eq + (call $ext_scratch_size) + (i32.const 8) + ) + ) + (call $ext_scratch_read + (i32.const 80) ;; The pointer where to store the contract address. + (i32.const 0) ;; Offset from the start of the scratch buffer. + (i32.const 8) ;; Count of bytes to copy. + ) + + ;; Calling the destination contract with non-empty input data should fail. + (call $assert + (i32.eq + (call $ext_call + (i32.const 80) ;; Pointer to destination address + (i32.const 8) ;; Length of destination address + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 0) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer + (i32.const 0) ;; Pointer to input data buffer address + (i32.const 1) ;; Length of input data buffer + ) + (i32.const 0x0100) + ) + ) + + ;; Call the destination contract regularly, forcing it to self-destruct. + (call $assert + (i32.eq + (call $ext_call + (i32.const 80) ;; Pointer to destination address + (i32.const 8) ;; Length of destination address + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 8) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer + (i32.const 0) ;; Pointer to input data buffer address + (i32.const 0) ;; Length of input data buffer + ) + (i32.const 0) + ) + ) + + ;; Calling the destination address with non-empty input data should now work since the + ;; contract has been removed. Also transfer a balance to the address so we can ensure this + ;; does not keep the contract alive. + (call $assert + (i32.eq + (call $ext_call + (i32.const 80) ;; Pointer to destination address + (i32.const 8) ;; Length of destination address + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 0) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer + (i32.const 0) ;; Pointer to input data buffer address + (i32.const 1) ;; Length of input data buffer + ) + (i32.const 0) + ) + ) + ) + + (data (i32.const 0) "\00\00\01") ;; Endowment to send when creating contract. + (data (i32.const 8) "") ;; Value to send when calling contract. + (data (i32.const 16) "") ;; The key to store the contract address under. +) diff --git a/frame/contracts/tests/dispatch_call.wat b/frame/contracts/tests/dispatch_call.wat new file mode 100644 index 0000000000000000000000000000000000000000..db0995bd6c79ad37a8b71b4db9d88a8682b308e8 --- /dev/null +++ b/frame/contracts/tests/dispatch_call.wat @@ -0,0 +1,14 @@ +(module + (import "env" "ext_dispatch_call" (func $ext_dispatch_call (param i32 i32))) + (import "env" "memory" (memory 1 1)) + + (func (export "call") + (call $ext_dispatch_call + (i32.const 8) ;; Pointer to the start of encoded call buffer + (i32.const 11) ;; Length of the buffer + ) + ) + (func (export "deploy")) + + (data (i32.const 8) "\00\00\03\00\00\00\00\00\00\00\C8") +) diff --git a/frame/contracts/tests/dispatch_call_then_trap.wat b/frame/contracts/tests/dispatch_call_then_trap.wat new file mode 100644 index 0000000000000000000000000000000000000000..ce949d68236f39846f82434ac45c9b825cedd698 --- /dev/null +++ b/frame/contracts/tests/dispatch_call_then_trap.wat @@ -0,0 +1,15 @@ +(module + (import "env" "ext_dispatch_call" (func $ext_dispatch_call (param i32 i32))) + (import "env" "memory" (memory 1 1)) + + (func (export "call") + (call $ext_dispatch_call + (i32.const 8) ;; Pointer to the start of encoded call buffer + (i32.const 11) ;; Length of the buffer + ) + (unreachable) ;; trap so that the top level transaction fails + ) + (func (export "deploy")) + + (data (i32.const 8) "\00\00\03\00\00\00\00\00\00\00\C8") +) diff --git a/frame/contracts/tests/drain.wat b/frame/contracts/tests/drain.wat new file mode 100644 index 0000000000000000000000000000000000000000..d08e1dd0d2981eb926478c081e8125a1ab3f6cbc --- /dev/null +++ b/frame/contracts/tests/drain.wat @@ -0,0 +1,54 @@ +(module + (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) + (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) + (import "env" "ext_balance" (func $ext_balance)) + (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32) (result i32))) + (import "env" "memory" (memory 1 1)) + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + (func (export "deploy")) + + (func (export "call") + ;; Send entire remaining balance to the 0 address. + (call $ext_balance) + + ;; Balance should be encoded as a u64. + (call $assert + (i32.eq + (call $ext_scratch_size) + (i32.const 8) + ) + ) + + ;; Read balance into memory. + (call $ext_scratch_read + (i32.const 8) ;; Pointer to write balance to + (i32.const 0) ;; Offset into scratch buffer + (i32.const 8) ;; Length of encoded balance + ) + + ;; Self-destruct by sending full balance to the 0 address. + (call $assert + (i32.eq + (call $ext_call + (i32.const 0) ;; Pointer to destination address + (i32.const 8) ;; Length of destination address + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 8) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer + (i32.const 0) ;; Pointer to input data buffer address + (i32.const 0) ;; Length of input data buffer + ) + (i32.const 0) + ) + ) + ) +) diff --git a/frame/contracts/tests/get_runtime_storage.wat b/frame/contracts/tests/get_runtime_storage.wat new file mode 100644 index 0000000000000000000000000000000000000000..6148f1c408c017c5096f055759d1ed66ba9b7104 --- /dev/null +++ b/frame/contracts/tests/get_runtime_storage.wat @@ -0,0 +1,74 @@ +(module + (import "env" "ext_get_runtime_storage" + (func $ext_get_runtime_storage (param i32 i32) (result i32)) + ) + (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) + (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) + (import "env" "ext_scratch_write" (func $ext_scratch_write (param i32 i32))) + (import "env" "memory" (memory 1 1)) + + (func (export "deploy")) + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + (func $call (export "call") + ;; Load runtime storage for the first key and assert that it exists. + (call $assert + (i32.eq + (call $ext_get_runtime_storage + (i32.const 16) + (i32.const 4) + ) + (i32.const 0) + ) + ) + + ;; assert $ext_scratch_size == 4 + (call $assert + (i32.eq + (call $ext_scratch_size) + (i32.const 4) + ) + ) + + ;; copy contents of the scratch buffer into the contract's memory. + (call $ext_scratch_read + (i32.const 4) ;; Pointer in memory to the place where to copy. + (i32.const 0) ;; Offset from the start of the scratch buffer. + (i32.const 4) ;; Count of bytes to copy. + ) + + ;; assert that contents of the buffer is equal to the i32 value of 0x14144020. + (call $assert + (i32.eq + (i32.load + (i32.const 4) + ) + (i32.const 0x14144020) + ) + ) + + ;; Load the second key and assert that it doesn't exist. + (call $assert + (i32.eq + (call $ext_get_runtime_storage + (i32.const 20) + (i32.const 4) + ) + (i32.const 1) + ) + ) + ) + + ;; The first key, 4 bytes long. + (data (i32.const 16) "\01\02\03\04") + ;; The second key, 4 bytes long. + (data (i32.const 20) "\02\03\04\05") +) diff --git a/frame/contracts/tests/restoration.wat b/frame/contracts/tests/restoration.wat new file mode 100644 index 0000000000000000000000000000000000000000..4e11f97d5a2ccd722a44211082cbd523a9bc4737 --- /dev/null +++ b/frame/contracts/tests/restoration.wat @@ -0,0 +1,56 @@ +(module + (import "env" "ext_set_storage" (func $ext_set_storage (param i32 i32 i32))) + (import "env" "ext_restore_to" (func $ext_restore_to (param i32 i32 i32 i32 i32 i32 i32 i32))) + (import "env" "memory" (memory 1 1)) + + (func (export "call") + (call $ext_restore_to + ;; Pointer and length of the encoded dest buffer. + (i32.const 256) + (i32.const 8) + ;; Pointer and length of the encoded code hash buffer + (i32.const 264) + (i32.const 32) + ;; Pointer and length of the encoded rent_allowance buffer + (i32.const 296) + (i32.const 8) + ;; Pointer and number of items in the delta buffer. + ;; This buffer specifies multiple keys for removal before restoration. + (i32.const 100) + (i32.const 1) + ) + ) + (func (export "deploy") + ;; Data to restore + (call $ext_set_storage + (i32.const 0) + (i32.const 0) + (i32.const 4) + ) + + ;; ACL + (call $ext_set_storage + (i32.const 100) + (i32.const 0) + (i32.const 4) + ) + ) + + ;; Data to restore + (data (i32.const 0) "\28") + + ;; Buffer that has ACL storage keys. + (data (i32.const 100) "\01") + + ;; Address of bob + (data (i32.const 256) "\02\00\00\00\00\00\00\00") + + ;; Code hash of SET_RENT + (data (i32.const 264) + "\c2\1c\41\10\a5\22\d8\59\1c\4c\77\35\dd\2d\bf\a1" + "\13\0b\50\93\76\9b\92\31\97\b7\c5\74\26\aa\38\2a" + ) + + ;; Rent allowance + (data (i32.const 296) "\32\00\00\00\00\00\00\00") +) diff --git a/frame/contracts/tests/return_from_start_fn.wat b/frame/contracts/tests/return_from_start_fn.wat new file mode 100644 index 0000000000000000000000000000000000000000..ac898d4d944e9f3b08e9d2a079a8ecbdc9c77658 --- /dev/null +++ b/frame/contracts/tests/return_from_start_fn.wat @@ -0,0 +1,27 @@ +(module + (import "env" "ext_return" (func $ext_return (param i32 i32))) + (import "env" "ext_deposit_event" (func $ext_deposit_event (param i32 i32 i32 i32))) + (import "env" "memory" (memory 1 1)) + + (start $start) + (func $start + (call $ext_deposit_event + (i32.const 0) ;; The topics buffer + (i32.const 0) ;; The topics buffer's length + (i32.const 8) ;; The data buffer + (i32.const 4) ;; The data buffer's length + ) + (call $ext_return + (i32.const 8) + (i32.const 4) + ) + (unreachable) + ) + + (func (export "call") + (unreachable) + ) + (func (export "deploy")) + + (data (i32.const 8) "\01\02\03\04") +) diff --git a/frame/contracts/tests/return_with_data.wat b/frame/contracts/tests/return_with_data.wat new file mode 100644 index 0000000000000000000000000000000000000000..8cc84006a0b00eff34536b5f861461cb045d570e --- /dev/null +++ b/frame/contracts/tests/return_with_data.wat @@ -0,0 +1,39 @@ +(module + (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) + (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) + (import "env" "ext_scratch_write" (func $ext_scratch_write (param i32 i32))) + (import "env" "memory" (memory 1 1)) + + ;; Deploy routine is the same as call. + (func (export "deploy") (result i32) + (call $call) + ) + + ;; Call reads the first 4 bytes (LE) as the exit status and returns the rest as output data. + (func $call (export "call") (result i32) + (local $buf_size i32) + (local $exit_status i32) + + ;; Find out the size of the scratch buffer + (set_local $buf_size (call $ext_scratch_size)) + + ;; Copy scratch buffer into this contract memory. + (call $ext_scratch_read + (i32.const 0) ;; The pointer where to store the scratch buffer contents, + (i32.const 0) ;; Offset from the start of the scratch buffer. + (get_local $buf_size) ;; Count of bytes to copy. + ) + + ;; Copy all but the first 4 bytes of the input data as the output data. + (call $ext_scratch_write + (i32.const 4) ;; Pointer to the data to return. + (i32.sub ;; Count of bytes to copy. + (get_local $buf_size) + (i32.const 4) + ) + ) + + ;; Return the first 4 bytes of the input data as the exit status. + (i32.load (i32.const 0)) + ) +) diff --git a/frame/contracts/tests/run_out_of_gas.wat b/frame/contracts/tests/run_out_of_gas.wat new file mode 100644 index 0000000000000000000000000000000000000000..52ee92539fd521d656235cc3f3feb555bcd12cb9 --- /dev/null +++ b/frame/contracts/tests/run_out_of_gas.wat @@ -0,0 +1,7 @@ +(module + (func (export "call") + (loop $inf (br $inf)) ;; just run out of gas + (unreachable) + ) + (func (export "deploy")) +) diff --git a/frame/contracts/tests/self_destruct.wat b/frame/contracts/tests/self_destruct.wat new file mode 100644 index 0000000000000000000000000000000000000000..464b5c663ea4a96f544a8f7c6534006c474c5aa1 --- /dev/null +++ b/frame/contracts/tests/self_destruct.wat @@ -0,0 +1,72 @@ +(module + (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) + (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) + (import "env" "ext_address" (func $ext_address)) + (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32) (result i32))) + (import "env" "ext_terminate" (func $ext_terminate (param i32 i32))) + (import "env" "memory" (memory 1 1)) + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + (func (export "deploy")) + + (func (export "call") + ;; If the input data is not empty, then recursively call self with empty input data. + ;; This should trap instead of self-destructing since a contract cannot be removed live in + ;; the execution stack cannot be removed. If the recursive call traps, then trap here as + ;; well. + (if (call $ext_scratch_size) + (then + (call $ext_address) + + ;; Expect address to be 8 bytes. + (call $assert + (i32.eq + (call $ext_scratch_size) + (i32.const 8) + ) + ) + + ;; Read own address into memory. + (call $ext_scratch_read + (i32.const 16) ;; Pointer to write address to + (i32.const 0) ;; Offset into scratch buffer + (i32.const 8) ;; Length of encoded address + ) + + ;; Recursively call self with empty input data. + (call $assert + (i32.eq + (call $ext_call + (i32.const 16) ;; Pointer to own address + (i32.const 8) ;; Length of own address + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 8) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer + (i32.const 0) ;; Pointer to input data buffer address + (i32.const 0) ;; Length of input data buffer + ) + (i32.const 0) + ) + ) + ) + (else + ;; Try to terminate and give balance to django. + (call $ext_terminate + (i32.const 32) ;; Pointer to beneficiary address + (i32.const 8) ;; Length of beneficiary address + ) + (unreachable) ;; ext_terminate never returns + ) + ) + ) + ;; Address of django + (data (i32.const 32) "\04\00\00\00\00\00\00\00") +) diff --git a/frame/contracts/tests/self_destructing_constructor.wat b/frame/contracts/tests/self_destructing_constructor.wat new file mode 100644 index 0000000000000000000000000000000000000000..b19d6e5b50daca2b37f20c7866134a02097c6285 --- /dev/null +++ b/frame/contracts/tests/self_destructing_constructor.wat @@ -0,0 +1,54 @@ +(module + (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) + (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) + (import "env" "ext_balance" (func $ext_balance)) + (import "env" "ext_call" (func $ext_call (param i32 i32 i64 i32 i32 i32 i32) (result i32))) + (import "env" "memory" (memory 1 1)) + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + (func (export "deploy") + ;; Send entire remaining balance to the 0 address. + (call $ext_balance) + + ;; Balance should be encoded as a u64. + (call $assert + (i32.eq + (call $ext_scratch_size) + (i32.const 8) + ) + ) + + ;; Read balance into memory. + (call $ext_scratch_read + (i32.const 8) ;; Pointer to write balance to + (i32.const 0) ;; Offset into scratch buffer + (i32.const 8) ;; Length of encoded balance + ) + + ;; Self-destruct by sending full balance to the 0 address. + (call $assert + (i32.eq + (call $ext_call + (i32.const 0) ;; Pointer to destination address + (i32.const 8) ;; Length of destination address + (i64.const 0) ;; How much gas to devote for the execution. 0 = all. + (i32.const 8) ;; Pointer to the buffer with value to transfer + (i32.const 8) ;; Length of the buffer with value to transfer + (i32.const 0) ;; Pointer to input data buffer address + (i32.const 0) ;; Length of input data buffer + ) + (i32.const 0) + ) + ) + ) + + (func (export "call")) +) diff --git a/frame/contracts/tests/set_rent.wat b/frame/contracts/tests/set_rent.wat new file mode 100644 index 0000000000000000000000000000000000000000..d1affa0d7415f4257597cfac518b5cc7a72631e6 --- /dev/null +++ b/frame/contracts/tests/set_rent.wat @@ -0,0 +1,101 @@ +(module + (import "env" "ext_dispatch_call" (func $ext_dispatch_call (param i32 i32))) + (import "env" "ext_set_storage" (func $ext_set_storage (param i32 i32 i32))) + (import "env" "ext_clear_storage" (func $ext_clear_storage (param i32))) + (import "env" "ext_set_rent_allowance" (func $ext_set_rent_allowance (param i32 i32))) + (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) + (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) + (import "env" "memory" (memory 1 1)) + + ;; insert a value of 4 bytes into storage + (func $call_0 + (call $ext_set_storage + (i32.const 1) + (i32.const 0) + (i32.const 4) + ) + ) + + ;; remove the value inserted by call_1 + (func $call_1 + (call $ext_clear_storage + (i32.const 1) + ) + ) + + ;; transfer 50 to ALICE + (func $call_2 + (call $ext_dispatch_call + (i32.const 68) + (i32.const 11) + ) + ) + + ;; do nothing + (func $call_else) + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + ;; Dispatch the call according to input size + (func (export "call") + (local $input_size i32) + (set_local $input_size + (call $ext_scratch_size) + ) + (block $IF_ELSE + (block $IF_2 + (block $IF_1 + (block $IF_0 + (br_table $IF_0 $IF_1 $IF_2 $IF_ELSE + (get_local $input_size) + ) + (unreachable) + ) + (call $call_0) + return + ) + (call $call_1) + return + ) + (call $call_2) + return + ) + (call $call_else) + ) + + ;; Set into storage a 4 bytes value + ;; Set call set_rent_allowance with input + (func (export "deploy") + (local $input_size i32) + (set_local $input_size + (call $ext_scratch_size) + ) + (call $ext_set_storage + (i32.const 0) + (i32.const 0) + (i32.const 4) + ) + (call $ext_scratch_read + (i32.const 0) + (i32.const 0) + (get_local $input_size) + ) + (call $ext_set_rent_allowance + (i32.const 0) + (get_local $input_size) + ) + ) + + ;; Encoding of 10 in balance + (data (i32.const 0) "\28") + + ;; Encoding of call transfer 50 to CHARLIE + (data (i32.const 68) "\00\00\03\00\00\00\00\00\00\00\C8") +) diff --git a/frame/contracts/tests/storage_size.wat b/frame/contracts/tests/storage_size.wat new file mode 100644 index 0000000000000000000000000000000000000000..8de9f42ee97483fed59b8d84724ef431c3777728 --- /dev/null +++ b/frame/contracts/tests/storage_size.wat @@ -0,0 +1,60 @@ +(module + (import "env" "ext_get_storage" (func $ext_get_storage (param i32) (result i32))) + (import "env" "ext_set_storage" (func $ext_set_storage (param i32 i32 i32))) + (import "env" "ext_scratch_size" (func $ext_scratch_size (result i32))) + (import "env" "ext_scratch_read" (func $ext_scratch_read (param i32 i32 i32))) + (import "env" "memory" (memory 16 16)) + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + (func (export "call") + ;; assert $ext_scratch_size == 8 + (call $assert + (i32.eq + (call $ext_scratch_size) + (i32.const 4) + ) + ) + + ;; copy contents of the scratch buffer into the contract's memory. + (call $ext_scratch_read + (i32.const 32) ;; Pointer in memory to the place where to copy. + (i32.const 0) ;; Offset from the start of the scratch buffer. + (i32.const 4) ;; Count of bytes to copy. + ) + + ;; place a garbage value in storage, the size of which is specified by the call input. + (call $ext_set_storage + (i32.const 0) ;; Pointer to storage key + (i32.const 0) ;; Pointer to value + (i32.load (i32.const 32)) ;; Size of value + ) + + (call $assert + (i32.eq + (call $ext_get_storage + (i32.const 0) ;; Pointer to storage key + ) + (i32.const 0) + ) + ) + + (call $assert + (i32.eq + (call $ext_scratch_size) + (i32.load (i32.const 32)) + ) + ) + ) + + (func (export "deploy")) + + (data (i32.const 0) "\01") ;; Storage key (32 B) +) diff --git a/frame/democracy/Cargo.toml b/frame/democracy/Cargo.toml index 788645764410b9bf35dcf969e1ee152a20716a64..83caa671cee958d28c68a06b744215b69a351def 100644 --- a/frame/democracy/Cargo.toml +++ b/frame/democracy/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-democracy" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,21 +8,24 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for democracy" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../benchmarking", optional = true } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } -pallet-scheduler = { version = "2.0.0-alpha.5", path = "../scheduler" } -sp-storage = { version = "2.0.0-alpha.5", path = "../../primitives/storage" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } +pallet-scheduler = { version = "2.0.0-dev", path = "../scheduler" } +sp-storage = { version = "2.0.0-dev", path = "../../primitives/storage" } hex-literal = "0.2.1" [features] @@ -42,6 +45,3 @@ runtime-benchmarks = [ "frame-system/runtime-benchmarks", "sp-runtime/runtime-benchmarks", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/democracy/src/benchmarking.rs b/frame/democracy/src/benchmarking.rs index 83f7ca795f3ac4dfc9652072a4829125669c281a..4459a1aaa5410d5845ccf806ef0a5a0573d3fb7b 100644 --- a/frame/democracy/src/benchmarking.rs +++ b/frame/democracy/src/benchmarking.rs @@ -19,8 +19,8 @@ use super::*; use frame_benchmarking::{benchmarks, account}; -use frame_support::traits::{Currency, Get, EnsureOrigin}; -use frame_system::{RawOrigin, Module as System, self}; +use frame_support::traits::{Currency, Get, EnsureOrigin, OnInitialize}; +use frame_system::{RawOrigin, Module as System, self, EventRecord}; use sp_runtime::traits::{Bounded, One}; use crate::Module as Democracy; @@ -33,6 +33,14 @@ const MAX_SECONDERS: u32 = 100; const MAX_VETOERS: u32 = 100; const MAX_BYTES: u32 = 16_384; +fn assert_last_event(generic_event: ::Event) { + let events = System::::events(); + let system_event: ::Event = generic_event.into(); + // compare to the last event record + let EventRecord { event, .. } = &events[events.len() - 1]; + assert_eq!(event, &system_event); +} + fn funded_account(name: &'static str, index: u32) -> T::AccountId { let caller: T::AccountId = account(name, index, SEED); T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); @@ -54,7 +62,7 @@ fn add_referendum(n: u32) -> Result { let vote_threshold = VoteThreshold::SimpleMajority; Democracy::::inject_referendum( - 0.into(), + T::LaunchPeriod::get(), proposal_hash, vote_threshold, 0.into(), @@ -70,7 +78,7 @@ fn add_referendum(n: u32) -> Result { Ok(referendum_index) } -fn account_vote() -> AccountVote> { +fn account_vote(b: BalanceOf) -> AccountVote> { let v = Vote { aye: true, conviction: Conviction::Locked1x, @@ -78,18 +86,18 @@ fn account_vote() -> AccountVote> { AccountVote::Standard { vote: v, - balance: BalanceOf::::one(), + balance: b, } } -fn open_activate_proxy(u: u32) -> Result { +fn open_activate_proxy(u: u32) -> Result<(T::AccountId, T::AccountId), &'static str> { let caller = funded_account::("caller", u); - let proxy = funded_account::("proxy", u); + let voter = funded_account::("voter", u); - Democracy::::open_proxy(RawOrigin::Signed(proxy.clone()).into(), caller.clone())?; - Democracy::::activate_proxy(RawOrigin::Signed(caller).into(), proxy.clone())?; + Democracy::::open_proxy(RawOrigin::Signed(caller.clone()).into(), voter.clone())?; + Democracy::::activate_proxy(RawOrigin::Signed(voter.clone()).into(), caller.clone())?; - Ok(proxy) + Ok((caller, voter)) } benchmarks! { @@ -103,10 +111,15 @@ benchmarks! { add_proposal::(i)?; } + assert_eq!(Democracy::::public_props().len(), p as usize, "Proposals not created."); + let caller = funded_account::("caller", 0); let proposal_hash: T::Hash = T::Hashing::hash_of(&p); let value = T::MinimumDeposit::get(); }: _(RawOrigin::Signed(caller), proposal_hash, value.into()) + verify { + assert_eq!(Democracy::::public_props().len(), (p + 1) as usize, "Proposals not created."); + } second { let s in 0 .. MAX_SECONDERS; @@ -120,95 +133,225 @@ benchmarks! { Democracy::::second(RawOrigin::Signed(seconder).into(), 0)?; } + let deposits = Democracy::::deposit_of(0).ok_or("Proposal not created")?; + assert_eq!(deposits.1.len(), (s + 1) as usize, "Seconds not recorded"); }: _(RawOrigin::Signed(caller), 0) + verify { + let deposits = Democracy::::deposit_of(0).ok_or("Proposal not created")?; + assert_eq!(deposits.1.len(), (s + 2) as usize, "`second` benchmark did not work"); + } - vote { + vote_new { let r in 1 .. MAX_REFERENDUMS; let caller = funded_account::("caller", 0); - let account_vote = account_vote::(); + let account_vote = account_vote::(100.into()); + // We need to create existing direct votes for i in 0 .. r { let ref_idx = add_referendum::(i)?; Democracy::::vote(RawOrigin::Signed(caller.clone()).into(), ref_idx, account_vote.clone())?; } + let votes = match VotingOf::::get(&caller) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), r as usize, "Votes were not recorded."); + + let referendum_index = add_referendum::(r)?; + + }: vote(RawOrigin::Signed(caller.clone()), referendum_index, account_vote) + verify { + let votes = match VotingOf::::get(&caller) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), (r + 1) as usize, "Vote was not recorded."); + } - let referendum_index = r - 1; + vote_existing { + let r in 1 .. MAX_REFERENDUMS; - }: _(RawOrigin::Signed(caller), referendum_index, account_vote) + let caller = funded_account::("caller", 0); + let account_vote = account_vote::(100.into()); + + // We need to create existing direct votes + for i in 0 ..=r { + let ref_idx = add_referendum::(i)?; + Democracy::::vote(RawOrigin::Signed(caller.clone()).into(), ref_idx, account_vote.clone())?; + } + let votes = match VotingOf::::get(&caller) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), (r + 1) as usize, "Votes were not recorded."); + + // Change vote from aye to nay + let nay = Vote { aye: false, conviction: Conviction::Locked1x }; + let new_vote = AccountVote::Standard { vote: nay, balance: 1000.into() }; + let referendum_index = Democracy::::referendum_count() - 1; + + // This tests when a user changes a vote + }: vote(RawOrigin::Signed(caller.clone()), referendum_index, new_vote) + verify { + let votes = match VotingOf::::get(&caller) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), (r + 1) as usize, "Vote was incorrectly added"); + let referendum_info = Democracy::::referendum_info(referendum_index) + .ok_or("referendum doesn't exist")?; + let tally = match referendum_info { + ReferendumInfo::Ongoing(r) => r.tally, + _ => return Err("referendum not ongoing"), + }; + assert_eq!(tally.nays, 1000.into(), "changed vote was not recorded"); + } - proxy_vote { + // Basically copy paste of `vote_new` + proxy_vote_new { let r in 1 .. MAX_REFERENDUMS; - let caller = funded_account::("caller", r); - let proxy = open_activate_proxy::(r)?; - let account_vote = account_vote::(); + let (caller, voter) = open_activate_proxy::(0)?; + let account_vote = account_vote::(100.into()); + // Populate existing direct votes for the voter, they can vote on their own behalf for i in 0 .. r { let ref_idx = add_referendum::(i)?; - Democracy::::vote(RawOrigin::Signed(caller.clone()).into(), ref_idx, account_vote.clone())?; + Democracy::::vote(RawOrigin::Signed(voter.clone()).into(), ref_idx, account_vote.clone())?; } - let referendum_index = r - 1; + let referendum_index = add_referendum::(r)?; + + }: proxy_vote(RawOrigin::Signed(caller), referendum_index, account_vote) + verify { + let votes = match VotingOf::::get(&voter) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), (r + 1) as usize, "Vote was not recorded."); + } - }: _(RawOrigin::Signed(proxy), referendum_index, account_vote) + // Basically copy paste of `vote_existing` + proxy_vote_existing { + let r in 1 .. MAX_REFERENDUMS; - emergency_cancel { - let u in 1 .. MAX_USERS; + let (caller, voter) = open_activate_proxy::(0)?; + let account_vote = account_vote::(100.into()); + + // We need to create existing direct votes + for i in 0 ..=r { + let ref_idx = add_referendum::(i)?; + Democracy::::vote(RawOrigin::Signed(voter.clone()).into(), ref_idx, account_vote.clone())?; + } + let votes = match VotingOf::::get(&voter) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), (r + 1) as usize, "Votes were not recorded."); + + // Change vote from aye to nay + let nay = Vote { aye: false, conviction: Conviction::Locked1x }; + let new_vote = AccountVote::Standard { vote: nay, balance: 1000.into() }; + let referendum_index = Democracy::::referendum_count() - 1; + + // This tests when a user changes a vote + }: proxy_vote(RawOrigin::Signed(caller.clone()), referendum_index, new_vote) + verify { + let votes = match VotingOf::::get(&voter) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), (r + 1) as usize, "Vote was incorrectly added"); + let referendum_info = Democracy::::referendum_info(referendum_index) + .ok_or("referendum doesn't exist")?; + let tally = match referendum_info { + ReferendumInfo::Ongoing(r) => r.tally, + _ => return Err("referendum not ongoing"), + }; + assert_eq!(tally.nays, 1000.into(), "changed vote was not recorded"); + } - let referendum_index = add_referendum::(u)?; + emergency_cancel { + let r in 1 .. MAX_REFERENDUMS; let origin = T::CancellationOrigin::successful_origin(); + + // Create and cancel a bunch of referendums + for i in 0 .. r { + let ref_idx = add_referendum::(i)?; + let call = Call::::emergency_cancel(ref_idx); + call.dispatch(origin.clone())?; + } + + // Lets now measure one more + let referendum_index = add_referendum::(r)?; let call = Call::::emergency_cancel(referendum_index); - }: { - let _ = call.dispatch(origin)?; + assert!(Democracy::::referendum_status(referendum_index).is_ok()); + }: { call.dispatch(origin)? } + verify { + // Referendum has been canceled + assert!(Democracy::::referendum_status(referendum_index).is_err()); } + // Worst case scenario, we external propose a previously blacklisted proposal external_propose { - let u in 1 .. MAX_USERS; + let p in 1 .. MAX_PROPOSALS; let origin = T::ExternalOrigin::successful_origin(); - let proposal_hash = T::Hashing::hash_of(&u); + let proposal_hash = T::Hashing::hash_of(&p); + // Add proposal to blacklist with block number 0 + Blacklist::::insert( + proposal_hash, + (T::BlockNumber::zero(), vec![T::AccountId::default()]) + ); + let call = Call::::external_propose(proposal_hash); - }: { - let _ = call.dispatch(origin)?; + }: { call.dispatch(origin)? } + verify { + // External proposal created + ensure!(>::exists(), "External proposal didn't work"); } external_propose_majority { - let u in 1 .. MAX_USERS; + let p in 1 .. MAX_PROPOSALS; let origin = T::ExternalMajorityOrigin::successful_origin(); - let proposal_hash = T::Hashing::hash_of(&u); + let proposal_hash = T::Hashing::hash_of(&p); let call = Call::::external_propose_majority(proposal_hash); - - }: { - let _ = call.dispatch(origin)?; + }: { call.dispatch(origin)? } + verify { + // External proposal created + ensure!(>::exists(), "External proposal didn't work"); } external_propose_default { - let u in 1 .. MAX_USERS; + let p in 1 .. MAX_PROPOSALS; let origin = T::ExternalDefaultOrigin::successful_origin(); - let proposal_hash = T::Hashing::hash_of(&u); + let proposal_hash = T::Hashing::hash_of(&p); let call = Call::::external_propose_default(proposal_hash); - - }: { - let _ = call.dispatch(origin)?; + }: { call.dispatch(origin)? } + verify { + // External proposal created + ensure!(>::exists(), "External proposal didn't work"); } fast_track { - let u in 1 .. MAX_USERS; + let p in 1 .. MAX_PROPOSALS; let origin_propose = T::ExternalDefaultOrigin::successful_origin(); - let proposal_hash: T::Hash = T::Hashing::hash_of(&u); + let proposal_hash: T::Hash = T::Hashing::hash_of(&p); Democracy::::external_propose_default(origin_propose, proposal_hash.clone())?; + // NOTE: Instant origin may invoke a little bit more logic, but may not always succeed. let origin_fast_track = T::FastTrackOrigin::successful_origin(); let voting_period = T::FastTrackVotingPeriod::get(); let delay = 0; let call = Call::::fast_track(proposal_hash, voting_period.into(), delay.into()); - }: { - let _ = call.dispatch(origin_fast_track)?; + }: { call.dispatch(origin_fast_track)? } + verify { + assert_eq!(Democracy::::referendum_count(), 1, "referendum not created") } veto_external { @@ -224,33 +367,105 @@ benchmarks! { for i in 0 .. v { vetoers.push(account("vetoer", i, SEED)); } + vetoers.sort(); Blacklist::::insert(proposal_hash, (T::BlockNumber::zero(), vetoers)); let call = Call::::veto_external(proposal_hash); let origin = T::VetoOrigin::successful_origin(); - }: { - let _ = call.dispatch(origin)?; + ensure!(NextExternal::::get().is_some(), "no external proposal"); + }: { call.dispatch(origin)? } + verify { + assert!(NextExternal::::get().is_none()); + let (_, new_vetoers) = >::get(&proposal_hash).ok_or("no blacklist")?; + assert_eq!(new_vetoers.len(), (v + 1) as usize, "vetoers not added"); } cancel_referendum { - let u in 1 .. MAX_USERS; - - let referendum_index = add_referendum::(u)?; + let r in 0 .. MAX_REFERENDUMS; + // Should have no effect on the execution time. + for i in 0..r { + add_referendum::(i)?; + } + let referendum_index = add_referendum::(r)?; }: _(RawOrigin::Root, referendum_index) cancel_queued { - let u in 1 .. MAX_USERS; - - let referendum_index = add_referendum::(u)?; + let r in 1 .. MAX_REFERENDUMS; + // Should have no effect on the execution time. + for i in 0..r { + add_referendum::(i)?; + } + let referendum_index = add_referendum::(r)?; }: _(RawOrigin::Root, referendum_index) - open_proxy { - let u in 1 .. MAX_USERS; + // Note that we have a separate benchmark for `launch_next` + on_initialize_external { + let r in 0 .. MAX_REFERENDUMS; - let caller: T::AccountId = funded_account::("caller", u); - let proxy: T::AccountId = funded_account::("proxy", u); + for i in 0..r { + add_referendum::(i)?; + } - }: _(RawOrigin::Signed(proxy), caller) + assert_eq!(Democracy::::referendum_count(), r, "referenda not created"); + + // Launch external + LastTabledWasExternal::put(false); + + let origin = T::ExternalMajorityOrigin::successful_origin(); + let proposal_hash = T::Hashing::hash_of(&r); + let call = Call::::external_propose_majority(proposal_hash); + call.dispatch(origin)?; + // External proposal created + ensure!(>::exists(), "External proposal didn't work"); + + let block_number = T::LaunchPeriod::get(); + + }: { Democracy::::on_initialize(block_number) } + verify { + // One extra because of next external + assert_eq!(Democracy::::referendum_count(), r + 1, "referenda not created"); + ensure!(!>::exists(), "External wasn't taken"); + + // All but the new next external should be finished + for i in 0 .. r { + if let Some(value) = ReferendumInfoOf::::get(i) { + match value { + ReferendumInfo::Finished { .. } => (), + ReferendumInfo::Ongoing(_) => return Err("Referendum was not finished"), + } + } + } + } + + on_initialize_public { + let r in 1 .. MAX_REFERENDUMS; + + for i in 0..r { + add_referendum::(i)?; + } + + assert_eq!(Democracy::::referendum_count(), r, "referenda not created"); + + // Launch public + LastTabledWasExternal::put(true); + + let block_number = T::LaunchPeriod::get(); + + }: { Democracy::::on_initialize(block_number) } + verify { + // One extra because of next public + assert_eq!(Democracy::::referendum_count(), r + 1, "referenda not created"); + + // All should be finished + for i in 0 .. r { + if let Some(value) = ReferendumInfoOf::::get(i) { + match value { + ReferendumInfo::Finished { .. } => (), + ReferendumInfo::Ongoing(_) => return Err("Referendum was not finished"), + } + } + } + } activate_proxy { let u in 1 .. MAX_USERS; @@ -258,51 +473,120 @@ benchmarks! { let caller: T::AccountId = funded_account::("caller", u); let proxy: T::AccountId = funded_account::("proxy", u); Democracy::::open_proxy(RawOrigin::Signed(proxy.clone()).into(), caller.clone())?; - - }: _(RawOrigin::Signed(caller), proxy) + }: _(RawOrigin::Signed(caller.clone()), proxy.clone()) + verify { + assert_eq!(Democracy::::proxy(proxy), Some(ProxyState::Active(caller))); + } close_proxy { let u in 1 .. MAX_USERS; - - let proxy = open_activate_proxy::(u)?; - - }: _(RawOrigin::Signed(proxy)) + let (caller, _) = open_activate_proxy::(u)?; + }: _(RawOrigin::Signed(caller.clone())) + verify { + assert_eq!(Democracy::::proxy(caller), None); + } deactivate_proxy { let u in 1 .. MAX_USERS; - - let caller = funded_account::("caller", u); - let proxy = open_activate_proxy::(u)?; - - }: _(RawOrigin::Signed(caller), proxy) + let (caller, voter) = open_activate_proxy::(u)?; + }: _(RawOrigin::Signed(voter.clone()), caller.clone()) + verify { + assert_eq!(Democracy::::proxy(caller), Some(ProxyState::Open(voter))); + } delegate { - let u in 1 .. MAX_USERS; + let r in 1 .. MAX_REFERENDUMS; - let caller = funded_account::("caller", u); - let d: T::AccountId = funded_account::("delegate", u); - let balance = 1u32; + let initial_balance: BalanceOf = 100.into(); + let delegated_balance: BalanceOf = 1000.into(); - }: _(RawOrigin::Signed(caller), d.into(), Conviction::Locked1x, balance.into()) + let caller = funded_account::("caller", 0); + // Caller will initially delegate to `old_delegate` + let old_delegate: T::AccountId = funded_account::("old_delegate", r); + Democracy::::delegate( + RawOrigin::Signed(caller.clone()).into(), + old_delegate.clone(), + Conviction::Locked1x, + delegated_balance, + )?; + let (target, balance) = match VotingOf::::get(&caller) { + Voting::Delegating { target, balance, .. } => (target, balance), + _ => return Err("Votes are not direct"), + }; + assert_eq!(target, old_delegate, "delegation target didn't work"); + assert_eq!(balance, delegated_balance, "delegation balance didn't work"); + // Caller will now switch to `new_delegate` + let new_delegate: T::AccountId = funded_account::("new_delegate", r); + let account_vote = account_vote::(initial_balance); + // We need to create existing direct votes for the `new_delegate` + for i in 0..r { + let ref_idx = add_referendum::(i)?; + Democracy::::vote(RawOrigin::Signed(new_delegate.clone()).into(), ref_idx, account_vote.clone())?; + } + let votes = match VotingOf::::get(&new_delegate) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), r as usize, "Votes were not recorded."); + }: _(RawOrigin::Signed(caller.clone()), new_delegate.clone(), Conviction::Locked1x, delegated_balance) + verify { + let (target, balance) = match VotingOf::::get(&caller) { + Voting::Delegating { target, balance, .. } => (target, balance), + _ => return Err("Votes are not direct"), + }; + assert_eq!(target, new_delegate, "delegation target didn't work"); + assert_eq!(balance, delegated_balance, "delegation balance didn't work"); + let delegations = match VotingOf::::get(&new_delegate) { + Voting::Direct { delegations, .. } => delegations, + _ => return Err("Votes are not direct"), + }; + assert_eq!(delegations.capital, delegated_balance, "delegation was not recorded."); + } undelegate { let r in 1 .. MAX_REFERENDUMS; - let other = funded_account::("other", 0); - let account_vote = account_vote::(); + let initial_balance: BalanceOf = 100.into(); + let delegated_balance: BalanceOf = 1000.into(); - for i in 0 .. r { + let caller = funded_account::("caller", 0); + // Caller will delegate + let the_delegate: T::AccountId = funded_account::("delegate", r); + Democracy::::delegate( + RawOrigin::Signed(caller.clone()).into(), + the_delegate.clone(), + Conviction::Locked1x, + delegated_balance, + )?; + let (target, balance) = match VotingOf::::get(&caller) { + Voting::Delegating { target, balance, .. } => (target, balance), + _ => return Err("Votes are not direct"), + }; + assert_eq!(target, the_delegate, "delegation target didn't work"); + assert_eq!(balance, delegated_balance, "delegation balance didn't work"); + // We need to create votes direct votes for the `delegate` + let account_vote = account_vote::(initial_balance); + for i in 0..r { let ref_idx = add_referendum::(i)?; - Democracy::::vote(RawOrigin::Signed(other.clone()).into(), ref_idx, account_vote.clone())?; + Democracy::::vote( + RawOrigin::Signed(the_delegate.clone()).into(), + ref_idx, + account_vote.clone() + )?; } - - let delegator = funded_account::("delegator", r); - let conviction = Conviction::Locked1x; - let balance = 1u32; - - Democracy::::delegate(RawOrigin::Signed(delegator.clone()).into(), other.clone().into(), conviction, balance.into())?; - - }: _(RawOrigin::Signed(delegator)) + let votes = match VotingOf::::get(&the_delegate) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), r as usize, "Votes were not recorded."); + }: _(RawOrigin::Signed(caller.clone())) + verify { + // Voting should now be direct + match VotingOf::::get(&caller) { + Voting::Direct { .. } => (), + _ => return Err("undelegation failed"), + } + } clear_public_proposals { let p in 0 .. MAX_PROPOSALS; @@ -317,131 +601,410 @@ benchmarks! { // Num of bytes in encoded proposal let b in 0 .. MAX_BYTES; - let caller = funded_account::("caller", b); - let encoded_proposal = vec![0; b as usize]; - }: _(RawOrigin::Signed(caller), encoded_proposal) + let caller = funded_account::("caller", 0); + let encoded_proposal = vec![1; b as usize]; + }: _(RawOrigin::Signed(caller), encoded_proposal.clone()) + verify { + let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); + match Preimages::::get(proposal_hash) { + Some(PreimageStatus::Available { .. }) => (), + _ => return Err("preimage not available") + } + } note_imminent_preimage { // Num of bytes in encoded proposal let b in 0 .. MAX_BYTES; // d + 1 to include the one we are testing - let encoded_proposal = vec![0; b as usize]; + let encoded_proposal = vec![1; b as usize]; let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); let block_number = T::BlockNumber::one(); Preimages::::insert(&proposal_hash, PreimageStatus::Missing(block_number)); - let caller = funded_account::("caller", b); - let encoded_proposal = vec![0; b as usize]; - }: _(RawOrigin::Signed(caller), encoded_proposal) + let caller = funded_account::("caller", 0); + let encoded_proposal = vec![1; b as usize]; + }: _(RawOrigin::Signed(caller), encoded_proposal.clone()) + verify { + let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); + match Preimages::::get(proposal_hash) { + Some(PreimageStatus::Available { .. }) => (), + _ => return Err("preimage not available") + } + } reap_preimage { // Num of bytes in encoded proposal let b in 0 .. MAX_BYTES; - let encoded_proposal = vec![0; b as usize]; + let encoded_proposal = vec![1; b as usize]; let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); - let caller = funded_account::("caller", b); - Democracy::::note_preimage(RawOrigin::Signed(caller.clone()).into(), encoded_proposal.clone())?; + let submitter = funded_account::("submitter", b); + Democracy::::note_preimage(RawOrigin::Signed(submitter.clone()).into(), encoded_proposal.clone())?; // We need to set this otherwise we get `Early` error. let block_number = T::VotingPeriod::get() + T::EnactmentPeriod::get() + T::BlockNumber::one(); System::::set_block_number(block_number.into()); - }: _(RawOrigin::Signed(caller), proposal_hash) + assert!(Preimages::::contains_key(proposal_hash)); - unlock { - let u in 1 .. MAX_USERS; + let caller = funded_account::("caller", 0); + }: _(RawOrigin::Signed(caller), proposal_hash.clone()) + verify { + let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); + assert!(!Preimages::::contains_key(proposal_hash)); + } - let caller = funded_account::("caller", u); - let locked_until = T::BlockNumber::zero(); - Locks::::insert(&caller, locked_until); + // Test when unlock will remove locks + unlock_remove { + let r in 1 .. MAX_REFERENDUMS; - T::Currency::extend_lock( - DEMOCRACY_ID, - &caller, - Bounded::max_value(), - WithdrawReason::Transfer.into() - ); + let locker = funded_account::("locker", 0); + // Populate votes so things are locked + let base_balance: BalanceOf = 100.into(); + let small_vote = account_vote::(base_balance); + // Vote and immediately unvote + for i in 0 .. r { + let ref_idx = add_referendum::(i)?; + Democracy::::vote(RawOrigin::Signed(locker.clone()).into(), ref_idx, small_vote.clone())?; + Democracy::::remove_vote(RawOrigin::Signed(locker.clone()).into(), ref_idx)?; + } + + let caller = funded_account::("caller", 0); + }: unlock(RawOrigin::Signed(caller), locker.clone()) + verify { + // Note that we may want to add a `get_lock` api to actually verify + let voting = VotingOf::::get(&locker); + assert_eq!(voting.locked_balance(), BalanceOf::::zero()); + } + + // Test when unlock will set a new value + unlock_set { + let r in 1 .. MAX_REFERENDUMS; - let other = caller.clone(); + let locker = funded_account::("locker", 0); + // Populate votes so things are locked + let base_balance: BalanceOf = 100.into(); + let small_vote = account_vote::(base_balance); + for i in 0 .. r { + let ref_idx = add_referendum::(i)?; + Democracy::::vote(RawOrigin::Signed(locker.clone()).into(), ref_idx, small_vote.clone())?; + } + + // Create a big vote so lock increases + let big_vote = account_vote::(base_balance * 10.into()); + let referendum_index = add_referendum::(r)?; + Democracy::::vote(RawOrigin::Signed(locker.clone()).into(), referendum_index, big_vote)?; + + let votes = match VotingOf::::get(&locker) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), (r + 1) as usize, "Votes were not recorded."); + + let voting = VotingOf::::get(&locker); + assert_eq!(voting.locked_balance(), base_balance * 10.into()); - }: _(RawOrigin::Signed(caller), other) + Democracy::::remove_vote(RawOrigin::Signed(locker.clone()).into(), referendum_index)?; + + let caller = funded_account::("caller", 0); + }: unlock(RawOrigin::Signed(caller), locker.clone()) + verify { + let votes = match VotingOf::::get(&locker) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), r as usize, "Vote was not removed"); + + let voting = VotingOf::::get(&locker); + // Note that we may want to add a `get_lock` api to actually verify + assert_eq!(voting.locked_balance(), base_balance); + } + + open_proxy { + let u in 1 .. MAX_USERS; + + let caller: T::AccountId = funded_account::("caller", u); + let proxy: T::AccountId = funded_account::("proxy", u); + + }: _(RawOrigin::Signed(proxy), caller) remove_vote { let r in 1 .. MAX_REFERENDUMS; let caller = funded_account::("caller", 0); - let account_vote = account_vote::(); + let account_vote = account_vote::(100.into()); for i in 0 .. r { let ref_idx = add_referendum::(i)?; Democracy::::vote(RawOrigin::Signed(caller.clone()).into(), ref_idx, account_vote.clone())?; } + let votes = match VotingOf::::get(&caller) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), r as usize, "Votes not created"); + let referendum_index = r - 1; - }: _(RawOrigin::Signed(caller), referendum_index) + }: _(RawOrigin::Signed(caller.clone()), referendum_index) + verify { + let votes = match VotingOf::::get(&caller) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), (r - 1) as usize, "Vote was not removed"); + } remove_other_vote { let r in 1 .. MAX_REFERENDUMS; let other = funded_account::("other", r); - let account_vote = account_vote::(); + let account_vote = account_vote::(100.into()); for i in 0 .. r { let ref_idx = add_referendum::(i)?; Democracy::::vote(RawOrigin::Signed(other.clone()).into(), ref_idx, account_vote.clone())?; } + let votes = match VotingOf::::get(&other) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), r as usize, "Votes not created"); + let referendum_index = r - 1; ReferendumInfoOf::::insert( referendum_index, ReferendumInfo::Finished { end: T::BlockNumber::zero(), approved: true } ); - let caller = funded_account::("caller", r); + let caller = funded_account::("caller", 0); System::::set_block_number(T::EnactmentPeriod::get() * 10u32.into()); - }: _(RawOrigin::Signed(caller), other, referendum_index) + }: _(RawOrigin::Signed(caller), other.clone(), referendum_index) + verify { + let votes = match VotingOf::::get(&other) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), (r - 1) as usize, "Vote was not removed"); + } + // This is a copy of delegate benchmark, but with `open_activate_proxy` proxy_delegate { - let u in 1 .. MAX_USERS; - - let other: T::AccountId = account("other", u, SEED); - let proxy = open_activate_proxy::(u)?; - let conviction = Conviction::Locked1x; - let balance = 1u32; + let r in 1 .. MAX_REFERENDUMS; - }: _(RawOrigin::Signed(proxy), other, conviction, balance.into()) + let initial_balance: BalanceOf = 100.into(); + let delegated_balance: BalanceOf = 1000.into(); + + let (caller, voter) = open_activate_proxy::(0)?; + + // Voter will initially delegate to `old_delegate` + let old_delegate: T::AccountId = funded_account::("old_delegate", r); + Democracy::::delegate( + RawOrigin::Signed(voter.clone()).into(), + old_delegate.clone(), + Conviction::Locked1x, + delegated_balance, + )?; + let (target, balance) = match VotingOf::::get(&voter) { + Voting::Delegating { target, balance, .. } => (target, balance), + _ => return Err("Votes are not direct"), + }; + assert_eq!(target, old_delegate, "delegation target didn't work"); + assert_eq!(balance, delegated_balance, "delegation balance didn't work"); + // Voter will now switch to `new_delegate` + let new_delegate: T::AccountId = funded_account::("new_delegate", r); + let account_vote = account_vote::(initial_balance); + // We need to create existing direct votes for the `new_delegate` + for i in 0..r { + let ref_idx = add_referendum::(i)?; + Democracy::::vote(RawOrigin::Signed(new_delegate.clone()).into(), ref_idx, account_vote.clone())?; + } + let votes = match VotingOf::::get(&new_delegate) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), r as usize, "Votes were not recorded."); + }: _(RawOrigin::Signed(caller.clone()), new_delegate.clone(), Conviction::Locked1x, delegated_balance) + verify { + let (target, balance) = match VotingOf::::get(&voter) { + Voting::Delegating { target, balance, .. } => (target, balance), + _ => return Err("Votes are not direct"), + }; + assert_eq!(target, new_delegate, "delegation target didn't work"); + assert_eq!(balance, delegated_balance, "delegation balance didn't work"); + let delegations = match VotingOf::::get(&new_delegate) { + Voting::Direct { delegations, .. } => delegations, + _ => return Err("Votes are not direct"), + }; + assert_eq!(delegations.capital, delegated_balance, "delegation was not recorded."); + } + // This is a copy of undelegate benchmark, but with `open_activate_proxy` proxy_undelegate { let r in 1 .. MAX_REFERENDUMS; - let other = funded_account::("other", 0); - let account_vote = account_vote::(); + let initial_balance: BalanceOf = 100.into(); + let delegated_balance: BalanceOf = 1000.into(); + + let (caller, voter) = open_activate_proxy::(0)?; + // Caller will delegate + let the_delegate: T::AccountId = funded_account::("delegate", r); + Democracy::::delegate( + RawOrigin::Signed(voter.clone()).into(), + the_delegate.clone(), + Conviction::Locked1x, + delegated_balance, + )?; + let (target, balance) = match VotingOf::::get(&voter) { + Voting::Delegating { target, balance, .. } => (target, balance), + _ => return Err("Votes are not direct"), + }; + assert_eq!(target, the_delegate, "delegation target didn't work"); + assert_eq!(balance, delegated_balance, "delegation balance didn't work"); + // We need to create votes direct votes for the `delegate` + let account_vote = account_vote::(initial_balance); + for i in 0..r { + let ref_idx = add_referendum::(i)?; + Democracy::::vote( + RawOrigin::Signed(the_delegate.clone()).into(), + ref_idx, + account_vote.clone() + )?; + } + let votes = match VotingOf::::get(&the_delegate) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), r as usize, "Votes were not recorded."); + }: _(RawOrigin::Signed(caller.clone())) + verify { + // Voting should now be direct + match VotingOf::::get(&voter) { + Voting::Direct { .. } => (), + _ => return Err("undelegation failed"), + } + } + + proxy_remove_vote { + let r in 1 .. MAX_REFERENDUMS; + + let (caller, voter) = open_activate_proxy::(0)?; + let account_vote = account_vote::(100.into()); for i in 0 .. r { let ref_idx = add_referendum::(i)?; - Democracy::::vote(RawOrigin::Signed(other.clone()).into(), ref_idx, account_vote.clone())?; + Democracy::::vote(RawOrigin::Signed(voter.clone()).into(), ref_idx, account_vote.clone())?; } - let proxy = open_activate_proxy::(r)?; - let conviction = Conviction::Locked1x; - let balance = 1u32; - Democracy::::proxy_delegate(RawOrigin::Signed(proxy.clone()).into(), other, conviction, balance.into())?; + let votes = match VotingOf::::get(&voter) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), r as usize, "Votes not created"); - }: _(RawOrigin::Signed(proxy)) + let referendum_index = r - 1; - proxy_remove_vote { - let u in 1 .. MAX_USERS; + }: _(RawOrigin::Signed(caller.clone()), referendum_index) + verify { + let votes = match VotingOf::::get(&voter) { + Voting::Direct { votes, .. } => votes, + _ => return Err("Votes are not direct"), + }; + assert_eq!(votes.len(), (r - 1) as usize, "Vote was not removed"); + } + + enact_proposal_execute { + // Num of bytes in encoded proposal + let b in 0 .. MAX_BYTES; - let referendum_index = add_referendum::(u)?; - let account_vote = account_vote::(); - let proxy = open_activate_proxy::(u)?; + let proposer = funded_account::("proposer", 0); + let raw_call = Call::note_preimage(vec![1; b as usize]); + let generic_call: T::Proposal = raw_call.into(); + let encoded_proposal = generic_call.encode(); + let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); + Democracy::::note_preimage(RawOrigin::Signed(proposer).into(), encoded_proposal)?; - Democracy::::proxy_vote(RawOrigin::Signed(proxy.clone()).into(), referendum_index, account_vote)?; + match Preimages::::get(proposal_hash) { + Some(PreimageStatus::Available { .. }) => (), + _ => return Err("preimage not available") + } + }: enact_proposal(RawOrigin::Root, proposal_hash, 0) + verify { + // Fails due to mismatched origin + assert_last_event::(RawEvent::Executed(0, false).into()); + } - }: _(RawOrigin::Signed(proxy), referendum_index) + enact_proposal_slash { + // Num of bytes in encoded proposal + let b in 0 .. MAX_BYTES; + + let proposer = funded_account::("proposer", 0); + // Random invalid bytes + let encoded_proposal = vec![200; b as usize]; + let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); + Democracy::::note_preimage(RawOrigin::Signed(proposer).into(), encoded_proposal)?; + + match Preimages::::get(proposal_hash) { + Some(PreimageStatus::Available { .. }) => (), + _ => return Err("preimage not available") + } + }: { + assert_eq!( + Democracy::::enact_proposal(RawOrigin::Root.into(), proposal_hash, 0), + Err(Error::::PreimageInvalid.into()) + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_propose::()); + assert_ok!(test_benchmark_second::()); + assert_ok!(test_benchmark_vote_new::()); + assert_ok!(test_benchmark_vote_existing::()); + assert_ok!(test_benchmark_proxy_vote_new::()); + assert_ok!(test_benchmark_proxy_vote_existing::()); + assert_ok!(test_benchmark_emergency_cancel::()); + assert_ok!(test_benchmark_external_propose::()); + assert_ok!(test_benchmark_external_propose_majority::()); + assert_ok!(test_benchmark_external_propose_default::()); + assert_ok!(test_benchmark_fast_track::()); + assert_ok!(test_benchmark_veto_external::()); + assert_ok!(test_benchmark_cancel_referendum::()); + assert_ok!(test_benchmark_cancel_queued::()); + assert_ok!(test_benchmark_on_initialize_external::()); + assert_ok!(test_benchmark_on_initialize_public::()); + assert_ok!(test_benchmark_open_proxy::()); + assert_ok!(test_benchmark_activate_proxy::()); + assert_ok!(test_benchmark_close_proxy::()); + assert_ok!(test_benchmark_deactivate_proxy::()); + assert_ok!(test_benchmark_delegate::()); + assert_ok!(test_benchmark_undelegate::()); + assert_ok!(test_benchmark_clear_public_proposals::()); + assert_ok!(test_benchmark_note_preimage::()); + assert_ok!(test_benchmark_note_imminent_preimage::()); + assert_ok!(test_benchmark_reap_preimage::()); + assert_ok!(test_benchmark_unlock_remove::()); + assert_ok!(test_benchmark_unlock_set::()); + assert_ok!(test_benchmark_remove_vote::()); + assert_ok!(test_benchmark_remove_other_vote::()); + assert_ok!(test_benchmark_proxy_delegate::()); + assert_ok!(test_benchmark_proxy_undelegate::()); + assert_ok!(test_benchmark_proxy_remove_vote::()); + assert_ok!(test_benchmark_enact_proposal_execute::()); + assert_ok!(test_benchmark_enact_proposal_slash::()); + }); + } } diff --git a/frame/democracy/src/lib.rs b/frame/democracy/src/lib.rs index b09f305c642477cc277653a5ddd33be68a4a9766..b60d658cde59d5954b617465251373bda4cd9b86 100644 --- a/frame/democracy/src/lib.rs +++ b/frame/democracy/src/lib.rs @@ -171,7 +171,7 @@ use sp_runtime::{ use codec::{Ref, Encode, Decode}; use frame_support::{ decl_module, decl_storage, decl_event, decl_error, ensure, Parameter, - weights::{SimpleDispatchInfo, Weight, WeighData}, + weights::{Weight, DispatchClass}, traits::{ Currency, ReservableCurrency, LockableCurrency, WithdrawReason, LockIdentifier, Get, OnUnbalanced, BalanceStatus, schedule::Named as ScheduleNamed, EnsureOrigin @@ -346,7 +346,7 @@ decl_storage! { /// Accounts for which there are locks in action which may be removed at some point in the /// future. The value is the block number at which the lock expires and may be removed. - pub Locks get(locks): map hasher(twox_64_concat) T::AccountId => Option; + pub Locks get(fn locks): map hasher(twox_64_concat) T::AccountId => Option; /// True if the last referendum tabled was submitted externally. False if it was a public /// proposal. @@ -528,7 +528,7 @@ decl_module! { fn on_runtime_upgrade() -> Weight { Self::migrate(); - SimpleDispatchInfo::default().weigh_data(()) + 0 } /// Propose a sensitive action to be taken. @@ -546,7 +546,7 @@ decl_module! { /// - P is the number proposals in the `PublicProps` vec. /// - Two DB changes, one DB entry. /// # - #[weight = SimpleDispatchInfo::FixedNormal(5_000_000)] + #[weight = 5_000_000_000] fn propose(origin, proposal_hash: T::Hash, #[compact] value: BalanceOf @@ -577,7 +577,7 @@ decl_module! { /// - S is the number of seconds a proposal already has. /// - One DB entry. /// # - #[weight = SimpleDispatchInfo::FixedNormal(5_000_000)] + #[weight = 5_000_000_000] fn second(origin, #[compact] proposal: PropIndex) { let who = ensure_signed(origin)?; let mut deposit = Self::deposit_of(proposal) @@ -600,7 +600,7 @@ decl_module! { /// - R is the number of referendums the voter has voted on. /// - One DB change, one DB entry. /// # - #[weight = SimpleDispatchInfo::FixedNormal(200_000)] + #[weight = 200_000_000] fn vote(origin, #[compact] ref_index: ReferendumIndex, vote: AccountVote>, @@ -621,7 +621,7 @@ decl_module! { /// - `O(1)`. /// - One DB change, one DB entry. /// # - #[weight = SimpleDispatchInfo::FixedNormal(200_000)] + #[weight = 200_000_000] fn proxy_vote(origin, #[compact] ref_index: ReferendumIndex, vote: AccountVote>, @@ -641,7 +641,7 @@ decl_module! { /// # /// - `O(1)`. /// # - #[weight = SimpleDispatchInfo::FixedOperational(500_000)] + #[weight = (500_000_000, DispatchClass::Operational)] fn emergency_cancel(origin, ref_index: ReferendumIndex) { T::CancellationOrigin::ensure_origin(origin)?; @@ -664,7 +664,7 @@ decl_module! { /// - `O(1)`. /// - One DB change. /// # - #[weight = SimpleDispatchInfo::FixedNormal(5_000_000)] + #[weight = 5_000_000_000] fn external_propose(origin, proposal_hash: T::Hash) { T::ExternalOrigin::ensure_origin(origin)?; ensure!(!>::exists(), Error::::DuplicateProposal); @@ -691,7 +691,7 @@ decl_module! { /// - `O(1)`. /// - One DB change. /// # - #[weight = SimpleDispatchInfo::FixedNormal(5_000_000)] + #[weight = 5_000_000_000] fn external_propose_majority(origin, proposal_hash: T::Hash) { T::ExternalMajorityOrigin::ensure_origin(origin)?; >::put((proposal_hash, VoteThreshold::SimpleMajority)); @@ -711,7 +711,7 @@ decl_module! { /// - `O(1)`. /// - One DB change. /// # - #[weight = SimpleDispatchInfo::FixedNormal(5_000_000)] + #[weight = 5_000_000_000] fn external_propose_default(origin, proposal_hash: T::Hash) { T::ExternalDefaultOrigin::ensure_origin(origin)?; >::put((proposal_hash, VoteThreshold::SuperMajorityAgainst)); @@ -736,7 +736,7 @@ decl_module! { /// - One DB change. /// - One extra DB entry. /// # - #[weight = SimpleDispatchInfo::FixedNormal(200_000)] + #[weight = 200_000_000] fn fast_track(origin, proposal_hash: T::Hash, voting_period: T::BlockNumber, @@ -787,7 +787,7 @@ decl_module! { /// be very large. /// - O(log v), v is number of `existing_vetoers` /// # - #[weight = SimpleDispatchInfo::FixedNormal(200_000)] + #[weight = 200_000_000] fn veto_external(origin, proposal_hash: T::Hash) { let who = T::VetoOrigin::ensure_origin(origin)?; @@ -820,7 +820,7 @@ decl_module! { /// # /// - `O(1)`. /// # - #[weight = SimpleDispatchInfo::FixedOperational(10_000)] + #[weight = (0, DispatchClass::Operational)] fn cancel_referendum(origin, #[compact] ref_index: ReferendumIndex) { ensure_root(origin)?; Self::internal_cancel_referendum(ref_index); @@ -836,7 +836,7 @@ decl_module! { /// - One DB change. /// - O(d) where d is the items in the dispatch queue. /// # - #[weight = SimpleDispatchInfo::FixedOperational(10_000)] + #[weight = (0, DispatchClass::Operational)] fn cancel_queued(origin, which: ReferendumIndex) { ensure_root(origin)?; T::Scheduler::cancel_named((DEMOCRACY_ID, which)) @@ -848,7 +848,7 @@ decl_module! { sp_runtime::print(e); } - SimpleDispatchInfo::default().weigh_data(()) + 0 } /// Specify a proxy that is already open to us. Called by the stash. @@ -862,7 +862,7 @@ decl_module! { /// # /// - One extra DB entry. /// # - #[weight = SimpleDispatchInfo::FixedNormal(100_000)] + #[weight = 100_000_000] fn activate_proxy(origin, proxy: T::AccountId) { let who = ensure_signed(origin)?; Proxy::::try_mutate(&proxy, |a| match a.take() { @@ -885,7 +885,7 @@ decl_module! { /// # /// - One DB clear. /// # - #[weight = SimpleDispatchInfo::FixedNormal(100_000)] + #[weight = 100_000_000] fn close_proxy(origin) { let who = ensure_signed(origin)?; Proxy::::mutate(&who, |a| { @@ -909,7 +909,7 @@ decl_module! { /// # /// - One DB clear. /// # - #[weight = SimpleDispatchInfo::FixedNormal(100_000)] + #[weight = 100_000_000] fn deactivate_proxy(origin, proxy: T::AccountId) { let who = ensure_signed(origin)?; Proxy::::try_mutate(&proxy, |a| match a.take() { @@ -942,7 +942,7 @@ decl_module! { /// /// # /// # - #[weight = SimpleDispatchInfo::FixedNormal(500_000)] + #[weight = 500_000_000] pub fn delegate(origin, to: T::AccountId, conviction: Conviction, balance: BalanceOf) { let who = ensure_signed(origin)?; Self::try_delegate(who, to, conviction, balance)?; @@ -961,7 +961,7 @@ decl_module! { /// # /// - O(1). /// # - #[weight = SimpleDispatchInfo::FixedNormal(500_000)] + #[weight = 500_000_000] fn undelegate(origin) { let who = ensure_signed(origin)?; Self::try_undelegate(who)?; @@ -975,7 +975,7 @@ decl_module! { /// - `O(1)`. /// - One DB clear. /// # - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] fn clear_public_proposals(origin) { ensure_root(origin)?; @@ -995,7 +995,7 @@ decl_module! { /// - Dependent on the size of `encoded_proposal` but protected by a /// required deposit. /// # - #[weight = SimpleDispatchInfo::FixedNormal(100_000)] + #[weight = 100_000_000] fn note_preimage(origin, encoded_proposal: Vec) { let who = ensure_signed(origin)?; let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); @@ -1030,7 +1030,7 @@ decl_module! { /// # /// - Dependent on the size of `encoded_proposal` and length of dispatch queue. /// # - #[weight = SimpleDispatchInfo::FixedNormal(100_000)] + #[weight = 100_000_000] fn note_imminent_preimage(origin, encoded_proposal: Vec) { let who = ensure_signed(origin)?; let proposal_hash = T::Hashing::hash(&encoded_proposal[..]); @@ -1066,7 +1066,7 @@ decl_module! { /// # /// - One DB clear. /// # - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] fn reap_preimage(origin, proposal_hash: T::Hash) { let who = ensure_signed(origin)?; let (provider, deposit, since, expiry) = >::get(&proposal_hash) @@ -1096,7 +1096,7 @@ decl_module! { /// # /// - `O(1)`. /// # - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] fn unlock(origin, target: T::AccountId) { ensure_signed(origin)?; Self::update_lock(&target); @@ -1115,7 +1115,7 @@ decl_module! { /// # /// - One extra DB entry. /// # - #[weight = SimpleDispatchInfo::FixedNormal(100_000)] + #[weight = 100_000_000] fn open_proxy(origin, target: T::AccountId) { let who = ensure_signed(origin)?; Proxy::::mutate(&who, |a| { @@ -1154,7 +1154,7 @@ decl_module! { /// # /// - `O(R + log R)` where R is the number of referenda that `target` has voted on. /// # - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] fn remove_vote(origin, index: ReferendumIndex) -> DispatchResult { let who = ensure_signed(origin)?; Self::try_remove_vote(&who, index, UnvoteScope::Any) @@ -1176,7 +1176,7 @@ decl_module! { /// # /// - `O(R + log R)` where R is the number of referenda that `target` has voted on. /// # - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] fn remove_other_vote(origin, target: T::AccountId, index: ReferendumIndex) -> DispatchResult { let who = ensure_signed(origin)?; let scope = if target == who { UnvoteScope::Any } else { UnvoteScope::OnlyExpired }; @@ -1207,7 +1207,7 @@ decl_module! { /// /// # /// # - #[weight = SimpleDispatchInfo::FixedNormal(500_000)] + #[weight = 500_000_000] pub fn proxy_delegate(origin, to: T::AccountId, conviction: Conviction, @@ -1231,7 +1231,7 @@ decl_module! { /// # /// - O(1). /// # - #[weight = SimpleDispatchInfo::FixedNormal(500_000)] + #[weight = 500_000_000] fn proxy_undelegate(origin) { let who = ensure_signed(origin)?; let target = Self::proxy(who).and_then(|a| a.as_active()).ok_or(Error::::NotProxy)?; @@ -1251,7 +1251,7 @@ decl_module! { /// # /// - `O(R + log R)` where R is the number of referenda that `target` has voted on. /// # - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] fn proxy_remove_vote(origin, index: ReferendumIndex) -> DispatchResult { let who = ensure_signed(origin)?; let target = Self::proxy(who).and_then(|a| a.as_active()).ok_or(Error::::NotProxy)?; @@ -1259,7 +1259,7 @@ decl_module! { } /// Enact a proposal from a referendum. For now we just make the weight be the maximum. - #[weight = SimpleDispatchInfo::MaxNormal] + #[weight = frame_system::Module::::max_extrinsic_weight(DispatchClass::Normal)] fn enact_proposal(origin, proposal_hash: T::Hash, index: ReferendumIndex) -> DispatchResult { ensure_root(origin)?; Self::do_enact_proposal(proposal_hash, index) diff --git a/frame/democracy/src/tests.rs b/frame/democracy/src/tests.rs index 8fca8fa4cfba25b1e6a9d9dab0d509c8dfe028ac..076eb66d6bb56d90ad1523c1f373644480ae825c 100644 --- a/frame/democracy/src/tests.rs +++ b/frame/democracy/src/tests.rs @@ -21,7 +21,7 @@ use std::cell::RefCell; use codec::Encode; use frame_support::{ impl_outer_origin, impl_outer_dispatch, assert_noop, assert_ok, parameter_types, - ord_parameter_types, traits::{Contains, OnInitialize}, weights::Weight, + impl_outer_event, ord_parameter_types, traits::{Contains, OnInitialize}, weights::Weight, }; use sp_core::H256; use sp_runtime::{ @@ -53,11 +53,25 @@ impl_outer_origin! { impl_outer_dispatch! { pub enum Call for Test where origin: Origin { + frame_system::System, pallet_balances::Balances, democracy::Democracy, } } +mod democracy { + pub use crate::Event; +} + +impl_outer_event! { + pub enum Event for Test { + system, + pallet_balances, + pallet_scheduler, + democracy, + } +} + // Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. #[derive(Clone, Eq, PartialEq, Debug)] pub struct Test; @@ -77,9 +91,12 @@ impl frame_system::Trait for Test { type AccountId = u64; type Lookup = IdentityLookup; type Header = Header; - type Event = (); + type Event = Event; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -93,14 +110,14 @@ parameter_types! { pub const MaximumWeight: u32 = 1000000; } impl pallet_scheduler::Trait for Test { - type Event = (); + type Event = Event; type Origin = Origin; type Call = Call; type MaximumWeight = MaximumWeight; } impl pallet_balances::Trait for Test { type Balance = u64; - type Event = (); + type Event = Event; type DustRemoval = (); type ExistentialDeposit = ExistentialDeposit; type AccountStore = System; @@ -126,6 +143,8 @@ impl Contains for OneToFive { fn sorted_members() -> Vec { vec![1, 2, 3, 4, 5] } + #[cfg(feature = "runtime-benchmarks")] + fn add(_m: &u64) {} } thread_local! { static PREIMAGE_BYTE_DEPOSIT: RefCell = RefCell::new(0); @@ -141,7 +160,7 @@ impl Get for InstantAllowed { } impl super::Trait for Test { type Proposal = Call; - type Event = (); + type Event = Event; type Currency = pallet_balances::Module; type EnactmentPeriod = EnactmentPeriod; type LaunchPeriod = LaunchPeriod; @@ -162,7 +181,7 @@ impl super::Trait for Test { type Scheduler = Scheduler; } -fn new_test_ext() -> sp_io::TestExternalities { +pub fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig::{ balances: vec![(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)], diff --git a/frame/elections-phragmen/Cargo.toml b/frame/elections-phragmen/Cargo.toml index fb219bbebc33d5a864e3d0beee08e32e07e2eab5..f9d681b7608e66dd1b615c5524ca8ea8740e796e 100644 --- a/frame/elections-phragmen/Cargo.toml +++ b/frame/elections-phragmen/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-elections-phragmen" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,22 +8,25 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME election pallet for PHRAGMEN" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-phragmen = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/phragmen" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-phragmen = { version = "2.0.0-dev", default-features = false, path = "../../primitives/phragmen" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } [dev-dependencies] -sp-io = { version = "2.0.0-alpha.5", path = "../../primitives/io" } +sp-io = { version = "2.0.0-dev", path = "../../primitives/io" } hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } -pallet-scheduler = { version = "2.0.0-alpha.5", path = "../scheduler" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -substrate-test-utils = { version = "2.0.0-alpha.5", path = "../../test-utils" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } +pallet-scheduler = { version = "2.0.0-dev", path = "../scheduler" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +substrate-test-utils = { version = "2.0.0-dev", path = "../../test-utils" } [features] default = ["std"] @@ -37,6 +40,3 @@ std = [ "sp-std/std", ] runtime-benchmarks = ["frame-support/runtime-benchmarks"] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs index 036a5f492c19c94c9f4ca980bb1de0846074bb37..d77bd750e80c633c5f515fcb52b936c8804682b8 100644 --- a/frame/elections-phragmen/src/lib.rs +++ b/frame/elections-phragmen/src/lib.rs @@ -88,17 +88,17 @@ use sp_runtime::{ }; use frame_support::{ decl_storage, decl_event, ensure, decl_module, decl_error, - weights::{SimpleDispatchInfo, Weight, WeighData}, storage::{StorageMap, IterableStorageMap}, + weights::{Weight, DispatchClass}, + storage::{StorageMap, IterableStorageMap}, traits::{ Currency, Get, LockableCurrency, LockIdentifier, ReservableCurrency, WithdrawReasons, ChangeMembers, OnUnbalanced, WithdrawReason, Contains, BalanceStatus, InitializeMembers, + ContainsLengthBound, } }; -use sp_phragmen::{build_support_map, ExtendedBalance}; +use sp_phragmen::{build_support_map, ExtendedBalance, VoteWeight, PhragmenResult}; use frame_system::{self as system, ensure_signed, ensure_root}; -const MODULE_ID: LockIdentifier = *b"phrelect"; - /// The maximum votes allowed per voter. pub const MAXIMUM_VOTE: usize = 16; @@ -110,6 +110,9 @@ pub trait Trait: frame_system::Trait { /// The overarching event type.c type Event: From> + Into<::Event>; + /// Identifier for the elections-phragmen pallet's lock + type ModuleId: Get; + /// The currency that people are electing with. type Currency: LockableCurrency + @@ -123,7 +126,7 @@ pub trait Trait: frame_system::Trait { /// Convert a balance into a number used for election calculation. /// This must fit into a `u64` but is allowed to be sensibly lossy. - type CurrencyToVote: Convert, u64> + Convert>; + type CurrencyToVote: Convert, VoteWeight> + Convert>; /// How much should be locked up in order to submit one's candidacy. type CandidacyBond: Get>; @@ -267,7 +270,7 @@ decl_module! { fn on_runtime_upgrade() -> Weight { migration::migrate::(); - SimpleDispatchInfo::default().weigh_data(()) + 0 } const CandidacyBond: BalanceOf = T::CandidacyBond::get(); @@ -275,6 +278,7 @@ decl_module! { const DesiredMembers: u32 = T::DesiredMembers::get(); const DesiredRunnersUp: u32 = T::DesiredRunnersUp::get(); const TermDuration: T::BlockNumber = T::TermDuration::get(); + const ModuleId: LockIdentifier = T::ModuleId::get(); /// Vote for a set of candidates for the upcoming round of election. /// @@ -291,7 +295,7 @@ decl_module! { /// Reads: O(1) /// Writes: O(V) given `V` votes. V is bounded by 16. /// # - #[weight = SimpleDispatchInfo::FixedNormal(100_000)] + #[weight = 100_000_000] fn vote(origin, votes: Vec, #[compact] value: BalanceOf) { let who = ensure_signed(origin)?; @@ -320,7 +324,7 @@ decl_module! { // lock T::Currency::set_lock( - MODULE_ID, + T::ModuleId::get(), &who, locked_balance, WithdrawReasons::except(WithdrawReason::TransactionPayment), @@ -336,7 +340,7 @@ decl_module! { /// Reads: O(1) /// Writes: O(1) /// # - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] fn remove_voter(origin) { let who = ensure_signed(origin)?; @@ -358,7 +362,7 @@ decl_module! { /// Reads: O(NLogM) given M current candidates and N votes for `target`. /// Writes: O(1) /// # - #[weight = SimpleDispatchInfo::FixedNormal(1_000_000)] + #[weight = 1_000_000_000] fn report_defunct_voter(origin, target: ::Source) { let reporter = ensure_signed(origin)?; let target = T::Lookup::lookup(target)?; @@ -401,7 +405,7 @@ decl_module! { /// Reads: O(LogN) Given N candidates. /// Writes: O(1) /// # - #[weight = SimpleDispatchInfo::FixedNormal(500_000)] + #[weight = 500_000_000] fn submit_candidacy(origin) { let who = ensure_signed(origin)?; @@ -428,7 +432,7 @@ decl_module! { /// - `origin` is a current member. In this case, the bond is unreserved and origin is /// removed as a member, consequently not being a candidate for the next round anymore. /// Similar to [`remove_voter`], if replacement runners exists, they are immediately used. - #[weight = SimpleDispatchInfo::FixedOperational(2_000_000)] + #[weight = (2_000_000_000, DispatchClass::Operational)] fn renounce_candidacy(origin) { let who = ensure_signed(origin)?; @@ -487,7 +491,7 @@ decl_module! { /// Reads: O(do_phragmen) /// Writes: O(do_phragmen) /// # - #[weight = SimpleDispatchInfo::FixedOperational(2_000_000)] + #[weight = (2_000_000_000, DispatchClass::Operational)] fn remove_member(origin, who: ::Source) -> DispatchResult { ensure_root(origin)?; let who = T::Lookup::lookup(who)?; @@ -510,7 +514,7 @@ decl_module! { print(e); } - SimpleDispatchInfo::default().weigh_data(()) + 0 } } } @@ -649,7 +653,7 @@ impl Module { fn do_remove_voter(who: &T::AccountId, unreserve: bool) { // remove storage and lock. Voting::::remove(who); - T::Currency::remove_lock(MODULE_ID, who); + T::Currency::remove_lock(T::ModuleId::get(), who); if unreserve { T::Currency::unreserve(who, T::VotingBond::get()); @@ -703,17 +707,28 @@ impl Module { // previous runners_up are also always candidates for the next round. candidates.append(&mut Self::runners_up_ids()); + // helper closures to deal with balance/stake. + let to_votes = |b: BalanceOf| -> VoteWeight { + , VoteWeight>>::convert(b) + }; + let to_balance = |e: ExtendedBalance| -> BalanceOf { + >>::convert(e) + }; + let stake_of = |who: &T::AccountId| -> VoteWeight { + to_votes(Self::locked_stake_of(who)) + }; + let voters_and_votes = Voting::::iter() - .map(|(voter, (stake, targets))| { (voter, stake, targets) }) + .map(|(voter, (stake, targets))| { (voter, to_votes(stake), targets) }) .collect::>(); - let maybe_phragmen_result = sp_phragmen::elect::<_, _, T::CurrencyToVote, Perbill>( + let maybe_phragmen_result = sp_phragmen::elect::( num_to_elect, 0, candidates, voters_and_votes.clone(), ); - if let Some(phragmen_result) = maybe_phragmen_result { + if let Some(PhragmenResult { winners, assignments }) = maybe_phragmen_result { let old_members_ids = >::take().into_iter() .map(|(m, _)| m) .collect::>(); @@ -726,26 +741,19 @@ impl Module { // vote are still considered by phragmen and when good candidates are scarce, then these // cheap ones might get elected. We might actually want to remove the filter and allow // zero-voted candidates to also make it to the membership set. - let new_set_with_approval = phragmen_result.winners; + let new_set_with_approval = winners; let new_set = new_set_with_approval .into_iter() .filter_map(|(m, a)| if a.is_zero() { None } else { Some(m) } ) .collect::>(); - let stake_of = |who: &T::AccountId| -> ExtendedBalance { - , u64>>::convert( - Self::locked_stake_of(who) - ) as ExtendedBalance - }; let staked_assignments = sp_phragmen::assignment_ratio_to_staked( - phragmen_result.assignments, + assignments, stake_of, ); let (support_map, _) = build_support_map::(&new_set, &staked_assignments); - let to_balance = |e: ExtendedBalance| - >>::convert(e); let new_set_with_stake = new_set .into_iter() .map(|ref m| { @@ -766,14 +774,14 @@ impl Module { // save the members, sorted based on account id. new_members.sort_by(|i, j| i.0.cmp(&j.0)); - let mut prime_votes: Vec<_> = new_members.iter().map(|c| (&c.0, BalanceOf::::zero())).collect(); + let mut prime_votes: Vec<_> = new_members.iter().map(|c| (&c.0, VoteWeight::zero())).collect(); for (_, stake, targets) in voters_and_votes.into_iter() { for (votes, who) in targets.iter() .enumerate() .map(|(votes, who)| ((MAXIMUM_VOTE - votes) as u32, who)) { if let Ok(i) = prime_votes.binary_search_by_key(&who, |k| k.0) { - prime_votes[i].1 += stake * votes.into(); + prime_votes[i].1 += stake * votes as VoteWeight; } } } @@ -873,6 +881,15 @@ impl Contains for Module { } } +impl ContainsLengthBound for Module { + fn min_len() -> usize { 0 } + + /// Implementation uses a parameter type so calling is cost-free. + fn max_len() -> usize { + Self::desired_members() as usize + } +} + #[cfg(test)] mod tests { use super::*; @@ -907,6 +924,9 @@ mod tests { type Event = Event; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -918,7 +938,7 @@ mod tests { parameter_types! { pub const ExistentialDeposit: u64 = 1; -} + } impl pallet_balances::Trait for Test { type Balance = u64; @@ -1015,7 +1035,12 @@ mod tests { } } + parameter_types!{ + pub const ElectionsPhragmenModuleId: LockIdentifier = *b"phrelect"; + } + impl Trait for Test { + type ModuleId = ElectionsPhragmenModuleId; type Event = Event; type Currency = Balances; type CurrencyToVote = CurrencyToVoteHandler; @@ -1083,7 +1108,7 @@ mod tests { self.genesis_members = members; self } - pub fn build(self) -> sp_io::TestExternalities { + pub fn build_and_execute(self, test: impl FnOnce() -> ()) { VOTING_BOND.with(|v| *v.borrow_mut() = self.voter_bond); TERM_DURATION.with(|v| *v.borrow_mut() = self.term_duration); DESIRED_RUNNERS_UP.with(|v| *v.borrow_mut() = self.desired_runners_up); @@ -1103,8 +1128,9 @@ mod tests { members: self.genesis_members }), }.build_storage().unwrap().into(); - ext.execute_with(|| System::set_block_number(1)); - ext + ext.execute_with(pre_conditions); + ext.execute_with(test); + ext.execute_with(post_conditions) } } @@ -1118,13 +1144,59 @@ mod tests { fn has_lock(who: &u64) -> u64 { let lock = Balances::locks(who)[0].clone(); - assert_eq!(lock.id, MODULE_ID); + assert_eq!(lock.id, ElectionsPhragmenModuleId::get()); lock.amount } + fn intersects(a: &[T], b: &[T]) -> bool { + a.iter().any(|e| b.contains(e)) + } + + fn ensure_members_sorted() { + let mut members = Elections::members().clone(); + members.sort(); + assert_eq!(Elections::members(), members); + } + + fn ensure_candidates_sorted() { + let mut candidates = Elections::candidates().clone(); + candidates.sort(); + assert_eq!(Elections::candidates(), candidates); + } + + fn ensure_members_has_approval_stake() { + // we filter members that have no approval state. This means that even we have more seats + // than candidates, we will never ever chose a member with no votes. + assert!( + Elections::members().iter().chain( + Elections::runners_up().iter() + ).all(|(_, s)| *s != Zero::zero()) + ); + } + + fn ensure_member_candidates_runners_up_disjoint() { + // members, candidates and runners-up must always be disjoint sets. + assert!(!intersects(&Elections::members_ids(), &Elections::candidates())); + assert!(!intersects(&Elections::members_ids(), &Elections::runners_up_ids())); + assert!(!intersects(&Elections::candidates(), &Elections::runners_up_ids())); + } + + fn pre_conditions() { + System::set_block_number(1); + ensure_members_sorted(); + ensure_candidates_sorted(); + } + + fn post_conditions() { + ensure_members_sorted(); + ensure_candidates_sorted(); + ensure_member_candidates_runners_up_disjoint(); + ensure_members_has_approval_stake(); + } + #[test] fn params_should_work() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_eq!(Elections::desired_members(), 2); assert_eq!(Elections::term_duration(), 5); assert_eq!(Elections::election_rounds(), 0); @@ -1143,7 +1215,7 @@ mod tests { #[test] fn genesis_members_should_work() { - ExtBuilder::default().genesis_members(vec![(1, 10), (2, 20)]).build().execute_with(|| { + ExtBuilder::default().genesis_members(vec![(1, 10), (2, 20)]).build_and_execute(|| { System::set_block_number(1); assert_eq!(Elections::members(), vec![(1, 10), (2, 20)]); @@ -1160,7 +1232,7 @@ mod tests { #[test] fn genesis_members_unsorted_should_work() { - ExtBuilder::default().genesis_members(vec![(2, 20), (1, 10)]).build().execute_with(|| { + ExtBuilder::default().genesis_members(vec![(2, 20), (1, 10)]).build_and_execute(|| { System::set_block_number(1); assert_eq!(Elections::members(), vec![(1, 10), (2, 20)]); @@ -1179,28 +1251,27 @@ mod tests { #[should_panic = "Genesis member does not have enough stake"] fn genesis_members_cannot_over_stake_0() { // 10 cannot lock 20 as their stake and extra genesis will panic. - ExtBuilder::default().genesis_members(vec![(1, 20), (2, 20)]).build(); + ExtBuilder::default().genesis_members(vec![(1, 20), (2, 20)]).build_and_execute(|| {}); } #[test] #[should_panic] fn genesis_members_cannot_over_stake_1() { // 10 cannot reserve 20 as voting bond and extra genesis will panic. - ExtBuilder::default().voter_bond(20).genesis_members(vec![(1, 10), (2, 20)]).build(); + ExtBuilder::default().voter_bond(20).genesis_members(vec![(1, 10), (2, 20)]).build_and_execute(|| {}); } #[test] #[should_panic = "Duplicate member in elections phragmen genesis: 2"] fn genesis_members_cannot_be_duplicate() { - ExtBuilder::default().genesis_members(vec![(1, 10), (2, 10), (2, 10)]).build(); + ExtBuilder::default().genesis_members(vec![(1, 10), (2, 10), (2, 10)]).build_and_execute(|| {}); } #[test] fn term_duration_zero_is_passive() { ExtBuilder::default() .term_duration(0) - .build() - .execute_with(|| + .build_and_execute(|| { assert_eq!(Elections::term_duration(), 0); assert_eq!(Elections::desired_members(), 2); @@ -1221,7 +1292,7 @@ mod tests { #[test] fn simple_candidate_submission_should_work() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_eq!(Elections::candidates(), Vec::::new()); assert!(Elections::is_candidate(&1).is_err()); assert!(Elections::is_candidate(&2).is_err()); @@ -1248,7 +1319,7 @@ mod tests { #[test] fn simple_candidate_submission_with_no_votes_should_work() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_eq!(Elections::candidates(), Vec::::new()); assert_ok!(Elections::submit_candidacy(Origin::signed(1))); @@ -1275,7 +1346,7 @@ mod tests { #[test] fn dupe_candidate_submission_should_not_work() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_eq!(Elections::candidates(), Vec::::new()); assert_ok!(Elections::submit_candidacy(Origin::signed(1))); assert_eq!(Elections::candidates(), vec![1]); @@ -1289,7 +1360,7 @@ mod tests { #[test] fn member_candidacy_submission_should_not_work() { // critically important to make sure that outgoing candidates and losers are not mixed up. - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::vote(Origin::signed(2), vec![5], 20)); @@ -1309,7 +1380,7 @@ mod tests { #[test] fn runner_candidate_submission_should_not_work() { - ExtBuilder::default().desired_runners_up(2).build().execute_with(|| { + ExtBuilder::default().desired_runners_up(2).build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(3))); @@ -1332,7 +1403,7 @@ mod tests { #[test] fn poor_candidate_submission_should_not_work() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_eq!(Elections::candidates(), Vec::::new()); assert_noop!( Elections::submit_candidacy(Origin::signed(7)), @@ -1343,7 +1414,7 @@ mod tests { #[test] fn simple_voting_should_work() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_eq!(Elections::candidates(), Vec::::new()); assert_eq!(balances(&2), (20, 0)); @@ -1357,7 +1428,7 @@ mod tests { #[test] fn can_vote_with_custom_stake() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_eq!(Elections::candidates(), Vec::::new()); assert_eq!(balances(&2), (20, 0)); @@ -1371,7 +1442,7 @@ mod tests { #[test] fn can_update_votes_and_stake() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_eq!(balances(&2), (20, 0)); assert_ok!(Elections::submit_candidacy(Origin::signed(5))); @@ -1392,7 +1463,7 @@ mod tests { #[test] fn cannot_vote_for_no_candidate() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_noop!( Elections::vote(Origin::signed(2), vec![], 20), Error::::UnableToVote, @@ -1402,7 +1473,7 @@ mod tests { #[test] fn can_vote_for_old_members_even_when_no_new_candidates() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); @@ -1420,7 +1491,7 @@ mod tests { #[test] fn prime_works() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(3))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(5))); @@ -1444,7 +1515,7 @@ mod tests { #[test] fn prime_votes_for_exiting_members_are_removed() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(3))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(5))); @@ -1469,7 +1540,7 @@ mod tests { #[test] fn cannot_vote_for_more_than_candidates() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); @@ -1482,7 +1553,7 @@ mod tests { #[test] fn cannot_vote_for_less_than_ed() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); @@ -1495,7 +1566,7 @@ mod tests { #[test] fn can_vote_for_more_than_total_balance_but_moot() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); @@ -1508,7 +1579,7 @@ mod tests { #[test] fn remove_voter_should_work() { - ExtBuilder::default().voter_bond(8).build().execute_with(|| { + ExtBuilder::default().voter_bond(8).build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::vote(Origin::signed(2), vec![5], 20)); @@ -1533,14 +1604,14 @@ mod tests { #[test] fn non_voter_remove_should_not_work() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_noop!(Elections::remove_voter(Origin::signed(3)), Error::::MustBeVoter); }); } #[test] fn dupe_remove_should_fail() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::vote(Origin::signed(2), vec![5], 20)); @@ -1553,7 +1624,7 @@ mod tests { #[test] fn removed_voter_should_not_be_counted() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(3))); @@ -1573,7 +1644,7 @@ mod tests { #[test] fn reporter_must_be_voter() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_noop!( Elections::report_defunct_voter(Origin::signed(1), 2), Error::::MustBeVoter, @@ -1583,7 +1654,7 @@ mod tests { #[test] fn can_detect_defunct_voter() { - ExtBuilder::default().desired_runners_up(2).build().execute_with(|| { + ExtBuilder::default().desired_runners_up(2).build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(6))); @@ -1622,7 +1693,7 @@ mod tests { #[test] fn report_voter_should_work_and_earn_reward() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); @@ -1653,7 +1724,7 @@ mod tests { #[test] fn report_voter_should_slash_when_bad_report() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); @@ -1682,7 +1753,7 @@ mod tests { #[test] fn simple_voting_rounds_should_work() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(3))); @@ -1717,7 +1788,7 @@ mod tests { #[test] fn defunct_voter_will_be_counted() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); // This guy's vote is pointless for this round. @@ -1745,7 +1816,7 @@ mod tests { #[test] fn only_desired_seats_are_chosen() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(3))); @@ -1766,7 +1837,7 @@ mod tests { #[test] fn phragmen_should_not_self_vote() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); @@ -1781,7 +1852,7 @@ mod tests { #[test] fn runners_up_should_be_kept() { - ExtBuilder::default().desired_runners_up(2).build().execute_with(|| { + ExtBuilder::default().desired_runners_up(2).build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(3))); @@ -1808,7 +1879,7 @@ mod tests { #[test] fn runners_up_should_be_next_candidates() { - ExtBuilder::default().desired_runners_up(2).build().execute_with(|| { + ExtBuilder::default().desired_runners_up(2).build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(3))); @@ -1835,7 +1906,7 @@ mod tests { #[test] fn runners_up_lose_bond_once_outgoing() { - ExtBuilder::default().desired_runners_up(1).build().execute_with(|| { + ExtBuilder::default().desired_runners_up(1).build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(2))); @@ -1863,7 +1934,7 @@ mod tests { #[test] fn members_lose_bond_once_outgoing() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_eq!(balances(&5), (50, 0)); assert_ok!(Elections::submit_candidacy(Origin::signed(5))); @@ -1889,7 +1960,7 @@ mod tests { #[test] fn losers_will_lose_the_bond() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(3))); @@ -1912,7 +1983,7 @@ mod tests { #[test] fn current_members_are_always_next_candidate() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); @@ -1948,7 +2019,7 @@ mod tests { fn election_state_is_uninterrupted() { // what I mean by uninterrupted: // given no input or stimulants the same members are re-elected. - ExtBuilder::default().desired_runners_up(2).build().execute_with(|| { + ExtBuilder::default().desired_runners_up(2).build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(3))); @@ -1981,7 +2052,7 @@ mod tests { #[test] fn remove_members_triggers_election() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); @@ -2007,7 +2078,7 @@ mod tests { #[test] fn seats_should_be_released_when_no_vote() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(3))); @@ -2041,7 +2112,7 @@ mod tests { #[test] fn incoming_outgoing_are_reported() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(5))); @@ -2088,7 +2159,7 @@ mod tests { #[test] fn invalid_votes_are_moot() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(3))); @@ -2106,7 +2177,7 @@ mod tests { #[test] fn members_are_sorted_based_on_id_runners_on_merit() { - ExtBuilder::default().desired_runners_up(2).build().execute_with(|| { + ExtBuilder::default().desired_runners_up(2).build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(3))); @@ -2128,7 +2199,7 @@ mod tests { #[test] fn candidates_are_sorted() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(3))); @@ -2144,7 +2215,7 @@ mod tests { #[test] fn runner_up_replacement_maintains_members_order() { - ExtBuilder::default().desired_runners_up(2).build().execute_with(|| { + ExtBuilder::default().desired_runners_up(2).build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(2))); @@ -2164,7 +2235,7 @@ mod tests { #[test] fn runner_up_replacement_works_when_out_of_order() { - ExtBuilder::default().desired_runners_up(2).build().execute_with(|| { + ExtBuilder::default().desired_runners_up(2).build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(3))); @@ -2185,7 +2256,7 @@ mod tests { #[test] fn can_renounce_candidacy_member_with_runners_bond_is_refunded() { - ExtBuilder::default().desired_runners_up(2).build().execute_with(|| { + ExtBuilder::default().desired_runners_up(2).build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(3))); @@ -2212,7 +2283,7 @@ mod tests { #[test] fn can_renounce_candidacy_member_without_runners_bond_is_refunded() { - ExtBuilder::default().desired_runners_up(2).build().execute_with(|| { + ExtBuilder::default().desired_runners_up(2).build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); @@ -2244,7 +2315,7 @@ mod tests { #[test] fn can_renounce_candidacy_runner() { - ExtBuilder::default().desired_runners_up(2).build().execute_with(|| { + ExtBuilder::default().desired_runners_up(2).build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_ok!(Elections::submit_candidacy(Origin::signed(4))); assert_ok!(Elections::submit_candidacy(Origin::signed(3))); @@ -2271,7 +2342,7 @@ mod tests { #[test] fn can_renounce_candidacy_candidate() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Elections::submit_candidacy(Origin::signed(5))); assert_eq!(balances(&5), (47, 3)); assert_eq!(Elections::candidates(), vec![5]); @@ -2284,7 +2355,7 @@ mod tests { #[test] fn wrong_renounce_candidacy_should_fail() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_noop!( Elections::renounce_candidacy(Origin::signed(5)), Error::::InvalidOrigin, @@ -2294,7 +2365,7 @@ mod tests { #[test] fn behavior_with_dupe_candidate() { - ExtBuilder::default().desired_runners_up(2).build().execute_with(|| { + ExtBuilder::default().desired_runners_up(2).build_and_execute(|| { >::put(vec![1, 1, 2, 3, 4]); assert_ok!(Elections::vote(Origin::signed(5), vec![1], 50)); diff --git a/frame/elections/Cargo.toml b/frame/elections/Cargo.toml index 6043ac4681e212afb462540552af02639c8af4bd..407b5ccfdb1769d503ad1c5270529f3c8cd75470 100644 --- a/frame/elections/Cargo.toml +++ b/frame/elections/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-elections" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,19 +8,22 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for elections" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] hex-literal = "0.2.1" -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } [features] default = ["std"] @@ -34,6 +37,3 @@ std = [ "sp-runtime/std", "frame-system/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/elections/src/lib.rs b/frame/elections/src/lib.rs index a8ea0b8c2e4555d8990ba485e749f3c49b11024c..4e3fa9c75d92fcf83c21a72eb69107abdad7cb7c 100644 --- a/frame/elections/src/lib.rs +++ b/frame/elections/src/lib.rs @@ -30,10 +30,10 @@ use sp_runtime::{ }; use frame_support::{ decl_storage, decl_event, ensure, decl_module, decl_error, - weights::{Weight, SimpleDispatchInfo, WeighData}, + weights::{Weight, DispatchClass}, traits::{ Currency, ExistenceRequirement, Get, LockableCurrency, LockIdentifier, BalanceStatus, - OnUnbalanced, ReservableCurrency, WithdrawReason, WithdrawReasons, ChangeMembers + OnUnbalanced, ReservableCurrency, WithdrawReason, WithdrawReasons, ChangeMembers, } }; use codec::{Encode, Decode}; @@ -126,8 +126,6 @@ pub enum CellStatus { Hole, } -const MODULE_ID: LockIdentifier = *b"py/elect"; - /// Number of voters grouped in one chunk. pub const VOTER_SET_SIZE: usize = 64; /// NUmber of approvals grouped in one chunk. @@ -149,6 +147,9 @@ const APPROVAL_FLAG_LEN: usize = 32; pub trait Trait: frame_system::Trait { type Event: From> + Into<::Event>; + /// Identifier for the elections pallet's lock + type ModuleId: Get; + /// The currency that people are electing with. type Currency: LockableCurrency @@ -379,6 +380,8 @@ decl_module! { /// The chunk size of the approval vector. const APPROVAL_SET_SIZE: u32 = APPROVAL_SET_SIZE as u32; + const ModuleId: LockIdentifier = T::ModuleId::get(); + fn deposit_event() = default; /// Set candidate approvals. Approval slots stay valid as long as candidates in those slots @@ -405,13 +408,13 @@ decl_module! { /// - Two extra DB entries, one DB change. /// - Argument `votes` is limited in length to number of candidates. /// # - #[weight = SimpleDispatchInfo::FixedNormal(2_500_000)] + #[weight = 2_500_000_000] fn set_approvals( origin, votes: Vec, #[compact] index: VoteIndex, hint: SetIndex, - #[compact] value: BalanceOf + #[compact] value: BalanceOf, ) -> DispatchResult { let who = ensure_signed(origin)?; Self::do_set_approvals(who, votes, index, hint, value) @@ -423,12 +426,12 @@ decl_module! { /// # /// - Same as `set_approvals` with one additional storage read. /// # - #[weight = SimpleDispatchInfo::FixedNormal(2_500_000)] + #[weight = 2_500_000_000] fn proxy_set_approvals(origin, votes: Vec, #[compact] index: VoteIndex, hint: SetIndex, - #[compact] value: BalanceOf + #[compact] value: BalanceOf, ) -> DispatchResult { let who = Self::proxy(ensure_signed(origin)?).ok_or(Error::::NotProxy)?; Self::do_set_approvals(who, votes, index, hint, value) @@ -446,13 +449,13 @@ decl_module! { /// - O(1). /// - Two fewer DB entries, one DB change. /// # - #[weight = SimpleDispatchInfo::FixedNormal(2_500_000)] + #[weight = 2_500_000_000] fn reap_inactive_voter( origin, #[compact] reporter_index: u32, who: ::Source, #[compact] who_index: u32, - #[compact] assumed_vote_index: VoteIndex + #[compact] assumed_vote_index: VoteIndex, ) { let reporter = ensure_signed(origin)?; let who = T::Lookup::lookup(who)?; @@ -494,7 +497,7 @@ decl_module! { ); T::Currency::remove_lock( - MODULE_ID, + T::ModuleId::get(), if valid { &who } else { &reporter } ); @@ -520,7 +523,7 @@ decl_module! { /// - O(1). /// - Two fewer DB entries, one DB change. /// # - #[weight = SimpleDispatchInfo::FixedNormal(1_250_000)] + #[weight = 1_250_000_000] fn retract_voter(origin, #[compact] index: u32) { let who = ensure_signed(origin)?; @@ -532,7 +535,7 @@ decl_module! { Self::remove_voter(&who, index); T::Currency::unreserve(&who, T::VotingBond::get()); - T::Currency::remove_lock(MODULE_ID, &who); + T::Currency::remove_lock(T::ModuleId::get(), &who); } /// Submit oneself for candidacy. @@ -548,7 +551,7 @@ decl_module! { /// - Independent of input. /// - Three DB changes. /// # - #[weight = SimpleDispatchInfo::FixedNormal(2_500_000)] + #[weight = 2_500_000_000] fn submit_candidacy(origin, #[compact] slot: u32) { let who = ensure_signed(origin)?; @@ -585,12 +588,12 @@ decl_module! { /// - O(voters) compute. /// - One DB change. /// # - #[weight = SimpleDispatchInfo::FixedNormal(10_000_000)] + #[weight = 10_000_000_000] fn present_winner( origin, candidate: ::Source, #[compact] total: BalanceOf, - #[compact] index: VoteIndex + #[compact] index: VoteIndex, ) -> DispatchResult { let who = ensure_signed(origin)?; ensure!( @@ -659,7 +662,7 @@ decl_module! { /// Set the desired member count; if lower than the current count, then seats will not be up /// election when they expire. If more, then a new vote will be started if one is not /// already in progress. - #[weight = SimpleDispatchInfo::FixedOperational(10_000)] + #[weight = (0, DispatchClass::Operational)] fn set_desired_seats(origin, #[compact] count: u32) { ensure_root(origin)?; DesiredSeats::put(count); @@ -669,7 +672,7 @@ decl_module! { /// /// Note: A tally should happen instantly (if not already in a presentation /// period) to fill the seat if removal means that the desired members are not met. - #[weight = SimpleDispatchInfo::FixedOperational(10_000)] + #[weight = (0, DispatchClass::Operational)] fn remove_member(origin, who: ::Source) { ensure_root(origin)?; let who = T::Lookup::lookup(who)?; @@ -684,7 +687,7 @@ decl_module! { /// Set the presentation duration. If there is currently a vote being presented for, will /// invoke `finalize_vote`. - #[weight = SimpleDispatchInfo::FixedOperational(10_000)] + #[weight = (0, DispatchClass::Operational)] fn set_presentation_duration(origin, #[compact] count: T::BlockNumber) { ensure_root(origin)?; >::put(count); @@ -692,7 +695,7 @@ decl_module! { /// Set the presentation duration. If there is current a vote being presented for, will /// invoke `finalize_vote`. - #[weight = SimpleDispatchInfo::FixedOperational(10_000)] + #[weight = (0, DispatchClass::Operational)] fn set_term_duration(origin, #[compact] count: T::BlockNumber) { ensure_root(origin)?; >::put(count); @@ -703,7 +706,7 @@ decl_module! { print("Guru meditation"); print(e); } - SimpleDispatchInfo::default().weigh_data(()) + 0 } } } @@ -892,7 +895,7 @@ impl Module { } T::Currency::set_lock( - MODULE_ID, + T::ModuleId::get(), &who, locked_balance, WithdrawReasons::all(), diff --git a/frame/elections/src/mock.rs b/frame/elections/src/mock.rs index 2898be26ca3001200d1806297b4695a655e212ea..c779f22a3203b17660ccaf572cd55e685a38855f 100644 --- a/frame/elections/src/mock.rs +++ b/frame/elections/src/mock.rs @@ -21,7 +21,7 @@ use std::cell::RefCell; use frame_support::{ StorageValue, StorageMap, parameter_types, assert_ok, - traits::{Get, ChangeMembers, Currency}, + traits::{Get, ChangeMembers, Currency, LockIdentifier}, weights::Weight, }; use sp_core::H256; @@ -50,6 +50,9 @@ impl frame_system::Trait for Test { type Event = Event; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -121,6 +124,10 @@ impl ChangeMembers for TestChangeMembers { } } +parameter_types!{ + pub const ElectionModuleId: LockIdentifier = *b"py/elect"; +} + impl elections::Trait for Test { type Event = Event; type Currency = Balances; @@ -138,6 +145,7 @@ impl elections::Trait for Test { type InactiveGracePeriod = InactiveGracePeriod; type VotingPeriod = VotingPeriod; type DecayRatio = DecayRatio; + type ModuleId = ElectionModuleId; } pub type Block = sp_runtime::generic::Block; diff --git a/frame/evm/Cargo.toml b/frame/evm/Cargo.toml index 95b3c24d8796e683c30c60d6a2cb3e3a5ffd4990..eaa5ae3e42baa6b514e4f2e8c50bb23032ebda26 100644 --- a/frame/evm/Cargo.toml +++ b/frame/evm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-evm" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,17 +8,20 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME EVM contracts pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -pallet-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../timestamp" } -pallet-balances = { version = "2.0.0-alpha.5", default-features = false, path = "../balances" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +pallet-timestamp = { version = "2.0.0-dev", default-features = false, path = "../timestamp" } +pallet-balances = { version = "2.0.0-dev", default-features = false, path = "../balances" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } primitive-types = { version = "0.7.0", default-features = false, features = ["rlp"] } rlp = { version = "0.4", default-features = false } evm = { version = "0.16", default-features = false } @@ -42,6 +45,3 @@ std = [ "evm/std", "pallet-timestamp/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/evm/src/lib.rs b/frame/evm/src/lib.rs index a50c545a461c1937d9d58eb06a86ff372558ca66..c7aa8b8d3a443697aad238a3e5ca061615b84740 100644 --- a/frame/evm/src/lib.rs +++ b/frame/evm/src/lib.rs @@ -25,11 +25,10 @@ pub use crate::backend::{Account, Log, Vicinity, Backend}; use sp_std::{vec::Vec, marker::PhantomData}; use frame_support::{ensure, decl_module, decl_storage, decl_event, decl_error}; -use frame_support::weights::{Weight, DispatchClass, FunctionOf}; -use frame_support::traits::{Currency, WithdrawReason, ExistenceRequirement}; +use frame_support::weights::{Weight, DispatchClass, FunctionOf, Pays}; +use frame_support::traits::{Currency, WithdrawReason, ExistenceRequirement, Get}; use frame_system::{self as system, ensure_signed}; use sp_runtime::ModuleId; -use frame_support::weights::SimpleDispatchInfo; use sp_core::{U256, H256, H160, Hasher}; use sp_runtime::{ DispatchResult, traits::{UniqueSaturatedInto, AccountIdConversion, SaturatedConversion}, @@ -39,8 +38,6 @@ use evm::{ExitReason, ExitSucceed, ExitError, Config}; use evm::executor::StackExecutor; use evm::backend::ApplyBackend; -const MODULE_ID: ModuleId = ModuleId(*b"py/ethvm"); - /// Type alias for currency balance. pub type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; @@ -120,6 +117,8 @@ static ISTANBUL_CONFIG: Config = Config::istanbul(); /// EVM module trait pub trait Trait: frame_system::Trait + pallet_timestamp::Trait { + /// The EVM's module id + type ModuleId: Get; /// Calculator for current gas price. type FeeCalculator: FeeCalculator; /// Convert account ID to H160; @@ -190,8 +189,10 @@ decl_module! { fn deposit_event() = default; + const ModuleId: ModuleId = T::ModuleId::get(); + /// Deposit balance from currency/balances module into EVM. - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] fn deposit_balance(origin, value: BalanceOf) { let sender = ensure_signed(origin)?; @@ -212,7 +213,7 @@ decl_module! { } /// Withdraw balance from EVM into currency/balances module. - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] fn withdraw_balance(origin, value: BalanceOf) { let sender = ensure_signed(origin)?; let address = T::ConvertAccountId::convert_account_id(&sender); @@ -236,7 +237,12 @@ decl_module! { } /// Issue an EVM call operation. This is similar to a message call transaction in Ethereum. - #[weight = FunctionOf(|(_, _, _, gas_limit, gas_price, _): (&H160, &Vec, &U256, &u32, &U256, &Option)| (*gas_price).saturated_into::().saturating_mul(*gas_limit), DispatchClass::Normal, true)] + #[weight = FunctionOf( + |(_, _, _, gas_limit, gas_price, _): (&H160, &Vec, &U256, &u32, &U256, &Option)| + (*gas_price).saturated_into::().saturating_mul(*gas_limit as Weight), + DispatchClass::Normal, + Pays::Yes, + )] fn call( origin, target: H160, @@ -267,7 +273,12 @@ decl_module! { /// Issue an EVM create operation. This is similar to a contract creation transaction in /// Ethereum. - #[weight = FunctionOf(|(_, _, gas_limit, gas_price, _): (&Vec, &U256, &u32, &U256, &Option)| (*gas_price).saturated_into::().saturating_mul(*gas_limit), DispatchClass::Normal, true)] + #[weight = FunctionOf( + |(_, _, gas_limit, gas_price, _): (&Vec, &U256, &u32, &U256, &Option)| + (*gas_price).saturated_into::().saturating_mul(*gas_limit as Weight), + DispatchClass::Normal, + Pays::Yes, + )] fn create( origin, init: Vec, @@ -302,7 +313,12 @@ decl_module! { } /// Issue an EVM create2 operation. - #[weight = FunctionOf(|(_, _, _, gas_limit, gas_price, _): (&Vec, &H256, &U256, &u32, &U256, &Option)| (*gas_price).saturated_into::().saturating_mul(*gas_limit), DispatchClass::Normal, true)] + #[weight = FunctionOf( + |(_, _, _, gas_limit, gas_price, _): (&Vec, &H256, &U256, &u32, &U256, &Option)| + (*gas_price).saturated_into::().saturating_mul(*gas_limit as Weight), + DispatchClass::Normal, + Pays::Yes, + )] fn create2( origin, init: Vec, @@ -347,7 +363,7 @@ impl Module { /// This actually does computation. If you need to keep using it, then make sure you cache the /// value and only call this once. pub fn account_id() -> T::AccountId { - MODULE_ID.into_account() + T::ModuleId::get().into_account() } /// Check whether an account is empty. diff --git a/frame/example-offchain-worker/Cargo.toml b/frame/example-offchain-worker/Cargo.toml index 6b3452d01894f30c46fcfefb8c81f7e0125ae9df..30381adb49f171c538e18e9af8e9c64ccf110a2c 100644 --- a/frame/example-offchain-worker/Cargo.toml +++ b/frame/example-offchain-worker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-example-offchain-worker" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "Unlicense" @@ -8,15 +8,18 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME example pallet for offchain worker" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } serde = { version = "1.0.101", optional = true } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } lite-json = { version = "0.1", default-features = false } [features] @@ -32,6 +35,3 @@ std = [ "sp-runtime/std", "sp-std/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/example-offchain-worker/src/lib.rs b/frame/example-offchain-worker/src/lib.rs index e64b3dfa7757c40c1d49bf4d1c99b6ecf881fbc7..e2c00990943340492eb0c81d13c60254340f6018 100644 --- a/frame/example-offchain-worker/src/lib.rs +++ b/frame/example-offchain-worker/src/lib.rs @@ -22,12 +22,12 @@ //! Run `cargo doc --package pallet-example-offchain-worker --open` to view this module's //! documentation. //! -//! - \[`pallet_example_offchain_worker::Trait`](./trait.Trait.html) -//! - \[`Call`](./enum.Call.html) -//! - \[`Module`](./struct.Module.html) +//! - [`pallet_example_offchain_worker::Trait`](./trait.Trait.html) +//! - [`Call`](./enum.Call.html) +//! - [`Module`](./struct.Module.html) //! //! -//! \## Overview +//! ## Overview //! //! In this example we are going to build a very simplistic, naive and definitely NOT //! production-ready oracle for BTC/USD price. @@ -40,22 +40,32 @@ //! one unsigned transaction floating in the network. #![cfg_attr(not(feature = "std"), no_std)] +use frame_system::{ + self as system, + ensure_signed, + ensure_none, + offchain::{ + AppCrypto, CreateSignedTransaction, SendUnsignedTransaction, + SignedPayload, SigningTypes, Signer, SubmitTransaction, + } +}; use frame_support::{ debug, dispatch::DispatchResult, decl_module, decl_storage, decl_event, traits::Get, - weights::SimpleDispatchInfo, }; -use frame_system::{self as system, ensure_signed, ensure_none, offchain}; use sp_core::crypto::KeyTypeId; use sp_runtime::{ + RuntimeDebug, offchain::{http, Duration, storage::StorageValueRef}, traits::Zero, transaction_validity::{ InvalidTransaction, ValidTransaction, TransactionValidity, TransactionSource, + TransactionPriority, }, }; -use sp_std::{vec, vec::Vec}; +use codec::{Encode, Decode}; +use sp_std::vec::Vec; use lite_json::json::JsonValue; #[cfg(test)] @@ -75,18 +85,25 @@ pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"btc!"); /// the types with this pallet-specific identifier. pub mod crypto { use super::KEY_TYPE; - use sp_runtime::app_crypto::{app_crypto, sr25519}; + use sp_runtime::{ + app_crypto::{app_crypto, sr25519}, + traits::Verify, + }; + use sp_core::sr25519::Signature as Sr25519Signature; app_crypto!(sr25519, KEY_TYPE); + + pub struct TestAuthId; + impl frame_system::offchain::AppCrypto<::Signer, Sr25519Signature> for TestAuthId { + type RuntimeAppPublic = Public; + type GenericSignature = sp_core::sr25519::Signature; + type GenericPublic = sp_core::sr25519::Public; + } } /// This pallet's configuration trait -pub trait Trait: frame_system::Trait { - /// The type to sign and submit transactions. - type SubmitSignedTransaction: - offchain::SubmitSignedTransaction::Call>; - /// The type to submit unsigned transactions. - type SubmitUnsignedTransaction: - offchain::SubmitUnsignedTransaction::Call>; +pub trait Trait: CreateSignedTransaction> { + /// The identifier type for an offchain worker. + type AuthorityId: AppCrypto; /// The overarching event type. type Event: From> + Into<::Event>; @@ -106,6 +123,27 @@ pub trait Trait: frame_system::Trait { /// /// This ensures that we only accept unsigned transactions once, every `UnsignedInterval` blocks. type UnsignedInterval: Get; + + /// A configuration for base priority of unsigned transactions. + /// + /// This is exposed so that it can be tuned for particular runtime, when + /// multiple pallets send unsigned transactions. + type UnsignedPriority: Get; +} + +/// Payload used by this example crate to hold price +/// data required to submit a transaction. +#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug)] +pub struct PricePayload { + block_number: BlockNumber, + price: u32, + public: Public, +} + +impl SignedPayload for PricePayload { + fn public(&self) -> T::Public { + self.public.clone() + } } decl_storage! { @@ -150,7 +188,7 @@ decl_module! { /// working and receives (and provides) meaningful data. /// This example is not focused on correctness of the oracle itself, but rather its /// purpose is to showcase offchain worker capabilities. - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] pub fn submit_price(origin, price: u32) -> DispatchResult { // Retrieve sender of the transaction. let who = ensure_signed(origin)?; @@ -175,7 +213,7 @@ decl_module! { /// /// This example is not focused on correctness of the oracle itself, but rather its /// purpose is to showcase offchain worker capabilities. - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] pub fn submit_price_unsigned(origin, _block_number: T::BlockNumber, price: u32) -> DispatchResult { @@ -189,6 +227,22 @@ decl_module! { Ok(()) } + #[weight = 0] + pub fn submit_price_unsigned_with_signed_payload( + origin, + price_payload: PricePayload, + _signature: T::Signature, + ) -> DispatchResult { + // This ensures that the function can only be called via unsigned transaction. + ensure_none(origin)?; + // Add the price to the on-chain list, but mark it as coming from an empty address. + Self::add_price(Default::default(), price_payload.price); + // now increment the block number at which we expect next unsigned transaction. + let current_block = >::block_number(); + >::put(current_block + T::UnsignedInterval::get()); + Ok(()) + } + /// Offchain Worker entry point. /// /// By implementing `fn offchain_worker` within `decl_module!` you declare a new offchain @@ -229,7 +283,9 @@ decl_module! { let should_send = Self::choose_transaction_type(block_number); let res = match should_send { TransactionType::Signed => Self::fetch_price_and_send_signed(), - TransactionType::Unsigned => Self::fetch_price_and_send_unsigned(block_number), + TransactionType::UnsignedForAny => Self::fetch_price_and_send_unsigned_for_any_account(block_number), + TransactionType::UnsignedForAll => Self::fetch_price_and_send_unsigned_for_all_accounts(block_number), + TransactionType::Raw => Self::fetch_price_and_send_raw_unsigned(block_number), TransactionType::None => Ok(()), }; if let Err(e) = res { @@ -241,7 +297,9 @@ decl_module! { enum TransactionType { Signed, - Unsigned, + UnsignedForAny, + UnsignedForAll, + Raw, None, } @@ -304,12 +362,11 @@ impl Module { // transactions in a row. If a strict order is desired, it's better to use // the storage entry for that. (for instance store both block number and a flag // indicating the type of next transaction to send). - let send_signed = block_number % 2.into() == Zero::zero(); - if send_signed { - TransactionType::Signed - } else { - TransactionType::Unsigned - } + let transaction_type = block_number % 3.into(); + if transaction_type == Zero::zero() { TransactionType::Signed } + else if transaction_type == T::BlockNumber::from(1) { TransactionType::UnsignedForAny } + else if transaction_type == T::BlockNumber::from(2) { TransactionType::UnsignedForAll } + else { TransactionType::Raw } }, // We are in the grace period, we should not send a transaction this time. Err(RECENTLY_SENT) => TransactionType::None, @@ -324,44 +381,43 @@ impl Module { /// A helper function to fetch the price and send signed transaction. fn fetch_price_and_send_signed() -> Result<(), &'static str> { - use system::offchain::SubmitSignedTransaction; - // Firstly we check if there are any accounts in the local keystore that are capable of - // signing the transaction. - // If not it doesn't even make sense to make external HTTP requests, since we won't be able - // to put the results back on-chain. - if !T::SubmitSignedTransaction::can_sign() { + use frame_system::offchain::SendSignedTransaction; + + let signer = Signer::::all_accounts(); + if !signer.can_sign() { return Err( "No local accounts available. Consider adding one via `author_insertKey` RPC." )? } - // Make an external HTTP request to fetch the current price. // Note this call will block until response is received. let price = Self::fetch_price().map_err(|_| "Failed to fetch price")?; - // Received price is wrapped into a call to `submit_price` public function of this pallet. - // This means that the transaction, when executed, will simply call that function passing - // `price` as an argument. - let call = Call::submit_price(price); - - // Using `SubmitSignedTransaction` associated type we create and submit a transaction + // Using `send_signed_transaction` associated type we create and submit a transaction // representing the call, we've just created. // Submit signed will return a vector of results for all accounts that were found in the // local keystore with expected `KEY_TYPE`. - let results = T::SubmitSignedTransaction::submit_signed(call); + let results = signer.send_signed_transaction( + |_account| { + // Received price is wrapped into a call to `submit_price` public function of this pallet. + // This means that the transaction, when executed, will simply call that function passing + // `price` as an argument. + Call::submit_price(price) + } + ); + for (acc, res) in &results { match res { - Ok(()) => debug::info!("[{:?}] Submitted price of {} cents", acc, price), - Err(e) => debug::error!("[{:?}] Failed to submit transaction: {:?}", acc, e), + Ok(()) => debug::info!("[{:?}] Submitted price of {} cents", acc.id, price), + Err(e) => debug::error!("[{:?}] Failed to submit transaction: {:?}", acc.id, e), } } Ok(()) } - /// A helper function to fetch the price and send unsigned transaction. - fn fetch_price_and_send_unsigned(block_number: T::BlockNumber) -> Result<(), &'static str> { - use system::offchain::SubmitUnsignedTransaction; + /// A helper function to fetch the price and send a raw unsigned transaction. + fn fetch_price_and_send_raw_unsigned(block_number: T::BlockNumber) -> Result<(), &'static str> { // Make sure we don't fetch the price if unsigned transaction is going to be rejected // anyway. let next_unsigned_at = >::get(); @@ -378,14 +434,101 @@ impl Module { // passing `price` as an argument. let call = Call::submit_price_unsigned(block_number, price); - // Now let's create an unsigned transaction out of this call and submit it to the pool. + // Now let's create a transaction out of this call and submit it to the pool. + // Here we showcase two ways to send an unsigned transaction / unsigned payload (raw) + // // By default unsigned transactions are disallowed, so we need to whitelist this case // by writing `UnsignedValidator`. Note that it's EXTREMELY important to carefuly // implement unsigned validation logic, as any mistakes can lead to opening DoS or spam // attack vectors. See validation logic docs for more details. - T::SubmitUnsignedTransaction::submit_unsigned(call) - .map_err(|()| "Unable to submit unsigned transaction.".into()) + // + SubmitTransaction::>::submit_unsigned_transaction(call.into()) + .map_err(|()| "Unable to submit unsigned transaction.")?; + Ok(()) + } + + /// A helper function to fetch the price, sign payload and send an unsigned transaction + fn fetch_price_and_send_unsigned_for_any_account(block_number: T::BlockNumber) -> Result<(), &'static str> { + // Make sure we don't fetch the price if unsigned transaction is going to be rejected + // anyway. + let next_unsigned_at = >::get(); + if next_unsigned_at > block_number { + return Err("Too early to send unsigned transaction") + } + + // Make an external HTTP request to fetch the current price. + // Note this call will block until response is received. + let price = Self::fetch_price().map_err(|_| "Failed to fetch price")?; + + // Received price is wrapped into a call to `submit_price_unsigned` public function of this + // pallet. This means that the transaction, when executed, will simply call that function + // passing `price` as an argument. + let call = Call::submit_price_unsigned(block_number, price); + + // Now let's create a transaction out of this call and submit it to the pool. + // Here we showcase two ways to send an unsigned transaction with a signed payload + SubmitTransaction::>::submit_unsigned_transaction(call.into()) + .map_err(|()| "Unable to submit unsigned transaction.")?; + + // -- Sign using any account + let (_, result) = Signer::::any_account().send_unsigned_transaction( + |account| PricePayload { + price, + block_number, + public: account.public.clone() + }, + |payload, signature| { + Call::submit_price_unsigned_with_signed_payload(payload, signature) + } + ).ok_or("No local accounts accounts available.")?; + result.map_err(|()| "Unable to submit transaction")?; + + Ok(()) + } + + /// A helper function to fetch the price, sign payload and send an unsigned transaction + fn fetch_price_and_send_unsigned_for_all_accounts(block_number: T::BlockNumber) -> Result<(), &'static str> { + // Make sure we don't fetch the price if unsigned transaction is going to be rejected + // anyway. + let next_unsigned_at = >::get(); + if next_unsigned_at > block_number { + return Err("Too early to send unsigned transaction") + } + + // Make an external HTTP request to fetch the current price. + // Note this call will block until response is received. + let price = Self::fetch_price().map_err(|_| "Failed to fetch price")?; + + // Received price is wrapped into a call to `submit_price_unsigned` public function of this + // pallet. This means that the transaction, when executed, will simply call that function + // passing `price` as an argument. + let call = Call::submit_price_unsigned(block_number, price); + + // Now let's create a transaction out of this call and submit it to the pool. + // Here we showcase two ways to send an unsigned transaction with a signed payload + SubmitTransaction::>::submit_unsigned_transaction(call.into()) + .map_err(|()| "Unable to submit unsigned transaction.")?; + + // -- Sign using all accounts + let transaction_results = Signer::::all_accounts() + .send_unsigned_transaction( + |account| PricePayload { + price, + block_number, + public: account.public.clone() + }, + |payload, signature| { + Call::submit_price_unsigned_with_signed_payload(payload, signature) + } + ); + for (_account_id, result) in transaction_results.into_iter() { + if result.is_err() { + return Err("Unable to submit transaction"); + } + } + + Ok(()) } /// Fetch current price and return the result in cents. @@ -500,6 +643,58 @@ impl Module { Some(prices.iter().fold(0_u32, |a, b| a.saturating_add(*b)) / prices.len() as u32) } } + + fn validate_transaction_parameters( + block_number: &T::BlockNumber, + new_price: &u32, + ) -> TransactionValidity { + // Now let's check if the transaction has any chance to succeed. + let next_unsigned_at = >::get(); + if &next_unsigned_at > block_number { + return InvalidTransaction::Stale.into(); + } + // Let's make sure to reject transactions from the future. + let current_block = >::block_number(); + if ¤t_block < block_number { + return InvalidTransaction::Future.into(); + } + + // We prioritize transactions that are more far away from current average. + // + // Note this doesn't make much sense when building an actual oracle, but this example + // is here mostly to show off offchain workers capabilities, not about building an + // oracle. + let avg_price = Self::average_price() + .map(|price| if &price > new_price { price - new_price } else { new_price - price }) + .unwrap_or(0); + + ValidTransaction::with_tag_prefix("ExampleOffchainWorker") + // We set base priority to 2**20 and hope it's included before any other + // transactions in the pool. Next we tweak the priority depending on how much + // it differs from the current average. (the more it differs the more priority it + // has). + .priority(T::UnsignedPriority::get().saturating_add(avg_price as _)) + // This transaction does not require anything else to go before into the pool. + // In theory we could require `previous_unsigned_at` transaction to go first, + // but it's not necessary in our case. + //.and_requires() + // We set the `provides` tag to be the same as `next_unsigned_at`. This makes + // sure only one transaction produced after `next_unsigned_at` will ever + // get to the transaction pool and will end up in the block. + // We can still have multiple transactions compete for the same "spot", + // and the one with higher priority will replace other one in the pool. + .and_provides(next_unsigned_at) + // The transaction is only valid for next 5 blocks. After that it's + // going to be revalidated by the pool. + .longevity(5) + // It's fine to propagate that transaction to other peers, which means it can be + // created even by nodes that don't produce blocks. + // Note that sometimes it's better to keep it for yourself (if you are the block + // producer), since for instance in some schemes others may copy your solution and + // claim a reward. + .propagate(true) + .build() + } } #[allow(deprecated)] // ValidateUnsigned @@ -516,53 +711,16 @@ impl frame_support::unsigned::ValidateUnsigned for Module { call: &Self::Call, ) -> TransactionValidity { // Firstly let's check that we call the right function. - if let Call::submit_price_unsigned(block_number, new_price) = call { - // Now let's check if the transaction has any chance to succeed. - let next_unsigned_at = >::get(); - if &next_unsigned_at > block_number { - return InvalidTransaction::Stale.into(); - } - // Let's make sure to reject transactions from the future. - let current_block = >::block_number(); - if ¤t_block < block_number { - return InvalidTransaction::Future.into(); + if let Call::submit_price_unsigned_with_signed_payload( + ref payload, ref signature + ) = call { + let signature_valid = SignedPayload::::verify::(payload, signature.clone()); + if !signature_valid { + return InvalidTransaction::BadProof.into(); } - - // We prioritize transactions that are more far away from current average. - // - // Note this doesn't make much sense when building an actual oracle, but this example - // is here mostly to show off offchain workers capabilities, not about building an - // oracle. - let avg_price = Self::average_price() - .map(|price| if &price > new_price { price - new_price } else { new_price - price }) - .unwrap_or(0); - - Ok(ValidTransaction { - // We set base priority to 2**20 to make sure it's included before any other - // transactions in the pool. Next we tweak the priority depending on how much - // it differs from the current average. (the more it differs the more priority it - // has). - priority: (1 << 20) + avg_price as u64, - // This transaction does not require anything else to go before into the pool. - // In theory we could require `previous_unsigned_at` transaction to go first, - // but it's not necessary in our case. - requires: vec![], - // We set the `provides` tag to be the same as `next_unsigned_at`. This makes - // sure only one transaction produced after `next_unsigned_at` will ever - // get to the transaction pool and will end up in the block. - // We can still have multiple transactions compete for the same "spot", - // and the one with higher priority will replace other one in the pool. - provides: vec![codec::Encode::encode(&(KEY_TYPE.0, next_unsigned_at))], - // The transaction is only valid for next 5 blocks. After that it's - // going to be revalidated by the pool. - longevity: 5, - // It's fine to propagate that transaction to other peers, which means it can be - // created even by nodes that don't produce blocks. - // Note that sometimes it's better to keep it for yourself (if you are the block - // producer), since for instance in some schemes others may copy your solution and - // claim a reward. - propagate: true, - }) + Self::validate_transaction_parameters(&payload.block_number, &payload.price) + } else if let Call::submit_price_unsigned(block_number, new_price) = call { + Self::validate_transaction_parameters(block_number, new_price) } else { InvalidTransaction::Call.into() } diff --git a/frame/example-offchain-worker/src/tests.rs b/frame/example-offchain-worker/src/tests.rs index f64503b0a92a84a33fb3d4949e2dd1c575882a8d..ad0ae01d10ea768bb4be9bcc5a48c6762b868110 100644 --- a/frame/example-offchain-worker/src/tests.rs +++ b/frame/example-offchain-worker/src/tests.rs @@ -16,21 +16,25 @@ use crate::*; -use codec::Decode; +use codec::{Encode, Decode}; use frame_support::{ assert_ok, impl_outer_origin, parameter_types, - weights::{GetDispatchInfo, Weight}, + weights::Weight, }; use sp_core::{ H256, offchain::{OffchainExt, TransactionPoolExt, testing}, + sr25519::Signature, testing::KeyStore, traits::KeystoreExt, }; use sp_runtime::{ Perbill, RuntimeAppPublic, testing::{Header, TestXt}, - traits::{BlakeTwo256, IdentityLookup, Extrinsic as ExtrinsicsT}, + traits::{ + BlakeTwo256, IdentityLookup, Extrinsic as ExtrinsicT, + IdentifyAccount, Verify, + }, }; impl_outer_origin! { @@ -40,7 +44,7 @@ impl_outer_origin! { // For testing the module, we construct most of a mock runtime. This means // first constructing a configuration type (`Test`) which `impl`s each of the // configuration traits of modules we want to use. -#[derive(Clone, Eq, PartialEq)] +#[derive(Clone, Eq, PartialEq, Encode, Decode)] pub struct Test; parameter_types! { pub const BlockHashCount: u64 = 250; @@ -61,6 +65,9 @@ impl frame_system::Trait for Test { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -71,22 +78,29 @@ impl frame_system::Trait for Test { } type Extrinsic = TestXt, ()>; -type SubmitTransaction = frame_system::offchain::TransactionSubmitter< - crypto::Public, - Test, - Extrinsic ->; - -impl frame_system::offchain::CreateTransaction for Test { - type Public = sp_core::sr25519::Public; - type Signature = sp_core::sr25519::Signature; - - fn create_transaction>( - call: ::Call, - _public: Self::Public, - _account: ::AccountId, - nonce: ::Index, - ) -> Option<(::Call, ::SignaturePayload)> { +type AccountId = <::Signer as IdentifyAccount>::AccountId; + +impl frame_system::offchain::SigningTypes for Test { + type Public = ::Signer; + type Signature = Signature; +} + +impl frame_system::offchain::SendTransactionTypes for Test where + Call: From, +{ + type OverarchingCall = Call; + type Extrinsic = Extrinsic; +} + +impl frame_system::offchain::CreateSignedTransaction for Test where + Call: From, +{ + fn create_transaction>( + call: Call, + _public: ::Signer, + _account: AccountId, + nonce: u64, + ) -> Option<(Call, ::SignaturePayload)> { Some((call, (nonce, ()))) } } @@ -94,15 +108,16 @@ impl frame_system::offchain::CreateTransaction for Test { parameter_types! { pub const GracePeriod: u64 = 5; pub const UnsignedInterval: u64 = 128; + pub const UnsignedPriority: u64 = 1 << 20; } impl Trait for Test { type Event = (); + type AuthorityId = crypto::TestAuthId; type Call = Call; - type SubmitSignedTransaction = SubmitTransaction; - type SubmitUnsignedTransaction = SubmitTransaction; type GracePeriod = GracePeriod; type UnsignedInterval = UnsignedInterval; + type UnsignedPriority = UnsignedPriority; } type Example = Module; @@ -169,34 +184,135 @@ fn should_submit_signed_transaction_on_chain() { } #[test] -fn should_submit_unsigned_transaction_on_chain() { +fn should_submit_unsigned_transaction_on_chain_for_any_account() { + const PHRASE: &str = "news slush supreme milk chapter athlete soap sausage put clutch what kitten"; let (offchain, offchain_state) = testing::TestOffchainExt::new(); let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + + let keystore = KeyStore::new(); + + keystore.write().sr25519_generate_new( + crate::crypto::Public::ID, + Some(&format!("{}/hunter1", PHRASE)) + ).unwrap(); + let mut t = sp_io::TestExternalities::default(); t.register_extension(OffchainExt::new(offchain)); t.register_extension(TransactionPoolExt::new(pool)); + t.register_extension(KeystoreExt(keystore.clone())); price_oracle_response(&mut offchain_state.write()); + let public_key = keystore.read() + .sr25519_public_keys(crate::crypto::Public::ID) + .get(0) + .unwrap() + .clone(); + + let price_payload = PricePayload { + block_number: 1, + price: 15523, + public: ::Public::from(public_key), + }; + + // let signature = price_payload.sign::().unwrap(); t.execute_with(|| { // when - Example::fetch_price_and_send_unsigned(1).unwrap(); + Example::fetch_price_and_send_unsigned_for_any_account(1).unwrap(); // then let tx = pool_state.write().transactions.pop().unwrap(); - assert!(pool_state.read().transactions.is_empty()); let tx = Extrinsic::decode(&mut &*tx).unwrap(); assert_eq!(tx.signature, None); - assert_eq!(tx.call, Call::submit_price_unsigned(1, 15523)); + if let Call::submit_price_unsigned_with_signed_payload(body, signature) = tx.call { + assert_eq!(body, price_payload); + + let signature_valid = ::Public, + ::BlockNumber + > as SignedPayload>::verify::(&price_payload, signature); + + assert!(signature_valid); + } + }); +} + +#[test] +fn should_submit_unsigned_transaction_on_chain_for_all_accounts() { + const PHRASE: &str = "news slush supreme milk chapter athlete soap sausage put clutch what kitten"; + let (offchain, offchain_state) = testing::TestOffchainExt::new(); + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + + let keystore = KeyStore::new(); + + keystore.write().sr25519_generate_new( + crate::crypto::Public::ID, + Some(&format!("{}/hunter1", PHRASE)) + ).unwrap(); + + let mut t = sp_io::TestExternalities::default(); + t.register_extension(OffchainExt::new(offchain)); + t.register_extension(TransactionPoolExt::new(pool)); + t.register_extension(KeystoreExt(keystore.clone())); + + price_oracle_response(&mut offchain_state.write()); + + let public_key = keystore.read() + .sr25519_public_keys(crate::crypto::Public::ID) + .get(0) + .unwrap() + .clone(); + + let price_payload = PricePayload { + block_number: 1, + price: 15523, + public: ::Public::from(public_key), + }; + + // let signature = price_payload.sign::().unwrap(); + t.execute_with(|| { + // when + Example::fetch_price_and_send_unsigned_for_all_accounts(1).unwrap(); + // then + let tx = pool_state.write().transactions.pop().unwrap(); + let tx = Extrinsic::decode(&mut &*tx).unwrap(); + assert_eq!(tx.signature, None); + if let Call::submit_price_unsigned_with_signed_payload(body, signature) = tx.call { + assert_eq!(body, price_payload); + + let signature_valid = ::Public, + ::BlockNumber + > as SignedPayload>::verify::(&price_payload, signature); + + assert!(signature_valid); + } }); } #[test] -fn weights_work() { - // must have a default weight. - let default_call = >::submit_price(10); - let info = default_call.get_dispatch_info(); - // aka. `let info = as GetDispatchInfo>::get_dispatch_info(&default_call);` - assert_eq!(info.weight, 10_000); +fn should_submit_raw_unsigned_transaction_on_chain() { + let (offchain, offchain_state) = testing::TestOffchainExt::new(); + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + + let keystore = KeyStore::new(); + + let mut t = sp_io::TestExternalities::default(); + t.register_extension(OffchainExt::new(offchain)); + t.register_extension(TransactionPoolExt::new(pool)); + t.register_extension(KeystoreExt(keystore)); + + price_oracle_response(&mut offchain_state.write()); + + t.execute_with(|| { + // when + Example::fetch_price_and_send_raw_unsigned(1).unwrap(); + // then + let tx = pool_state.write().transactions.pop().unwrap(); + assert!(pool_state.read().transactions.is_empty()); + let tx = Extrinsic::decode(&mut &*tx).unwrap(); + assert_eq!(tx.signature, None); + assert_eq!(tx.call, Call::submit_price_unsigned(1, 15523)); + }); } fn price_oracle_response(state: &mut testing::OffchainState) { diff --git a/frame/example/Cargo.toml b/frame/example/Cargo.toml index 4053cc0b1a51a47d17a23566afc36c1798b34988..d12b1e7c83f88c2437fa0287ceef9af9f32c5a62 100644 --- a/frame/example/Cargo.toml +++ b/frame/example/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-example" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "Unlicense" @@ -8,20 +8,23 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME example pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -pallet-balances = { version = "2.0.0-alpha.5", default-features = false, path = "../balances" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +pallet-balances = { version = "2.0.0-dev", default-features = false, path = "../balances" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } -frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core", default-features = false } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core", default-features = false } [features] default = ["std"] @@ -37,6 +40,3 @@ std = [ "sp-std/std" ] runtime-benchmarks = ["frame-benchmarking"] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/example/src/lib.rs b/frame/example/src/lib.rs index e8ce89a863aebbfb6a14d572df1f6a25eab24941..78ff803d37aaf5a52b21b6adc9f1a6c7aefb1dea 100644 --- a/frame/example/src/lib.rs +++ b/frame/example/src/lib.rs @@ -256,16 +256,15 @@ use sp_std::marker::PhantomData; use frame_support::{ dispatch::DispatchResult, decl_module, decl_storage, decl_event, - weights::{ - SimpleDispatchInfo, DispatchInfo, DispatchClass, ClassifyDispatch, WeighData, Weight, - PaysFee, - }, + weights::{DispatchClass, ClassifyDispatch, WeighData, Weight, PaysFee, Pays}, }; use sp_std::prelude::*; use frame_system::{self as system, ensure_signed, ensure_root}; use codec::{Encode, Decode}; use sp_runtime::{ - traits::{SignedExtension, Bounded, SaturatedConversion}, + traits::{ + SignedExtension, Bounded, SaturatedConversion, DispatchInfoOf, + }, transaction_validity::{ ValidTransaction, TransactionValidityError, InvalidTransaction, TransactionValidity, }, @@ -306,8 +305,8 @@ impl ClassifyDispatch<(&BalanceOf,)> for WeightFor } impl PaysFee<(&BalanceOf,)> for WeightForSetDummy { - fn pays_fee(&self, _target: (&BalanceOf,)) -> bool { - true + fn pays_fee(&self, _target: (&BalanceOf,)) -> Pays { + Pays::Yes } } @@ -467,7 +466,7 @@ decl_module! { // weight (a numeric representation of pure execution time and difficulty) of the // transaction and the latter demonstrates the [`DispatchClass`] of the call. A higher // weight means a larger transaction (less of which can be placed in a single block). - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] fn accumulate_dummy(origin, increase_by: T::Balance) -> DispatchResult { // This is a public call, so we ensure that the origin is some signed account. let _sender = ensure_signed(origin)?; @@ -516,13 +515,12 @@ decl_module! { // The signature could also look like: `fn on_initialize()`. // This function could also very well have a weight annotation, similar to any other. The - // only difference being that if it is not annotated, the default is - // `SimpleDispatchInfo::zero()`, which resolves into no weight. + // only difference is that it mut be returned, not annotated. fn on_initialize(_n: T::BlockNumber) -> Weight { // Anything that needs to be done at the start of the block. // We don't do anything here. - SimpleDispatchInfo::default().weigh_data(()) + 0 } // The signature could also look like: `fn on_finalize()` @@ -619,7 +617,6 @@ impl SignedExtension for WatchDummy { // other pallets. type Call = Call; type AdditionalSigned = (); - type DispatchInfo = DispatchInfo; type Pre = (); fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } @@ -628,7 +625,7 @@ impl SignedExtension for WatchDummy { &self, _who: &Self::AccountId, call: &Self::Call, - _info: Self::DispatchInfo, + _info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { // if the transaction is too big, just drop it. @@ -671,26 +668,41 @@ mod benchmarking { // This will measure the execution time of `set_dummy` for b in [1..1000] range. set_dummy { let b in ...; - let caller = account("caller", 0, 0); - }: set_dummy (RawOrigin::Signed(caller), b.into()) + }: set_dummy (RawOrigin::Root, b.into()) // This will measure the execution time of `set_dummy` for b in [1..10] range. another_set_dummy { let b in 1 .. 10; - let caller = account("caller", 0, 0); - }: set_dummy (RawOrigin::Signed(caller), b.into()) + }: set_dummy (RawOrigin::Root, b.into()) // This will measure the execution time of sorting a vector. sort_vector { let x in 0 .. 10000; let mut m = Vec::::new(); - for i in 0..x { + for i in (0..x).rev() { m.push(i); } }: { m.sort(); } } + + #[cfg(test)] + mod tests { + use super::*; + use crate::tests::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_accumulate_dummy::()); + assert_ok!(test_benchmark_set_dummy::()); + assert_ok!(test_benchmark_another_set_dummy::()); + assert_ok!(test_benchmark_sort_vector::()); + }); + } + } } #[cfg(test)] @@ -698,7 +710,7 @@ mod tests { use super::*; use frame_support::{ - assert_ok, impl_outer_origin, parameter_types, weights::GetDispatchInfo, + assert_ok, impl_outer_origin, parameter_types, weights::{DispatchInfo, GetDispatchInfo}, traits::{OnInitialize, OnFinalize} }; use sp_core::H256; @@ -738,6 +750,9 @@ mod tests { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -764,7 +779,7 @@ mod tests { // This function basically just builds a genesis storage key/value store according to // our desired mockup. - fn new_test_ext() -> sp_io::TestExternalities { + pub fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); // We use default for brevity, but you can configure as desired if needed. pallet_balances::GenesisConfig::::default().assimilate_storage(&mut t).unwrap(); @@ -814,13 +829,13 @@ mod tests { let info = DispatchInfo::default(); assert_eq!( - WatchDummy::(PhantomData).validate(&1, &call, info, 150) + WatchDummy::(PhantomData).validate(&1, &call, &info, 150) .unwrap() .priority, Bounded::max_value(), ); assert_eq!( - WatchDummy::(PhantomData).validate(&1, &call, info, 250), + WatchDummy::(PhantomData).validate(&1, &call, &info, 250), InvalidTransaction::ExhaustsResources.into(), ); }) @@ -828,11 +843,11 @@ mod tests { #[test] fn weights_work() { - // must have a default weight. + // must have a defined weight. let default_call = >::accumulate_dummy(10); let info = default_call.get_dispatch_info(); // aka. `let info = as GetDispatchInfo>::get_dispatch_info(&default_call);` - assert_eq!(info.weight, 10_000); + assert_eq!(info.weight, 0); // must have a custom weight of `100 * arg = 2000` let custom_call = >::set_dummy(20); diff --git a/frame/executive/Cargo.toml b/frame/executive/Cargo.toml index 3c494199cb2374e07a45b1b888d3af52545681b8..3e0e1c938a89ab01d44d6beb7e4575e450fc8b14 100644 --- a/frame/executive/Cargo.toml +++ b/frame/executive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-executive" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,22 +8,27 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME executives engine" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } serde = { version = "1.0.101", optional = true } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-tracing = { version = "2.0.0-dev", default-features = false, path = "../../primitives/tracing" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } [dev-dependencies] hex-literal = "0.2.1" -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-io ={ path = "../../primitives/io", version = "2.0.0-alpha.5"} -pallet-indices = { version = "2.0.0-alpha.5", path = "../indices" } -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } -pallet-transaction-payment = { version = "2.0.0-alpha.5", path = "../transaction-payment" } -sp-version = { version = "2.0.0-alpha.5", path = "../../primitives/version" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-io ={ path = "../../primitives/io", version = "2.0.0-dev"} +pallet-indices = { version = "2.0.0-dev", path = "../indices" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } +pallet-transaction-payment = { version = "2.0.0-dev", path = "../transaction-payment" } +sp-version = { version = "2.0.0-dev", path = "../../primitives/version" } [features] default = ["std"] @@ -33,8 +38,6 @@ std = [ "frame-system/std", "serde", "sp-runtime/std", + "sp-tracing/std", "sp-std/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index d30b66e0837df9af76f7d5ae4da3231b95690c9f..742397b62476364e7a3c366fdb1339c0a31f57e4 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -60,19 +60,57 @@ //! # pub type AllModules = u64; //! # pub enum Runtime {}; //! # use sp_runtime::transaction_validity::{ -//! TransactionValidity, UnknownTransaction, TransactionSource, +//! # TransactionValidity, UnknownTransaction, TransactionSource, //! # }; //! # use sp_runtime::traits::ValidateUnsigned; //! # impl ValidateUnsigned for Runtime { -//! # type Call = (); +//! # type Call = (); //! # -//! # fn validate_unsigned(_source: TransactionSource, _call: &Self::Call) -> TransactionValidity { -//! # UnknownTransaction::NoUnsignedValidator.into() -//! # } +//! # fn validate_unsigned(_source: TransactionSource, _call: &Self::Call) -> TransactionValidity { +//! # UnknownTransaction::NoUnsignedValidator.into() +//! # } //! # } //! /// Executive: handles dispatch to the various modules. //! pub type Executive = executive::Executive; //! ``` +//! +//! ### Custom `OnRuntimeUpgrade` logic +//! +//! You can add custom logic that should be called in your runtime on a runtime upgrade. This is +//! done by setting an optional generic parameter. The custom logic will be called before +//! the on runtime upgrade logic of all modules is called. +//! +//! ``` +//! # use sp_runtime::generic; +//! # use frame_executive as executive; +//! # pub struct UncheckedExtrinsic {}; +//! # pub struct Header {}; +//! # type Context = frame_system::ChainContext; +//! # pub type Block = generic::Block; +//! # pub type Balances = u64; +//! # pub type AllModules = u64; +//! # pub enum Runtime {}; +//! # use sp_runtime::transaction_validity::{ +//! # TransactionValidity, UnknownTransaction, TransactionSource, +//! # }; +//! # use sp_runtime::traits::ValidateUnsigned; +//! # impl ValidateUnsigned for Runtime { +//! # type Call = (); +//! # +//! # fn validate_unsigned(_source: TransactionSource, _call: &Self::Call) -> TransactionValidity { +//! # UnknownTransaction::NoUnsignedValidator.into() +//! # } +//! # } +//! struct CustomOnRuntimeUpgrade; +//! impl frame_support::traits::OnRuntimeUpgrade for CustomOnRuntimeUpgrade { +//! fn on_runtime_upgrade() -> frame_support::weights::Weight { +//! // Do whatever you want. +//! 0 +//! } +//! } +//! +//! pub type Executive = executive::Executive; +//! ``` #![cfg_attr(not(feature = "std"), no_std)] @@ -102,8 +140,19 @@ pub type CheckedOf = >::Checked; pub type CallOf = as Applyable>::Call; pub type OriginOf = as Dispatchable>::Origin; -pub struct Executive( - PhantomData<(System, Block, Context, UnsignedValidator, AllModules)> +/// Main entry point for certain runtime actions as e.g. `execute_block`. +/// +/// Generic parameters: +/// - `System`: Something that implements `frame_system::Trait` +/// - `Block`: The block type of the runtime +/// - `Context`: The context that is used when checking an extrinsic. +/// - `UnsignedValidator`: The unsigned transaction validator of the runtime. +/// - `AllModules`: Tuple that contains all modules. Will be used to call e.g. `on_initialize`. +/// - `OnRuntimeUpgrade`: Custom logic that should be called after a runtime upgrade. Modules are +/// already called by `AllModules`. It will be called before all modules will +/// be called. +pub struct Executive( + PhantomData<(System, Block, Context, UnsignedValidator, AllModules, OnRuntimeUpgrade)> ); impl< @@ -116,13 +165,15 @@ impl< OnInitialize + OnFinalize + OffchainWorker, -> ExecuteBlock for Executive + COnRuntimeUpgrade: OnRuntimeUpgrade, +> ExecuteBlock for + Executive where Block::Extrinsic: Checkable + Codec, CheckedOf: - Applyable + + Applyable + GetDispatchInfo, - CallOf: Dispatchable, + CallOf: Dispatchable, OriginOf: From>, UnsignedValidator: ValidateUnsigned>, { @@ -141,13 +192,14 @@ impl< OnInitialize + OnFinalize + OffchainWorker, -> Executive + COnRuntimeUpgrade: OnRuntimeUpgrade, +> Executive where Block::Extrinsic: Checkable + Codec, CheckedOf: - Applyable + + Applyable + GetDispatchInfo, - CallOf: Dispatchable, + CallOf: Dispatchable, OriginOf: From>, UnsignedValidator: ValidateUnsigned>, { @@ -180,8 +232,9 @@ where ) { if Self::runtime_upgraded() { // System is not part of `AllModules`, so we need to call this manually. - as OnRuntimeUpgrade>::on_runtime_upgrade(); - let weight = ::on_runtime_upgrade(); + let mut weight = as OnRuntimeUpgrade>::on_runtime_upgrade(); + weight = weight.saturating_add(COnRuntimeUpgrade::on_runtime_upgrade()); + weight = weight.saturating_add(::on_runtime_upgrade()); >::register_extra_weight_unchecked(weight); } >::initialize( @@ -192,8 +245,9 @@ where frame_system::InitKind::Full, ); as OnInitialize>::on_initialize(*block_number); - let weight = >::on_initialize(*block_number); - >::register_extra_weight_unchecked(weight); + let weight = >::on_initialize(*block_number) + .saturating_add(>::get()); + >::register_extra_weight_unchecked(weight); frame_system::Module::::note_finished_initialize(); } @@ -237,9 +291,13 @@ where // any initial checks Self::initial_checks(&block); + let batching_safeguard = sp_runtime::SignatureBatching::start(); // execute extrinsics let (header, extrinsics) = block.deconstruct(); Self::execute_extrinsics_with_book_keeping(extrinsics, *header.number()); + if !sp_runtime::SignatureBatching::verify(batching_safeguard) { + panic!("Signature verification failed."); + } // any final checks Self::final_checks(&header); @@ -307,7 +365,7 @@ where // Decode parameters and dispatch let dispatch_info = xt.get_dispatch_info(); - let r = Applyable::apply::(xt, dispatch_info, encoded_len)?; + let r = Applyable::apply::(xt, &dispatch_info, encoded_len)?; >::note_applied_extrinsic(&r, encoded_len as u32, dispatch_info); @@ -344,11 +402,20 @@ where source: TransactionSource, uxt: Block::Extrinsic, ) -> TransactionValidity { - let encoded_len = uxt.using_encoded(|d| d.len()); - let xt = uxt.check(&Default::default())?; + use sp_tracing::tracing_span; - let dispatch_info = xt.get_dispatch_info(); - xt.validate::(source, dispatch_info, encoded_len) + sp_tracing::enter_span!("validate_transaction"); + + let encoded_len = tracing_span!{ "using_encoded"; uxt.using_encoded(|d| d.len()) }; + + let xt = tracing_span!{ "check"; uxt.check(&Default::default())? }; + + let dispatch_info = tracing_span!{ "dispatch_info"; xt.get_dispatch_info() }; + + tracing_span! { + "validate"; + xt.validate::(source, &dispatch_info, encoded_len) + } } /// Start an offchain worker and generate extrinsics. @@ -385,35 +452,36 @@ mod tests { use sp_core::H256; use sp_runtime::{ generic::Era, Perbill, DispatchError, testing::{Digest, Header, Block}, - traits::{Header as HeaderT, BlakeTwo256, IdentityLookup, ConvertInto}, + traits::{Header as HeaderT, BlakeTwo256, IdentityLookup, Convert, ConvertInto}, transaction_validity::{InvalidTransaction, UnknownTransaction, TransactionValidityError}, }; use frame_support::{ impl_outer_event, impl_outer_origin, parameter_types, impl_outer_dispatch, - weights::Weight, + weights::{Weight, RuntimeDbWeight}, traits::{Currency, LockIdentifier, LockableCurrency, WithdrawReasons, WithdrawReason}, }; use frame_system::{self as system, Call as SystemCall, ChainContext, LastRuntimeUpgradeInfo}; use pallet_balances::Call as BalancesCall; use hex_literal::hex; + const TEST_KEY: &[u8] = &*b":test:key:"; mod custom { - use frame_support::weights::{SimpleDispatchInfo, Weight}; + use frame_support::weights::{Weight, DispatchClass}; pub trait Trait: frame_system::Trait {} frame_support::decl_module! { pub struct Module for enum Call where origin: T::Origin { - #[weight = SimpleDispatchInfo::FixedNormal(100)] + #[weight = 100] fn some_function(origin) { // NOTE: does not make any different. let _ = frame_system::ensure_signed(origin); } - #[weight = SimpleDispatchInfo::FixedOperational(200)] + #[weight = (200, DispatchClass::Operational)] fn some_root_operation(origin) { let _ = frame_system::ensure_root(origin); } - #[weight = SimpleDispatchInfo::InsecureFreeNormal] + #[weight = 0] fn some_unsigned_message(origin) { let _ = frame_system::ensure_none(origin); } @@ -428,6 +496,11 @@ mod tests { fn on_finalize() { println!("on_finalize(?)"); } + + fn on_runtime_upgrade() -> Weight { + sp_io::storage::set(super::TEST_KEY, "module".as_bytes()); + 0 + } } } } @@ -462,6 +535,12 @@ mod tests { pub const MaximumBlockWeight: Weight = 1024; pub const MaximumBlockLength: u32 = 2 * 1024; pub const AvailableBlockRatio: Perbill = Perbill::one(); + pub const BlockExecutionWeight: Weight = 10; + pub const ExtrinsicBaseWeight: Weight = 5; + pub const DbWeight: RuntimeDbWeight = RuntimeDbWeight { + read: 10, + write: 100, + }; } impl frame_system::Trait for Runtime { type Origin = Origin; @@ -476,19 +555,24 @@ mod tests { type Event = MetaEvent; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = DbWeight; + type BlockExecutionWeight = BlockExecutionWeight; + type ExtrinsicBaseWeight = ExtrinsicBaseWeight; type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = RuntimeVersion; type ModuleToIndex = (); - type AccountData = pallet_balances::AccountData; + type AccountData = pallet_balances::AccountData; type OnNewAccount = (); type OnKilledAccount = (); } + + type Balance = u64; parameter_types! { - pub const ExistentialDeposit: u64 = 1; + pub const ExistentialDeposit: Balance = 1; } impl pallet_balances::Trait for Runtime { - type Balance = u64; + type Balance = Balance; type Event = MetaEvent; type DustRemoval = (); type ExistentialDeposit = ExistentialDeposit; @@ -496,13 +580,11 @@ mod tests { } parameter_types! { - pub const TransactionBaseFee: u64 = 10; - pub const TransactionByteFee: u64 = 0; + pub const TransactionByteFee: Balance = 0; } impl pallet_transaction_payment::Trait for Runtime { type Currency = Balances; type OnTransactionPayment = (); - type TransactionBaseFee = TransactionBaseFee; type TransactionByteFee = TransactionByteFee; type WeightToFee = ConvertInto; type FeeMultiplierUpdate = (); @@ -543,13 +625,33 @@ mod tests { frame_system::CheckEra, frame_system::CheckNonce, frame_system::CheckWeight, - pallet_transaction_payment::ChargeTransactionPayment + pallet_transaction_payment::ChargeTransactionPayment, ); type AllModules = (System, Balances, Custom); type TestXt = sp_runtime::testing::TestXt; - type Executive = super::Executive, ChainContext, Runtime, AllModules>; - fn extra(nonce: u64, fee: u64) -> SignedExtra { + // Will contain `true` when the custom runtime logic was called. + const CUSTOM_ON_RUNTIME_KEY: &[u8] = &*b":custom:on_runtime"; + + struct CustomOnRuntimeUpgrade; + impl OnRuntimeUpgrade for CustomOnRuntimeUpgrade { + fn on_runtime_upgrade() -> Weight { + sp_io::storage::set(TEST_KEY, "custom_upgrade".as_bytes()); + sp_io::storage::set(CUSTOM_ON_RUNTIME_KEY, &true.encode()); + 0 + } + } + + type Executive = super::Executive< + Runtime, + Block, + ChainContext, + Runtime, + AllModules, + CustomOnRuntimeUpgrade + >; + + fn extra(nonce: u64, fee: Balance) -> SignedExtra { ( frame_system::CheckEra::from(Era::Immortal), frame_system::CheckNonce::from(nonce), @@ -558,7 +660,7 @@ mod tests { ) } - fn sign_extra(who: u64, nonce: u64, fee: u64) -> Option<(u64, SignedExtra)> { + fn sign_extra(who: u64, nonce: u64, fee: Balance) -> Option<(u64, SignedExtra)> { Some((who, extra(nonce, fee))) } @@ -569,7 +671,8 @@ mod tests { balances: vec![(1, 211)], }.assimilate_storage(&mut t).unwrap(); let xt = TestXt::new(Call::Balances(BalancesCall::transfer(2, 69)), sign_extra(1, 0, 0)); - let weight = xt.get_dispatch_info().weight as u64; + let weight = xt.get_dispatch_info().weight + ::ExtrinsicBaseWeight::get(); + let fee: Balance = ::WeightToFee::convert(weight); let mut t = sp_io::TestExternalities::new(t); t.execute_with(|| { Executive::initialize_block(&Header::new( @@ -581,12 +684,12 @@ mod tests { )); let r = Executive::apply_extrinsic(xt); assert!(r.is_ok()); - assert_eq!(>::total_balance(&1), 142 - 10 - weight); + assert_eq!(>::total_balance(&1), 142 - fee); assert_eq!(>::total_balance(&2), 69); }); } - fn new_test_ext(balance_factor: u64) -> sp_io::TestExternalities { + fn new_test_ext(balance_factor: Balance) -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)], @@ -601,7 +704,7 @@ mod tests { header: Header { parent_hash: [69u8; 32].into(), number: 1, - state_root: hex!("489ae9b57a19bb4733a264dc64bbcae9b140a904657a681ed3bb5fbbe8cf412b").into(), + state_root: hex!("409fb5a14aeb8b8c59258b503396a56dee45a0ee28a78de3e622db957425e275").into(), extrinsics_root: hex!("03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314").into(), digest: Digest { logs: vec![], }, }, @@ -669,8 +772,10 @@ mod tests { let xt = TestXt::new(Call::Balances(BalancesCall::transfer(33, 0)), sign_extra(1, 0, 0)); let encoded = xt.encode(); let encoded_len = encoded.len() as Weight; - let limit = AvailableBlockRatio::get() * MaximumBlockWeight::get() - 175; - let num_to_exhaust_block = limit / encoded_len; + // Block execution weight + on_initialize weight + let base_block_weight = 175 + ::BlockExecutionWeight::get(); + let limit = AvailableBlockRatio::get() * MaximumBlockWeight::get() - base_block_weight; + let num_to_exhaust_block = limit / (encoded_len + 5); t.execute_with(|| { Executive::initialize_block(&Header::new( 1, @@ -679,8 +784,8 @@ mod tests { [69u8; 32].into(), Digest::default(), )); - // Initial block weight form the custom module. - assert_eq!(>::all_extrinsics_weight(), 175); + // Base block execution weight + `on_initialize` weight from the custom module. + assert_eq!(>::all_extrinsics_weight(), base_block_weight); for nonce in 0..=num_to_exhaust_block { let xt = TestXt::new( @@ -691,7 +796,8 @@ mod tests { assert!(res.is_ok()); assert_eq!( >::all_extrinsics_weight(), - encoded_len * (nonce + 1) + 175, + //--------------------- on_initialize + block_execution + extrinsic_base weight + (encoded_len + 5) * (nonce + 1) + base_block_weight, ); assert_eq!(>::extrinsic_index(), Some(nonce as u32 + 1)); } else { @@ -717,7 +823,10 @@ mod tests { assert!(Executive::apply_extrinsic(x2.clone()).unwrap().is_ok()); // default weight for `TestXt` == encoded length. - assert_eq!(>::all_extrinsics_weight(), (3 * len) as Weight); + assert_eq!( + >::all_extrinsics_weight(), + 3 * (len as Weight + ::ExtrinsicBaseWeight::get()), + ); assert_eq!(>::all_extrinsics_len(), 3 * len); let _ = >::finalize(); @@ -747,7 +856,7 @@ mod tests { let execute_with_lock = |lock: WithdrawReasons| { let mut t = new_test_ext(1); t.execute_with(|| { - as LockableCurrency>::set_lock( + as LockableCurrency>::set_lock( id, &1, 110, @@ -757,7 +866,9 @@ mod tests { Call::System(SystemCall::remark(vec![1u8])), sign_extra(1, 0, 0), ); - let weight = xt.get_dispatch_info().weight as u64; + let weight = xt.get_dispatch_info().weight + + ::ExtrinsicBaseWeight::get(); + let fee: Balance = ::WeightToFee::convert(weight); Executive::initialize_block(&Header::new( 1, H256::default(), @@ -769,7 +880,7 @@ mod tests { if lock == WithdrawReasons::except(WithdrawReason::TransactionPayment) { assert!(Executive::apply_extrinsic(xt).unwrap().is_ok()); // tx fee has been deducted. - assert_eq!(>::total_balance(&1), 111 - 10 - weight); + assert_eq!(>::total_balance(&1), 111 - fee); } else { assert_eq!( Executive::apply_extrinsic(xt), @@ -789,9 +900,10 @@ mod tests { new_test_ext(1).execute_with(|| { Executive::initialize_block(&Header::new_from_number(1)); - // NOTE: might need updates over time if system and balance introduce new weights. For - // now only accounts for the custom module. - assert_eq!(>::all_extrinsics_weight(), 150 + 25); + // NOTE: might need updates over time if new weights are introduced. + // For now it only accounts for the base block execution weight and + // the `on_initialize` weight defined in the custom test module. + assert_eq!(>::all_extrinsics_weight(), 175 + 10); }) } @@ -866,4 +978,26 @@ mod tests { assert_eq!(result, last.was_upgraded(¤t)); } } + + #[test] + fn custom_runtime_upgrade_is_called_before_modules() { + new_test_ext(1).execute_with(|| { + // Make sure `on_runtime_upgrade` is called. + RUNTIME_VERSION.with(|v| *v.borrow_mut() = sp_version::RuntimeVersion { + spec_version: 1, + ..Default::default() + }); + + Executive::initialize_block(&Header::new( + 1, + H256::default(), + H256::default(), + [69u8; 32].into(), + Digest::default(), + )); + + assert_eq!(&sp_io::storage::get(TEST_KEY).unwrap()[..], *b"module"); + assert_eq!(sp_io::storage::get(CUSTOM_ON_RUNTIME_KEY).unwrap(), true.encode()); + }); + } } diff --git a/frame/finality-tracker/Cargo.toml b/frame/finality-tracker/Cargo.toml index e261fae05f5afd4bda7e69a0fc26e3d052890488..1b11fbea5ad585a1a34bfa08c22b31d9c8729869 100644 --- a/frame/finality-tracker/Cargo.toml +++ b/frame/finality-tracker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-finality-tracker" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,21 +9,24 @@ repository = "https://github.com/paritytech/substrate/" description = "FRAME Pallet that tracks the last finalized block, as perceived by block authors." documentation = "https://docs.rs/pallet-finality-tracker" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", default-features = false, features = ["derive"] } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-finality-tracker = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/finality-tracker" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../../primitives/inherents" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-finality-tracker = { version = "2.0.0-dev", default-features = false, path = "../../primitives/finality-tracker" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } impl-trait-for-tuples = "0.1.3" [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } [features] default = ["std"] @@ -37,6 +40,3 @@ std = [ "sp-finality-tracker/std", "sp-inherents/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/finality-tracker/src/lib.rs b/frame/finality-tracker/src/lib.rs index 8200543ffa17136173346a5bf0fb6f9ece5277e2..ac306e268998ff925248ff21bc4c72960330ba0a 100644 --- a/frame/finality-tracker/src/lib.rs +++ b/frame/finality-tracker/src/lib.rs @@ -23,6 +23,7 @@ use sp_runtime::traits::{One, Zero, SaturatedConversion}; use sp_std::{prelude::*, result, cmp, vec}; use frame_support::{decl_module, decl_storage, decl_error, ensure}; use frame_support::traits::Get; +use frame_support::weights::{DispatchClass}; use frame_system::{ensure_none, Trait as SystemTrait}; use sp_finality_tracker::{INHERENT_IDENTIFIER, FinalizedInherentData}; @@ -76,7 +77,7 @@ decl_module! { /// Hint that the author of this block thinks the best finalized /// block is the given number. - #[weight = frame_support::weights::SimpleDispatchInfo::FixedMandatory(10_000)] + #[weight = (0, DispatchClass::Mandatory)] fn final_hint(origin, #[compact] hint: T::BlockNumber) { ensure_none(origin)?; ensure!(!::Update::exists(), Error::::AlreadyUpdated); @@ -211,7 +212,9 @@ mod tests { traits::{BlakeTwo256, IdentityLookup, Header as HeaderT}, }; use frame_support::{ - assert_ok, impl_outer_origin, parameter_types, weights::Weight, traits::OnFinalize + assert_ok, impl_outer_origin, parameter_types, + weights::Weight, + traits::OnFinalize, }; use frame_system as system; use std::cell::RefCell; @@ -260,6 +263,9 @@ mod tests { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); diff --git a/frame/generic-asset/Cargo.toml b/frame/generic-asset/Cargo.toml index b531a0ed9af24be72d5146a0d06ada267521a8e8..c19a7884b3eadb802d25f73fbb4f46ccf90e601c 100644 --- a/frame/generic-asset/Cargo.toml +++ b/frame/generic-asset/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-generic-asset" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Centrality Developers "] edition = "2018" license = "GPL-3.0" @@ -8,17 +8,20 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for generic asset management" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-io ={ version = "2.0.0-alpha.5", path = "../../primitives/io" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } +sp-io ={ version = "2.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } [features] default = ["std"] @@ -30,6 +33,3 @@ std =[ "frame-support/std", "frame-system/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/generic-asset/src/lib.rs b/frame/generic-asset/src/lib.rs index b16666cb6b7e1c5dd96616a38035d141a70f371d..f2507669e537866bc58a54eb1451c4d4ddbc185c 100644 --- a/frame/generic-asset/src/lib.rs +++ b/frame/generic-asset/src/lib.rs @@ -360,14 +360,14 @@ decl_module! { fn deposit_event() = default; /// Create a new kind of asset. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn create(origin, options: AssetOptions) -> DispatchResult { let origin = ensure_signed(origin)?; Self::create_asset(None, Some(origin), options) } /// Transfer some liquid free balance to another account. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn transfer(origin, #[compact] asset_id: T::AssetId, to: T::AccountId, #[compact] amount: T::Balance) { let origin = ensure_signed(origin)?; ensure!(!amount.is_zero(), Error::::ZeroAmount); @@ -377,7 +377,7 @@ decl_module! { /// Updates permission for a given `asset_id` and an account. /// /// The `origin` must have `update` permission. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn update_permission( origin, #[compact] asset_id: T::AssetId, @@ -400,7 +400,7 @@ decl_module! { /// Mints an asset, increases its total issuance. /// The origin must have `mint` permissions. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn mint(origin, #[compact] asset_id: T::AssetId, to: T::AccountId, amount: T::Balance) -> DispatchResult { let who = ensure_signed(origin)?; Self::mint_free(&asset_id, &who, &to, &amount)?; @@ -410,7 +410,7 @@ decl_module! { /// Burns an asset, decreases its total issuance. /// The `origin` must have `burn` permissions. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn burn(origin, #[compact] asset_id: T::AssetId, to: T::AccountId, amount: T::Balance) -> DispatchResult { let who = ensure_signed(origin)?; Self::burn_free(&asset_id, &who, &to, &amount)?; @@ -420,7 +420,7 @@ decl_module! { /// Can be used to create reserved tokens. /// Requires Root call. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn create_reserved( origin, asset_id: T::AssetId, @@ -1124,6 +1124,9 @@ impl frame_system::Trait for ElevatedTrait { type Event = (); type BlockHashCount = T::BlockHashCount; type MaximumBlockWeight = T::MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = T::MaximumBlockLength; type AvailableBlockRatio = T::AvailableBlockRatio; type Version = T::Version; diff --git a/frame/generic-asset/src/mock.rs b/frame/generic-asset/src/mock.rs index c805b793bc7b4e358e8c8f2b4b5045a8c14dd360..3e3bd892d56880f2a3d512e1120cf71590f513c0 100644 --- a/frame/generic-asset/src/mock.rs +++ b/frame/generic-asset/src/mock.rs @@ -57,6 +57,9 @@ impl frame_system::Trait for Test { type Header = Header; type Event = TestEvent; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type BlockHashCount = BlockHashCount; diff --git a/frame/grandpa/Cargo.toml b/frame/grandpa/Cargo.toml index 206b563bd9555c9f163d82473788a8baf3ecc275..2e1dbc4deba4ceb9e86233fe020748bee49e94ec 100644 --- a/frame/grandpa/Cargo.toml +++ b/frame/grandpa/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-grandpa" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,21 +8,24 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for GRANDPA finality gadget" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -sp-finality-grandpa = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/finality-grandpa" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-staking = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/staking" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -pallet-session = { version = "2.0.0-alpha.5", default-features = false, path = "../session" } -pallet-finality-tracker = { version = "2.0.0-alpha.5", default-features = false, path = "../finality-tracker" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-finality-grandpa = { version = "2.0.0-dev", default-features = false, path = "../../primitives/finality-grandpa" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "2.0.0-dev", default-features = false, path = "../../primitives/staking" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +pallet-session = { version = "2.0.0-dev", default-features = false, path = "../session" } +pallet-finality-tracker = { version = "2.0.0-dev", default-features = false, path = "../finality-tracker" } [dev-dependencies] -sp-io ={ version = "2.0.0-alpha.5", path = "../../primitives/io" } +sp-io ={ version = "2.0.0-dev", path = "../../primitives/io" } [features] default = ["std"] @@ -39,6 +42,3 @@ std = [ "pallet-session/std", "pallet-finality-tracker/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/grandpa/src/lib.rs b/frame/grandpa/src/lib.rs index 030699b52587262eef974a923fd4074a7fb043e3..aa5db8849fe3b79c1894461c2ce45f12da8a70cc 100644 --- a/frame/grandpa/src/lib.rs +++ b/frame/grandpa/src/lib.rs @@ -184,7 +184,7 @@ decl_module! { fn deposit_event() = default; /// Report some misbehavior. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn report_misbehavior(origin, _report: Vec) { ensure_signed(origin)?; // FIXME: https://github.com/paritytech/substrate/issues/1112 diff --git a/frame/grandpa/src/mock.rs b/frame/grandpa/src/mock.rs index 8b94becd5aa6a123f846325b20e6e9632e8d67a6..3ef78e757118480fe4528c97aa86b38ef26ab822 100644 --- a/frame/grandpa/src/mock.rs +++ b/frame/grandpa/src/mock.rs @@ -61,6 +61,9 @@ impl frame_system::Trait for Test { type Event = TestEvent; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/identity/Cargo.toml b/frame/identity/Cargo.toml index 22b385d06d76f36677a50b2afcbe51ad564e970f..f20a8c983df24f126bd64da20590e46dc6f56517 100644 --- a/frame/identity/Cargo.toml +++ b/frame/identity/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-identity" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,20 +8,23 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME identity management pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } enumflags2 = { version = "0.6.2" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../benchmarking", optional = true } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } [features] default = ["std"] @@ -36,6 +39,3 @@ std = [ "frame-system/std", ] runtime-benchmarks = ["frame-benchmarking"] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/identity/src/benchmarking.rs b/frame/identity/src/benchmarking.rs index b5236e62191ae08bb8a301ce1f9dd85649edfbe3..81a9f3e1340cf82458adfd5f71562ecc76afbdef 100644 --- a/frame/identity/src/benchmarking.rs +++ b/frame/identity/src/benchmarking.rs @@ -27,9 +27,6 @@ use sp_runtime::traits::Bounded; use crate::Module as Identity; -// The maximum number of identity registrars we will test. -const MAX_REGISTRARS: u32 = 50; - // Support Functions fn account(name: &'static str, index: u32) -> T::AccountId { let entropy = (name, index).using_encoded(blake2_256); @@ -53,9 +50,9 @@ fn add_registrars(r: u32) -> Result<(), &'static str> { Ok(()) } -// Adds `s` sub-accounts to the identity of `who`. Each wil have 32 bytes of raw data added to it. -// This additionally returns the vector of sub-accounts to it can be modified if needed. -fn add_sub_accounts(who: &T::AccountId, s: u32) -> Result, &'static str> { +// Create `s` sub-accounts for the identity of `who` and return them. +// Each will have 32 bytes of raw data added to it. +fn create_sub_accounts(who: &T::AccountId, s: u32) -> Result, &'static str> { let mut subs = Vec::new(); let who_origin = RawOrigin::Signed(who.clone()); let data = Data::Raw(vec![0; 32]); @@ -70,9 +67,18 @@ fn add_sub_accounts(who: &T::AccountId, s: u32) -> Result(1); Identity::::set_identity(who_origin.clone().into(), info)?; + Ok(subs) +} + +// Adds `s` sub-accounts to the identity of `who`. Each will have 32 bytes of raw data added to it. +// This additionally returns the vector of sub-accounts so it can be modified if needed. +fn add_sub_accounts(who: &T::AccountId, s: u32) -> Result, &'static str> { + let who_origin = RawOrigin::Signed(who.clone()); + let subs = create_sub_accounts::(who, s)?; + Identity::::set_subs(who_origin.into(), subs.clone())?; - return Ok(subs) + Ok(subs) } // This creates an `IdentityInfo` object with `num_fields` extra fields. @@ -98,7 +104,9 @@ fn create_identity_info(num_fields: u32) -> IdentityInfo { benchmarks! { // These are the common parameters along with their instancing. _ { - let r in 1 .. MAX_REGISTRARS => add_registrars::(r)?; + let r in 1 .. T::MaxRegistrars::get() => add_registrars::(r)?; + // extra parameter for the set_subs bench for previous sub accounts + let p in 1 .. T::MaxSubAccounts::get() => (); let s in 1 .. T::MaxSubAccounts::get() => { // Give them s many sub accounts let caller = account::("caller", 0); @@ -114,7 +122,7 @@ benchmarks! { } add_registrar { - let r in ...; + let r in 1 .. T::MaxRegistrars::get() - 1 => add_registrars::(r)?; }: _(RawOrigin::Root, account::("registrar", r + 1)) set_identity { @@ -153,16 +161,13 @@ benchmarks! { set_subs { let caller = account::("caller", 0); - // Give them s many sub accounts. - let s in 1 .. T::MaxSubAccounts::get() - 1 => { - let _ = add_sub_accounts::(&caller, s)?; + // Give them p many previous sub accounts. + let p in 1 .. T::MaxSubAccounts::get() => { + let _ = add_sub_accounts::(&caller, p)?; }; - - let mut subs = Module::::subs(&caller); - - // Create an s + 1 sub account. - let data = Data::Raw(vec![0; 32]); - subs.push((account::("sub", s + 1), data)); + // Create a new subs vec with s sub accounts + let s in 1 .. T::MaxSubAccounts::get() => (); + let subs = create_sub_accounts::(&caller, s)?; }: _(RawOrigin::Signed(caller), subs) @@ -210,7 +215,7 @@ benchmarks! { set_fee { let caller = account::("caller", 0); - let r in ...; + let r in 1 .. T::MaxRegistrars::get() - 1 => add_registrars::(r)?; Identity::::add_registrar(RawOrigin::Root.into(), caller.clone())?; }: _(RawOrigin::Signed(caller), r, 10.into()) @@ -219,7 +224,7 @@ benchmarks! { let caller = account::("caller", 0); let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); - let r in ...; + let r in 1 .. T::MaxRegistrars::get() - 1 => add_registrars::(r)?; Identity::::add_registrar(RawOrigin::Root.into(), caller.clone())?; }: _(RawOrigin::Signed(caller), r, account::("new", 0)) @@ -228,7 +233,7 @@ benchmarks! { let caller = account::("caller", 0); let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); - let r in ...; + let r in 1 .. T::MaxRegistrars::get() - 1 => add_registrars::(r)?; Identity::::add_registrar(RawOrigin::Root.into(), caller.clone())?; let fields = IdentityFields( @@ -247,7 +252,7 @@ benchmarks! { let caller = account::("caller", 0); let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); - let r in ...; + let r in 1 .. T::MaxRegistrars::get() - 1 => add_registrars::(r)?; // For this x, it's the user identity that gts the fields, not the caller. let x in _ .. _ => { let info = create_identity_info::(x); @@ -280,3 +285,27 @@ benchmarks! { } }: _(RawOrigin::Root, caller_lookup) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_add_registrar::()); + assert_ok!(test_benchmark_set_identity::()); + assert_ok!(test_benchmark_set_subs::()); + assert_ok!(test_benchmark_clear_identity::()); + assert_ok!(test_benchmark_request_judgement::()); + assert_ok!(test_benchmark_cancel_request::()); + assert_ok!(test_benchmark_set_fee::()); + assert_ok!(test_benchmark_set_account_id::()); + assert_ok!(test_benchmark_set_fields::()); + assert_ok!(test_benchmark_provide_judgement::()); + assert_ok!(test_benchmark_kill_identity::()); + }); + } +} diff --git a/frame/identity/src/lib.rs b/frame/identity/src/lib.rs index e18689001bec65145043179ea3673c2ce65b1a33..ad13d536c01d7249d682224201ac73fde5180623 100644 --- a/frame/identity/src/lib.rs +++ b/frame/identity/src/lib.rs @@ -69,12 +69,13 @@ use sp_std::prelude::*; use sp_std::{fmt::Debug, ops::Add, iter::once}; use enumflags2::BitFlags; use codec::{Encode, Decode}; -use sp_runtime::{DispatchResult, RuntimeDebug}; +use sp_runtime::{DispatchError, RuntimeDebug}; use sp_runtime::traits::{StaticLookup, Zero, AppendZerosInput}; use frame_support::{ decl_module, decl_event, decl_storage, ensure, decl_error, + dispatch::DispatchResultWithPostInfo, traits::{Currency, ReservableCurrency, OnUnbalanced, Get, BalanceStatus, EnsureOrigin}, - weights::SimpleDispatchInfo, + weights::Weight, }; use frame_system::{self as system, ensure_signed, ensure_root}; @@ -108,6 +109,10 @@ pub trait Trait: frame_system::Trait { /// required to access an identity, but can be pretty high. type MaxAdditionalFields: Get; + /// Maxmimum number of registrars allowed in the system. Needed to bound the complexity + /// of, e.g., updating judgements. + type MaxRegistrars: Get; + /// What to do with slashed funds. type Slashed: OnUnbalanced>; @@ -451,12 +456,132 @@ decl_error! { InvalidTarget, /// Too many additional fields. TooManyFields, + /// Maximum amount of registrars reached. Cannot add any more. + TooManyRegistrars, +} } + +/// Functions for calcuating the weight of dispatchables. +mod weight_for { + use frame_support::weights::{RuntimeDbWeight, Weight}; + + /// Weight calculation for `set_identity`. + pub(crate) fn set_identity( + db: RuntimeDbWeight, + judgements: impl Into, + extra_fields: impl Into + ) -> Weight { + db.reads_writes(1, 1) + + 150_000_000 // constant + + 700_000 * judgements.into() // R + + 3_000_000 * extra_fields.into() // X + } + + /// Weight calculation for `set_subs`. + pub(crate) fn set_subs( + db: RuntimeDbWeight, + old_subs: impl Into + Copy, + subs: impl Into + Copy + ) -> Weight { + db.reads(1) // storage-exists (`IdentityOf::contains_key`) + + db.reads_writes(1, old_subs.into()) // `SubsOf::get` read + P old DB deletions + + db.writes(subs.into() + 1) // S + 1 new DB writes + + 130_000_000 // constant + + 5_200_000 * old_subs.into() // P + + 7_300_000 * subs.into() // S + } + + /// Weight calculation for `clear_identity`. + pub(crate) fn clear_identity( + db: RuntimeDbWeight, + judgements: impl Into, + subs: impl Into + Copy, + extra_fields: impl Into + ) -> Weight { + db.reads_writes(2, subs.into() + 2) // S + 2 deletions + + 160_000_000 // constant + + 500_000 * judgements.into() // R + + 5_400_000 * subs.into() // S + + 2_000_000 * extra_fields.into() // X + } + + /// Weight calculation for `request_judgement`. + pub(crate) fn request_judgement( + db: RuntimeDbWeight, + judgements: impl Into, + extra_fields: impl Into + ) -> Weight { + db.reads_writes(2, 1) + + 180_000_000 // constant + + 950_000 * judgements.into() // R + + 3_400_000 * extra_fields.into() // X + } + + /// Weight calculation for `cancel_request`. + pub(crate) fn cancel_request( + db: RuntimeDbWeight, + judgements: impl Into, + extra_fields: impl Into + ) -> Weight { + db.reads_writes(1, 1) + + 150_000_000 // constant + + 600_000 * judgements.into() // R + + 3_600_000 * extra_fields.into() // X + } + + /// Weight calculation for `provide_judgement`. + pub(crate) fn provide_judgement( + db: RuntimeDbWeight, + judgements: impl Into, + extra_fields: impl Into + ) -> Weight { + db.reads_writes(2, 1) + + 120_000_000 // constant + + 1_100_000 * judgements.into() // R + + 3_500_000 * extra_fields.into()// X + } + + /// Weight calculation for `kill_identity`. + pub(crate) fn kill_identity( + db: RuntimeDbWeight, + judgements: impl Into, + subs: impl Into + Copy, + extra_fields: impl Into + ) -> Weight { + db.reads_writes(3, subs.into() + 3) // 2 `take`s + S deletions + + db.reads_writes(1, 1) // balance ops + + 170_000_000 // constant + + 1_200_000 * judgements.into() // R + + 5_400_000 * subs.into() // S + + 2_300_000 * extra_fields.into() // X + } } decl_module! { // Simple declaration of the `Module` type. Lets the macro know what it's working on. pub struct Module for enum Call where origin: T::Origin { + /// The amount held on deposit for a registered identity. + const BasicDeposit: BalanceOf = T::BasicDeposit::get(); + + /// The amount held on deposit per additional field for a registered identity. + const FieldDeposit: BalanceOf = T::FieldDeposit::get(); + + /// The amount held on deposit for a registered subaccount. This should account for the fact + /// that one storage item's value will increase by the size of an account ID, and there will be + /// another trie item whose value is the size of an account ID plus 32 bytes. + const SubAccountDeposit: BalanceOf = T::SubAccountDeposit::get(); + + /// The maximum number of sub-accounts allowed per identified account. + const MaxSubAccounts: u32 = T::MaxSubAccounts::get(); + + /// Maximum number of additional fields that may be stored in an ID. Needed to bound the I/O + /// required to access an identity, but can be pretty high. + const MaxAdditionalFields: u32 = T::MaxAdditionalFields::get(); + + /// Maxmimum number of registrars allowed in the system. Needed to bound the complexity + /// of, e.g., updating judgements. + const MaxRegistrars: u32 = T::MaxRegistrars::get(); + type Error = Error; fn deposit_event() = default; @@ -470,22 +595,32 @@ decl_module! { /// Emits `RegistrarAdded` if successful. /// /// # - /// - `O(R)` where `R` registrar-count (governance-bounded). + /// - `O(R)` where `R` registrar-count (governance-bounded and code-bounded). /// - One storage mutation (codec `O(R)`). /// - One event. + /// - Benchmarks: + /// - 78.71 + R * 0.965 µs (min squares analysis) + /// - 94.28 + R * 0.991 µs (min squares analysis) /// # - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] - fn add_registrar(origin, account: T::AccountId) { + #[weight = T::DbWeight::get().reads_writes(1, 1) + + 95_000_000 // constant + + 1_000_000 * T::MaxRegistrars::get() as Weight // R + ] + fn add_registrar(origin, account: T::AccountId) -> DispatchResultWithPostInfo { T::RegistrarOrigin::try_origin(origin) .map(|_| ()) .or_else(ensure_root)?; - let i = >::mutate(|r| { - r.push(Some(RegistrarInfo { account, fee: Zero::zero(), fields: Default::default() })); - (r.len() - 1) as RegistrarIndex - }); + let (i, registrar_count) = >::try_mutate(|registrars| -> Result<(RegistrarIndex, usize), DispatchError> { + ensure!((registrars.len() as u32) < T::MaxRegistrars::get(), Error::::TooManyRegistrars); + registrars.push(Some(RegistrarInfo { account, fee: Zero::zero(), fields: Default::default() })); + Ok(((registrars.len() - 1) as RegistrarIndex, registrars.len())) + })?; Self::deposit_event(RawEvent::RegistrarAdded(i)); + + Ok(Some(T::DbWeight::get().reads_writes(1, 1) + + 95_000_000 + 1_000_000 * registrar_count as Weight).into()) } /// Set an account's identity information and reserve the appropriate deposit. @@ -493,21 +628,29 @@ decl_module! { /// If the account already has identity information, the deposit is taken as part payment /// for the new deposit. /// - /// The dispatch origin for this call must be _Signed_ and the sender must have a registered - /// identity. + /// The dispatch origin for this call must be _Signed_. /// /// - `info`: The identity information. /// /// Emits `IdentitySet` if successful. /// /// # - /// - `O(X + X' + R)` where `X` additional-field-count (deposit-bounded and code-bounded). - /// - At most two balance operations. + /// - `O(X + X' + R)` + /// - where `X` additional-field-count (deposit-bounded and code-bounded) + /// - where `R` judgements-count (registrar-count-bounded) + /// - One balance reserve operation. /// - One storage mutation (codec-read `O(X' + R)`, codec-write `O(X + R)`). /// - One event. + /// - Benchmarks: + /// - 136.6 + R * 0.62 + X * 2.62 µs (min squares analysis) + /// - 146.2 + R * 0.372 + X * 2.98 µs (min squares analysis) /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] - fn set_identity(origin, info: IdentityInfo) { + #[weight = weight_for::set_identity( + T::DbWeight::get(), + T::MaxRegistrars::get(), // R + T::MaxAdditionalFields::get() // X + )] + fn set_identity(origin, info: IdentityInfo) -> DispatchResultWithPostInfo { let sender = ensure_signed(origin)?; let extra_fields = info.additional.len() as u32; ensure!(extra_fields <= T::MaxAdditionalFields::get(), Error::::TooManyFields); @@ -532,8 +675,15 @@ decl_module! { let _ = T::Currency::unreserve(&sender, old_deposit - id.deposit); } + let judgements = id.judgements.len() as Weight; >::insert(&sender, id); Self::deposit_event(RawEvent::IdentitySet(sender)); + + Ok(Some(weight_for::set_identity( + T::DbWeight::get(), + judgements, // R + extra_fields as Weight // X + )).into()) } /// Set the sub-accounts of the sender. @@ -544,16 +694,28 @@ decl_module! { /// The dispatch origin for this call must be _Signed_ and the sender must have a registered /// identity. /// - /// - `subs`: The identity's sub-accounts. + /// - `subs`: The identity's (new) sub-accounts. /// /// # - /// - `O(S)` where `S` subs-count (hard- and deposit-bounded). - /// - At most two balance operations. - /// - At most O(2 * S + 1) storage mutations; codec complexity `O(1 * S + S * 1)`); - /// one storage-exists. + /// - `O(P + S)` + /// - where `P` old-subs-count (hard- and deposit-bounded). + /// - where `S` subs-count (hard- and deposit-bounded). + /// - At most one balance operations. + /// - DB: + /// - `P + S` storage mutations (codec complexity `O(1)`) + /// - One storage read (codec complexity `O(P)`). + /// - One storage write (codec complexity `O(S)`). + /// - One storage-exists (`IdentityOf::contains_key`). + /// - Benchmarks: + /// - 115.2 + P * 5.11 + S * 6.67 µs (min squares analysis) + /// - 121 + P * 4.852 + S * 7.111 µs (min squares analysis) /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] - fn set_subs(origin, subs: Vec<(T::AccountId, Data)>) { + #[weight = weight_for::set_subs( + T::DbWeight::get(), + T::MaxSubAccounts::get(), // P + subs.len() as Weight // S + )] + fn set_subs(origin, subs: Vec<(T::AccountId, Data)>) -> DispatchResultWithPostInfo { let sender = ensure_signed(origin)?; ensure!(>::contains_key(&sender), Error::::NotFound); ensure!(subs.len() <= T::MaxSubAccounts::get() as usize, Error::::TooManySubAccounts); @@ -576,15 +738,22 @@ decl_module! { >::insert(&id, (sender.clone(), name)); id }).collect::>(); + let new_subs = ids.len() as Weight; if ids.is_empty() { >::remove(&sender); } else { >::insert(&sender, (new_deposit, ids)); } + + Ok(Some(weight_for::set_subs( + T::DbWeight::get(), + old_ids.len() as Weight, // P + new_subs // S + )).into()) } - /// Clear an account's identity info and all sub-account and return all deposits. + /// Clear an account's identity info and all sub-accounts and return all deposits. /// /// Payment: All reserved balances on the account are returned. /// @@ -594,17 +763,29 @@ decl_module! { /// Emits `IdentityCleared` if successful. /// /// # - /// - `O(R + S + X)`. - /// - One balance-reserve operation. - /// - `S + 2` storage deletions. + /// - `O(R + S + X)` + /// - where `R` registrar-count (governance-bounded). + /// - where `S` subs-count (hard- and deposit-bounded). + /// - where `X` additional-field-count (deposit-bounded and code-bounded). + /// - One balance-unreserve operation. + /// - `2` storage reads and `S + 2` storage deletions. /// - One event. + /// - Benchmarks: + /// - 152.3 + R * 0.306 + S * 4.967 + X * 1.697 µs (min squares analysis) + /// - 139.5 + R * 0.466 + S * 5.304 + X * 1.895 µs (min squares analysis) /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] - fn clear_identity(origin) { + #[weight = weight_for::clear_identity( + T::DbWeight::get(), + T::MaxRegistrars::get(), // R + T::MaxSubAccounts::get(), // S + T::MaxAdditionalFields::get() // X + )] + fn clear_identity(origin) -> DispatchResultWithPostInfo { let sender = ensure_signed(origin)?; let (subs_deposit, sub_ids) = >::take(&sender); - let deposit = >::take(&sender).ok_or(Error::::NotNamed)?.total_deposit() + let id = >::take(&sender).ok_or(Error::::NotNamed)?; + let deposit = id.total_deposit() + subs_deposit; for sub in sub_ids.iter() { >::remove(sub); @@ -613,6 +794,13 @@ decl_module! { let _ = T::Currency::unreserve(&sender, deposit.clone()); Self::deposit_event(RawEvent::IdentityCleared(sender, deposit)); + + Ok(Some(weight_for::clear_identity( + T::DbWeight::get(), + id.judgements.len() as Weight, // R + sub_ids.len() as Weight, // S + id.info.additional.len() as Weight // X + )).into()) } /// Request a judgement from a registrar. @@ -627,7 +815,7 @@ decl_module! { /// - `max_fee`: The maximum fee that may be paid. This should just be auto-populated as: /// /// ```nocompile - /// Self::registrars(reg_index).unwrap().fee + /// Self::registrars().get(reg_index).unwrap().fee /// ``` /// /// Emits `JudgementRequested` if successful. @@ -637,12 +825,19 @@ decl_module! { /// - One balance-reserve operation. /// - Storage: 1 read `O(R)`, 1 mutate `O(X + R)`. /// - One event. + /// - Benchmarks: + /// - 154 + R * 0.932 + X * 3.302 µs (min squares analysis) + /// - 172.9 + R * 0.69 + X * 3.304 µs (min squares analysis) /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = weight_for::request_judgement( + T::DbWeight::get(), + T::MaxRegistrars::get(), // R + T::MaxAdditionalFields::get() // X + )] fn request_judgement(origin, #[compact] reg_index: RegistrarIndex, #[compact] max_fee: BalanceOf, - ) { + ) -> DispatchResultWithPostInfo { let sender = ensure_signed(origin)?; let registrars = >::get(); let registrar = registrars.get(reg_index as usize).and_then(Option::as_ref) @@ -662,9 +857,13 @@ decl_module! { T::Currency::reserve(&sender, registrar.fee)?; + let judgements = id.judgements.len() as Weight; + let extra_fields = id.info.additional.len() as Weight; >::insert(&sender, id); Self::deposit_event(RawEvent::JudgementRequested(sender, reg_index)); + + Ok(Some(weight_for::request_judgement(T::DbWeight::get(), judgements, extra_fields)).into()) } /// Cancel a previous request. @@ -683,9 +882,16 @@ decl_module! { /// - One balance-reserve operation. /// - One storage mutation `O(R + X)`. /// - One event. + /// - Benchmarks: + /// - 135.3 + R * 0.574 + X * 3.394 µs (min squares analysis) + /// - 144.3 + R * 0.316 + X * 3.53 µs (min squares analysis) /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] - fn cancel_request(origin, reg_index: RegistrarIndex) { + #[weight = weight_for::cancel_request( + T::DbWeight::get(), + T::MaxRegistrars::get(), // R + T::MaxAdditionalFields::get() // X + )] + fn cancel_request(origin, reg_index: RegistrarIndex) -> DispatchResultWithPostInfo { let sender = ensure_signed(origin)?; let mut id = >::get(&sender).ok_or(Error::::NoIdentity)?; @@ -698,9 +904,13 @@ decl_module! { }; let _ = T::Currency::unreserve(&sender, fee); + let judgements = id.judgements.len() as Weight; + let extra_fields = id.info.additional.len() as Weight; >::insert(&sender, id); Self::deposit_event(RawEvent::JudgementUnrequested(sender, reg_index)); + + Ok(Some(weight_for::request_judgement(T::DbWeight::get(), judgements, extra_fields)).into()) } /// Set the fee required for a judgement to be requested from a registrar. @@ -714,20 +924,29 @@ decl_module! { /// # /// - `O(R)`. /// - One storage mutation `O(R)`. + /// - Benchmarks: + /// - 23.81 + R * 0.774 µs (min squares analysis) /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = T::DbWeight::get().reads_writes(1, 1) + + 24_000_000 // constant + + 780_000 * T::MaxRegistrars::get() as Weight // R + ] fn set_fee(origin, #[compact] index: RegistrarIndex, #[compact] fee: BalanceOf, - ) -> DispatchResult { + ) -> DispatchResultWithPostInfo { let who = ensure_signed(origin)?; - >::mutate(|rs| + let registrars = >::mutate(|rs| -> Result { rs.get_mut(index as usize) .and_then(|x| x.as_mut()) .and_then(|r| if r.account == who { r.fee = fee; Some(()) } else { None }) - .ok_or_else(|| Error::::InvalidIndex.into()) - ) + .ok_or_else(|| DispatchError::from(Error::::InvalidIndex))?; + Ok(rs.len()) + })?; + Ok(Some(T::DbWeight::get().reads_writes(1, 1) + + 24_000_000 + 780_000 * registrars as Weight // R + ).into()) } /// Change the account associated with a registrar. @@ -741,20 +960,28 @@ decl_module! { /// # /// - `O(R)`. /// - One storage mutation `O(R)`. + /// - Benchmark: 24.59 + R * 0.832 µs (min squares analysis) /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = T::DbWeight::get().reads_writes(1, 1) + + 25_000_000 // constant + + 850_000 * T::MaxRegistrars::get() as Weight // R + ] fn set_account_id(origin, #[compact] index: RegistrarIndex, new: T::AccountId, - ) -> DispatchResult { + ) -> DispatchResultWithPostInfo { let who = ensure_signed(origin)?; - >::mutate(|rs| + let registrars = >::mutate(|rs| -> Result { rs.get_mut(index as usize) .and_then(|x| x.as_mut()) .and_then(|r| if r.account == who { r.account = new; Some(()) } else { None }) - .ok_or_else(|| Error::::InvalidIndex.into()) - ) + .ok_or_else(|| DispatchError::from(Error::::InvalidIndex))?; + Ok(rs.len()) + })?; + Ok(Some(T::DbWeight::get().reads_writes(1, 1) + + 25_000_000 + 850_000 * registrars as Weight // R + ).into()) } /// Set the field information for a registrar. @@ -768,20 +995,28 @@ decl_module! { /// # /// - `O(R)`. /// - One storage mutation `O(R)`. + /// - Benchmark: 22.85 + R * 0.853 µs (min squares analysis) /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = T::DbWeight::get().reads_writes(1, 1) + + 23_000_000 // constant + + 860_000 * T::MaxRegistrars::get() as Weight // R + ] fn set_fields(origin, #[compact] index: RegistrarIndex, fields: IdentityFields, - ) -> DispatchResult { + ) -> DispatchResultWithPostInfo { let who = ensure_signed(origin)?; - >::mutate(|rs| + let registrars = >::mutate(|rs| -> Result { rs.get_mut(index as usize) .and_then(|x| x.as_mut()) .and_then(|r| if r.account == who { r.fields = fields; Some(()) } else { None }) - .ok_or_else(|| Error::::InvalidIndex.into()) - ) + .ok_or_else(|| DispatchError::from(Error::::InvalidIndex))?; + Ok(rs.len()) + })?; + Ok(Some(T::DbWeight::get().reads_writes(1, 1) + + 23_000_000 + 860_000 * registrars as Weight // R + ).into()) } /// Provide a judgement for an account's identity. @@ -802,13 +1037,18 @@ decl_module! { /// - Up to one account-lookup operation. /// - Storage: 1 read `O(R)`, 1 mutate `O(R + X)`. /// - One event. + /// - Benchmark: 110.7 + R * 1.066 + X * 3.402 µs (min squares analysis) /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = weight_for::provide_judgement( + T::DbWeight::get(), + T::MaxRegistrars::get(), // R + T::MaxAdditionalFields::get() // X + )] fn provide_judgement(origin, #[compact] reg_index: RegistrarIndex, target: ::Source, judgement: Judgement>, - ) { + ) -> DispatchResultWithPostInfo { let sender = ensure_signed(origin)?; let target = T::Lookup::lookup(target)?; ensure!(!judgement.has_deposit(), Error::::InvalidJudgement); @@ -829,8 +1069,13 @@ decl_module! { } Err(position) => id.judgements.insert(position, item), } + + let judgements = id.judgements.len() as Weight; + let extra_fields = id.info.additional.len() as Weight; >::insert(&target, id); Self::deposit_event(RawEvent::JudgementGiven(target, reg_index)); + + Ok(Some(weight_for::provide_judgement(T::DbWeight::get(), judgements, extra_fields)).into()) } /// Remove an account's identity and sub-account information and slash the deposits. @@ -851,9 +1096,15 @@ decl_module! { /// - One balance-reserve operation. /// - `S + 2` storage mutations. /// - One event. + /// - Benchmark: 167.4 + R * 1.107 + S * 5.343 + X * 2.294 µs (min squares analysis) /// # - #[weight = SimpleDispatchInfo::FixedNormal(100_000)] - fn kill_identity(origin, target: ::Source) { + #[weight = weight_for::kill_identity( + T::DbWeight::get(), + T::MaxRegistrars::get(), // R + T::MaxSubAccounts::get(), // S + T::MaxAdditionalFields::get() // X + )] + fn kill_identity(origin, target: ::Source) -> DispatchResultWithPostInfo { T::ForceOrigin::try_origin(origin) .map(|_| ()) .or_else(ensure_root)?; @@ -862,8 +1113,8 @@ decl_module! { let target = T::Lookup::lookup(target)?; // Grab their deposit (and check that they have one). let (subs_deposit, sub_ids) = >::take(&target); - let deposit = >::take(&target).ok_or(Error::::NotNamed)?.total_deposit() - + subs_deposit; + let id = >::take(&target).ok_or(Error::::NotNamed)?; + let deposit = id.total_deposit() + subs_deposit; for sub in sub_ids.iter() { >::remove(sub); } @@ -871,6 +1122,13 @@ decl_module! { T::Slashed::on_unbalanced(T::Currency::slash_reserved(&target, deposit).0); Self::deposit_event(RawEvent::IdentityKilled(target, deposit)); + + Ok(Some(weight_for::kill_identity( + T::DbWeight::get(), + id.judgements.len() as Weight, // R + sub_ids.len() as Weight, // S + id.info.additional.len() as Weight // X + )).into()) } } } @@ -930,6 +1188,9 @@ mod tests { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -954,6 +1215,7 @@ mod tests { pub const SubAccountDeposit: u64 = 10; pub const MaxSubAccounts: u32 = 2; pub const MaxAdditionalFields: u32 = 2; + pub const MaxRegistrars: u32 = 20; } ord_parameter_types! { pub const One: u64 = 1; @@ -968,6 +1230,7 @@ mod tests { type SubAccountDeposit = SubAccountDeposit; type MaxSubAccounts = MaxSubAccounts; type MaxAdditionalFields = MaxAdditionalFields; + type MaxRegistrars = MaxRegistrars; type RegistrarOrigin = EnsureSignedBy; type ForceOrigin = EnsureSignedBy; } @@ -977,7 +1240,7 @@ mod tests { // This function basically just builds a genesis storage key/value store according to // our desired mockup. - fn new_test_ext() -> sp_io::TestExternalities { + pub fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); // We use default for brevity, but you can configure as desired if needed. pallet_balances::GenesisConfig:: { @@ -1024,6 +1287,20 @@ mod tests { }); } + #[test] + fn amount_of_registrars_is_limited() { + new_test_ext().execute_with(|| { + for i in 1..MaxRegistrars::get() + 1 { + assert_ok!(Identity::add_registrar(Origin::signed(1), i as u64)); + } + let last_registrar = MaxRegistrars::get() as u64 + 1; + assert_noop!( + Identity::add_registrar(Origin::signed(1), last_registrar), + Error::::TooManyRegistrars + ); + }); + } + #[test] fn registration_should_work() { new_test_ext().execute_with(|| { diff --git a/frame/im-online/Cargo.toml b/frame/im-online/Cargo.toml index dabcc45ef166af7fdf5c47e6a86bbc41732017a0..55e5a49d586e5ff1554eec5dbacae7d6668f18f5 100644 --- a/frame/im-online/Cargo.toml +++ b/frame/im-online/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-im-online" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,21 +8,24 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME's I'm online pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/application-crypto" } -pallet-authorship = { version = "2.0.0-alpha.5", default-features = false, path = "../authorship" } +sp-application-crypto = { version = "2.0.0-dev", default-features = false, path = "../../primitives/application-crypto" } +pallet-authorship = { version = "2.0.0-dev", default-features = false, path = "../authorship" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } serde = { version = "1.0.101", optional = true } -pallet-session = { version = "2.0.0-alpha.5", default-features = false, path = "../session" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-staking = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/staking" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +pallet-session = { version = "2.0.0-dev", default-features = false, path = "../session" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "2.0.0-dev", default-features = false, path = "../../primitives/staking" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } -frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../benchmarking", optional = true } [features] default = ["std", "pallet-session/historical"] @@ -41,6 +44,3 @@ std = [ "frame-system/std", ] runtime-benchmarks = ["frame-benchmarking"] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/im-online/src/benchmarking.rs b/frame/im-online/src/benchmarking.rs index 973bd0c36147c6c4c7e38eba55ef2fc490a0d85b..dae2f719b614cc2317d65f7f231fb53c6663d170 100644 --- a/frame/im-online/src/benchmarking.rs +++ b/frame/im-online/src/benchmarking.rs @@ -23,7 +23,7 @@ use super::*; use frame_system::RawOrigin; use frame_benchmarking::benchmarks; use sp_core::offchain::{OpaquePeerId, OpaqueMultiaddr}; -use sp_runtime::traits::{ValidateUnsigned, Zero}; +use sp_runtime::traits::{ValidateUnsigned, Zero, Dispatchable}; use sp_runtime::transaction_validity::TransactionSource; use crate::Module as ImOnline; @@ -49,6 +49,7 @@ pub fn create_heartbeat(k: u32, e: u32) -> network_state, session_index: 0, authority_index: k-1, + validators_len: keys.len() as u32, }; let encoded_heartbeat = input_heartbeat.encode(); @@ -75,50 +76,30 @@ benchmarks! { }: { ImOnline::::validate_unsigned(TransactionSource::InBlock, &call)?; } + + validate_unsigned_and_then_heartbeat { + let k in 1 .. MAX_KEYS; + let e in 1 .. MAX_EXTERNAL_ADDRESSES; + let (input_heartbeat, signature) = create_heartbeat::(k, e)?; + let call = Call::heartbeat(input_heartbeat, signature); + }: { + ImOnline::::validate_unsigned(TransactionSource::InBlock, &call)?; + call.dispatch(RawOrigin::None.into())?; + } } #[cfg(test)] mod tests { - use crate::*; - use super::SelectedBenchmark; - use crate::mock::*; + use super::*; + use crate::mock::{new_test_ext, Runtime}; use frame_support::assert_ok; #[test] - fn test_heartbeat_benchmark() { + fn test_benchmarks() { new_test_ext().execute_with(|| { - let k = 10; - - assert_eq!(ReceivedHeartbeats::iter_prefix(0).count(), 0); - - let selected_benchmark = SelectedBenchmark::heartbeat; - let c = vec![(frame_benchmarking::BenchmarkParameter::k, k)]; - let closure_to_benchmark = - >::instance( - &selected_benchmark, - &c - ).unwrap(); - - assert_ok!(closure_to_benchmark()); - - assert_eq!(ReceivedHeartbeats::iter_prefix(0).count(), 1); - }); - } - - #[test] - fn test_validate_unsigned_benchmark() { - new_test_ext().execute_with(|| { - let k = 10; - - let selected_benchmark = SelectedBenchmark::validate_unsigned; - let c = vec![(frame_benchmarking::BenchmarkParameter::k, k)]; - let closure_to_benchmark = - >::instance( - &selected_benchmark, - &c - ).unwrap(); - - assert_ok!(closure_to_benchmark()); + assert_ok!(test_benchmark_heartbeat::()); + assert_ok!(test_benchmark_validate_unsigned::()); + assert_ok!(test_benchmark_validate_unsigned_and_then_heartbeat::()); }); } } diff --git a/frame/im-online/src/lib.rs b/frame/im-online/src/lib.rs index cbce3095b69b54757ae9f6a0ccd11c454901a2bd..e280a890545c8d115726585abfa311608a392da0 100644 --- a/frame/im-online/src/lib.rs +++ b/frame/im-online/src/lib.rs @@ -50,7 +50,7 @@ //! //! decl_module! { //! pub struct Module for enum Call where origin: T::Origin { -//! #[weight = frame_support::weights::SimpleDispatchInfo::default()] +//! #[weight = 0] //! pub fn is_online(origin, authority_index: u32) -> dispatch::DispatchResult { //! let _sender = ensure_signed(origin)?; //! let _is_online = >::is_online(authority_index); @@ -94,9 +94,13 @@ use sp_staking::{ use frame_support::{ decl_module, decl_event, decl_storage, Parameter, debug, decl_error, traits::Get, + weights::Weight, }; use frame_system::{self as system, ensure_none}; -use frame_system::offchain::SubmitUnsignedTransaction; +use frame_system::offchain::{ + SendTransactionTypes, + SubmitTransaction, +}; pub mod sr25519 { mod app_sr25519 { @@ -217,21 +221,17 @@ pub struct Heartbeat pub session_index: SessionIndex, /// An index of the authority on the list of validators. pub authority_index: AuthIndex, + /// The length of session validator set + pub validators_len: u32, } -pub trait Trait: frame_system::Trait + pallet_session::historical::Trait { +pub trait Trait: SendTransactionTypes> + pallet_session::historical::Trait { /// The identifier type for an authority. type AuthorityId: Member + Parameter + RuntimeAppPublic + Default + Ord; /// The overarching event type. type Event: From> + Into<::Event>; - /// A dispatchable call type. - type Call: From>; - - /// A transaction submitter. - type SubmitTransaction: SubmitUnsignedTransaction::Call>; - /// An expected duration of the session. /// /// This parameter is used to determine the longevity of `heartbeat` transaction @@ -247,6 +247,12 @@ pub trait Trait: frame_system::Trait + pallet_session::historical::Trait { IdentificationTuple, UnresponsivenessOffence>, >; + + /// A configuration for base priority of unsigned transactions. + /// + /// This is exposed so that it can be tuned for particular runtime, when + /// multiple pallets send unsigned transactions. + type UnsignedPriority: Get; } decl_event!( @@ -310,13 +316,30 @@ decl_module! { fn deposit_event() = default; - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + /// # + /// - Complexity: `O(K + E)` where K is length of `Keys` and E is length of + /// `Heartbeat.network_state.external_address` + /// + /// - `O(K)`: decoding of length `K` + /// - `O(E)`: decoding/encoding of length `E` + /// - DbReads: pallet_session `Validators`, pallet_session `CurrentIndex`, `Keys`, + /// `ReceivedHeartbeats` + /// - DbWrites: `ReceivedHeartbeats` + /// # + // NOTE: the weight include cost of validate_unsigned as it is part of the cost to import + // block with such an extrinsic. + #[weight = (310_000_000 + T::DbWeight::get().reads_writes(4, 1)) + .saturating_add(750_000.saturating_mul(heartbeat.validators_len as Weight)) + .saturating_add( + 1_200_000.saturating_mul(heartbeat.network_state.external_addresses.len() as Weight) + ) + ] fn heartbeat( origin, heartbeat: Heartbeat, // since signature verification is done in `validate_unsigned` // we can skip doing it here again. - _signature: ::Signature + _signature: ::Signature, ) { ensure_none(origin)?; @@ -436,9 +459,17 @@ impl Module { } let session_index = >::current_index(); + let validators_len = >::validators().len() as u32; + Ok(Self::local_authority_keys() .map(move |(authority_index, key)| - Self::send_single_heartbeat(authority_index, key, session_index, block_number) + Self::send_single_heartbeat( + authority_index, + key, + session_index, + block_number, + validators_len, + ) )) } @@ -447,7 +478,8 @@ impl Module { authority_index: u32, key: T::AuthorityId, session_index: SessionIndex, - block_number: T::BlockNumber + block_number: T::BlockNumber, + validators_len: u32, ) -> OffchainResult { // A helper function to prepare heartbeat call. let prepare_heartbeat = || -> OffchainResult> { @@ -458,8 +490,11 @@ impl Module { network_state, session_index, authority_index, + validators_len, }; + let signature = key.sign(&heartbeat_data.encode()).ok_or(OffchainErr::FailedSigning)?; + Ok(Call::heartbeat(heartbeat_data, signature)) }; @@ -484,7 +519,7 @@ impl Module { call, ); - T::SubmitTransaction::submit_unsigned(call) + SubmitTransaction::>::submit_unsigned_transaction(call.into()) .map_err(|_| OffchainErr::SubmitTransaction)?; Ok(()) @@ -493,9 +528,18 @@ impl Module { } fn local_authority_keys() -> impl Iterator { - // we run only when a local authority key is configured + // on-chain storage + // + // At index `idx`: + // 1. A (ImOnline) public key to be used by a validator at index `idx` to send im-online + // heartbeats. let authorities = Keys::::get(); + + // local keystore + // + // All `ImOnline` public (+private) keys currently in the local keystore. let mut local_keys = T::AuthorityId::all(); + local_keys.sort(); authorities.into_iter() @@ -557,6 +601,11 @@ impl Module { Keys::::put(keys); } } + + #[cfg(test)] + fn set_keys(keys: Vec) { + Keys::::put(&keys) + } } impl sp_runtime::BoundToRuntimeAppPublic for Module { @@ -623,6 +672,9 @@ impl pallet_session::OneSessionHandler for Module { } } +/// Invalid transaction custom error. Returned when validators_len field in heartbeat is incorrect. +const INVALID_VALIDATORS_LEN: u8 = 10; + impl frame_support::unsigned::ValidateUnsigned for Module { type Call = Call; @@ -644,6 +696,9 @@ impl frame_support::unsigned::ValidateUnsigned for Module { // verify that the incoming (unverified) pubkey is actually an authority id let keys = Keys::::get(); + if keys.len() as u32 != heartbeat.validators_len { + return InvalidTransaction::Custom(INVALID_VALIDATORS_LEN).into(); + } let authority_id = match keys.get(heartbeat.authority_index as usize) { Some(id) => id, None => return InvalidTransaction::BadProof.into(), @@ -658,13 +713,14 @@ impl frame_support::unsigned::ValidateUnsigned for Module { return InvalidTransaction::BadProof.into(); } - Ok(ValidTransaction { - priority: TransactionPriority::max_value(), - requires: vec![], - provides: vec![(current_session, authority_id).encode()], - longevity: TryInto::::try_into(T::SessionDuration::get() / 2.into()).unwrap_or(64_u64), - propagate: true, - }) + ValidTransaction::with_tag_prefix("ImOnline") + .priority(T::UnsignedPriority::get()) + .and_provides((current_session, authority_id)) + .longevity(TryInto::::try_into( + T::SessionDuration::get() / 2.into() + ).unwrap_or(64_u64)) + .propagate(true) + .build() } else { InvalidTransaction::Call.into() } @@ -679,11 +735,11 @@ pub struct UnresponsivenessOffence { /// /// It acts as a time measure for unresponsiveness reports and effectively will always point /// at the end of the session. - session_index: SessionIndex, + pub session_index: SessionIndex, /// The size of the validator set in current session/era. - validator_set_count: u32, + pub validator_set_count: u32, /// Authorities that were unresponsive during the current era. - offenders: Vec, + pub offenders: Vec, } impl Offence for UnresponsivenessOffence { diff --git a/frame/im-online/src/mock.rs b/frame/im-online/src/mock.rs index 3dc0543d8881b7d4154fe9895448038c99a1ad3b..7ba9dd19f91d325e9f25266f703504a4d12640be 100644 --- a/frame/im-online/src/mock.rs +++ b/frame/im-online/src/mock.rs @@ -27,7 +27,6 @@ use sp_runtime::testing::{Header, UintAuthorityId, TestXt}; use sp_runtime::traits::{IdentityLookup, BlakeTwo256, ConvertInto}; use sp_core::H256; use frame_support::{impl_outer_origin, impl_outer_dispatch, parameter_types, weights::Weight}; - use frame_system as system; impl_outer_origin!{ pub enum Origin for Runtime {} @@ -40,7 +39,11 @@ impl_outer_dispatch! { } thread_local! { - pub static VALIDATORS: RefCell>> = RefCell::new(Some(vec![1, 2, 3])); + pub static VALIDATORS: RefCell>> = RefCell::new(Some(vec![ + 1, + 2, + 3, + ])); } pub struct TestSessionManager; @@ -68,7 +71,6 @@ impl pallet_session::historical::SessionManager for TestSessionManager /// An extrinsic type used for tests. pub type Extrinsic = TestXt; -type SubmitTransaction = frame_system::offchain::TransactionSubmitter<(), Call, Extrinsic>; type IdentificationTuple = (u64, u64); type Offence = crate::UnresponsivenessOffence; @@ -90,7 +92,6 @@ pub fn new_test_ext() -> sp_io::TestExternalities { t.into() } - #[derive(Clone, PartialEq, Eq, Debug)] pub struct Runtime; @@ -114,6 +115,9 @@ impl frame_system::Trait for Runtime { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -160,13 +164,23 @@ impl pallet_authorship::Trait for Runtime { type EventHandler = ImOnline; } +parameter_types! { + pub const UnsignedPriority: u64 = 1 << 20; +} + impl Trait for Runtime { type AuthorityId = UintAuthorityId; type Event = (); - type Call = Call; - type SubmitTransaction = SubmitTransaction; type ReportUnresponsiveness = OffenceHandler; type SessionDuration = Period; + type UnsignedPriority = UnsignedPriority; +} + +impl frame_system::offchain::SendTransactionTypes for Runtime where + Call: From, +{ + type OverarchingCall = Call; + type Extrinsic = Extrinsic; } /// Im Online module. @@ -178,5 +192,7 @@ pub fn advance_session() { let now = System::block_number().max(1); System::set_block_number(now + 1); Session::rotate_session(); + let keys = Session::validators().into_iter().map(UintAuthorityId).collect(); + ImOnline::set_keys(keys); assert_eq!(Session::current_index(), (now / Period::get()) as u32); } diff --git a/frame/im-online/src/tests.rs b/frame/im-online/src/tests.rs index c7bf2afcca629f74765c1519abdd063e63dae1f1..2578b5114e21a866cddfa4f7eea04b29c4124602 100644 --- a/frame/im-online/src/tests.rs +++ b/frame/im-online/src/tests.rs @@ -27,7 +27,7 @@ use sp_core::offchain::{ testing::{TestOffchainExt, TestTransactionPoolExt}, }; use frame_support::{dispatch, assert_noop}; -use sp_runtime::testing::UintAuthorityId; +use sp_runtime::{testing::UintAuthorityId, transaction_validity::TransactionValidityError}; #[test] fn test_unresponsiveness_slash_fraction() { @@ -61,15 +61,15 @@ fn should_report_offline_validators() { let block = 1; System::set_block_number(block); // buffer new validators - Session::rotate_session(); + advance_session(); // enact the change and buffer another one let validators = vec![1, 2, 3, 4, 5, 6]; VALIDATORS.with(|l| *l.borrow_mut() = Some(validators.clone())); - Session::rotate_session(); + advance_session(); // when // we end current session and start the next one - Session::rotate_session(); + advance_session(); // then let offences = OFFENCES.with(|l| l.replace(vec![])); @@ -87,9 +87,9 @@ fn should_report_offline_validators() { // should not report when heartbeat is sent for (idx, v) in validators.into_iter().take(4).enumerate() { - let _ = heartbeat(block, 3, idx as u32, v.into()).unwrap(); + let _ = heartbeat(block, 3, idx as u32, v.into(), Session::validators()).unwrap(); } - Session::rotate_session(); + advance_session(); // then let offences = OFFENCES.with(|l| l.replace(vec![])); @@ -111,6 +111,7 @@ fn heartbeat( session_index: u32, authority_index: u32, id: UintAuthorityId, + validators: Vec, ) -> dispatch::DispatchResult { use frame_support::unsigned::ValidateUnsigned; @@ -122,15 +123,20 @@ fn heartbeat( }, session_index, authority_index, + validators_len: validators.len() as u32, }; let signature = id.sign(&heartbeat.encode()).unwrap(); ImOnline::pre_dispatch(&crate::Call::heartbeat(heartbeat.clone(), signature.clone())) - .map_err(|e| <&'static str>::from(e))?; + .map_err(|e| match e { + TransactionValidityError::Invalid(InvalidTransaction::Custom(INVALID_VALIDATORS_LEN)) => + "invalid validators len", + e @ _ => <&'static str>::from(e), + })?; ImOnline::heartbeat( Origin::system(frame_system::RawOrigin::None), heartbeat, - signature + signature, ) } @@ -152,7 +158,7 @@ fn should_mark_online_validator_when_heartbeat_is_received() { assert!(!ImOnline::is_online(2)); // when - let _ = heartbeat(1, 2, 0, 1.into()).unwrap(); + let _ = heartbeat(1, 2, 0, 1.into(), Session::validators()).unwrap(); // then assert!(ImOnline::is_online(0)); @@ -160,7 +166,7 @@ fn should_mark_online_validator_when_heartbeat_is_received() { assert!(!ImOnline::is_online(2)); // and when - let _ = heartbeat(1, 2, 2, 3.into()).unwrap(); + let _ = heartbeat(1, 2, 2, 3.into(), Session::validators()).unwrap(); // then assert!(ImOnline::is_online(0)); @@ -170,11 +176,11 @@ fn should_mark_online_validator_when_heartbeat_is_received() { } #[test] -fn late_heartbeat_should_fail() { +fn late_heartbeat_and_invalid_keys_len_should_fail() { new_test_ext().execute_with(|| { advance_session(); // given - VALIDATORS.with(|l| *l.borrow_mut() = Some(vec![1, 2, 4, 4, 5, 6])); + VALIDATORS.with(|l| *l.borrow_mut() = Some(vec![1, 2, 3, 4, 5, 6])); assert_eq!(Session::validators(), Vec::::new()); // enact the change and buffer another one advance_session(); @@ -183,8 +189,11 @@ fn late_heartbeat_should_fail() { assert_eq!(Session::validators(), vec![1, 2, 3]); // when - assert_noop!(heartbeat(1, 3, 0, 1.into()), "Transaction is outdated"); - assert_noop!(heartbeat(1, 1, 0, 1.into()), "Transaction is outdated"); + assert_noop!(heartbeat(1, 3, 0, 1.into(), Session::validators()), "Transaction is outdated"); + assert_noop!(heartbeat(1, 1, 0, 1.into(), Session::validators()), "Transaction is outdated"); + + // invalid validators_len + assert_noop!(heartbeat(1, 2, 0, 1.into(), vec![]), "invalid validators len"); }); } @@ -220,7 +229,7 @@ fn should_generate_heartbeats() { // check stuff about the transaction. let ex: Extrinsic = Decode::decode(&mut &*transaction).unwrap(); let heartbeat = match ex.call { - crate::mock::Call::ImOnline(crate::Call::heartbeat(h, _)) => h, + crate::mock::Call::ImOnline(crate::Call::heartbeat(h, ..)) => h, e => panic!("Unexpected call: {:?}", e), }; @@ -229,6 +238,7 @@ fn should_generate_heartbeats() { network_state: sp_io::offchain::network_state().unwrap(), session_index: 2, authority_index: 2, + validators_len: 3, }); }); } @@ -248,7 +258,7 @@ fn should_cleanup_received_heartbeats_on_session_end() { assert_eq!(Session::validators(), vec![1, 2, 3]); // send an heartbeat from authority id 0 at session 2 - let _ = heartbeat(1, 2, 0, 1.into()).unwrap(); + let _ = heartbeat(1, 2, 0, 1.into(), Session::validators()).unwrap(); // the heartbeat is stored assert!(!ImOnline::received_heartbeats(&2, &0).is_none()); @@ -315,7 +325,7 @@ fn should_not_send_a_report_if_already_online() { ImOnline::note_uncle(3, 0); // when - UintAuthorityId::set_all_keys(vec![0]); // all authorities use pallet_session key 0 + UintAuthorityId::set_all_keys(vec![1, 2, 3]); // we expect error, since the authority is already online. let mut res = ImOnline::send_heartbeats(4).unwrap(); assert_eq!(res.next().unwrap().unwrap(), ()); @@ -330,7 +340,7 @@ fn should_not_send_a_report_if_already_online() { // check stuff about the transaction. let ex: Extrinsic = Decode::decode(&mut &*transaction).unwrap(); let heartbeat = match ex.call { - crate::mock::Call::ImOnline(crate::Call::heartbeat(h, _)) => h, + crate::mock::Call::ImOnline(crate::Call::heartbeat(h, ..)) => h, e => panic!("Unexpected call: {:?}", e), }; @@ -339,6 +349,7 @@ fn should_not_send_a_report_if_already_online() { network_state: sp_io::offchain::network_state().unwrap(), session_index: 2, authority_index: 0, + validators_len: 3, }); }); } diff --git a/frame/indices/Cargo.toml b/frame/indices/Cargo.toml index f28f393642bedac28e1644f35046d6ad9c1b4a2e..515c0b478ddb426ca754c383a686a0e9a77a715a 100644 --- a/frame/indices/Cargo.toml +++ b/frame/indices/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-indices" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,19 +8,22 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME indices management pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-keyring = { version = "2.0.0-alpha.5", optional = true, path = "../../primitives/keyring" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-keyring = { version = "2.0.0-dev", optional = true, path = "../../primitives/keyring" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } [features] default = ["std"] @@ -35,6 +38,3 @@ std = [ "sp-runtime/std", "frame-system/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/indices/src/lib.rs b/frame/indices/src/lib.rs index d2ba664d425ffce1ef0aac0745fea825d7183a42..4861197eda371199abbd2906ea9635d528402b01 100644 --- a/frame/indices/src/lib.rs +++ b/frame/indices/src/lib.rs @@ -25,7 +25,7 @@ use sp_runtime::traits::{ StaticLookup, Member, LookupError, Zero, One, BlakeTwo256, Hash, Saturating, AtLeast32Bit }; use frame_support::{Parameter, decl_module, decl_error, decl_event, decl_storage, ensure}; -use frame_support::weights::{Weight, SimpleDispatchInfo, WeighData}; +use frame_support::weights::Weight; use frame_support::dispatch::DispatchResult; use frame_support::traits::{Currency, ReservableCurrency, Get, BalanceStatus::Reserved}; use frame_support::storage::migration::take_storage_value; @@ -102,7 +102,7 @@ decl_module! { fn on_initialize() -> Weight { Self::migrations(); - SimpleDispatchInfo::default().weigh_data(()) + 0 } /// Assign an previously unassigned index. @@ -121,7 +121,7 @@ decl_module! { /// - One reserve operation. /// - One event. /// # - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn claim(origin, index: T::AccountIndex) { let who = ensure_signed(origin)?; @@ -149,7 +149,7 @@ decl_module! { /// - One transfer operation. /// - One event. /// # - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn transfer(origin, new: T::AccountId, index: T::AccountIndex) { let who = ensure_signed(origin)?; ensure!(who != new, Error::::NotTransfer); @@ -180,7 +180,7 @@ decl_module! { /// - One reserve operation. /// - One event. /// # - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn free(origin, index: T::AccountIndex) { let who = ensure_signed(origin)?; @@ -209,7 +209,7 @@ decl_module! { /// - Up to one reserve operation. /// - One event. /// # - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn force_transfer(origin, new: T::AccountId, index: T::AccountIndex) { ensure_root(origin)?; diff --git a/frame/indices/src/mock.rs b/frame/indices/src/mock.rs index 355b3cc792c9456a0537c263d0f79a5ec58e7042..aa7057e61a1dce71904cc89c46410a7069059574 100644 --- a/frame/indices/src/mock.rs +++ b/frame/indices/src/mock.rs @@ -61,6 +61,9 @@ impl frame_system::Trait for Test { type Event = MetaEvent; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/membership/Cargo.toml b/frame/membership/Cargo.toml index 41e56b584f550332fe1b91f5763fa31765086d75..742cc124a295a9cb2a8d664aa88eb3883f5834f6 100644 --- a/frame/membership/Cargo.toml +++ b/frame/membership/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-membership" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,17 +8,20 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME membership management pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } [features] default = ["std"] @@ -31,6 +34,3 @@ std = [ "frame-support/std", "frame-system/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/membership/src/lib.rs b/frame/membership/src/lib.rs index 8f086fa2f32114a6b2e17bc8d5f228cfee070d08..faf2be8e11e5eb77f77bd2406f4cc6aec7379407 100644 --- a/frame/membership/src/lib.rs +++ b/frame/membership/src/lib.rs @@ -26,7 +26,6 @@ use sp_std::prelude::*; use frame_support::{ decl_module, decl_storage, decl_event, decl_error, traits::{ChangeMembers, InitializeMembers, EnsureOrigin}, - weights::SimpleDispatchInfo, }; use frame_system::{self as system, ensure_root, ensure_signed}; @@ -118,7 +117,7 @@ decl_module! { /// Add a member `who` to the set. /// /// May only be called from `AddOrigin` or root. - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = 50_000_000] fn add_member(origin, who: T::AccountId) { T::AddOrigin::try_origin(origin) .map(|_| ()) @@ -137,7 +136,7 @@ decl_module! { /// Remove a member `who` from the set. /// /// May only be called from `RemoveOrigin` or root. - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = 50_000_000] fn remove_member(origin, who: T::AccountId) { T::RemoveOrigin::try_origin(origin) .map(|_| ()) @@ -159,7 +158,7 @@ decl_module! { /// May only be called from `SwapOrigin` or root. /// /// Prime membership is *not* passed from `remove` to `add`, if extant. - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = 50_000_000] fn swap_member(origin, remove: T::AccountId, add: T::AccountId) { T::SwapOrigin::try_origin(origin) .map(|_| ()) @@ -188,7 +187,7 @@ decl_module! { /// pass `members` pre-sorted. /// /// May only be called from `ResetOrigin` or root. - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = 50_000_000] fn reset_members(origin, members: Vec) { T::ResetOrigin::try_origin(origin) .map(|_| ()) @@ -211,7 +210,7 @@ decl_module! { /// May only be called from `Signed` origin of a current member. /// /// Prime membership is passed from the origin account to `new`, if extant. - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = 50_000_000] fn change_key(origin, new: T::AccountId) { let remove = ensure_signed(origin)?; @@ -239,7 +238,7 @@ decl_module! { } /// Set the prime member. Must be a current member. - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = 50_000_000] fn set_prime(origin, who: T::AccountId) { T::PrimeOrigin::try_origin(origin) .map(|_| ()) @@ -250,7 +249,7 @@ decl_module! { } /// Remove the prime member if it exists. - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = 50_000_000] fn clear_prime(origin) { T::PrimeOrigin::try_origin(origin) .map(|_| ()) @@ -315,6 +314,9 @@ mod tests { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/metadata/Cargo.toml b/frame/metadata/Cargo.toml index f965e1dddcb09687f4cdc193c62f09e60f4d12e5..73418be9b2b69295cd8cfd088be1161b0cf8e6b8 100644 --- a/frame/metadata/Cargo.toml +++ b/frame/metadata/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-metadata" -version = "11.0.0-alpha.5" +version = "11.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,11 +8,14 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Decodable variant of the RuntimeMetadata." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } [features] default = ["std"] @@ -22,6 +25,3 @@ std = [ "sp-core/std", "serde", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/nicks/Cargo.toml b/frame/nicks/Cargo.toml index ea88021d2528b8897c02a721ffc6fcf64564358d..fcb64731051e0e8d85894571232942e5f6c5f08c 100644 --- a/frame/nicks/Cargo.toml +++ b/frame/nicks/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-nicks" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,18 +8,21 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for nick management" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } [features] default = ["std"] @@ -32,6 +35,3 @@ std = [ "frame-support/std", "frame-system/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/nicks/src/lib.rs b/frame/nicks/src/lib.rs index ae005e2500b85ab3da72aa94746681d1eaa01f3a..8ded9408d92cb1386b979825e83f736d26648436 100644 --- a/frame/nicks/src/lib.rs +++ b/frame/nicks/src/lib.rs @@ -45,7 +45,6 @@ use sp_runtime::{ use frame_support::{ decl_module, decl_event, decl_storage, ensure, decl_error, traits::{Currency, EnsureOrigin, ReservableCurrency, OnUnbalanced, Get}, - weights::SimpleDispatchInfo, }; use frame_system::{self as system, ensure_signed, ensure_root}; @@ -141,7 +140,7 @@ decl_module! { /// - One storage read/write. /// - One event. /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = 50_000_000] fn set_name(origin, name: Vec) { let sender = ensure_signed(origin)?; @@ -171,7 +170,7 @@ decl_module! { /// - One storage read/write. /// - One event. /// # - #[weight = SimpleDispatchInfo::FixedNormal(70_000)] + #[weight = 70_000_000] fn clear_name(origin) { let sender = ensure_signed(origin)?; @@ -195,7 +194,7 @@ decl_module! { /// - One storage read/write. /// - One event. /// # - #[weight = SimpleDispatchInfo::FixedNormal(70_000)] + #[weight = 70_000_000] fn kill_name(origin, target: ::Source) { T::ForceOrigin::try_origin(origin) .map(|_| ()) @@ -223,7 +222,7 @@ decl_module! { /// - One storage read/write. /// - One event. /// # - #[weight = SimpleDispatchInfo::FixedNormal(70_000)] + #[weight = 70_000_000] fn force_name(origin, target: ::Source, name: Vec) { T::ForceOrigin::try_origin(origin) .map(|_| ()) @@ -282,6 +281,9 @@ mod tests { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/offences/Cargo.toml b/frame/offences/Cargo.toml index b858c03ba5ac0b58c038422e8d077ba7c278ddc6..e0759325feba7db1591f38e34823248dc6990e98 100644 --- a/frame/offences/Cargo.toml +++ b/frame/offences/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-offences" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,19 +8,22 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME offences pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -pallet-balances = { version = "2.0.0-alpha.5", default-features = false, path = "../balances" } +pallet-balances = { version = "2.0.0-dev", default-features = false, path = "../balances" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } serde = { version = "1.0.101", optional = true } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-staking = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/staking" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "2.0.0-dev", default-features = false, path = "../../primitives/staking" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-io = { version = "2.0.0-alpha.5", path = "../../primitives/io" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } +sp-io = { version = "2.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } [features] default = ["std"] @@ -34,6 +37,4 @@ std = [ "frame-support/std", "frame-system/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +runtime-benchmarks = [] diff --git a/frame/offences/benchmarking/Cargo.toml b/frame/offences/benchmarking/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..de9d68dc8048587eb6a168acaeb8b0e42cf4614d --- /dev/null +++ b/frame/offences/benchmarking/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "pallet-offences-benchmarking" +version = "2.0.0-dev" +authors = ["Parity Technologies "] +edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME offences pallet benchmarking" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } + +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/std" } +sp-staking = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/staking" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/runtime" } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../../benchmarking" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../../system" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../../support" } +pallet-im-online = { version = "2.0.0-dev", default-features = false, path = "../../im-online" } +pallet-offences = { version = "2.0.0-dev", default-features = false, features = ["runtime-benchmarks"], path = "../../offences" } +pallet-staking = { version = "2.0.0-dev", default-features = false, features = ["runtime-benchmarks"], path = "../../staking" } +pallet-session = { version = "2.0.0-dev", default-features = false, path = "../../session" } +sp-io = { path = "../../../primitives/io", default-features = false, version = "2.0.0-dev"} + + +[features] +default = ["std"] +std = [ + "sp-runtime/std", + "sp-std/std", + "sp-staking/std", + "frame-benchmarking/std", + "frame-support/std", + "frame-system/std", + "pallet-offences/std", + "pallet-im-online/std", + "pallet-staking/std", + "pallet-session/std", +] diff --git a/frame/offences/benchmarking/src/lib.rs b/frame/offences/benchmarking/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..a88714a89a7faf82176d0e1feed568db42f8b285 --- /dev/null +++ b/frame/offences/benchmarking/src/lib.rs @@ -0,0 +1,175 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Offences pallet benchmarking. + +#![cfg_attr(not(feature = "std"), no_std)] + +use sp_std::prelude::*; +use sp_std::vec; + +use frame_system::RawOrigin; +use frame_benchmarking::{benchmarks, account}; +use frame_support::traits::{Currency, OnInitialize}; + +use sp_runtime::{Perbill, traits::{Convert, StaticLookup}}; +use sp_staking::offence::ReportOffence; + +use pallet_im_online::{Trait as ImOnlineTrait, Module as ImOnline, UnresponsivenessOffence}; +use pallet_offences::{Trait as OffencesTrait, Module as Offences}; +use pallet_staking::{ + Module as Staking, Trait as StakingTrait, RewardDestination, ValidatorPrefs, + Exposure, IndividualExposure, ElectionStatus +}; +use pallet_session::Trait as SessionTrait; +use pallet_session::historical::{Trait as HistoricalTrait, IdentificationTuple}; + +const SEED: u32 = 0; + +const MAX_USERS: u32 = 1000; +const MAX_REPORTERS: u32 = 100; +const MAX_OFFENDERS: u32 = 100; +const MAX_NOMINATORS: u32 = 100; +const MAX_DEFERRED_OFFENCES: u32 = 100; + +pub struct Module(Offences); + +pub trait Trait: SessionTrait + StakingTrait + OffencesTrait + ImOnlineTrait + HistoricalTrait {} + +fn create_offender(n: u32, nominators: u32) -> Result { + let stash: T::AccountId = account("stash", n, SEED); + let controller: T::AccountId = account("controller", n, SEED); + let controller_lookup: ::Source = T::Lookup::unlookup(controller.clone()); + let reward_destination = RewardDestination::Staked; + let amount = T::Currency::minimum_balance(); + + Staking::::bond( + RawOrigin::Signed(stash.clone()).into(), + controller_lookup.clone(), + amount.clone(), + reward_destination.clone(), + )?; + + let validator_prefs = ValidatorPrefs { + commission: Perbill::from_percent(50), + }; + Staking::::validate(RawOrigin::Signed(controller.clone()).into(), validator_prefs)?; + + let mut individual_exposures = vec![]; + + // Create n nominators + for i in 0 .. nominators { + let nominator_stash: T::AccountId = account("nominator stash", n * MAX_NOMINATORS + i, SEED); + let nominator_controller: T::AccountId = account("nominator controller", n * MAX_NOMINATORS + i, SEED); + let nominator_controller_lookup: ::Source = T::Lookup::unlookup(nominator_controller.clone()); + + Staking::::bond( + RawOrigin::Signed(nominator_stash.clone()).into(), + nominator_controller_lookup.clone(), + amount, + reward_destination, + )?; + + let selected_validators: Vec<::Source> = vec![controller_lookup.clone()]; + Staking::::nominate(RawOrigin::Signed(nominator_controller.clone()).into(), selected_validators)?; + + individual_exposures.push(IndividualExposure { + who: nominator_controller.clone(), + value: amount.clone(), + }); + } + + let exposure = Exposure { + total: amount.clone() * n.into(), + own: amount, + others: individual_exposures, + }; + let current_era = 0u32; + Staking::::add_era_stakers(current_era.into(), stash.clone().into(), exposure); + + Ok(controller) +} + +fn make_offenders(num_offenders: u32, num_nominators: u32) -> Result>, &'static str> { + let mut offenders: Vec = vec![]; + + for i in 0 .. num_offenders { + let offender = create_offender::(i, num_nominators)?; + offenders.push(offender); + } + + Ok(offenders.iter() + .map(|id| + ::ValidatorIdOf::convert(id.clone()) + .expect("failed to get validator id from account id")) + .map(|validator_id| + ::FullIdentificationOf::convert(validator_id.clone()) + .map(|full_id| (validator_id, full_id)) + .expect("failed to convert validator id to full identification")) + .collect::>>()) +} + +benchmarks! { + _ { + let u in 1 .. MAX_USERS => (); + let r in 1 .. MAX_REPORTERS => (); + let o in 1 .. MAX_OFFENDERS => (); + let n in 1 .. MAX_NOMINATORS => (); + let d in 1 .. MAX_DEFERRED_OFFENCES => (); + } + + report_offence { + let r in ...; + let o in ...; + let n in ...; + + let mut reporters = vec![]; + + for i in 0 .. r { + let reporter = account("reporter", i, SEED); + reporters.push(reporter); + } + + let offenders = make_offenders::(o, n).expect("failed to create offenders"); + let keys = ImOnline::::keys(); + + let offence = UnresponsivenessOffence { + session_index: 0, + validator_set_count: keys.len() as u32, + offenders, + }; + + }: { + let _ = ::ReportUnresponsiveness::report_offence(reporters, offence); + } + + on_initialize { + let d in ...; + + Staking::::put_election_status(ElectionStatus::Closed); + + let mut deferred_offences = vec![]; + + for i in 0 .. d { + deferred_offences.push((vec![], vec![], 0u32)); + } + + Offences::::set_deferred_offences(deferred_offences); + + }: { + Offences::::on_initialize(u.into()); + } +} diff --git a/frame/offences/src/lib.rs b/frame/offences/src/lib.rs index 0ba7cd87f24552b9e14c5227b3683d1246b434f8..1ad188a7a8364bb225c7575088cab252276b2f4b 100644 --- a/frame/offences/src/lib.rs +++ b/frame/offences/src/lib.rs @@ -27,7 +27,7 @@ mod tests; use sp_std::vec::Vec; use frame_support::{ decl_module, decl_event, decl_storage, Parameter, debug, - weights::{Weight, SimpleDispatchInfo, WeighData}, + weights::Weight, }; use sp_runtime::{traits::Hash, Perbill}; use sp_staking::{ @@ -44,7 +44,7 @@ type OpaqueTimeSlot = Vec; type ReportIdOf = ::Hash; /// Type of data stored as a deferred offence -type DeferredOffenceOf = ( +pub type DeferredOffenceOf = ( Vec::AccountId, ::IdentificationTuple>>, Vec, SessionIndex, @@ -69,7 +69,7 @@ decl_storage! { /// Deferred reports that have been rejected by the offence handler and need to be submitted /// at a later time. - DeferredOffences get(deferred_offences): Vec>; + DeferredOffences get(fn deferred_offences): Vec>; /// A vector of reports of the same kind that happened at the same time slot. ConcurrentReportsIndex: @@ -104,7 +104,7 @@ decl_module! { ConcurrentReportsIndex::::remove_all(); ReportsByKindIndex::remove_all(); - SimpleDispatchInfo::default().weigh_data(()) + 0 } fn on_initialize(now: T::BlockNumber) -> Weight { @@ -125,7 +125,7 @@ decl_module! { }) } - SimpleDispatchInfo::default().weigh_data(()) + 0 } } } @@ -249,6 +249,11 @@ impl Module { None } } + + #[cfg(feature = "runtime-benchmarks")] + pub fn set_deferred_offences(offences: Vec>) { + >::put(offences); + } } struct TriageOutcome { diff --git a/frame/offences/src/mock.rs b/frame/offences/src/mock.rs index e464200396ebde9be263ee75ea610813203d34c1..595d091ea445556261e320803e06ad2d4e792938 100644 --- a/frame/offences/src/mock.rs +++ b/frame/offences/src/mock.rs @@ -100,6 +100,9 @@ impl frame_system::Trait for Runtime { type Event = TestEvent; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/randomness-collective-flip/Cargo.toml b/frame/randomness-collective-flip/Cargo.toml index acd1c216884c036715f0f52b997cb0b78d5c6669..08d715899fb89189f83e3806ade882e0a01fb482 100644 --- a/frame/randomness-collective-flip/Cargo.toml +++ b/frame/randomness-collective-flip/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-randomness-collective-flip" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,17 +8,20 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME randomness collective flip pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] safe-mix = { version = "1.0", default-features = false } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-io = { version = "2.0.0-alpha.5", path = "../../primitives/io" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-io = { version = "2.0.0-dev", path = "../../primitives/io" } [features] default = ["std"] @@ -30,6 +33,3 @@ std = [ "sp-runtime/std", "sp-std/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/randomness-collective-flip/src/lib.rs b/frame/randomness-collective-flip/src/lib.rs index fdc465b4dc3bb0e9163bc750595a290138a7b24d..29068ea91ff81e17de7df6ea7dbea793772feabd 100644 --- a/frame/randomness-collective-flip/src/lib.rs +++ b/frame/randomness-collective-flip/src/lib.rs @@ -19,7 +19,8 @@ //! The Randomness Collective Flip module provides a [`random`](./struct.Module.html#method.random) //! function that generates low-influence random values based on the block hashes from the previous //! `81` blocks. Low-influence randomness can be useful when defending against relatively weak -//! adversaries. +//! adversaries. Using this pallet as a randomness source is advisable primarily in low-security +//! situations like testing. //! //! ## Public Functions //! @@ -35,15 +36,15 @@ //! ### Example - Get random seed for the current block //! //! ``` -//! use frame_support::{decl_module, dispatch, traits::Randomness, weights::SimpleDispatchInfo}; +//! use frame_support::{decl_module, dispatch, traits::Randomness}; //! //! pub trait Trait: frame_system::Trait {} //! //! decl_module! { //! pub struct Module for enum Call where origin: T::Origin { -//! #[weight = SimpleDispatchInfo::default()] +//! #[weight = 0] //! pub fn random_module_example(origin) -> dispatch::DispatchResult { -//! let _random_seed = >::random_seed(); +//! let _random_value = >::random(&b"my context"[..]); //! Ok(()) //! } //! } @@ -57,7 +58,7 @@ use sp_std::{prelude::*, convert::TryInto}; use sp_runtime::traits::Hash; use frame_support::{ decl_module, decl_storage, traits::Randomness, - weights::{Weight, SimpleDispatchInfo, WeighData} + weights::Weight }; use safe_mix::TripletMix; use codec::Encode; @@ -83,7 +84,7 @@ decl_module! { values[index] = parent_hash; }); - SimpleDispatchInfo::default().weigh_data(()) + 0 } } } @@ -98,17 +99,6 @@ decl_storage! { } impl Randomness for Module { - /// Get a low-influence "random" value. - /// - /// Being a deterministic block chain, real randomness is difficult to come by. This gives you - /// something that approximates it. `subject` is a context identifier and allows you to get a - /// different result to other callers of this function; use it like - /// `random(&b"my context"[..])`. This is initially implemented through a low-influence - /// "triplet mix" convolution of previous block hash values. In the future it will be generated - /// from a secure verifiable random function (VRF). - /// - /// ### Security Notes - /// /// This randomness uses a low-influence function, drawing upon the block hashes from the /// previous 81 blocks. Its result for any given subject will be known far in advance by anyone /// observing the chain. Any block producer has significant influence over their block hashes @@ -116,22 +106,6 @@ impl Randomness for Module { /// block producer's influence over the randomness, but increases the influence of small /// colluding groups of recent block producers. /// - /// Some BABE blocks have VRF outputs where the block producer has exactly one bit of influence, - /// either they make the block or they do not make the block and thus someone else makes the - /// next block. Yet, this randomness is not fresh in all BABE blocks. - /// - /// If that is an insufficient security guarantee then two things can be used to improve this - /// randomness: - /// - /// - Name, in advance, the block number whose random value will be used; ensure your module - /// retains a buffer of previous random values for its subject and then index into these in - /// order to obviate the ability of your user to look up the parent hash and choose when to - /// transact based upon it. - /// - Require your user to first commit to an additional value by first posting its hash. - /// Require them to reveal the value to determine the final result, hashing it with the - /// output of this random function. This reduces the ability of a cabal of block producers - /// from conspiring against individuals. - /// /// WARNING: Hashing the result of this function will remove any low-influence properties it has /// and mean that all bits of the resulting value are entirely manipulatable by the author of /// the parent block, who can determine the value of `parent_hash`. @@ -195,6 +169,9 @@ mod tests { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); diff --git a/frame/recovery/Cargo.toml b/frame/recovery/Cargo.toml index 3347014f6e8cafc53a28626aad5124e35fa25b6a..de422678c370d5e84b0525de89b83c8bc0fcc283 100644 --- a/frame/recovery/Cargo.toml +++ b/frame/recovery/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-recovery" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,19 +8,22 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME account recovery pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } enumflags2 = { version = "0.6.2" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } [features] default = ["std"] @@ -33,6 +36,3 @@ std = [ "frame-support/std", "frame-system/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/recovery/src/lib.rs b/frame/recovery/src/lib.rs index c055f2bd97c67777893d493feb4d51c3934048ed..008461e50392b6ed144c6137fe0d37c5ed75f8fb 100644 --- a/frame/recovery/src/lib.rs +++ b/frame/recovery/src/lib.rs @@ -159,7 +159,7 @@ use codec::{Encode, Decode}; use frame_support::{ decl_module, decl_event, decl_storage, decl_error, ensure, - Parameter, RuntimeDebug, weights::{GetDispatchInfo, SimpleDispatchInfo, FunctionOf}, + Parameter, RuntimeDebug, weights::{GetDispatchInfo, FunctionOf, Pays}, traits::{Currency, ReservableCurrency, Get, BalanceStatus}, dispatch::PostDispatchInfo, }; @@ -338,7 +338,7 @@ decl_module! { #[weight = FunctionOf( |args: (&T::AccountId, &Box<::Call>)| args.1.get_dispatch_info().weight + 10_000, |args: (&T::AccountId, &Box<::Call>)| args.1.get_dispatch_info().class, - true + Pays::Yes, )] fn as_recovered(origin, account: T::AccountId, @@ -365,7 +365,7 @@ decl_module! { /// - One storage write O(1) /// - One event /// # - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] fn set_recovered(origin, lost: T::AccountId, rescuer: T::AccountId) { ensure_root(origin)?; // Create the recovery storage item. @@ -400,7 +400,7 @@ decl_module! { /// /// Total Complexity: O(F + X) /// # - #[weight = SimpleDispatchInfo::FixedNormal(100_000)] + #[weight = 100_000_000] fn create_recovery(origin, friends: Vec, threshold: u16, @@ -460,7 +460,7 @@ decl_module! { /// /// Total Complexity: O(F + X) /// # - #[weight = SimpleDispatchInfo::FixedNormal(100_000)] + #[weight = 100_000_000] fn initiate_recovery(origin, account: T::AccountId) { let who = ensure_signed(origin)?; // Check that the account is recoverable @@ -506,7 +506,7 @@ decl_module! { /// /// Total Complexity: O(F + logF + V + logV) /// # - #[weight = SimpleDispatchInfo::FixedNormal(100_000)] + #[weight = 100_000_000] fn vouch_recovery(origin, lost: T::AccountId, rescuer: T::AccountId) { let who = ensure_signed(origin)?; // Get the recovery configuration for the lost account. @@ -545,7 +545,7 @@ decl_module! { /// /// Total Complexity: O(F + V) /// # - #[weight = SimpleDispatchInfo::FixedNormal(100_000)] + #[weight = 100_000_000] fn claim_recovery(origin, account: T::AccountId) { let who = ensure_signed(origin)?; // Get the recovery configuration for the lost account @@ -590,7 +590,7 @@ decl_module! { /// /// Total Complexity: O(V + X) /// # - #[weight = SimpleDispatchInfo::FixedNormal(30_000)] + #[weight = 30_000_000] fn close_recovery(origin, rescuer: T::AccountId) { let who = ensure_signed(origin)?; // Take the active recovery process started by the rescuer for this account. @@ -622,11 +622,11 @@ decl_module! { /// /// Total Complexity: O(F + X) /// # - #[weight = SimpleDispatchInfo::FixedNormal(30_000)] + #[weight = 30_000_000] fn remove_recovery(origin) { let who = ensure_signed(origin)?; // Check there are no active recoveries - let mut active_recoveries = >::iter_prefix(&who); + let mut active_recoveries = >::iter_prefix_values(&who); ensure!(active_recoveries.next().is_none(), Error::::StillActive); // Take the recovery configuration for this account. let recovery_config = >::take(&who).ok_or(Error::::NotRecoverable)?; @@ -647,7 +647,7 @@ decl_module! { /// # /// - One storage mutation to check account is recovered by `who`. O(1) /// # - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn cancel_recovered(origin, account: T::AccountId) { let who = ensure_signed(origin)?; // Check `who` is allowed to make a call on behalf of `account` diff --git a/frame/recovery/src/mock.rs b/frame/recovery/src/mock.rs index 9327ece572212af48e739eac3725925592c9bc95..648321a0ae77b541ec64643da6a5d38b83ce8892 100644 --- a/frame/recovery/src/mock.rs +++ b/frame/recovery/src/mock.rs @@ -75,6 +75,9 @@ impl frame_system::Trait for Test { type Event = TestEvent; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/scheduler/Cargo.toml b/frame/scheduler/Cargo.toml index 531de95867bbbaace5557f7b0e2d7e3d057088b8..8a511cd3894c8fa24f8d821e1f7841596a96a96c 100644 --- a/frame/scheduler/Cargo.toml +++ b/frame/scheduler/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-scheduler" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "Unlicense" @@ -11,15 +11,15 @@ description = "FRAME example pallet" [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } -frame-benchmarking = { version = "2.0.0-alpha.4", default-features = false, path = "../benchmarking" } -frame-support = { version = "2.0.0-alpha.4", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.4", default-features = false, path = "../system" } -sp-runtime = { version = "2.0.0-alpha.4", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0-alpha.4", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.4", default-features = false, path = "../../primitives/io" } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../benchmarking" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.4", path = "../../primitives/core", default-features = false } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core", default-features = false } [features] default = ["std"] diff --git a/frame/scheduler/src/lib.rs b/frame/scheduler/src/lib.rs index f70204cb1a6b7201c89173cbe0dba6d5f10236ab..a18a48da0807e489ff74b85a7ca723acd7f86809 100644 --- a/frame/scheduler/src/lib.rs +++ b/frame/scheduler/src/lib.rs @@ -250,7 +250,7 @@ mod tests { use frame_support::{ impl_outer_event, impl_outer_origin, impl_outer_dispatch, parameter_types, assert_ok, traits::{OnInitialize, OnFinalize, schedule::{Anon, Named}}, - weights::{DispatchClass, FunctionOf} + weights::{DispatchClass, FunctionOf, Pays} }; use sp_core::H256; // The testing primitives are very useful for avoiding having to work with signatures @@ -293,7 +293,7 @@ mod tests { #[weight = FunctionOf( |args: (&u32, &Weight)| *args.1, |_: (&u32, &Weight)| DispatchClass::Normal, - true + Pays::Yes, )] fn log(origin, i: u32, weight: Weight) { ensure_root(origin)?; @@ -348,6 +348,9 @@ mod tests { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/scored-pool/Cargo.toml b/frame/scored-pool/Cargo.toml index b878c5bb47559d3d8eab8cb842d8eb00182abed7..ae8def3dd391b0e43ac4cd156eb70d6241cce4c0 100644 --- a/frame/scored-pool/Cargo.toml +++ b/frame/scored-pool/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-scored-pool" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,18 +8,21 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for scored pools" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } [features] default = ["std"] @@ -32,6 +35,3 @@ std = [ "frame-support/std", "frame-system/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/scored-pool/src/lib.rs b/frame/scored-pool/src/lib.rs index d162f42c3f745bff162fe00eead33a6026fe227e..46eb73293b2fa9c7ad80b91a27602999456d3b8c 100644 --- a/frame/scored-pool/src/lib.rs +++ b/frame/scored-pool/src/lib.rs @@ -61,7 +61,7 @@ //! //! decl_module! { //! pub struct Module for enum Call where origin: T::Origin { -//! #[weight = frame_support::weights::SimpleDispatchInfo::default()] +//! #[weight = 0] //! pub fn candidate(origin) -> dispatch::DispatchResult { //! let who = ensure_signed(origin)?; //! @@ -97,7 +97,7 @@ use sp_std::{ use frame_support::{ decl_module, decl_storage, decl_event, ensure, decl_error, traits::{EnsureOrigin, ChangeMembers, InitializeMembers, Currency, Get, ReservableCurrency}, - weights::{Weight, SimpleDispatchInfo, WeighData}, + weights::Weight, }; use frame_system::{self as system, ensure_root, ensure_signed}; use sp_runtime::{ @@ -191,8 +191,8 @@ decl_storage! { >::insert(who, true); }); - /// Sorts the `Pool` by score in a descending order. Entities which - /// have a score of `None` are sorted to the beginning of the vec. + // Sorts the `Pool` by score in a descending order. Entities which + // have a score of `None` are sorted to the beginning of the vec. pool.sort_by_key(|(_, maybe_score)| Reverse(maybe_score.unwrap_or_default()) ); @@ -252,7 +252,7 @@ decl_module! { let pool = >::get(); >::refresh_members(pool, ChangeReceiver::MembershipChanged); } - SimpleDispatchInfo::default().weigh_data(()) + 0 } /// Add `origin` to the pool of candidates. @@ -266,7 +266,7 @@ decl_module! { /// /// The `index` parameter of this function must be set to /// the index of the transactor in the `Pool`. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn submit_candidacy(origin) { let who = ensure_signed(origin)?; ensure!(!>::contains_key(&who), Error::::AlreadyInPool); @@ -296,7 +296,7 @@ decl_module! { /// /// The `index` parameter of this function must be set to /// the index of the transactor in the `Pool`. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn withdraw_candidacy( origin, index: u32 @@ -316,7 +316,7 @@ decl_module! { /// /// The `index` parameter of this function must be set to /// the index of `dest` in the `Pool`. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn kick( origin, dest: ::Source, @@ -341,7 +341,7 @@ decl_module! { /// /// The `index` parameter of this function must be set to /// the index of the `dest` in the `Pool`. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn score( origin, dest: ::Source, @@ -382,7 +382,7 @@ decl_module! { /// (this happens each `Period`). /// /// May only be called from root. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn change_member_count(origin, count: u32) { ensure_root(origin)?; >::put(&count); diff --git a/frame/scored-pool/src/mock.rs b/frame/scored-pool/src/mock.rs index a28b7891370eafbdc47260ef602ae2a69660c119..6d914c60aae2c90cccada321d783d379ee590866 100644 --- a/frame/scored-pool/src/mock.rs +++ b/frame/scored-pool/src/mock.rs @@ -66,6 +66,9 @@ impl frame_system::Trait for Test { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/session/Cargo.toml b/frame/session/Cargo.toml index f12a8b4a71f2fc8df7f81ac50098c32ffd9cd7aa..b3ca1ad596cca70f1bc8a40c7a6018e3bcf5855e 100644 --- a/frame/session/Cargo.toml +++ b/frame/session/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-session" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,22 +8,25 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME sessions pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-staking = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/staking" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -pallet-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../timestamp" } -sp-trie = { optional = true, path = "../../primitives/trie", default-features = false, version = "2.0.0-alpha.5"} -sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.5"} +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "2.0.0-dev", default-features = false, path = "../../primitives/staking" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +pallet-timestamp = { version = "2.0.0-dev", default-features = false, path = "../timestamp" } +sp-trie = { optional = true, path = "../../primitives/trie", default-features = false, version = "2.0.0-dev"} +sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-dev"} impl-trait-for-tuples = "0.1.3" [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-application-crypto = { version = "2.0.0-alpha.5", path = "../../primitives/application-crypto" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-application-crypto = { version = "2.0.0-dev", path = "../../primitives/application-crypto" } lazy_static = "1.4.0" [features] @@ -40,6 +43,3 @@ std = [ "sp-trie/std", "sp-io/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/session/benchmarking/Cargo.toml b/frame/session/benchmarking/Cargo.toml index 181fb37bfdbc10025ba7d9a1baa1840662c3fe79..a3994ab3790aeea6a621382850991ed00228d171 100644 --- a/frame/session/benchmarking/Cargo.toml +++ b/frame/session/benchmarking/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-session-benchmarking" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,13 +8,26 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME sessions pallet benchmarking" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/runtime" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../../system" } -frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../../benchmarking" } -pallet-staking = { version = "2.0.0-alpha.5", default-features = false, features = ["runtime-benchmarks"], path = "../../staking" } -pallet-session = { version = "2.0.0-alpha.5", default-features = false, path = "../../session" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/runtime" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../../system" } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../../benchmarking" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../../support" } +pallet-staking = { version = "2.0.0-dev", default-features = false, features = ["runtime-benchmarks"], path = "../../staking" } +pallet-session = { version = "2.0.0-dev", default-features = false, path = "../../session" } + +[dev-dependencies] +serde = { version = "1.0.101" } +codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +pallet-staking-reward-curve = { version = "2.0.0-dev", path = "../../staking/reward-curve" } +sp-io ={ path = "../../../primitives/io", version = "2.0.0-dev"} +pallet-timestamp = { version = "2.0.0-dev", path = "../../timestamp" } +pallet-balances = { version = "2.0.0-dev", path = "../../balances" } [features] default = ["std"] @@ -23,9 +36,7 @@ std = [ "sp-runtime/std", "frame-system/std", "frame-benchmarking/std", + "frame-support/std", "pallet-staking/std", "pallet-session/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/session/benchmarking/src/lib.rs b/frame/session/benchmarking/src/lib.rs index db925bd72ee45e204eca8faeee4a97d0d4d6a09d..3b91c2fdc5194f56086b139eb33a71efc468e050 100644 --- a/frame/session/benchmarking/src/lib.rs +++ b/frame/session/benchmarking/src/lib.rs @@ -19,6 +19,8 @@ #![cfg_attr(not(feature = "std"), no_std)] +mod mock; + use sp_std::prelude::*; use sp_std::vec; @@ -42,16 +44,33 @@ benchmarks! { set_keys { let n in 1 .. MAX_NOMINATIONS as u32; - let validator = create_validator_with_nominators::(n, MAX_NOMINATIONS as u32)?; + let v_stash = create_validator_with_nominators::(n, MAX_NOMINATIONS as u32)?; + let v_controller = pallet_staking::Module::::bonded(&v_stash).ok_or("not stash")?; let keys = T::Keys::default(); let proof: Vec = vec![0,1,2,3]; - }: _(RawOrigin::Signed(validator), keys, proof) + }: _(RawOrigin::Signed(v_controller), keys, proof) purge_keys { let n in 1 .. MAX_NOMINATIONS as u32; - let validator = create_validator_with_nominators::(n, MAX_NOMINATIONS as u32)?; + let v_stash = create_validator_with_nominators::(n, MAX_NOMINATIONS as u32)?; + let v_controller = pallet_staking::Module::::bonded(&v_stash).ok_or("not stash")?; let keys = T::Keys::default(); let proof: Vec = vec![0,1,2,3]; - Session::::set_keys(RawOrigin::Signed(validator.clone()).into(), keys, proof)?; - }: _(RawOrigin::Signed(validator)) + Session::::set_keys(RawOrigin::Signed(v_controller.clone()).into(), keys, proof)?; + }: _(RawOrigin::Signed(v_controller)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mock::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_set_keys::()); + assert_ok!(test_benchmark_purge_keys::()); + }); + } } diff --git a/frame/session/benchmarking/src/mock.rs b/frame/session/benchmarking/src/mock.rs new file mode 100644 index 0000000000000000000000000000000000000000..d488fe4eac214db6aeb150d70b7cbc9668588288 --- /dev/null +++ b/frame/session/benchmarking/src/mock.rs @@ -0,0 +1,191 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Mock file for staking fuzzing. + +#![cfg(test)] + +use sp_runtime::traits::{Convert, SaturatedConversion, IdentityLookup}; +use frame_support::{impl_outer_origin, impl_outer_dispatch, parameter_types}; + +type AccountId = u64; +type AccountIndex = u32; +type BlockNumber = u64; +type Balance = u64; + +type System = frame_system::Module; +type Balances = pallet_balances::Module; +type Staking = pallet_staking::Module; +type Session = pallet_session::Module; + +impl_outer_origin! { + pub enum Origin for Test where system = frame_system {} +} + +impl_outer_dispatch! { + pub enum Call for Test where origin: Origin { + pallet_staking::Staking, + } +} + +pub struct CurrencyToVoteHandler; +impl Convert for CurrencyToVoteHandler { + fn convert(x: u64) -> u64 { + x + } +} +impl Convert for CurrencyToVoteHandler { + fn convert(x: u128) -> u64 { + x.saturated_into() + } +} + +#[derive(Clone, Eq, PartialEq, Debug)] +pub struct Test; + +impl frame_system::Trait for Test { + type Origin = Origin; + type Index = AccountIndex; + type BlockNumber = BlockNumber; + type Call = Call; + type Hash = sp_core::H256; + type Hashing = ::sp_runtime::traits::BlakeTwo256; + type AccountId = AccountId; + type Lookup = IdentityLookup; + type Header = sp_runtime::testing::Header; + type Event = (); + type BlockHashCount = (); + type MaximumBlockWeight = (); + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); + type AvailableBlockRatio = (); + type MaximumBlockLength = (); + type Version = (); + type ModuleToIndex = (); + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (Balances,); +} +parameter_types! { + pub const ExistentialDeposit: Balance = 10; +} +impl pallet_balances::Trait for Test { + type Balance = Balance; + type Event = (); + type DustRemoval = (); + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; +} + +parameter_types! { + pub const MinimumPeriod: u64 = 5; +} +impl pallet_timestamp::Trait for Test { + type Moment = u64; + type OnTimestampSet = (); + type MinimumPeriod = MinimumPeriod; +} +impl pallet_session::historical::Trait for Test { + type FullIdentification = pallet_staking::Exposure; + type FullIdentificationOf = pallet_staking::ExposureOf; +} + +sp_runtime::impl_opaque_keys! { + pub struct SessionKeys { + pub foo: sp_runtime::testing::UintAuthorityId, + } +} + +pub struct TestSessionHandler; +impl pallet_session::SessionHandler for TestSessionHandler { + const KEY_TYPE_IDS: &'static [sp_runtime::KeyTypeId] = &[]; + + fn on_genesis_session(_validators: &[(AccountId, Ks)]) {} + + fn on_new_session( + _: bool, + _: &[(AccountId, Ks)], + _: &[(AccountId, Ks)], + ) {} + + fn on_disabled(_: usize) {} +} + +impl pallet_session::Trait for Test { + type SessionManager = pallet_session::historical::NoteHistoricalRoot; + type Keys = SessionKeys; + type ShouldEndSession = pallet_session::PeriodicSessions<(), ()>; + type NextSessionRotation = pallet_session::PeriodicSessions<(), ()>; + type SessionHandler = TestSessionHandler; + type Event = (); + type ValidatorId = AccountId; + type ValidatorIdOf = pallet_staking::StashOf; + type DisabledValidatorsThreshold = (); +} +pallet_staking_reward_curve::build! { + const I_NPOS: sp_runtime::curve::PiecewiseLinear<'static> = curve!( + min_inflation: 0_025_000, + max_inflation: 0_100_000, + ideal_stake: 0_500_000, + falloff: 0_050_000, + max_piece_count: 40, + test_precision: 0_005_000, + ); +} +parameter_types! { + pub const RewardCurve: &'static sp_runtime::curve::PiecewiseLinear<'static> = &I_NPOS; + pub const MaxNominatorRewardedPerValidator: u32 = 64; + pub const UnsignedPriority: u64 = 1 << 20; +} + +pub type Extrinsic = sp_runtime::testing::TestXt; + +impl frame_system::offchain::SendTransactionTypes for Test where + Call: From, +{ + type OverarchingCall = Call; + type Extrinsic = Extrinsic; +} + +impl pallet_staking::Trait for Test { + type Currency = Balances; + type UnixTime = pallet_timestamp::Module; + type CurrencyToVote = CurrencyToVoteHandler; + type RewardRemainder = (); + type Event = (); + type Slash = (); + type Reward = (); + type SessionsPerEra = (); + type SlashDeferDuration = (); + type SlashCancelOrigin = frame_system::EnsureRoot; + type BondingDuration = (); + type SessionInterface = Self; + type RewardCurve = RewardCurve; + type NextNewSession = Session; + type ElectionLookahead = (); + type Call = Call; + type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; + type UnsignedPriority = UnsignedPriority; + type MaxIterations = (); +} + +impl crate::Trait for Test {} + +pub fn new_test_ext() -> sp_io::TestExternalities { + let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + sp_io::TestExternalities::new(t) +} diff --git a/frame/session/src/lib.rs b/frame/session/src/lib.rs index 9346b060fa48a8afd22d01dc1d88d6a6b743a8e4..8e252211319fbf8652148399ea75eea0a450eae1 100644 --- a/frame/session/src/lib.rs +++ b/frame/session/src/lib.rs @@ -110,7 +110,7 @@ use frame_support::{ Get, FindAuthor, ValidatorRegistration, EstimateNextSessionRotation, EstimateNextNewSession, }, dispatch::{self, DispatchResult, DispatchError}, - weights::{Weight, SimpleDispatchInfo, WeighData}, + weights::Weight, }; use frame_system::{self as system, ensure_signed}; @@ -350,6 +350,8 @@ pub trait Trait: frame_system::Trait { type ValidatorId: Member + Parameter; /// A conversion from account ID to validator ID. + /// + /// Its cost must be at most one storage read. type ValidatorIdOf: Convert>; /// Indicator for when to end the session. @@ -493,12 +495,16 @@ decl_module! { /// The dispatch origin of this function must be signed. /// /// # - /// - O(log n) in number of accounts. - /// - One extra DB entry. - /// - Increases system account refs by one on success iff there were previously no keys set. - /// In this case, purge_keys will need to be called before the account can be removed. + /// - Complexity: `O(1)` + /// Actual cost depends on the number of length of `T::Keys::key_ids()` which is fixed. + /// - DbReads: `origin account`, `T::ValidatorIdOf`, `NextKeys` + /// - DbWrites: `origin account`, `NextKeys` + /// - DbReads per key id: `KeyOwner` + /// - DbWrites per key id: `KeyOwner` /// # - #[weight = SimpleDispatchInfo::FixedNormal(150_000)] + #[weight = 200_000_000 + + T::DbWeight::get().reads(2 + T::Keys::key_ids().len() as Weight) + + T::DbWeight::get().writes(1 + T::Keys::key_ids().len() as Weight)] pub fn set_keys(origin, keys: T::Keys, proof: Vec) -> dispatch::DispatchResult { let who = ensure_signed(origin)?; @@ -515,11 +521,14 @@ decl_module! { /// The dispatch origin of this function must be signed. /// /// # - /// - O(N) in number of key types. - /// - Removes N + 1 DB entries. - /// - Reduces system account refs by one on success. + /// - Complexity: `O(1)` in number of key types. + /// Actual cost depends on the number of length of `T::Keys::key_ids()` which is fixed. + /// - DbReads: `T::ValidatorIdOf`, `NextKeys`, `origin account` + /// - DbWrites: `NextKeys`, `origin account` + /// - DbWrites per key id: `KeyOwnder` /// # - #[weight = SimpleDispatchInfo::FixedNormal(150_000)] + #[weight = 120_000_000 + + T::DbWeight::get().reads_writes(2, 1 + T::Keys::key_ids().len() as Weight)] pub fn purge_keys(origin) { let who = ensure_signed(origin)?; Self::do_purge_keys(&who)?; @@ -530,9 +539,13 @@ decl_module! { fn on_initialize(n: T::BlockNumber) -> Weight { if T::ShouldEndSession::should_end_session(n) { Self::rotate_session(); + T::MaximumBlockWeight::get() + } else { + // NOTE: the non-database part of the weight for `should_end_session(n)` is + // included as weight for empty block, the database part is expected to be in + // cache. + 0 } - - SimpleDispatchInfo::default().weigh_data(()) } } } diff --git a/frame/session/src/mock.rs b/frame/session/src/mock.rs index dd28d357491dafb1a7f39e8195a75ea8117bad84..9e9b8776589d29b88fc58738bfe77b45f997ec2e 100644 --- a/frame/session/src/mock.rs +++ b/frame/session/src/mock.rs @@ -150,6 +150,16 @@ pub fn reset_before_session_end_called() { BEFORE_SESSION_END_CALLED.with(|b| *b.borrow_mut() = false); } +pub fn new_test_ext() -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + GenesisConfig:: { + keys: NEXT_VALIDATORS.with(|l| + l.borrow().iter().cloned().map(|i| (i, i, UintAuthorityId(i).into())).collect() + ), + }.assimilate_storage(&mut t).unwrap(); + sp_io::TestExternalities::new(t) +} + #[derive(Clone, Eq, PartialEq)] pub struct Test; @@ -174,6 +184,9 @@ impl frame_system::Trait for Test { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); diff --git a/frame/session/src/tests.rs b/frame/session/src/tests.rs index 4e95d91cc7ef8e35b0e94327cc002d1de944f43e..abfd9f738b614e2d9078ddfe02c07a1f3d862a32 100644 --- a/frame/session/src/tests.rs +++ b/frame/session/src/tests.rs @@ -21,21 +21,11 @@ use frame_support::{traits::OnInitialize, assert_ok}; use sp_core::crypto::key_types::DUMMY; use sp_runtime::testing::UintAuthorityId; use mock::{ - NEXT_VALIDATORS, SESSION_CHANGED, TEST_SESSION_CHANGED, authorities, force_new_session, - set_next_validators, set_session_length, session_changed, Test, Origin, System, Session, - reset_before_session_end_called, before_session_end_called, + SESSION_CHANGED, TEST_SESSION_CHANGED, authorities, force_new_session, + set_next_validators, set_session_length, session_changed, Origin, System, Session, + reset_before_session_end_called, before_session_end_called, new_test_ext, }; -fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - GenesisConfig:: { - keys: NEXT_VALIDATORS.with(|l| - l.borrow().iter().cloned().map(|i| (i, i, UintAuthorityId(i).into())).collect() - ), - }.assimilate_storage(&mut t).unwrap(); - sp_io::TestExternalities::new(t) -} - fn initialize_block(block: u64) { SESSION_CHANGED.with(|l| *l.borrow_mut() = false); System::set_block_number(block); diff --git a/frame/society/Cargo.toml b/frame/society/Cargo.toml index be419fb63fe2d0800dde5802e9ce1cf02de52b14..d79eb78b7d664dbd88da1c931730678aab6d7c4f 100644 --- a/frame/society/Cargo.toml +++ b/frame/society/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-society" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,19 +8,22 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME society pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.5"} -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-dev"} +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } rand_chacha = { version = "0.2", default-features = false } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } [features] default = ["std"] @@ -38,6 +41,3 @@ runtime-benchmarks = [ "sp-runtime/runtime-benchmarks", "frame-system/runtime-benchmarks", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/society/src/lib.rs b/frame/society/src/lib.rs index 2061c21d9c5f992cb6a6254918a9133d14483ca6..fcc0c66ebdbb463db4b320c388cc9eb32e1165d5 100644 --- a/frame/society/src/lib.rs +++ b/frame/society/src/lib.rs @@ -260,7 +260,7 @@ use sp_runtime::{Percent, ModuleId, RuntimeDebug, } }; use frame_support::{decl_error, decl_module, decl_storage, decl_event, ensure, dispatch::DispatchResult}; -use frame_support::weights::{SimpleDispatchInfo, Weight, WeighData}; +use frame_support::weights::Weight; use frame_support::traits::{ Currency, ReservableCurrency, Randomness, Get, ChangeMembers, BalanceStatus, ExistenceRequirement::AllowDeath, EnsureOrigin @@ -269,13 +269,14 @@ use frame_system::{self as system, ensure_signed, ensure_root}; type BalanceOf = <>::Currency as Currency<::AccountId>>::Balance; -const MODULE_ID: ModuleId = ModuleId(*b"py/socie"); - /// The module's configuration trait. pub trait Trait: system::Trait { /// The overarching event type. type Event: From> + Into<::Event>; + /// The societies's module id + type ModuleId: Get; + /// The currency type used for bidding. type Currency: ReservableCurrency; @@ -402,18 +403,18 @@ impl BidKind { decl_storage! { trait Store for Module, I: Instance=DefaultInstance> as Society { /// The first member. - pub Founder get(founder) build(|config: &GenesisConfig| config.members.first().cloned()): + pub Founder get(fn founder) build(|config: &GenesisConfig| config.members.first().cloned()): Option; /// A hash of the rules of this society concerning membership. Can only be set once and /// only by the founder. - pub Rules get(rules): Option; + pub Rules get(fn rules): Option; /// The current set of candidates; bidders that are attempting to become members. - pub Candidates get(candidates): Vec>>; + pub Candidates get(fn candidates): Vec>>; /// The set of suspended candidates. - pub SuspendedCandidates get(suspended_candidate): + pub SuspendedCandidates get(fn suspended_candidate): map hasher(twox_64_concat) T::AccountId => Option<(BalanceOf, BidKind>)>; @@ -421,7 +422,7 @@ decl_storage! { pub Pot get(fn pot) config(): BalanceOf; /// The most primary from the most recently approved members. - pub Head get(head) build(|config: &GenesisConfig| config.members.first().cloned()): + pub Head get(fn head) build(|config: &GenesisConfig| config.members.first().cloned()): Option; /// The current set of members, ordered. @@ -491,6 +492,9 @@ decl_module! { /// The number of blocks between membership challenges. const ChallengePeriod: T::BlockNumber = T::ChallengePeriod::get(); + /// The societies's module id + const ModuleId: ModuleId = T::ModuleId::get(); + // Used for handling module events. fn deposit_event() = default; @@ -527,7 +531,7 @@ decl_module! { /// /// Total Complexity: O(M + B + C + logM + logB + X) /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = 50_000_000] pub fn bid(origin, value: BalanceOf) -> DispatchResult { let who = ensure_signed(origin)?; ensure!(!>::contains_key(&who), Error::::Suspended); @@ -566,7 +570,7 @@ decl_module! { /// /// Total Complexity: O(B + X) /// # - #[weight = SimpleDispatchInfo::FixedNormal(20_000)] + #[weight = 20_000_000] pub fn unbid(origin, pos: u32) -> DispatchResult { let who = ensure_signed(origin)?; @@ -636,7 +640,7 @@ decl_module! { /// /// Total Complexity: O(M + B + C + logM + logB + X) /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = 50_000_000] pub fn vouch(origin, who: T::AccountId, value: BalanceOf, tip: BalanceOf) -> DispatchResult { let voucher = ensure_signed(origin)?; // Check user is not suspended. @@ -677,7 +681,7 @@ decl_module! { /// /// Total Complexity: O(B) /// # - #[weight = SimpleDispatchInfo::FixedNormal(20_000)] + #[weight = 20_000_000] pub fn unvouch(origin, pos: u32) -> DispatchResult { let voucher = ensure_signed(origin)?; ensure!(Self::vouching(&voucher) == Some(VouchingStatus::Vouching), Error::::NotVouching); @@ -715,7 +719,7 @@ decl_module! { /// /// Total Complexity: O(M + logM + C) /// # - #[weight = SimpleDispatchInfo::FixedNormal(30_000)] + #[weight = 30_000_000] pub fn vote(origin, candidate: ::Source, approve: bool) { let voter = ensure_signed(origin)?; let candidate = T::Lookup::lookup(candidate)?; @@ -746,7 +750,7 @@ decl_module! { /// /// Total Complexity: O(M + logM) /// # - #[weight = SimpleDispatchInfo::FixedNormal(20_000)] + #[weight = 20_000_000] pub fn defender_vote(origin, approve: bool) { let voter = ensure_signed(origin)?; let members = >::get(); @@ -778,7 +782,7 @@ decl_module! { /// /// Total Complexity: O(M + logM + P + X) /// # - #[weight = SimpleDispatchInfo::FixedNormal(30_000)] + #[weight = 30_000_000] pub fn payout(origin) { let who = ensure_signed(origin)?; @@ -820,7 +824,7 @@ decl_module! { /// /// Total Complexity: O(1) /// # - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] fn found(origin, founder: T::AccountId, max_members: u32, rules: Vec) { T::FounderSetOrigin::ensure_origin(origin)?; ensure!(!>::exists(), Error::::AlreadyFounded); @@ -847,7 +851,7 @@ decl_module! { /// /// Total Complexity: O(1) /// # - #[weight = SimpleDispatchInfo::FixedNormal(20_000)] + #[weight = 20_000_000] fn unfound(origin) { let founder = ensure_signed(origin)?; ensure!(Founder::::get() == Some(founder.clone()), Error::::NotFounder); @@ -889,7 +893,7 @@ decl_module! { /// /// Total Complexity: O(M + logM + B) /// # - #[weight = SimpleDispatchInfo::FixedNormal(30_000)] + #[weight = 30_000_000] fn judge_suspended_member(origin, who: T::AccountId, forgive: bool) { T::SuspensionJudgementOrigin::ensure_origin(origin)?; ensure!(>::contains_key(&who), Error::::NotSuspended); @@ -960,7 +964,7 @@ decl_module! { /// /// Total Complexity: O(M + logM + B + X) /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = 50_000_000] fn judge_suspended_candidate(origin, who: T::AccountId, judgement: Judgement) { T::SuspensionJudgementOrigin::ensure_origin(origin)?; if let Some((value, kind)) = >::get(&who) { @@ -1020,7 +1024,7 @@ decl_module! { /// /// Total Complexity: O(1) /// # - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] fn set_max_members(origin, max: u32) { ensure_root(origin)?; ensure!(max > 1, Error::::MaxMembers); @@ -1046,7 +1050,7 @@ decl_module! { Self::rotate_challenge(&mut members); } - SimpleDispatchInfo::default().weigh_data(()) + 0 } } } @@ -1570,7 +1574,7 @@ impl, I: Instance> Module { /// This actually does computation. If you need to keep using it, then make sure you cache the /// value and only call this once. pub fn account_id() -> T::AccountId { - MODULE_ID.into_account() + T::ModuleId::get().into_account() } /// The account ID of the payouts pot. This is where payouts are made from. @@ -1578,7 +1582,7 @@ impl, I: Instance> Module { /// This actually does computation. If you need to keep using it, then make sure you cache the /// value and only call this once. pub fn payouts() -> T::AccountId { - MODULE_ID.into_sub_account(b"payouts") + T::ModuleId::get().into_sub_account(b"payouts") } /// Return the duration of the lock, in blocks, with the given number of members. diff --git a/frame/society/src/mock.rs b/frame/society/src/mock.rs index a66a5e6e047d44230b35f62fcf1a78043b5e1085..121ec59555f0e73497600ef5ff653db443163202 100644 --- a/frame/society/src/mock.rs +++ b/frame/society/src/mock.rs @@ -55,6 +55,7 @@ parameter_types! { pub const AvailableBlockRatio: Perbill = Perbill::one(); pub const ExistentialDeposit: u64 = 1; + pub const SocietyModuleId: ModuleId = ModuleId(*b"py/socie"); } ord_parameter_types! { @@ -75,6 +76,9 @@ impl frame_system::Trait for Test { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -106,6 +110,7 @@ impl Trait for Test { type FounderSetOrigin = EnsureSignedBy; type SuspensionJudgementOrigin = EnsureSignedBy; type ChallengePeriod = ChallengePeriod; + type ModuleId = SocietyModuleId; } pub type Society = Module; diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index aac5616e4ea18cd2ff64d9dc0a4fe0da46e24c00..354b8dd30635f154b738917b70435ef147523303 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-staking" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,38 +8,41 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet staking" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-phragmen = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/phragmen" } -sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.5"} -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-staking = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/staking" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -pallet-session = { version = "2.0.0-alpha.5", features = ["historical"], path = "../session", default-features = false } -pallet-authorship = { version = "2.0.0-alpha.5", default-features = false, path = "../authorship" } -sp-application-crypto = { version = "2.0.0-alpha.4", default-features = false, path = "../../primitives/application-crypto" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-phragmen = { version = "2.0.0-dev", default-features = false, path = "../../primitives/phragmen" } +sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-dev"} +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-staking = { version = "2.0.0-dev", default-features = false, path = "../../primitives/staking" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +pallet-session = { version = "2.0.0-dev", features = ["historical"], path = "../session", default-features = false } +pallet-authorship = { version = "2.0.0-dev", default-features = false, path = "../authorship" } +sp-application-crypto = { version = "2.0.0-dev", default-features = false, path = "../../primitives/application-crypto" } static_assertions = "1.1.0" # Optional imports for tesing-utils feature -pallet-indices = { version = "2.0.0-alpha.4", optional = true, path = "../indices", default-features = false } -sp-core = { version = "2.0.0-alpha.4", optional = true, path = "../../primitives/core", default-features = false } +pallet-indices = { version = "2.0.0-dev", optional = true, path = "../indices", default-features = false } +sp-core = { version = "2.0.0-dev", optional = true, path = "../../primitives/core", default-features = false } rand = { version = "0.7.3", optional = true, default-features = false } # Optional imports for benchmarking -frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../benchmarking", optional = true } rand_chacha = { version = "0.2", default-features = false, optional = true } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-storage = { version = "2.0.0-alpha.5", path = "../../primitives/storage" } -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } -pallet-timestamp = { version = "2.0.0-alpha.5", path = "../timestamp" } -pallet-staking-reward-curve = { version = "2.0.0-alpha.5", path = "../staking/reward-curve" } -substrate-test-utils = { version = "2.0.0-alpha.5", path = "../../test-utils" } -frame-benchmarking = { version = "2.0.0-alpha.5", path = "../benchmarking" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-storage = { version = "2.0.0-dev", path = "../../primitives/storage" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } +pallet-timestamp = { version = "2.0.0-dev", path = "../timestamp" } +pallet-staking-reward-curve = { version = "2.0.0-dev", path = "../staking/reward-curve" } +substrate-test-utils = { version = "2.0.0-dev", path = "../../test-utils" } +frame-benchmarking = { version = "2.0.0-dev", path = "../benchmarking" } rand_chacha = { version = "0.2" } parking_lot = "0.10.0" env_logger = "0.7.1" @@ -72,6 +75,3 @@ runtime-benchmarks = [ "rand_chacha", "frame-benchmarking", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/staking/fuzz/.gitignore b/frame/staking/fuzz/.gitignore deleted file mode 100644 index 572e03bdf321b6cc3a99488183436905cefd086d..0000000000000000000000000000000000000000 --- a/frame/staking/fuzz/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ - -target -corpus -artifacts diff --git a/frame/staking/fuzz/Cargo.toml b/frame/staking/fuzz/Cargo.toml deleted file mode 100644 index a78fbf17dc8a3a1dd3c8572167d8d1d27692059a..0000000000000000000000000000000000000000 --- a/frame/staking/fuzz/Cargo.toml +++ /dev/null @@ -1,38 +0,0 @@ -[package] -name = "pallet-staking-fuzz" -version = "0.0.0" -authors = ["Automatically generated"] -publish = false -edition = "2018" - -[package.metadata] -cargo-fuzz = true - -[dependencies] -libfuzzer-sys = "0.3" -codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -pallet-staking = { version = "2.0.0-alpha.2", path = "..", features = ["testing-utils"] } -pallet-staking-reward-curve = { version = "2.0.0-alpha.2", path = "../reward-curve" } -pallet-session = { version = "2.0.0-alpha.2", path = "../../session" } -pallet-indices = { version = "2.0.0-alpha.2", path = "../../indices" } -pallet-balances = { version = "2.0.0-alpha.2", path = "../../balances" } -pallet-timestamp = { version = "2.0.0-alpha.2", path = "../../timestamp" } -frame-system = { version = "2.0.0-alpha.2", path = "../../system" } -frame-support = { version = "2.0.0-alpha.2", path = "../../support" } -sp-std = { version = "2.0.0-alpha.2", path = "../../../primitives/std" } -sp-io ={ version = "2.0.0-alpha.2", path = "../../../primitives/io" } -sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" } -sp-phragmen = { version = "2.0.0-alpha.2", path = "../../../primitives/phragmen" } -sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } -rand = "0.7.3" - -# Prevent this from interfering with workspaces -[workspace] -members = ["."] - -[[bin]] -name = "submit_solution" -path = "fuzz_targets/submit_solution.rs" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/staking/fuzz/fuzz_targets/submit_solution.rs b/frame/staking/fuzz/fuzz_targets/submit_solution.rs deleted file mode 100644 index 5d1fcf1d7ea850dd033b89e41514cd672848ab24..0000000000000000000000000000000000000000 --- a/frame/staking/fuzz/fuzz_targets/submit_solution.rs +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Fuzzing for staking pallet. - -#![no_main] -use libfuzzer_sys::fuzz_target; -use mock::Test; -use pallet_staking::testing_utils::{ - self, USER, get_seq_phragmen_solution, get_weak_solution, setup_chain_stakers, - set_validator_count, signed_account, -}; -use frame_support::assert_ok; -use sp_runtime::{traits::Dispatchable, DispatchError}; - -mod mock; - -#[repr(u32)] -#[allow(dead_code)] -#[derive(Debug, Clone, Copy)] -enum Mode { - /// Initial submission. This will be rather cheap. - InitialSubmission, - /// A better submission that will replace the previous ones. This is the most expensive. - StrongerSubmission, - /// A weak submission that will be rejected. This will be rather cheap. - WeakerSubmission, -} - -pub fn new_test_ext() -> Result { - frame_system::GenesisConfig::default().build_storage::().map(Into::into) -} - -fuzz_target!(|do_reduce: bool| { - let ext = new_test_ext(); - let mode: Mode = unsafe { std::mem::transmute(testing_utils::random(0, 2)) }; - let num_validators = testing_utils::random(50, 500); - let num_nominators = testing_utils::random(200, 2000); - let edge_per_voter = testing_utils::random(1, 16); - let to_elect = testing_utils::random(10, num_validators); - - println!("+++ instance with params {} / {} / {} / {:?} / {}", - num_nominators, - num_validators, - edge_per_voter, - mode, - to_elect, - ); - - ext.unwrap_or_default().execute_with(|| { - // initial setup - set_validator_count::(to_elect); - setup_chain_stakers::( - num_validators, - num_nominators, - edge_per_voter, - ); - - println!("++ Chain setup done."); - - // stuff to submit - let (winners, compact, score) = match mode { - Mode::InitialSubmission => { - /* No need to setup anything */ - get_seq_phragmen_solution::(do_reduce) - }, - Mode::StrongerSubmission => { - let (winners, compact, score) = get_weak_solution::(false); - assert_ok!( - >::submit_election_solution( - signed_account::(USER), - winners, - compact, - score, - ) - ); - get_seq_phragmen_solution::(do_reduce) - }, - Mode::WeakerSubmission => { - let (winners, compact, score) = get_seq_phragmen_solution::(do_reduce); - assert_ok!( - >::submit_election_solution( - signed_account::(USER), - winners, - compact, - score, - ) - ); - get_weak_solution::(false) - } - }; - - println!("++ Submission ready."); - - // must have chosen correct number of winners. - assert_eq!(winners.len() as u32, >::validator_count()); - - // final call and origin - let call = pallet_staking::Call::::submit_election_solution( - winners, - compact, - score, - ); - let caller = signed_account::(USER); - - // actually submit - match mode { - Mode::WeakerSubmission => { - assert_eq!( - call.dispatch(caller.into()).unwrap_err(), - DispatchError::Module { index: 0, error: 11, message: Some("PhragmenWeakSubmission") }, - ); - }, - _ => assert_ok!(call.dispatch(caller.into())), - }; - }) -}); diff --git a/frame/staking/fuzzer/.gitignore b/frame/staking/fuzzer/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..3ebcb104d4a50a19959dc7ff2bc06ee6bb48b31f --- /dev/null +++ b/frame/staking/fuzzer/.gitignore @@ -0,0 +1,2 @@ +hfuzz_target +hfuzz_workspace diff --git a/frame/staking/fuzz/Cargo.lock b/frame/staking/fuzzer/Cargo.lock similarity index 85% rename from frame/staking/fuzz/Cargo.lock rename to frame/staking/fuzzer/Cargo.lock index f6e8cfa08d6675144979b3a5ef10eaa8f9dcf0d3..a45f33fdce2492109446030c1a082e7cc32f2f19 100644 --- a/frame/staking/fuzz/Cargo.lock +++ b/frame/staking/fuzzer/Cargo.lock @@ -28,11 +28,31 @@ dependencies = [ "memchr", ] +[[package]] +name = "alga" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f823d037a7ec6ea2197046bafd4ae150e6bc36f9ca347404f46a46823fa84f2" +dependencies = [ + "approx", + "num-complex", + "num-traits", +] + +[[package]] +name = "approx" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0e60b75072ecd4168020818c0107f2857bb6c4e64252d8d3983f6263b40a5c3" +dependencies = [ + "num-traits", +] + [[package]] name = "arbitrary" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16971f2f0ce65c5cf2a1546cc6a0af102ecb11e265ddaa9433fb3e5bfdf676a4" +checksum = "75153c95fdedd7db9732dfbfc3702324a1627eec91ba56e37cd0ac78314ab2ed" [[package]] name = "arrayref" @@ -69,9 +89,9 @@ checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" [[package]] name = "backtrace" -version = "0.3.45" +version = "0.3.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad235dabf00f36301792cfe82499880ba54c6486be094d1047b02bacb67c14e8" +checksum = "b1e692897359247cc6bb902933361652380af0f1b7651ae5c5013407f30e109e" dependencies = [ "backtrace-sys", "cfg-if", @@ -81,9 +101,9 @@ dependencies = [ [[package]] name = "backtrace-sys" -version = "0.1.33" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17b52e737c40a7d75abca20b29a19a0eb7ba9fc72c5a72dd282a0a3c2c0dc35" +checksum = "7de8aba10a69c8e8d7622c5710229485ec32e9d55fdad160ea559c086fdcd118" dependencies = [ "cc", "libc", @@ -109,9 +129,13 @@ checksum = "5da9b3d9f6f585199287a473f4f8dfab6566cf827d15c00c219f53c645687ead" [[package]] name = "bitvec" -version = "0.15.2" +version = "0.17.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a993f74b4c99c1908d156b8d2e0fb6277736b0ecbd833982fd1241d39b2766a6" +checksum = "41262f11d771fd4a61aa3ce019fca363b4b6c282fca9da2a31186d3965a47a5c" +dependencies = [ + "either", + "radium", +] [[package]] name = "blake2-rfc" @@ -146,9 +170,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.2.0" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f359dc14ff8911330a51ef78022d376f25ed00248912803b58f00cb1c27f742" +checksum = "12ae9db68ad7fac5fe51304d20f016c911539251075a214f8e663babefa35187" [[package]] name = "byte-slice-cast" @@ -168,15 +192,6 @@ version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" -[[package]] -name = "c2-chacha" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" -dependencies = [ - "ppv-lite86", -] - [[package]] name = "cc" version = "1.0.50" @@ -249,19 +264,6 @@ dependencies = [ "subtle 1.0.0", ] -[[package]] -name = "curve25519-dalek" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b7dcd30ba50cdf88b55b033456138b7c0ac4afdc436d82e1b79f370f24cc66d" -dependencies = [ - "byteorder", - "clear_on_drop", - "digest", - "rand_core 0.3.1", - "subtle 2.2.2", -] - [[package]] name = "curve25519-dalek" version = "2.0.0" @@ -272,14 +274,14 @@ dependencies = [ "digest", "rand_core 0.5.1", "subtle 2.2.2", - "zeroize 1.1.0", + "zeroize", ] [[package]] name = "derive_more" -version = "0.99.3" +version = "0.99.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a806e96c59a76a5ba6e18735b6cf833344671e61e7863f2edb5c518ea2cac95c" +checksum = "e2323f3f47db9a0e77ce7a300605d8d2098597fc451ed1a97bb1f6411bb550a7" dependencies = [ "proc-macro2", "quote", @@ -302,11 +304,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978710b352437433c97b2bff193f2fb1dfd58a093f863dd95e225a19baa599a2" dependencies = [ "clear_on_drop", - "curve25519-dalek 2.0.0", + "curve25519-dalek", "rand 0.7.3", "sha2", ] +[[package]] +name = "either" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" + [[package]] name = "environmental" version = "1.1.1" @@ -343,12 +351,11 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "fixed-hash" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3367952ceb191f4ab95dd5685dc163ac539e36202f9fcfd0cb22f9f9c542fefc" +checksum = "32529fc42e86ec06e5047092082aab9ad459b070c5d2a76b14f4f5ce70bf2e84" dependencies = [ "byteorder", - "libc", "rand 0.7.3", "rustc-hex", "static_assertions", @@ -356,10 +363,11 @@ dependencies = [ [[package]] name = "frame-benchmarking" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "frame-support", "frame-system", + "linregress", "parity-scale-codec", "sp-api", "sp-io", @@ -370,7 +378,7 @@ dependencies = [ [[package]] name = "frame-metadata" -version = "11.0.0-alpha.3" +version = "11.0.0-alpha.5" dependencies = [ "parity-scale-codec", "serde", @@ -380,7 +388,7 @@ dependencies = [ [[package]] name = "frame-support" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "bitmask", "frame-metadata", @@ -403,7 +411,7 @@ dependencies = [ [[package]] name = "frame-support-procedural" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "frame-support-procedural-tools", "proc-macro2", @@ -413,7 +421,7 @@ dependencies = [ [[package]] name = "frame-support-procedural-tools" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "frame-support-procedural-tools-derive", "proc-macro-crate", @@ -424,7 +432,7 @@ dependencies = [ [[package]] name = "frame-support-procedural-tools-derive" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "proc-macro2", "quote", @@ -433,7 +441,7 @@ dependencies = [ [[package]] name = "frame-system" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "frame-support", "impl-trait-for-tuples", @@ -492,6 +500,7 @@ dependencies = [ "futures-core", "futures-task", "futures-util", + "num_cpus", ] [[package]] @@ -597,6 +606,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "hermit-abi" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "725cf19794cf90aa94e65050cb4191ff5d8fa87a498383774c47b332e3af952e" +dependencies = [ + "libc", +] + [[package]] name = "hex" version = "0.4.2" @@ -670,9 +688,9 @@ checksum = "f65877bf7d44897a473350b1046277941cee20b263397e90869c50b6e766088b" [[package]] name = "js-sys" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cb931d43e71f560c81badb0191596562bafad2be06a3f9025b845c847c60df5" +checksum = "6a27d435371a2fa5b6d2b028a74bbdb1234f308da363226a2854ca3ff8ba7055" dependencies = [ "wasm-bindgen", ] @@ -691,20 +709,26 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb147597cdf94ed43ab7a9038716637d2d1bf2bc571da995d0028dec06bd3018" +checksum = "dea0c0405123bba743ee3f91f49b1c7cfb684eef0da0a50110f758ccf24cdff0" [[package]] name = "libfuzzer-sys" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb789afcc589a08928d1e466087445ab740a0f70a2ee23d9349a0e3723d65e1b" +checksum = "8d718794b8e23533b9069bd2c4597d69e41cc7ab1c02700a502971aca0cdcf24" dependencies = [ "arbitrary", "cc", ] +[[package]] +name = "libm" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" + [[package]] name = "libsecp256k1" version = "0.3.5" @@ -721,6 +745,17 @@ dependencies = [ "typenum", ] +[[package]] +name = "linregress" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9290cf6f928576eeb9c096c6fad9d8d452a0a1a70a2bbffa6e36064eedc0aac9" +dependencies = [ + "failure", + "nalgebra", + "statrs", +] + [[package]] name = "lock_api" version = "0.3.3" @@ -739,6 +774,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "matrixmultiply" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4f7ec66360130972f34830bfad9ef05c6610a43938a467bcc9ab9369ab3478f" +dependencies = [ + "rawpointer", +] + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -753,9 +797,9 @@ checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" [[package]] name = "memory-db" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "198831fe8722331a395bc199a5d08efbc197497ef354cb4c77b969c02ffc0fc4" +checksum = "f58381b20ebe2c578e75dececd9da411414903415349548ccc46aac3209cdfbc" dependencies = [ "ahash", "hash-db", @@ -771,14 +815,31 @@ checksum = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" [[package]] name = "merlin" -version = "1.3.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0942b357c1b4d0dc43ba724674ec89c3218e6ca2b3e8269e7cb53bcecd2f6e" +checksum = "c6feca46f4fa3443a01769d768727f10c10a20fdb65e52dc16a81f0c8269bb78" dependencies = [ "byteorder", "keccak", - "rand_core 0.4.2", - "zeroize 1.1.0", + "rand_core 0.5.1", + "zeroize", +] + +[[package]] +name = "nalgebra" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaa9fddbc34c8c35dd2108515587b8ce0cab396f17977b8c738568e4edb521a2" +dependencies = [ + "alga", + "approx", + "generic-array", + "matrixmultiply", + "num-complex", + "num-rational", + "num-traits", + "rand 0.6.5", + "typenum", ] [[package]] @@ -798,6 +859,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" +dependencies = [ + "autocfg 1.0.0", + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.42" @@ -810,9 +881,9 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da4dc79f9e6c81bef96148c8f6b8e72ad4541caa4a24373e900a36da07de03a3" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" dependencies = [ "autocfg 1.0.0", "num-bigint", @@ -827,6 +898,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" dependencies = [ "autocfg 1.0.0", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" +dependencies = [ + "hermit-abi", + "libc", ] [[package]] @@ -846,7 +928,7 @@ checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" [[package]] name = "pallet-authorship" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "frame-support", "frame-system", @@ -862,7 +944,7 @@ dependencies = [ [[package]] name = "pallet-balances" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "frame-benchmarking", "frame-support", @@ -876,7 +958,7 @@ dependencies = [ [[package]] name = "pallet-indices" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "frame-support", "frame-system", @@ -891,7 +973,7 @@ dependencies = [ [[package]] name = "pallet-session" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "frame-support", "frame-system", @@ -908,7 +990,7 @@ dependencies = [ [[package]] name = "pallet-staking" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "frame-support", "frame-system", @@ -952,7 +1034,7 @@ dependencies = [ [[package]] name = "pallet-staking-reward-curve" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -962,7 +1044,7 @@ dependencies = [ [[package]] name = "pallet-timestamp" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "frame-benchmarking", "frame-support", @@ -980,7 +1062,7 @@ dependencies = [ name = "parity-scale-codec" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f509c5e67ca0605ee17dcd3f91ef41cadd685c75a298fb6261b781a5acb3f910" +checksum = "329c8f7f4244ddb5c37c103641027a76c530e65e8e4b8240b29f81ea40508b17" dependencies = [ "arrayvec 0.5.1", "bitvec", @@ -991,7 +1073,7 @@ dependencies = [ [[package]] name = "parity-scale-codec-derive" -version = "1.3.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a0ec292e92e8ec7c58e576adacc1e3f399c597c8f263c42f18420abe58e7245" dependencies = [ @@ -1003,9 +1085,9 @@ dependencies = [ [[package]] name = "parity-util-mem" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef1476e40bf8f5c6776e9600983435821ca86eb9819d74a6207cca69d091406a" +checksum = "6e42755f26e5ea21a6a819d9e63cbd70713e9867a2b767ec2cc65ca7659532c5" dependencies = [ "cfg-if", "impl-trait-for-tuples", @@ -1078,15 +1160,15 @@ dependencies = [ "cloudabi", "libc", "redox_syscall", - "smallvec 1.3.0", + "smallvec 1.2.0", "winapi", ] [[package]] name = "paste" -version = "0.1.7" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e1afe738d71b1ebab5f1207c055054015427dbfc7bbe9ee1266894156ec046" +checksum = "092d791bf7847f70bbd49085489fba25fc2c193571752bff9e36e74e72403932" dependencies = [ "paste-impl", "proc-macro-hack", @@ -1094,9 +1176,9 @@ dependencies = [ [[package]] name = "paste-impl" -version = "0.1.7" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d4dc4a7f6f743211c5aab239640a65091535d97d43d92a52bca435a640892bb" +checksum = "406c23fb4c45cc6f68a9bbabb8ec7bd6f8cfcbd17e9e8f72c2460282f8325729" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -1128,9 +1210,9 @@ checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" [[package]] name = "primitive-types" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4336f4f5d5524fa60bcbd6fe626f9223d8142a50e7053e979acdf0da41ab975" +checksum = "e5e4b9943a2da369aec5e96f7c10ebc74fcf434d39590d974b0a3460e6f67fbb" dependencies = [ "fixed-hash", "impl-codec", @@ -1149,26 +1231,21 @@ dependencies = [ [[package]] name = "proc-macro-hack" -version = "0.5.11" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "fcfdefadc3d57ca21cf17990a28ef4c0f7c61383a28cb7604cf4a18e6ede1420" [[package]] name = "proc-macro-nested" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" +checksum = "8e946095f9d3ed29ec38de908c22f95d9ac008e424c7bcae54c75a79c527c694" [[package]] name = "proc-macro2" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c09721c6781493a2a492a96b5a5bf19b65917fe6728884e7c44dd0c60ca3435" +checksum = "df246d292ff63439fea9bc8c0a270bed0e390d5ebd4db4ba15aba81111b5abe3" dependencies = [ "unicode-xid", ] @@ -1182,6 +1259,25 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "def50a86306165861203e7f84ecffbbdfdea79f0e51039b33de1e952358c47ac" + +[[package]] +name = "rand" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c618c47cd3ebd209790115ab837de41425723956ad3ce2e6a7f09890947cacb9" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "winapi", +] + [[package]] name = "rand" version = "0.6.5" @@ -1209,7 +1305,7 @@ checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ "getrandom", "libc", - "rand_chacha 0.2.1", + "rand_chacha 0.2.2", "rand_core 0.5.1", "rand_hc 0.2.0", ] @@ -1226,11 +1322,11 @@ dependencies = [ [[package]] name = "rand_chacha" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" dependencies = [ - "c2-chacha", + "ppv-lite86", "rand_core 0.5.1", ] @@ -1329,6 +1425,12 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rdrand" version = "0.4.0" @@ -1346,9 +1448,9 @@ checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" [[package]] name = "regex" -version = "1.3.4" +version = "1.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322cf97724bea3ee221b78fe25ac9c46114ebb51747ad5babd51a2fc6a8235a8" +checksum = "7f6946991529684867e47d86474e3a6d0c0ab9b82d5821e314b1ede31fa3a4b3" dependencies = [ "aho-corasick", "memchr", @@ -1358,9 +1460,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.16" +version = "0.6.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1132f845907680735a84409c3bebc64d1364a5683ffbce899550cd09d5eaefc1" +checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" [[package]] name = "rustc-demangle" @@ -1391,19 +1493,20 @@ dependencies = [ [[package]] name = "schnorrkel" -version = "0.8.5" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eacd8381b3c37840c9c9f40472af529e49975bdcbc24f83c31059fd6539023d3" +checksum = "021b403afe70d81eea68f6ea12f6b3c9588e5d536a94c3bf80f15e7faa267862" dependencies = [ - "curve25519-dalek 1.2.3", - "failure", + "arrayref", + "arrayvec 0.5.1", + "curve25519-dalek", + "getrandom", "merlin", - "rand 0.6.5", - "rand_core 0.4.2", - "rand_os", + "rand 0.7.3", + "rand_core 0.5.1", "sha2", "subtle 2.2.2", - "zeroize 0.9.3", + "zeroize", ] [[package]] @@ -1435,18 +1538,18 @@ checksum = "a0eddf2e8f50ced781f288c19f18621fa72a3779e3cb58dbf23b07469b0abeb4" [[package]] name = "serde" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" +checksum = "e707fbbf255b8fc8c3b99abb91e7257a622caeb20a9818cbadbeeede4e0932ff" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" +checksum = "ac5d00fc561ba2724df6758a17de23df5914f20e41cb00f94d5b7ae42fffaff8" dependencies = [ "proc-macro2", "quote", @@ -1482,13 +1585,13 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.3.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" [[package]] name = "sp-api" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "hash-db", "parity-scale-codec", @@ -1502,7 +1605,7 @@ dependencies = [ [[package]] name = "sp-api-proc-macro" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "blake2-rfc", "proc-macro-crate", @@ -1513,7 +1616,7 @@ dependencies = [ [[package]] name = "sp-application-crypto" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "parity-scale-codec", "serde", @@ -1524,7 +1627,7 @@ dependencies = [ [[package]] name = "sp-arithmetic" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "integer-sqrt", "num-traits", @@ -1536,7 +1639,7 @@ dependencies = [ [[package]] name = "sp-authorship" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "parity-scale-codec", "sp-inherents", @@ -1546,12 +1649,13 @@ dependencies = [ [[package]] name = "sp-core" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "base58", "blake2-rfc", "byteorder", "ed25519-dalek", + "futures", "hash-db", "hash256-std-hasher", "hex", @@ -1566,7 +1670,6 @@ dependencies = [ "primitive-types", "rand 0.7.3", "regex", - "rustc-hex", "schnorrkel", "serde", "sha2", @@ -1580,12 +1683,12 @@ dependencies = [ "tiny-keccak", "twox-hash", "wasmi", - "zeroize 1.1.0", + "zeroize", ] [[package]] name = "sp-debug-derive" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "proc-macro2", "quote", @@ -1594,7 +1697,7 @@ dependencies = [ [[package]] name = "sp-externalities" -version = "0.8.0-alpha.3" +version = "0.8.0-alpha.5" dependencies = [ "environmental", "sp-std", @@ -1603,7 +1706,7 @@ dependencies = [ [[package]] name = "sp-inherents" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "derive_more", "parity-scale-codec", @@ -1614,7 +1717,7 @@ dependencies = [ [[package]] name = "sp-io" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "hash-db", "libsecp256k1", @@ -1631,7 +1734,7 @@ dependencies = [ [[package]] name = "sp-keyring" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "lazy_static", "sp-core", @@ -1641,7 +1744,7 @@ dependencies = [ [[package]] name = "sp-panic-handler" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "backtrace", "log", @@ -1649,11 +1752,10 @@ dependencies = [ [[package]] name = "sp-phragmen" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "parity-scale-codec", "serde", - "sp-core", "sp-phragmen-compact", "sp-runtime", "sp-std", @@ -1671,7 +1773,7 @@ dependencies = [ [[package]] name = "sp-runtime" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "hash256-std-hasher", "impl-trait-for-tuples", @@ -1691,7 +1793,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "parity-scale-codec", "primitive-types", @@ -1704,7 +1806,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface-proc-macro" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "Inflector", "proc-macro-crate", @@ -1715,7 +1817,7 @@ dependencies = [ [[package]] name = "sp-staking" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "parity-scale-codec", "sp-runtime", @@ -1724,7 +1826,7 @@ dependencies = [ [[package]] name = "sp-state-machine" -version = "0.8.0-alpha.3" +version = "0.8.0-alpha.5" dependencies = [ "hash-db", "log", @@ -1742,11 +1844,11 @@ dependencies = [ [[package]] name = "sp-std" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" [[package]] name = "sp-storage" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "impl-serde 0.2.3", "serde", @@ -1756,7 +1858,7 @@ dependencies = [ [[package]] name = "sp-timestamp" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -1769,7 +1871,7 @@ dependencies = [ [[package]] name = "sp-trie" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "hash-db", "memory-db", @@ -1782,7 +1884,7 @@ dependencies = [ [[package]] name = "sp-version" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "impl-serde 0.2.3", "parity-scale-codec", @@ -1793,7 +1895,7 @@ dependencies = [ [[package]] name = "sp-wasm-interface" -version = "2.0.0-alpha.3" +version = "2.0.0-alpha.5" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -1807,6 +1909,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "statrs" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10102ac8d55e35db2b3fafc26f81ba8647da2e15879ab686a67e6d19af2685e8" +dependencies = [ + "rand 0.5.6", +] + [[package]] name = "strum" version = "0.16.0" @@ -1830,9 +1941,9 @@ dependencies = [ [[package]] name = "substrate-bip39" -version = "0.3.1" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be511be555a3633e71739a79e4ddff6a6aaa6579fa6114182a51d72c3eb93c5" +checksum = "c004e8166d6e0aa3a9d5fa673e5b7098ff25f930de1013a21341988151e681bb" dependencies = [ "hmac", "pbkdf2", @@ -1854,9 +1965,9 @@ checksum = "7c65d530b10ccaeac294f349038a597e435b18fb456aadd0840a623f83b9e941" [[package]] name = "syn" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "123bd9499cfb380418d509322d7a6d52e5315f064fe4b3ad18a53d6b92c07859" +checksum = "0df0eb663f387145cab623dea85b09c2c5b4b0aef44e945d928e682fce71bb03" dependencies = [ "proc-macro2", "quote", @@ -1886,9 +1997,9 @@ dependencies = [ [[package]] name = "tiny-bip39" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6848cd8f566953ce1e8faeba12ee23cbdbb0437754792cd857d44628b5685e3" +checksum = "b0165e045cc2ae1660270ca65e1676dbaab60feb0f91b10f7d0665e9b47e31f2" dependencies = [ "failure", "hmac", @@ -1958,7 +2069,7 @@ dependencies = [ "hashbrown", "log", "rustc-hex", - "smallvec 1.3.0", + "smallvec 1.2.0", ] [[package]] @@ -2003,7 +2114,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" dependencies = [ - "smallvec 1.3.0", + "smallvec 1.2.0", ] [[package]] @@ -2026,9 +2137,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasm-bindgen" -version = "0.2.59" +version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3557c397ab5a8e347d434782bcd31fc1483d927a6826804cec05cc792ee2519d" +checksum = "2cc57ce05287f8376e998cbddfb4c8cb43b84a7ec55cf4551d7c00eef317a47f" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2036,9 +2147,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.59" +version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0da9c9a19850d3af6df1cb9574970b566d617ecfaf36eb0b706b6f3ef9bd2f8" +checksum = "d967d37bf6c16cca2973ca3af071d0a2523392e4a594548155d89a678f4237cd" dependencies = [ "bumpalo", "lazy_static", @@ -2051,9 +2162,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "457414a91863c0ec00090dba537f88ab955d93ca6555862c29b6d860990b8a8a" +checksum = "7add542ea1ac7fdaa9dc25e031a6af33b7d63376292bd24140c637d00d1c312a" dependencies = [ "cfg-if", "js-sys", @@ -2063,9 +2174,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.59" +version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6fde1d36e75a714b5fe0cffbb78978f222ea6baebb726af13c78869fdb4205" +checksum = "8bd151b63e1ea881bb742cd20e1d6127cef28399558f3b5d415289bc41eee3a4" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2073,9 +2184,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.59" +version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25bda4168030a6412ea8a047e27238cadf56f0e53516e1e83fec0a8b7c786f6d" +checksum = "d68a5b36eef1be7868f668632863292e37739656a80fc4b9acec7b0bd35a4931" dependencies = [ "proc-macro2", "quote", @@ -2086,9 +2197,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.59" +version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc9f36ad51f25b0219a3d4d13b90eb44cd075dff8b6280cca015775d7acaddd8" +checksum = "daf76fe7d25ac79748a37538b7daeed1c7a6867c92d3245c12c6222e4a20d639" [[package]] name = "wasm-timer" @@ -2131,9 +2242,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721c6263e2c66fd44501cc5efbfa2b7dfa775d13e4ea38c46299646ed1f9c70a" +checksum = "2d6f51648d8c56c366144378a33290049eafdd784071077f6fe37dae64c1c4cb" dependencies = [ "js-sys", "wasm-bindgen", @@ -2161,12 +2272,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "zeroize" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45af6a010d13e4cf5b54c94ba5a2b2eba5596b9e46bf5875612d332a1f2b3f86" - [[package]] name = "zeroize" version = "1.1.0" diff --git a/frame/staking/fuzzer/Cargo.toml b/frame/staking/fuzzer/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..34e956a36724ebdc3b2dafbda70da3f2a70d5b7f --- /dev/null +++ b/frame/staking/fuzzer/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "pallet-staking-fuzz" +version = "0.0.0" +authors = ["Automatically generated"] +publish = false +edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet staking fuzzing" + +[dependencies] +honggfuzz = "0.5" +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } +pallet-staking = { version = "2.0.0-alpha.5", path = "..", features = ["testing-utils"] } +pallet-staking-reward-curve = { version = "2.0.0-alpha.5", path = "../reward-curve" } +pallet-session = { version = "2.0.0-alpha.5", path = "../../session" } +pallet-indices = { version = "2.0.0-alpha.5", path = "../../indices" } +pallet-balances = { version = "2.0.0-alpha.5", path = "../../balances" } +pallet-timestamp = { version = "2.0.0-alpha.5", path = "../../timestamp" } +frame-system = { version = "2.0.0-alpha.5", path = "../../system" } +frame-support = { version = "2.0.0-alpha.5", path = "../../support" } +sp-std = { version = "2.0.0-alpha.5", path = "../../../primitives/std" } +sp-io ={ version = "2.0.0-alpha.5", path = "../../../primitives/io" } +sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } +sp-phragmen = { version = "2.0.0-alpha.5", path = "../../../primitives/phragmen" } +sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } + +[[bin]] +name = "submit_solution" +path = "src/submit_solution.rs" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/staking/fuzz/fuzz_targets/mock.rs b/frame/staking/fuzzer/src/mock.rs similarity index 88% rename from frame/staking/fuzz/fuzz_targets/mock.rs rename to frame/staking/fuzzer/src/mock.rs index 4bb3437f92368b35b7bfd2ef4f4529214972e6ca..1819970e6061e62ef54537887bcbc50efca1d260 100644 --- a/frame/staking/fuzz/fuzz_targets/mock.rs +++ b/frame/staking/fuzzer/src/mock.rs @@ -24,11 +24,11 @@ type AccountIndex = u32; type BlockNumber = u64; type Balance = u64; -type System = frame_system::Module; -type Balances = pallet_balances::Module; -type Staking = pallet_staking::Module; -type Indices = pallet_indices::Module; -type Session = pallet_session::Module; +pub type System = frame_system::Module; +pub type Balances = pallet_balances::Module; +pub type Staking = pallet_staking::Module; +pub type Indices = pallet_indices::Module; +pub type Session = pallet_session::Module; impl_outer_origin! { pub enum Origin for Test where system = frame_system {} @@ -57,6 +57,9 @@ pub struct Test; impl frame_system::Trait for Test { type Origin = Origin; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type Index = AccountIndex; type BlockNumber = BlockNumber; type Call = Call; @@ -150,18 +153,21 @@ pallet_staking_reward_curve::build! { parameter_types! { pub const RewardCurve: &'static sp_runtime::curve::PiecewiseLinear<'static> = &I_NPOS; pub const MaxNominatorRewardedPerValidator: u32 = 64; + pub const MaxIterations: u32 = 20; } pub type Extrinsic = sp_runtime::testing::TestXt; -type SubmitTransaction = frame_system::offchain::TransactionSubmitter< - sp_runtime::testing::UintAuthorityId, - Test, - Extrinsic, ->; + +impl frame_system::offchain::SendTransactionTypes for Test where + Call: From, +{ + type OverarchingCall = Call; + type Extrinsic = Extrinsic; +} impl pallet_staking::Trait for Test { type Currency = Balances; - type Time = pallet_timestamp::Module; + type UnixTime = pallet_timestamp::Module; type CurrencyToVote = CurrencyToVoteHandler; type RewardRemainder = (); type Event = (); @@ -176,7 +182,7 @@ impl pallet_staking::Trait for Test { type NextNewSession = Session; type ElectionLookahead = (); type Call = Call; - type SubmitTransaction = SubmitTransaction; - type KeyType = sp_runtime::testing::UintAuthorityId; + type MaxIterations = MaxIterations; type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; + type UnsignedPriority = (); } diff --git a/frame/staking/fuzzer/src/submit_solution.rs b/frame/staking/fuzzer/src/submit_solution.rs new file mode 100644 index 0000000000000000000000000000000000000000..90d8fc56efc504f343b9baed34e49f52fd77f6bd --- /dev/null +++ b/frame/staking/fuzzer/src/submit_solution.rs @@ -0,0 +1,152 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Fuzzing for staking pallet. + +use honggfuzz::fuzz; + +use mock::Test; +use pallet_staking::testing_utils::{ + USER, get_seq_phragmen_solution, get_weak_solution, setup_chain_stakers, + set_validator_count, signed_account, +}; +use frame_support::assert_ok; +use sp_runtime::{traits::Dispatchable, DispatchError}; + +mod mock; + +#[repr(u32)] +#[allow(dead_code)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Mode { + /// Initial submission. This will be rather cheap. + InitialSubmission, + /// A better submission that will replace the previous ones. This is the most expensive. + StrongerSubmission, + /// A weak submission that will be rejected. This will be rather cheap. + WeakerSubmission, +} + +pub fn new_test_ext() -> Result { + frame_system::GenesisConfig::default().build_storage::().map(Into::into) +} + +fn main() { + let to_range = |x: u32, a: u32, b: u32| { + let collapsed = x % b; + if collapsed >= a { + collapsed + } else { + collapsed + a + } + }; + loop { + fuzz!(|data: (u32, u32, u32, u32, u32)| { + let (mut num_validators, mut num_nominators, mut edge_per_voter, mut to_elect, mode_u32) = data; + let ext = new_test_ext(); + let mode: Mode = unsafe { std::mem::transmute(mode_u32) }; + num_validators = to_range(num_validators, 50, 1000); + num_nominators = to_range(num_nominators, 50, 2000); + edge_per_voter = to_range(edge_per_voter, 1, 16); + to_elect = to_range(to_elect, 20, num_validators); + let do_reduce = true; + + println!("+++ instance with params {} / {} / {} / {:?}({}) / {}", + num_nominators, + num_validators, + edge_per_voter, + mode, + mode_u32, + to_elect, + ); + + ext.unwrap_or_default().execute_with(|| { + // initial setup + set_validator_count::(to_elect); + pallet_staking::testing_utils::init_active_era(); + setup_chain_stakers::( + num_validators, + num_nominators, + edge_per_voter, + ); + + println!("++ Chain setup done."); + + // stuff to submit + let (winners, compact, score) = match mode { + Mode::InitialSubmission => { + /* No need to setup anything */ + get_seq_phragmen_solution::(do_reduce) + }, + Mode::StrongerSubmission => { + let (winners, compact, score) = get_weak_solution::(false); + println!("Weak on chain score = {:?}", score); + assert_ok!( + >::submit_election_solution( + signed_account::(USER), + winners, + compact, + score, + pallet_staking::testing_utils::active_era::(), + ) + ); + get_seq_phragmen_solution::(do_reduce) + }, + Mode::WeakerSubmission => { + let (winners, compact, score) = get_seq_phragmen_solution::(do_reduce); + println!("Strong on chain score = {:?}", score); + assert_ok!( + >::submit_election_solution( + signed_account::(USER), + winners, + compact, + score, + pallet_staking::testing_utils::active_era::(), + ) + ); + get_weak_solution::(false) + } + }; + + println!("++ Submission ready. Score = {:?}", score); + + // must have chosen correct number of winners. + assert_eq!(winners.len() as u32, >::validator_count()); + + // final call and origin + let call = pallet_staking::Call::::submit_election_solution( + winners, + compact, + score, + pallet_staking::testing_utils::active_era::(), + ); + let caller = signed_account::(USER); + + // actually submit + match mode { + Mode::WeakerSubmission => { + assert_eq!( + call.dispatch(caller.into()).unwrap_err().error, + DispatchError::Module { index: 0, error: 15, message: Some("PhragmenWeakSubmission") }, + ); + }, + // NOTE: so exhaustive pattern doesn't work here.. maybe some rust issue? or due to `#[repr(u32)]`? + Mode::InitialSubmission | Mode::StrongerSubmission => assert!(call.dispatch(caller.into()).is_ok()), + }; + }) + }); + } +} diff --git a/frame/staking/reward-curve/Cargo.toml b/frame/staking/reward-curve/Cargo.toml index b3b749e96cd51fa050dfb6233f0c99eadf6ad3fb..b11d1cc0493389473a78e527b14d14ec68a1d765 100644 --- a/frame/staking/reward-curve/Cargo.toml +++ b/frame/staking/reward-curve/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-staking-reward-curve" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,6 +8,9 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Reward Curve for FRAME staking pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [lib] proc-macro = true @@ -18,7 +21,4 @@ proc-macro2 = "1.0.6" proc-macro-crate = "0.1.4" [dev-dependencies] -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } diff --git a/frame/staking/reward-curve/src/lib.rs b/frame/staking/reward-curve/src/lib.rs index d000afc49b515ea4ea2d20daf1c65b9c5ea4cf1a..5a3d88bb537a876e3d89dbad715ba11523c5b983 100644 --- a/frame/staking/reward-curve/src/lib.rs +++ b/frame/staking/reward-curve/src/lib.rs @@ -268,10 +268,14 @@ impl INPoS { } } + // calculates x from: + // y = i_0 + (i_ideal * x_ideal - i_0) * 2^((x_ideal - x)/d) + // See web3 docs for the details fn compute_opposite_after_x_ideal(&self, y: u32) -> u32 { if y == self.i_0 { return u32::max_value(); } + // Note: the log term calculated here represents a per_million value let log = log2(self.i_ideal_times_x_ideal - self.i_0, y - self.i_0); let term: u32 = ((self.d as u64 * log as u64) / 1_000_000).try_into().unwrap(); diff --git a/frame/staking/reward-curve/src/log.rs b/frame/staking/reward-curve/src/log.rs index e0929a95970139450c0e953f6df7e8849e5ae727..28acd5deed2bbf7c879f30307bf14075752f506d 100644 --- a/frame/staking/reward-curve/src/log.rs +++ b/frame/staking/reward-curve/src/log.rs @@ -1,48 +1,65 @@ use std::convert::TryInto; -/// Return Per-million value. +/// Simple u32 power of 2 function - simply uses a bit shift +macro_rules! pow2 { + ($n:expr) => { + 1_u32 << $n + } +} + +/// Returns the k_th per_million taylor term for a log2 function +fn taylor_term(k: u32, y_num: u128, y_den: u128) -> u32 { + let _2_div_ln_2: u128 = 2_885_390u128; + + if k == 0 { + (_2_div_ln_2 * (y_num).pow(1) / (y_den).pow(1)).try_into().unwrap() + } else { + let mut res = _2_div_ln_2 * (y_num).pow(3) / (y_den).pow(3); + for _ in 1..k { + res = res * (y_num).pow(2) / (y_den).pow(2); + } + res /= 2 * k as u128 + 1; + + res.try_into().unwrap() + } +} + +/// Performs a log2 operation using a rational fraction +/// +/// result = log2(p/q) where p/q is bound to [1, 1_000_000] +/// Where: +/// * q represents the numerator of the rational fraction input +/// * p represents the denominator of the rational fraction input +/// * result represents a per-million output of log2 pub fn log2(p: u32, q: u32) -> u32 { - assert!(p >= q); + assert!(p >= q); // keep p/q bound to [1, inf) assert!(p <= u32::max_value()/2); // This restriction should not be mandatory. But function is only tested and used for this. assert!(p <= 1_000_000); assert!(q <= 1_000_000); + // log2(1) = 0 if p == q { return 0 } + // find the power of 2 where q * 2^n <= p < q * 2^(n+1) let mut n = 0u32; - while !(p >= (1u32 << n)*q) || !(p < (1u32 << (n+1))*q) { + while !(p >= pow2!(n) * q) || !(p < pow2!(n + 1) * q) { n += 1; + assert!(n < 32); // cannot represent 2^32 in u32 } - assert!(p < (1u32 << (n+1)) * q); - - let y_num: u32 = (p - (1u32 << n) * q).try_into().unwrap(); - let y_den: u32 = (p + (1u32 << n) * q).try_into().unwrap(); - - let _2_div_ln_2 = 2_885_390u32; + assert!(p < pow2!(n + 1) * q); - let taylor_term = |k: u32| -> u32 { - if k == 0 { - (_2_div_ln_2 as u128 * (y_num as u128).pow(1) / (y_den as u128).pow(1)) - .try_into().unwrap() - } else { - let mut res = _2_div_ln_2 as u128 * (y_num as u128).pow(3) / (y_den as u128).pow(3); - for _ in 1..k { - res = res * (y_num as u128).pow(2) / (y_den as u128).pow(2); - } - res /= 2 * k as u128 + 1; - - res.try_into().unwrap() - } - }; + let y_num: u32 = (p - pow2!(n) * q).try_into().unwrap(); + let y_den: u32 = (p + pow2!(n) * q).try_into().unwrap(); + // Loop through each Taylor series coefficient until it reaches 10^-6 let mut res = n * 1_000_000u32; let mut k = 0; loop { - let term = taylor_term(k); + let term = taylor_term(k, y_num.into(), y_den.into()); if term == 0 { break } @@ -68,3 +85,43 @@ fn test_log() { } } } + +#[test] +#[should_panic] +fn test_log_p_must_be_greater_than_q() { + let p: u32 = 1_000; + let q: u32 = 1_001; + let _ = log2(p, q); +} + +#[test] +#[should_panic] +fn test_log_p_upper_bound() { + let p: u32 = 1_000_001; + let q: u32 = 1_000_000; + let _ = log2(p, q); +} + +#[test] +#[should_panic] +fn test_log_q_limit() { + let p: u32 = 1_000_000; + let q: u32 = 0; + let _ = log2(p, q); +} + +#[test] +fn test_log_of_one_boundary() { + let p: u32 = 1_000_000; + let q: u32 = 1_000_000; + assert_eq!(log2(p, q), 0); +} + +#[test] +fn test_log_of_largest_input() { + let p: u32 = 1_000_000; + let q: u32 = 1; + let expected = 19_931_568; + let tolerance = 100; + assert!((log2(p, q) as i32 - expected as i32).abs() < tolerance); +} \ No newline at end of file diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 60d0c132376f29d71aaf8a632621fce1e7a881e3..2686623aa1eae85655beaf9a917679d1e483319f 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -98,8 +98,8 @@ pub fn create_validators_with_nominators_for_era(v: u32, n: u32) -> Re Ok(()) } -// This function generates one validator being nominated by n nominators. -// It starts an era and creates pending payouts. +// This function generates one validator being nominated by n nominators, and returns +//the validator stash account. It also starts an era and creates pending payouts. pub fn create_validator_with_nominators(n: u32, upper_bound: u32) -> Result { let mut points_total = 0; let mut points_individual = Vec::new(); @@ -114,7 +114,7 @@ pub fn create_validator_with_nominators(n: u32, upper_bound: u32) -> R let stash_lookup: ::Source = T::Lookup::unlookup(v_stash.clone()); points_total += 10; - points_individual.push((v_stash, 10)); + points_individual.push((v_stash.clone(), 10)); // Give the validator n nominators, but keep total users in the system the same. for i in 0 .. upper_bound { @@ -144,7 +144,7 @@ pub fn create_validator_with_nominators(n: u32, upper_bound: u32) -> R let total_payout = T::Currency::minimum_balance() * 1000.into(); >::insert(current_era, total_payout); - Ok(v_controller) + Ok(v_stash) } benchmarks! { @@ -368,16 +368,10 @@ benchmarks! { #[cfg(test)] mod tests { - use crate::*; - use crate::mock::*; + use super::*; + use crate::mock::{ExtBuilder, Test, Balances, Staking, Origin}; use frame_support::assert_ok; - use crate::benchmarking::{ - create_validators_with_nominators_for_era, - create_validator_with_nominators, - SelectedBenchmark, - }; - #[test] fn create_validators_with_nominators_for_era_works() { ExtBuilder::default().has_stakers(false).build().execute_with(|| { @@ -399,19 +393,16 @@ mod tests { ExtBuilder::default().has_stakers(false).build().execute_with(|| { let n = 10; - let validator = create_validator_with_nominators::( + let validator_stash = create_validator_with_nominators::( n, MAX_NOMINATIONS as u32, ).unwrap(); let current_era = CurrentEra::get().unwrap(); - let controller = validator; - let ledger = Staking::ledger(&controller).unwrap(); - let stash = ledger.stash; - let original_free_balance = Balances::free_balance(&stash); - assert_ok!(Staking::payout_stakers(Origin::signed(1337), stash, current_era)); - let new_free_balance = Balances::free_balance(&stash); + let original_free_balance = Balances::free_balance(&validator_stash); + assert_ok!(Staking::payout_stakers(Origin::signed(1337), validator_stash, current_era)); + let new_free_balance = Balances::free_balance(&validator_stash); assert!(original_free_balance < new_free_balance); }); @@ -434,4 +425,33 @@ mod tests { assert_ok!(closure_to_benchmark()); }); } + + #[test] + fn test_benchmarks() { + ExtBuilder::default().has_stakers(false).build().execute_with(|| { + assert_ok!(test_benchmark_bond::()); + assert_ok!(test_benchmark_bond_extra::()); + assert_ok!(test_benchmark_unbond::()); + assert_ok!(test_benchmark_withdraw_unbonded::()); + assert_ok!(test_benchmark_validate::()); + assert_ok!(test_benchmark_nominate::()); + assert_ok!(test_benchmark_chill::()); + assert_ok!(test_benchmark_set_payee::()); + assert_ok!(test_benchmark_set_controller::()); + assert_ok!(test_benchmark_set_validator_count::()); + assert_ok!(test_benchmark_force_no_eras::()); + assert_ok!(test_benchmark_force_new_era::()); + assert_ok!(test_benchmark_force_new_era_always::()); + assert_ok!(test_benchmark_set_invulnerables::()); + assert_ok!(test_benchmark_force_unstake::()); + assert_ok!(test_benchmark_cancel_deferred_slash::()); + assert_ok!(test_benchmark_payout_stakers::()); + assert_ok!(test_benchmark_rebond::()); + assert_ok!(test_benchmark_set_history_depth::()); + assert_ok!(test_benchmark_reap_stash::()); + assert_ok!(test_benchmark_new_era::()); + assert_ok!(test_benchmark_do_slash::()); + assert_ok!(test_benchmark_payout_all::()); + }); + } } diff --git a/frame/staking/src/inflation.rs b/frame/staking/src/inflation.rs index d20741d9bc44d656134bfe9297268f41ea3b52b8..63d008a197c6465b3e937dccf36319842be375d6 100644 --- a/frame/staking/src/inflation.rs +++ b/frame/staking/src/inflation.rs @@ -21,10 +21,11 @@ use sp_runtime::{Perbill, traits::AtLeast32Bit, curve::PiecewiseLinear}; -/// The total payout to all validators (and their nominators) per era. +/// The total payout to all validators (and their nominators) per era and maximum payout. /// /// Defined as such: -/// `payout = yearly_inflation(npos_token_staked / total_tokens) * total_tokens / era_per_year` +/// `staker-payout = yearly_inflation(npos_token_staked / total_tokens) * total_tokens / era_per_year` +/// `maximum-payout = max_yearly_inflation * total_tokens / era_per_year` /// /// `era_duration` is expressed in millisecond. pub fn compute_total_payout( diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index b566d616a80ee9c35a702668a6edfdf6d48d1f24..67240d8d34b2a212186b7a9c0eeaa51d08c17d29 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -158,7 +158,7 @@ //! decl_module! { //! pub struct Module for enum Call where origin: T::Origin { //! /// Reward a validator. -//! #[weight = frame_support::weights::SimpleDispatchInfo::default()] +//! #[weight = 0] //! pub fn reward_myself(origin) -> dispatch::DispatchResult { //! let reported = ensure_signed(origin)?; //! >::reward_by_ids(vec![(reported, 10)]); @@ -171,6 +171,22 @@ //! //! ## Implementation Details //! +//! ### Era payout +//! +//! The era payout is computed using yearly inflation curve defined at +//! [`T::RewardCurve`](./trait.Trait.html#associatedtype.RewardCurve) as such: +//! +//! ```nocompile +//! staker_payout = yearly_inflation(npos_token_staked / total_tokens) * total_tokens / era_per_year +//! ``` +//! This payout is used to reward stakers as defined in next section +//! +//! ```nocompile +//! remaining_payout = max_yearly_inflation * total_tokens / era_per_year - staker_payout +//! ``` +//! The remaining reward is send to the configurable end-point +//! [`T::RewardRemainder`](./trait.Trait.html#associatedtype.RewardRemainder). +//! //! ### Reward Calculation //! //! Validators and nominators are rewarded at the end of each era. The total reward of an era is @@ -274,7 +290,7 @@ use sp_std::{ use codec::{HasCompact, Encode, Decode}; use frame_support::{ decl_module, decl_event, decl_storage, ensure, decl_error, debug, - weights::{SimpleDispatchInfo, Weight}, + weights::{Weight, DispatchClass}, storage::IterableStorageMap, dispatch::{IsSubType, DispatchResult}, traits::{ @@ -288,11 +304,11 @@ use sp_runtime::{ curve::PiecewiseLinear, traits::{ Convert, Zero, StaticLookup, CheckedSub, Saturating, SaturatedConversion, AtLeast32Bit, - SignedExtension, + Dispatchable, }, transaction_validity::{ TransactionValidityError, TransactionValidity, ValidTransaction, InvalidTransaction, - TransactionSource, + TransactionSource, TransactionPriority, }, }; use sp_staking::{ @@ -303,11 +319,11 @@ use sp_staking::{ use sp_runtime::{Serialize, Deserialize}; use frame_system::{ self as system, ensure_signed, ensure_root, ensure_none, - offchain::SubmitUnsignedTransaction, + offchain::SendTransactionTypes, }; use sp_phragmen::{ ExtendedBalance, Assignment, PhragmenScore, PhragmenResult, build_support_map, evaluate_support, - elect, generate_compact_solution_type, is_score_better, VotingLimit, SupportMap, + elect, generate_compact_solution_type, is_score_better, VotingLimit, SupportMap, VoteWeight, }; const DEFAULT_MINIMUM_VALIDATOR_COUNT: u32 = 4; @@ -315,6 +331,18 @@ const STAKING_ID: LockIdentifier = *b"staking "; pub const MAX_UNLOCKING_CHUNKS: usize = 32; pub const MAX_NOMINATIONS: usize = ::LIMIT; +// syntactic sugar for logging +#[cfg(feature = "std")] +const LOG_TARGET: &'static str = "staking"; +macro_rules! log { + ($level:tt, $patter:expr $(, $values:expr)* $(,)?) => { + debug::native::$level!( + target: LOG_TARGET, + $patter $(, $values)* + ) + }; +} + /// Data type used to index nominators in the compact type pub type NominatorIndex = u32; @@ -342,7 +370,7 @@ generate_compact_solution_type!(pub GenericCompactAssignments, 16); #[derive(Encode, Decode, RuntimeDebug)] pub struct ActiveEraInfo { /// Index of era. - index: EraIndex, + pub index: EraIndex, /// Moment of start expresed as millisecond from `$UNIX_EPOCH`. /// /// Start can be none if start hasn't been set for the era yet, @@ -583,10 +611,10 @@ pub struct Nominations { #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Encode, Decode, RuntimeDebug)] pub struct IndividualExposure { /// The stash account of the nominator in question. - who: AccountId, + pub who: AccountId, /// Amount of funds exposed. #[codec(compact)] - value: Balance, + pub value: Balance, } /// A snapshot of the stake backing a single validator in the system. @@ -714,7 +742,7 @@ impl SessionInterface<::AccountId> for T whe } } -pub trait Trait: frame_system::Trait { +pub trait Trait: frame_system::Trait + SendTransactionTypes> { /// The staking balance. type Currency: LockableCurrency; @@ -724,14 +752,14 @@ pub trait Trait: frame_system::Trait { /// is not used. type UnixTime: UnixTime; - /// Convert a balance into a number used for election calculation. - /// This must fit into a `u64` but is allowed to be sensibly lossy. - /// TODO: #1377 - /// The backward convert should be removed as the new Phragmen API returns ratio. - /// The post-processing needs it but will be moved to off-chain. TODO: #2908 - type CurrencyToVote: Convert, u64> + Convert>; + /// Convert a balance into a number used for election calculation. This must fit into a `u64` + /// but is allowed to be sensibly lossy. The `u64` is used to communicate with the + /// [`sp_phragmen`] crate which accepts u64 numbers and does operations in 128. Consequently, + /// the backward convert is used convert the u128s from phragmen back to a [`BalanceOf`]. + type CurrencyToVote: Convert, VoteWeight> + Convert>; /// Tokens have been minted and are unused for validator-reward. + /// See [Era payout](./index.html#era-payout). type RewardRemainder: OnUnbalanced>; /// The overarching event type. @@ -760,7 +788,8 @@ pub trait Trait: frame_system::Trait { /// Interface for interacting with a session module. type SessionInterface: self::SessionInterface; - /// The NPoS reward curve to use. + /// The NPoS reward curve used to define yearly inflation. + /// See [Era payout](./index.html#era-payout). type RewardCurve: Get<&'static PiecewiseLinear<'static>>; /// Something that can estimate the next session change, accurately or as a best effort guess. @@ -772,16 +801,23 @@ pub trait Trait: frame_system::Trait { type ElectionLookahead: Get; /// The overarching call type. - type Call: From> + IsSubType, Self> + Clone; + type Call: Dispatchable + From> + IsSubType, Self> + Clone; - /// A transaction submitter. - type SubmitTransaction: SubmitUnsignedTransaction::Call>; + /// Maximum number of equalise iterations to run in the offchain submission. If set to 0, + /// equalize will not be executed at all. + type MaxIterations: Get; - /// The maximum number of nominators rewarded for each validator. + /// The maximum number of nominator rewarded for each validator. /// /// For each validator only the `$MaxNominatorRewardedPerValidator` biggest stakers can claim /// their reward. This used to limit the i/o cost for the nominator payout. type MaxNominatorRewardedPerValidator: Get; + + /// A configuration for base priority of unsigned transactions. + /// + /// This is exposed so that it can be tuned for particular runtime, when + /// multiple pallets send unsigned transactions. + type UnsignedPriority: Get; } /// Mode of era-forcing. @@ -992,7 +1028,8 @@ decl_storage! { /// solutions to be submitted. pub EraElectionStatus get(fn era_election_status): ElectionStatus; - /// True if the current planned session is final. + /// True if the current **planned** session is final. Note that this does not take era + /// forcing into account. pub IsCurrentSessionFinal get(fn is_current_session_final): bool = false; /// True if network has been upgraded to this version. @@ -1040,6 +1077,9 @@ decl_storage! { decl_event!( pub enum Event where Balance = BalanceOf, ::AccountId { + /// The era payout has been set; the first balance is the validator-payout; the second is + /// the remainder from the maximum amount of reward. + EraPayout(EraIndex, Balance, Balance), /// The staker has been rewarded by this amount. `AccountId` is the stash account. Reward(AccountId, Balance), /// One validator (and its nominators) has been slashed by the given amount. @@ -1122,6 +1162,8 @@ decl_error! { PhragmenBogusEdge, /// The claimed score does not match with the one computed from the data. PhragmenBogusScore, + /// The call is not allowed at the given time due to restrictions of election period. + CallNotAllowed, } } @@ -1146,7 +1188,8 @@ decl_module! { if // if we don't have any ongoing offchain compute. Self::era_election_status().is_closed() && - Self::is_current_session_final() + // either current session final based on the plan, or we're forcing. + (Self::is_current_session_final() || Self::will_era_be_forced()) { if let Some(next_session_change) = T::NextNewSession::estimate_next_new_session(now){ if let Some(remaining) = next_session_change.checked_sub(&now) { @@ -1158,26 +1201,14 @@ decl_module! { >::put( ElectionStatus::::Open(now) ); - debug::native::info!( - target: "staking", - "Election window is Open({:?}). Snapshot created", - now, - ); + log!(info, "💸 Election window is Open({:?}). Snapshot created", now); } else { - debug::native::warn!( - target: "staking", - "Failed to create snapshot at {:?}. Election window will remain closed.", - now, - ); + log!(warn, "💸 Failed to create snapshot at {:?}.", now); } - } } } else { - debug::native::warn!( - target: "staking", - "estimate_next_new_session() failed to execute. Election status cannot be changed.", - ); + log!(warn, "💸 Estimating next session change failed."); } } @@ -1193,23 +1224,12 @@ decl_module! { if Self::era_election_status().is_open_at(now) { let offchain_status = set_check_offchain_execution_status::(now); if let Err(why) = offchain_status { - debug::native::warn!( - target: "staking", - "skipping offchain worker in open election window due to [{}]", - why, - ); + log!(debug, "skipping offchain worker in open election window due to [{}]", why); } else { if let Err(e) = compute_offchain_election::() { - debug::native::warn!( - target: "staking", - "Error in phragmen offchain worker: {:?}", - e, - ); + log!(warn, "💸 Error in phragmen offchain worker: {:?}", e); } else { - debug::native::debug!( - target: "staking", - "Executed offchain worker thread without errors. Transaction submitted to the pool.", - ); + log!(debug, "Executed offchain worker thread without errors."); } } } @@ -1229,8 +1249,12 @@ decl_module! { fn on_runtime_upgrade() -> Weight { // For Kusama the type hasn't actually changed as Moment was u64 and was the number of // millisecond since unix epoch. - StorageVersion::put(Releases::V3_0_0); - Self::migrate_last_reward_to_claimed_rewards(); + StorageVersion::mutate(|v| { + if matches!(v, Releases::V2_0_0) { + Self::migrate_last_reward_to_claimed_rewards(); + } + *v = Releases::V3_0_0; + }); 0 } @@ -1251,8 +1275,8 @@ decl_module! { /// NOTE: Two of the storage writes (`Self::bonded`, `Self::payee`) are _never_ cleaned /// unless the `origin` falls below _existential deposit_ and gets removed as dust. /// # - #[weight = SimpleDispatchInfo::FixedNormal(500_000)] - fn bond(origin, + #[weight = 500_000_000] + pub fn bond(origin, controller: ::Source, #[compact] value: BalanceOf, payee: RewardDestination, @@ -1305,7 +1329,8 @@ decl_module! { /// Unlike [`bond`] or [`unbond`] this function does not impose any limitation on the amount /// that can be added. /// - /// The dispatch origin for this call must be _Signed_ by the stash, not the controller. + /// The dispatch origin for this call must be _Signed_ by the stash, not the controller and + /// it can be only called when [`EraElectionStatus`] is `Closed`. /// /// Emits `Bonded`. /// @@ -1314,8 +1339,9 @@ decl_module! { /// - O(1). /// - One DB entry. /// # - #[weight = SimpleDispatchInfo::FixedNormal(500_000)] + #[weight = 500_000_000] fn bond_extra(origin, #[compact] max_additional: BalanceOf) { + ensure!(Self::era_election_status().is_closed(), Error::::CallNotAllowed); let stash = ensure_signed(origin)?; let controller = Self::bonded(&stash).ok_or(Error::::NotStash)?; @@ -1344,6 +1370,7 @@ decl_module! { /// to be called first to remove some of the chunks (if possible). /// /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// And, it can be only called when [`EraElectionStatus`] is `Closed`. /// /// Emits `Unbonded`. /// @@ -1358,8 +1385,9 @@ decl_module! { /// `withdraw_unbonded`. /// - One DB entry. /// - #[weight = SimpleDispatchInfo::FixedNormal(400_000)] + #[weight = 400_000_000] fn unbond(origin, #[compact] value: BalanceOf) { + ensure!(Self::era_election_status().is_closed(), Error::::CallNotAllowed); let controller = ensure_signed(origin)?; let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; ensure!( @@ -1392,6 +1420,7 @@ decl_module! { /// whatever it wants. /// /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// And, it can be only called when [`EraElectionStatus`] is `Closed`. /// /// Emits `Withdrawn`. /// @@ -1404,8 +1433,9 @@ decl_module! { /// - Contains a limited number of reads, yet the size of which could be large based on `ledger`. /// - Writes are limited to the `origin` account key. /// # - #[weight = SimpleDispatchInfo::FixedNormal(400_000)] + #[weight = 400_000_000] fn withdraw_unbonded(origin) { + ensure!(Self::era_election_status().is_closed(), Error::::CallNotAllowed); let controller = ensure_signed(origin)?; let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; let (stash, old_total) = (ledger.stash.clone(), ledger.total); @@ -1439,14 +1469,16 @@ decl_module! { /// Effects will be felt at the beginning of the next era. /// /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// And, it can be only called when [`EraElectionStatus`] is `Closed`. /// /// # /// - Independent of the arguments. Insignificant complexity. /// - Contains a limited number of reads. /// - Writes are limited to the `origin` account key. /// # - #[weight = SimpleDispatchInfo::FixedNormal(750_000)] - fn validate(origin, prefs: ValidatorPrefs) { + #[weight = 750_000_000] + pub fn validate(origin, prefs: ValidatorPrefs) { + ensure!(Self::era_election_status().is_closed(), Error::::CallNotAllowed); let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; let stash = &ledger.stash; @@ -1456,17 +1488,20 @@ decl_module! { /// Declare the desire to nominate `targets` for the origin controller. /// - /// Effects will be felt at the beginning of the next era. + /// Effects will be felt at the beginning of the next era. This can only be called when + /// [`EraElectionStatus`] is `Closed`. /// /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// And, it can be only called when [`EraElectionStatus`] is `Closed`. /// /// # /// - The transaction's complexity is proportional to the size of `targets`, /// which is capped at CompactAssignments::LIMIT. /// - Both the reads and writes follow a similar pattern. /// # - #[weight = SimpleDispatchInfo::FixedNormal(750_000)] - fn nominate(origin, targets: Vec<::Source>) { + #[weight = 750_000_000] + pub fn nominate(origin, targets: Vec<::Source>) { + ensure!(Self::era_election_status().is_closed(), Error::::CallNotAllowed); let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; let stash = &ledger.stash; @@ -1492,14 +1527,16 @@ decl_module! { /// Effects will be felt at the beginning of the next era. /// /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// And, it can be only called when [`EraElectionStatus`] is `Closed`. /// /// # /// - Independent of the arguments. Insignificant complexity. /// - Contains one read. /// - Writes are limited to the `origin` account key. /// # - #[weight = SimpleDispatchInfo::FixedNormal(500_000)] + #[weight = 500_000_000] fn chill(origin) { + ensure!(Self::era_election_status().is_closed(), Error::::CallNotAllowed); let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; Self::chill_stash(&ledger.stash); @@ -1516,7 +1553,7 @@ decl_module! { /// - Contains a limited number of reads. /// - Writes are limited to the `origin` account key. /// # - #[weight = SimpleDispatchInfo::FixedNormal(500_000)] + #[weight = 500_000_000] fn set_payee(origin, payee: RewardDestination) { let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; @@ -1535,7 +1572,7 @@ decl_module! { /// - Contains a limited number of reads. /// - Writes are limited to the `origin` account key. /// # - #[weight = SimpleDispatchInfo::FixedNormal(750_000)] + #[weight = 750_000_000] fn set_controller(origin, controller: ::Source) { let stash = ensure_signed(origin)?; let old_controller = Self::bonded(&stash).ok_or(Error::::NotStash)?; @@ -1551,10 +1588,8 @@ decl_module! { } } - // ----- Root calls. - /// The ideal number of validators. - #[weight = SimpleDispatchInfo::FixedNormal(5_000)] + #[weight = 5_000_000] fn set_validator_count(origin, #[compact] new: u32) { ensure_root(origin)?; ValidatorCount::put(new); @@ -1565,7 +1600,7 @@ decl_module! { /// # /// - No arguments. /// # - #[weight = SimpleDispatchInfo::FixedNormal(5_000)] + #[weight = 5_000_000] fn force_no_eras(origin) { ensure_root(origin)?; ForceEra::put(Forcing::ForceNone); @@ -1577,21 +1612,21 @@ decl_module! { /// # /// - No arguments. /// # - #[weight = SimpleDispatchInfo::FixedNormal(5_000)] + #[weight = 5_000_000] fn force_new_era(origin) { ensure_root(origin)?; ForceEra::put(Forcing::ForceNew); } /// Set the validators who cannot be slashed (if any). - #[weight = SimpleDispatchInfo::FixedNormal(5_000)] + #[weight = 5_000_000] fn set_invulnerables(origin, validators: Vec) { ensure_root(origin)?; >::put(validators); } /// Force a current staker to become completely unstaked, immediately. - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + #[weight = 0] fn force_unstake(origin, stash: T::AccountId) { ensure_root(origin)?; @@ -1607,7 +1642,7 @@ decl_module! { /// # /// - One storage write /// # - #[weight = SimpleDispatchInfo::FixedNormal(5_000)] + #[weight = 5_000_000] fn force_new_era_always(origin) { ensure_root(origin)?; ForceEra::put(Forcing::ForceAlways); @@ -1620,7 +1655,7 @@ decl_module! { /// # /// - One storage write. /// # - #[weight = SimpleDispatchInfo::FixedNormal(1_000_000)] + #[weight = 1_000_000_000] fn cancel_deferred_slash(origin, era: EraIndex, slash_indices: Vec) { T::SlashCancelOrigin::try_origin(origin) .map(|_| ()) @@ -1671,12 +1706,12 @@ decl_module! { /// maximum number of validators that may be nominated by a single nominator, it is /// bounded only economically (all nominators are required to place a minimum stake). /// # - #[weight = SimpleDispatchInfo::FixedNormal(500_000)] + #[weight = 500_000_000] fn payout_nominator(origin, era: EraIndex, validators: Vec<(T::AccountId, u32)>) -> DispatchResult { - let who = ensure_signed(origin)?; - Self::do_payout_nominator(who, era, validators) + let ctrl = ensure_signed(origin)?; + Self::do_payout_nominator(ctrl, era, validators) } /// **This extrinsic will be removed after `MigrationEra + HistoryDepth` has passed, giving @@ -1698,10 +1733,10 @@ decl_module! { /// - Time complexity: O(1). /// - Contains a limited number of reads and writes. /// # - #[weight = SimpleDispatchInfo::FixedNormal(500_000)] + #[weight = 500_000_000] fn payout_validator(origin, era: EraIndex) -> DispatchResult { - let who = ensure_signed(origin)?; - Self::do_payout_validator(who, era) + let ctrl = ensure_signed(origin)?; + Self::do_payout_validator(ctrl, era) } /// Pay out all the stakers behind a single validator for a single era. @@ -1713,30 +1748,34 @@ decl_module! { /// The origin of this call must be _Signed_. Any account can call this function, even if /// it is not one of the stakers. /// + /// This can only be called when [`EraElectionStatus`] is `Closed`. + /// /// # /// - Time complexity: at most O(MaxNominatorRewardedPerValidator). /// - Contains a limited number of reads and writes. /// # - #[weight = SimpleDispatchInfo::FixedNormal(500_000)] + #[weight = 500_000_000] fn payout_stakers(origin, validator_stash: T::AccountId, era: EraIndex) -> DispatchResult { + ensure!(Self::era_election_status().is_closed(), Error::::CallNotAllowed); ensure_signed(origin)?; Self::do_payout_stakers(validator_stash, era) } /// Rebond a portion of the stash scheduled to be unlocked. /// + /// The dispatch origin must be signed by the controller, and it can be only called when + /// [`EraElectionStatus`] is `Closed`. + /// /// # /// - Time complexity: O(1). Bounded by `MAX_UNLOCKING_CHUNKS`. /// - Storage changes: Can't increase storage, only decrease it. /// # - #[weight = SimpleDispatchInfo::FixedNormal(500_000)] + #[weight = 500_000_000] fn rebond(origin, #[compact] value: BalanceOf) { + ensure!(Self::era_election_status().is_closed(), Error::::CallNotAllowed); let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - ensure!( - !ledger.unlocking.is_empty(), - Error::::NoUnlockChunk, - ); + ensure!(!ledger.unlocking.is_empty(), Error::::NoUnlockChunk); let ledger = ledger.rebond(value); Self::update_ledger(&controller, &ledger); @@ -1745,7 +1784,7 @@ decl_module! { /// Set history_depth value. /// /// Origin must be root. - #[weight = SimpleDispatchInfo::FixedOperational(500_000)] + #[weight = (500_000_000, DispatchClass::Operational)] fn set_history_depth(origin, #[compact] new_history_depth: EraIndex) { ensure_root(origin)?; if let Some(current_era) = Self::current_era() { @@ -1767,7 +1806,7 @@ decl_module! { /// This can be called from any origin. /// /// - `stash`: The stash account to reap. Its balance must be zero. - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn reap_stash(_origin, stash: T::AccountId) { ensure!(T::Currency::total_balance(&stash).is_zero(), Error::::FundedTarget); Self::kill_stash(&stash)?; @@ -1838,7 +1877,7 @@ decl_module! { /// - Memory: O(n + m) reads to map index to `AccountId` for un-compact. /// /// - Storage: O(e) accountid reads from `Nomination` to read correct nominations. - /// - Storage: O(e) calls into `slashable_balance_of_extended` to convert ratio to staked. + /// - Storage: O(e) calls into `slashable_balance_of_vote_weight` to convert ratio to staked. /// /// - Memory: build_support_map. O(e). /// - Memory: evaluate_support: O(E). @@ -1848,7 +1887,7 @@ decl_module! { /// /// The weight of this call is 1/10th of the blocks total weight. /// # - #[weight = SimpleDispatchInfo::FixedNormal(100_000_000)] + #[weight = 100_000_000_000] pub fn submit_election_solution( origin, winners: Vec, @@ -1871,7 +1910,7 @@ decl_module! { /// Note that this must pass the [`ValidateUnsigned`] check which only allows transactions /// from the local node to be included. In other words, only the block author can include a /// transaction in the block. - #[weight = SimpleDispatchInfo::FixedNormal(100_000_000)] + #[weight = 100_000_000_000] pub fn submit_election_solution_unsigned( origin, winners: Vec, @@ -1938,11 +1977,11 @@ impl Module { Self::bonded(stash).and_then(Self::ledger).map(|l| l.active).unwrap_or_default() } - /// internal impl of [`slashable_balance_of`] that returns [`ExtendedBalance`]. - fn slashable_balance_of_extended(stash: &T::AccountId) -> ExtendedBalance { - , u64>>::convert( + /// internal impl of [`slashable_balance_of`] that returns [`VoteWeight`]. + fn slashable_balance_of_vote_weight(stash: &T::AccountId) -> VoteWeight { + , VoteWeight>>::convert( Self::slashable_balance_of(stash) - ) as ExtendedBalance + ) } /// Dump the list of validators and nominators into vectors and keep them on-chain. @@ -1959,9 +1998,9 @@ impl Module { num_validators > MAX_VALIDATORS || num_nominators.saturating_add(num_validators) > MAX_NOMINATORS { - debug::native::warn!( - target: "staking", - "Snapshot size too big [{} <> {}][{} <> {}].", + log!( + warn, + "💸 Snapshot size too big [{} <> {}][{} <> {}].", num_validators, MAX_VALIDATORS, num_nominators, @@ -1984,7 +2023,7 @@ impl Module { >::kill(); } - fn do_payout_nominator(who: T::AccountId, era: EraIndex, validators: Vec<(T::AccountId, u32)>) + fn do_payout_nominator(ctrl: T::AccountId, era: EraIndex, validators: Vec<(T::AccountId, u32)>) -> DispatchResult { // validators len must not exceed `MAX_NOMINATIONS` to avoid querying more validator @@ -2007,7 +2046,12 @@ impl Module { let era_payout = >::get(&era) .ok_or_else(|| Error::::InvalidEraToReward)?; - let mut nominator_ledger = >::get(&who).ok_or_else(|| Error::::NotController)?; + let mut nominator_ledger = >::get(&ctrl).ok_or_else(|| Error::::NotController)?; + + ensure!( + Self::era_election_status().is_closed() || Self::payee(&nominator_ledger.stash) != RewardDestination::Staked, + Error::::CallNotAllowed, + ); nominator_ledger.claimed_rewards.retain(|&x| x >= current_era.saturating_sub(history_depth)); match nominator_ledger.claimed_rewards.binary_search(&era) { @@ -2015,7 +2059,7 @@ impl Module { Err(pos) => nominator_ledger.claimed_rewards.insert(pos, era), } - >::insert(&who, &nominator_ledger); + >::insert(&ctrl, &nominator_ledger); let mut reward = Perbill::zero(); let era_reward_points = >::get(&era); @@ -2051,13 +2095,13 @@ impl Module { } if let Some(imbalance) = Self::make_payout(&nominator_ledger.stash, reward * era_payout) { - Self::deposit_event(RawEvent::Reward(who, imbalance.peek())); + Self::deposit_event(RawEvent::Reward(ctrl, imbalance.peek())); } Ok(()) } - fn do_payout_validator(who: T::AccountId, era: EraIndex) -> DispatchResult { + fn do_payout_validator(ctrl: T::AccountId, era: EraIndex) -> DispatchResult { // If migrate_era is not populated, then you should use `payout_stakers` let migrate_era = MigrateEra::get().ok_or(Error::::InvalidEraToReward)?; // This payout mechanism will only work for eras before the migration. @@ -2073,7 +2117,12 @@ impl Module { let era_payout = >::get(&era) .ok_or_else(|| Error::::InvalidEraToReward)?; - let mut ledger = >::get(&who).ok_or_else(|| Error::::NotController)?; + let mut ledger = >::get(&ctrl).ok_or_else(|| Error::::NotController)?; + + ensure!( + Self::era_election_status().is_closed() || Self::payee(&ledger.stash) != RewardDestination::Staked, + Error::::CallNotAllowed, + ); ledger.claimed_rewards.retain(|&x| x >= current_era.saturating_sub(history_depth)); match ledger.claimed_rewards.binary_search(&era) { @@ -2081,7 +2130,7 @@ impl Module { Err(pos) => ledger.claimed_rewards.insert(pos, era), } - >::insert(&who, &ledger); + >::insert(&ctrl, &ledger); let era_reward_points = >::get(&era); let commission = Self::eras_validator_prefs(&era, &ledger.stash).commission; @@ -2105,7 +2154,7 @@ impl Module { ); if let Some(imbalance) = Self::make_payout(&ledger.stash, reward * era_payout) { - Self::deposit_event(RawEvent::Reward(who, imbalance.peek())); + Self::deposit_event(RawEvent::Reward(ctrl, imbalance.peek())); } Ok(()) @@ -2115,7 +2164,7 @@ impl Module { validator_stash: T::AccountId, era: EraIndex, ) -> DispatchResult { - /* Validate input data */ + // Validate input data let current_era = CurrentEra::get().ok_or(Error::::InvalidEraToReward)?; ensure!(era <= current_era, Error::::InvalidEraToReward); let history_depth = Self::history_depth(); @@ -2303,7 +2352,7 @@ impl Module { ); // check current era. - if let Some(current_era) = Self::active_era().map(|e| e.index) { + if let Some(current_era) = Self::current_era() { ensure!( current_era == era, Error::::PhragmenEarlySubmission, @@ -2371,11 +2420,7 @@ impl Module { validator_at, ).map_err(|e| { // log the error since it is not propagated into the runtime error. - debug::native::warn!( - target: "staking", - "un-compacting solution failed due to {:?}", - e, - ); + log!(warn, "💸 un-compacting solution failed due to {:?}", e); Error::::PhragmenBogusCompact })?; @@ -2390,10 +2435,7 @@ impl Module { // all of the indices must map to either a validator or a nominator. If this is ever // not the case, then the locking system of staking is most likely faulty, or we // have bigger problems. - debug::native::error!( - target: "staking", - "detected an error in the staking locking and snapshot." - ); + log!(error, "💸 detected an error in the staking locking and snapshot."); // abort. return Err(Error::::PhragmenBogusNominator); } @@ -2438,7 +2480,7 @@ impl Module { // convert into staked assignments. let staked_assignments = sp_phragmen::assignment_ratio_to_staked( assignments, - Self::slashable_balance_of_extended, + Self::slashable_balance_of_vote_weight, ); // build the support map thereof in order to evaluate. @@ -2457,9 +2499,9 @@ impl Module { // At last, alles Ok. Exposures and store the result. let exposures = Self::collect_exposure(supports); - debug::native::info!( - target: "staking", - "A better solution (with compute {:?}) has been validated and stored on chain.", + log!( + info, + "💸 A better solution (with compute {:?}) has been validated and stored on chain.", compute, ); @@ -2553,16 +2595,20 @@ impl Module { let now_as_millis_u64 = T::UnixTime::now().as_millis().saturated_into::(); let era_duration = now_as_millis_u64 - active_era_start; - let (total_payout, _max_payout) = inflation::compute_total_payout( + let (validator_payout, max_payout) = inflation::compute_total_payout( &T::RewardCurve::get(), Self::eras_total_stake(&active_era.index), T::Currency::total_issuance(), // Duration of era; more than u64::MAX is rewarded as u64::MAX. era_duration.saturated_into::(), ); + let rest = max_payout.saturating_sub(validator_payout); + + Self::deposit_event(RawEvent::EraPayout(active_era.index, validator_payout, rest)); // Set ending era reward. - >::insert(&active_era.index, total_payout); + >::insert(&active_era.index, validator_payout); + T::RewardRemainder::on_unbalanced(T::Currency::issue(rest)); } } @@ -2640,9 +2686,9 @@ impl Module { // emit event Self::deposit_event(RawEvent::StakingElection(compute)); - debug::native::info!( - target: "staking", - "new validator set of size {:?} has been elected via {:?} for era {:?}", + log!( + info, + "💸 new validator set of size {:?} has been elected via {:?} for era {:?}", elected_stashes.len(), compute, current_era, @@ -2693,7 +2739,7 @@ impl Module { let staked_assignments = sp_phragmen::assignment_ratio_to_staked( assignments, - Self::slashable_balance_of_extended, + Self::slashable_balance_of_vote_weight, ); let (supports, _) = build_support_map::( @@ -2729,11 +2775,11 @@ impl Module { /// /// No storage item is updated. fn do_phragmen() -> Option> { - let mut all_nominators: Vec<(T::AccountId, BalanceOf, Vec)> = Vec::new(); + let mut all_nominators: Vec<(T::AccountId, VoteWeight, Vec)> = Vec::new(); let mut all_validators = Vec::new(); for (validator, _) in >::iter() { // append self vote - let self_vote = (validator.clone(), Self::slashable_balance_of(&validator), vec![validator.clone()]); + let self_vote = (validator.clone(), Self::slashable_balance_of_vote_weight(&validator), vec![validator.clone()]); all_nominators.push(self_vote); all_validators.push(validator); } @@ -2753,11 +2799,11 @@ impl Module { (nominator, targets) }); all_nominators.extend(nominator_votes.map(|(n, ns)| { - let s = Self::slashable_balance_of(&n); + let s = Self::slashable_balance_of_vote_weight(&n); (n, s, ns) })); - elect::<_, _, T::CurrencyToVote, Accuracy>( + elect::<_, Accuracy>( Self::validator_count() as usize, Self::minimum_validator_count().max(1) as usize, all_validators, @@ -2878,6 +2924,23 @@ impl Module { _ => ForceEra::put(Forcing::ForceNew), } } + + fn will_era_be_forced() -> bool { + match ForceEra::get() { + Forcing::ForceAlways | Forcing::ForceNew => true, + Forcing::ForceNone | Forcing::NotForcing => false, + } + } + + #[cfg(feature = "runtime-benchmarks")] + pub fn add_era_stakers(current_era: EraIndex, controller: T::AccountId, exposure: Exposure>) { + >::insert(¤t_era, &controller, &exposure); + } + + #[cfg(feature = "runtime-benchmarks")] + pub fn put_election_status(status: ElectionStatus::) { + >::put(status); + } } /// In this implementation `new_session(session)` must be called before `end_session(session-1)` @@ -3096,82 +3159,9 @@ impl ReportOffence } } -/// Disallows any transactions that change the election result to be submitted after the election -/// window is open. -#[derive(Encode, Decode, Clone, Eq, PartialEq)] -pub struct LockStakingStatus(sp_std::marker::PhantomData); - -impl sp_std::fmt::Debug for LockStakingStatus { - fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - write!(f, "LockStakingStatus") - } -} - -impl LockStakingStatus { - /// Create new `LockStakingStatus`. - pub fn new() -> Self { - Self(sp_std::marker::PhantomData) - } -} - -impl Default for LockStakingStatus { - fn default() -> Self { - Self::new() - } -} - -impl SignedExtension for LockStakingStatus { - const IDENTIFIER: &'static str = "LockStakingStatus"; - type AccountId = T::AccountId; - type Call = ::Call; - type AdditionalSigned = (); - type DispatchInfo = frame_support::weights::DispatchInfo; - type Pre = (); - - fn additional_signed(&self) -> Result<(), TransactionValidityError> { Ok(()) } - - fn validate( - &self, - _who: &Self::AccountId, - call: &Self::Call, - _info: Self::DispatchInfo, - _len: usize, - ) -> TransactionValidity { - if let Some(inner_call) = call.is_sub_type() { - if let ElectionStatus::Open(_) = >::era_election_status() { - match inner_call { - Call::::set_payee(..) | - Call::::set_controller(..) | - Call::::set_validator_count(..) | - Call::::force_no_eras(..) | - Call::::force_new_era(..) | - Call::::set_invulnerables(..) | - Call::::force_unstake(..) | - Call::::force_new_era_always(..) | - Call::::cancel_deferred_slash(..) | - Call::::set_history_depth(..) | - Call::::reap_stash(..) | - Call::::submit_election_solution(..) | - Call::::submit_election_solution_unsigned(..) => { - // These calls are allowed. Nothing. - } - _ => { - return Err(InvalidTransaction::Stale.into()); - } - } - } - } - - Ok(Default::default()) - } -} - impl From> for InvalidTransaction { fn from(e: Error) -> Self { - match e { - >::PhragmenEarlySubmission => InvalidTransaction::Future, - _ => InvalidTransaction::Custom(e.as_u8()), - } + InvalidTransaction::Custom(e.as_u8()) } } @@ -3191,48 +3181,36 @@ impl frame_support::unsigned::ValidateUnsigned for Module { match source { TransactionSource::Local | TransactionSource::InBlock => { /* allowed */ } _ => { - debug::native::debug!( - target: "staking", - "rejecting unsigned transaction because it is not local/in-block." - ); + log!(debug, "rejecting unsigned transaction because it is not local/in-block."); return InvalidTransaction::Call.into(); } } if let Err(e) = Self::pre_dispatch_checks(*score, *era) { - debug::native::debug!( - target: "staking", - "validate unsigned failed due to {:?}.", - e, - ); - let invalid: InvalidTransaction = e.into(); - return invalid.into(); + log!(debug, "validate unsigned pre dispatch checks failed due to {:?}.", e); + return InvalidTransaction::from(e).into(); } - debug::native::debug!( - target: "staking", - "Validated an unsigned transaction from the local node for era {}.", - era, - ); + log!(debug, "validateUnsigned succeeded for a solution at era {}.", era); - Ok(ValidTransaction { + ValidTransaction::with_tag_prefix("StakingOffchain") // The higher the score[0], the better a solution is. - priority: score[0].saturated_into(), - // no requires. - requires: vec![], + .priority(T::UnsignedPriority::get().saturating_add(score[0].saturated_into())) // Defensive only. A single solution can exist in the pool per era. Each validator // will run OCW at most once per era, hence there should never exist more than one // transaction anyhow. - provides: vec![("StakingOffchain", era).encode()], + .and_provides(era) // Note: this can be more accurate in the future. We do something like // `era_end_block - current_block` but that is not needed now as we eagerly run // offchain workers now and the above should be same as `T::ElectionLookahead` // without the need to query more storage in the validation phase. If we randomize // offchain worker, then we might re-consider this. - longevity: TryInto::::try_into(T::ElectionLookahead::get()).unwrap_or(DEFAULT_LONGEVITY), + .longevity(TryInto::::try_into( + T::ElectionLookahead::get()).unwrap_or(DEFAULT_LONGEVITY) + ) // We don't propagate this. This can never the validated at a remote node. - propagate: false, - }) + .propagate(false) + .build() } else { InvalidTransaction::Call.into() } diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index f8123416534ab44be108776e004cde7bba16e9bb..a34f3425564cdd1b3cf4ec3ceb3194c305540e2e 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -29,10 +29,10 @@ use frame_support::{ traits::{Currency, Get, FindAuthor, OnFinalize, OnInitialize}, weights::Weight, }; -use frame_system::offchain::TransactionSubmitter; use sp_io; use sp_phragmen::{ build_support_map, evaluate_support, reduce, ExtendedBalance, StakedAssignment, PhragmenScore, + VoteWeight, }; use crate::*; @@ -42,28 +42,29 @@ const INIT_TIMESTAMP: u64 = 30_000; pub(crate) type AccountId = u64; pub(crate) type AccountIndex = u64; pub(crate) type BlockNumber = u64; -pub(crate) type Balance = u64; +pub(crate) type Balance = u128; /// Simple structure that exposes how u64 currency can be represented as... u64. pub struct CurrencyToVoteHandler; -impl Convert for CurrencyToVoteHandler { - fn convert(x: u64) -> u64 { - x +impl Convert for CurrencyToVoteHandler { + fn convert(x: Balance) -> u64 { + x.saturated_into() } } -impl Convert for CurrencyToVoteHandler { - fn convert(x: u128) -> u64 { - x.saturated_into() +impl Convert for CurrencyToVoteHandler { + fn convert(x: u128) -> Balance { + x } } thread_local! { static SESSION: RefCell<(Vec, HashSet)> = RefCell::new(Default::default()); static SESSION_PER_ERA: RefCell = RefCell::new(3); - static EXISTENTIAL_DEPOSIT: RefCell = RefCell::new(0); + static EXISTENTIAL_DEPOSIT: RefCell = RefCell::new(0); static SLASH_DEFER_DURATION: RefCell = RefCell::new(0); static ELECTION_LOOKAHEAD: RefCell = RefCell::new(0); static PERIOD: RefCell = RefCell::new(1); + static MAX_ITERATIONS: RefCell = RefCell::new(0); } /// Another session handler struct to test on_disabled. @@ -104,8 +105,8 @@ pub fn is_disabled(controller: AccountId) -> bool { } pub struct ExistentialDeposit; -impl Get for ExistentialDeposit { - fn get() -> u64 { +impl Get for ExistentialDeposit { + fn get() -> Balance { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow()) } } @@ -143,6 +144,13 @@ impl Get for SlashDeferDuration { } } +pub struct MaxIterations; +impl Get for MaxIterations { + fn get() -> u32 { + MAX_ITERATIONS.with(|v| *v.borrow()) + } +} + impl_outer_origin! { pub enum Origin for Test where system = frame_system {} } @@ -172,8 +180,8 @@ impl_outer_event! { /// Author of block is always 11 pub struct Author11; -impl FindAuthor for Author11 { - fn find_author<'a, I>(_digests: I) -> Option +impl FindAuthor for Author11 { + fn find_author<'a, I>(_digests: I) -> Option where I: 'a + IntoIterator, { Some(11) @@ -203,11 +211,14 @@ impl frame_system::Trait for Test { type Event = MetaEvent; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); type ModuleToIndex = (); - type AccountData = pallet_balances::AccountData; + type AccountData = pallet_balances::AccountData; type OnNewAccount = (); type OnKilledAccount = (); } @@ -272,13 +283,29 @@ parameter_types! { pub const BondingDuration: EraIndex = 3; pub const RewardCurve: &'static PiecewiseLinear<'static> = &I_NPOS; pub const MaxNominatorRewardedPerValidator: u32 = 64; + pub const UnsignedPriority: u64 = 1 << 20; +} + +thread_local! { + pub static REWARD_REMAINDER_UNBALANCED: RefCell = RefCell::new(0); +} + +pub struct RewardRemainderMock; + +impl OnUnbalanced> for RewardRemainderMock { + fn on_nonzero_unbalanced(amount: NegativeImbalanceOf) { + REWARD_REMAINDER_UNBALANCED.with(|v| { + *v.borrow_mut() += amount.peek(); + }); + drop(amount); + } } impl Trait for Test { type Currency = Balances; type UnixTime = Timestamp; type CurrencyToVote = CurrencyToVoteHandler; - type RewardRemainder = (); + type RewardRemainder = RewardRemainderMock; type Event = MetaEvent; type Slash = (); type Reward = (); @@ -291,12 +318,19 @@ impl Trait for Test { type NextNewSession = Session; type ElectionLookahead = ElectionLookahead; type Call = Call; - type SubmitTransaction = SubmitTransaction; + type MaxIterations = MaxIterations; type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; + type UnsignedPriority = UnsignedPriority; +} + +impl frame_system::offchain::SendTransactionTypes for Test where + Call: From, +{ + type OverarchingCall = Call; + type Extrinsic = Extrinsic; } pub type Extrinsic = TestXt; -type SubmitTransaction = TransactionSubmitter<(), Test, Extrinsic>; pub struct ExtBuilder { session_length: BlockNumber, @@ -312,6 +346,7 @@ pub struct ExtBuilder { num_validators: Option, invulnerables: Vec, has_stakers: bool, + max_offchain_iterations: u32, } impl Default for ExtBuilder { @@ -330,12 +365,13 @@ impl Default for ExtBuilder { num_validators: None, invulnerables: vec![], has_stakers: true, + max_offchain_iterations: 0, } } } impl ExtBuilder { - pub fn existential_deposit(mut self, existential_deposit: u64) -> Self { + pub fn existential_deposit(mut self, existential_deposit: Balance) -> Self { self.existential_deposit = existential_deposit; self } @@ -367,7 +403,7 @@ impl ExtBuilder { self.num_validators = Some(num_validators); self } - pub fn invulnerables(mut self, invulnerables: Vec) -> Self { + pub fn invulnerables(mut self, invulnerables: Vec) -> Self { self.invulnerables = invulnerables; self } @@ -387,6 +423,10 @@ impl ExtBuilder { self.has_stakers = has; self } + pub fn max_offchain_iterations(mut self, iterations: u32) -> Self { + self.max_offchain_iterations = iterations; + self + } pub fn offchain_phragmen_ext(self) -> Self { self.session_per_era(4) .session_length(5) @@ -398,6 +438,7 @@ impl ExtBuilder { SESSION_PER_ERA.with(|v| *v.borrow_mut() = self.session_per_era); ELECTION_LOOKAHEAD.with(|v| *v.borrow_mut() = self.election_lookahead); PERIOD.with(|v| *v.borrow_mut() = self.session_length); + MAX_ITERATIONS.with(|v| *v.borrow_mut() = self.max_offchain_iterations); } pub fn build(self) -> sp_io::TestExternalities { let _ = env_logger::try_init(); @@ -413,7 +454,7 @@ impl ExtBuilder { let num_validators = self.num_validators.unwrap_or(self.validator_count); let validators = (0..num_validators) - .map(|x| ((x + 1) * 10 + 1) as u64) + .map(|x| ((x + 1) * 10 + 1) as AccountId) .collect::>(); let _ = pallet_balances::GenesisConfig:: { @@ -471,7 +512,7 @@ impl ExtBuilder { keys: validators.iter().map(|x| ( *x, *x, - SessionKeys { other: UintAuthorityId(*x) } + SessionKeys { other: UintAuthorityId(*x as u64) } )).collect(), }.assimilate_storage(&mut storage); @@ -491,6 +532,11 @@ impl ExtBuilder { ext } + pub fn build_and_execute(self, test: impl FnOnce() -> ()) { + let mut ext = self.build(); + ext.execute_with(test); + ext.execute_with(post_conditions); + } } pub type System = frame_system::Module; @@ -499,61 +545,92 @@ pub type Session = pallet_session::Module; pub type Timestamp = pallet_timestamp::Module; pub type Staking = Module; -pub fn active_era() -> EraIndex { - Staking::active_era().unwrap().index +pub(crate) fn current_era() -> EraIndex { + Staking::current_era().unwrap() } -pub fn check_exposure_all(era: EraIndex) { - ErasStakers::::iter_prefix(era).for_each(check_exposure) +fn post_conditions() { + check_nominators(); + check_exposures(); + check_ledgers(); } -pub fn check_nominator_all(era: EraIndex) { - >::iter() - .for_each(|(acc, _)| check_nominator_exposure(era, acc)); +pub(crate) fn active_era() -> EraIndex { + Staking::active_era().unwrap().index } -/// Check for each selected validator: expo.total = Sum(expo.other) + expo.own -pub fn check_exposure(expo: Exposure) { - assert_eq!( - expo.total as u128, - expo.own as u128 + expo.others.iter().map(|e| e.value as u128).sum::(), - "wrong total exposure", - ); +fn check_ledgers() { + // check the ledger of all stakers. + Bonded::::iter().for_each(|(_, ctrl)| assert_ledger_consistent(ctrl)) } -/// Check that for each nominator: slashable_balance > sum(used_balance) -/// Note: we might not consume all of a nominator's balance, but we MUST NOT over spend it. -pub fn check_nominator_exposure(era: EraIndex, stash: AccountId) { - assert_is_stash(stash); - let mut sum = 0; - Session::validators() - .iter() - .map(|v| Staking::eras_stakers(era, v)) - .for_each(|e| e.others.iter().filter(|i| i.who == stash).for_each(|i| sum += i.value)); - let nominator_stake = Staking::slashable_balance_of(&stash); - // a nominator cannot over-spend. - assert!( - nominator_stake >= sum, - "failed: Nominator({}) stake({}) >= sum divided({})", - stash, - nominator_stake, - sum, - ); +fn check_exposures() { + // a check per validator to ensure the exposure struct is always sane. + let era = active_era(); + ErasStakers::::iter_prefix_values(era).for_each(|expo| { + assert_eq!( + expo.total as u128, + expo.own as u128 + expo.others.iter().map(|e| e.value as u128).sum::(), + "wrong total exposure.", + ); + }) } -pub fn assert_is_stash(acc: AccountId) { - assert!(Staking::bonded(&acc).is_some(), "Not a stash."); +fn check_nominators() { + // a check per nominator to ensure their entire stake is correctly distributed. Will only kick- + // in if the nomination was submitted before the current era. + let era = active_era(); + >::iter() + .filter_map(|(nominator, nomination)| + if nomination.submitted_in > era { + Some(nominator) + } else { + None + }) + .for_each(|nominator| { + // must be bonded. + assert_is_stash(nominator); + let mut sum = 0; + Session::validators() + .iter() + .map(|v| Staking::eras_stakers(era, v)) + .for_each(|e| { + let individual = e.others.iter().filter(|e| e.who == nominator).collect::>(); + let len = individual.len(); + match len { + 0 => { /* not supporting this validator at all. */ }, + 1 => sum += individual[0].value, + _ => panic!("nominator cannot back a validator more than once."), + }; + }); + + let nominator_stake = Staking::slashable_balance_of(&nominator); + // a nominator cannot over-spend. + assert!( + nominator_stake >= sum, + "failed: Nominator({}) stake({}) >= sum divided({})", + nominator, + nominator_stake, + sum, + ); + + let diff = nominator_stake - sum; + assert!(diff < 100); + }); } -pub fn assert_ledger_consistent(stash: AccountId) { - assert_is_stash(stash); - let ledger = Staking::ledger(stash - 1).unwrap(); +fn assert_is_stash(acc: AccountId) { + assert!(Staking::bonded(&acc).is_some(), "Not a stash."); +} +fn assert_ledger_consistent(ctrl: AccountId) { + // ensures ledger.total == ledger.active + sum(ledger.unlocking). + let ledger = Staking::ledger(ctrl).expect("Not a controller."); let real_total: Balance = ledger.unlocking.iter().fold(ledger.active, |a, c| a + c.value); assert_eq!(real_total, ledger.total); } -pub fn bond_validator(stash: u64, ctrl: u64, val: u64) { +pub(crate) fn bond_validator(stash: AccountId, ctrl: AccountId, val: Balance) { let _ = Balances::make_free_balance_be(&stash, val); let _ = Balances::make_free_balance_be(&ctrl, val); assert_ok!(Staking::bond( @@ -568,7 +645,12 @@ pub fn bond_validator(stash: u64, ctrl: u64, val: u64) { )); } -pub fn bond_nominator(stash: u64, ctrl: u64, val: u64, target: Vec) { +pub(crate) fn bond_nominator( + stash: AccountId, + ctrl: AccountId, + val: Balance, + target: Vec, +) { let _ = Balances::make_free_balance_be(&stash, val); let _ = Balances::make_free_balance_be(&ctrl, val); assert_ok!(Staking::bond( @@ -580,7 +662,7 @@ pub fn bond_nominator(stash: u64, ctrl: u64, val: u64, target: Vec) { assert_ok!(Staking::nominate(Origin::signed(ctrl), target)); } -pub fn run_to_block(n: BlockNumber) { +pub(crate) fn run_to_block(n: BlockNumber) { Staking::on_finalize(System::block_number()); for b in System::block_number() + 1..=n { System::set_block_number(b); @@ -592,12 +674,12 @@ pub fn run_to_block(n: BlockNumber) { } } -pub fn advance_session() { +pub(crate) fn advance_session() { let current_index = Session::current_index(); start_session(current_index + 1); } -pub fn start_session(session_index: SessionIndex) { +pub(crate) fn start_session(session_index: SessionIndex) { assert_eq!(>::get(), 1, "start_session can only be used with session length 1."); for i in Session::current_index()..session_index { Staking::on_finalize(System::block_number()); @@ -610,12 +692,16 @@ pub fn start_session(session_index: SessionIndex) { assert_eq!(Session::current_index(), session_index); } -pub fn start_era(era_index: EraIndex) { +// This start and activate the era given. +// Because the mock use pallet-session which delays session by one, this will be one session after +// the election happened, not the first session after the election has happened. +pub(crate) fn start_era(era_index: EraIndex) { start_session((era_index * >::get()).into()); assert_eq!(Staking::current_era().unwrap(), era_index); + assert_eq!(Staking::active_era().unwrap().index, era_index); } -pub fn current_total_payout_for_duration(duration: u64) -> u64 { +pub(crate) fn current_total_payout_for_duration(duration: u64) -> Balance { inflation::compute_total_payout( ::RewardCurve::get(), Staking::eras_total_stake(Staking::active_era().unwrap().index), @@ -624,7 +710,7 @@ pub fn current_total_payout_for_duration(duration: u64) -> u64 { ).0 } -pub fn reward_all_elected() { +pub(crate) fn reward_all_elected() { let rewards = ::SessionInterface::validators() .into_iter() .map(|v| (v, 1)); @@ -632,14 +718,14 @@ pub fn reward_all_elected() { >::reward_by_ids(rewards) } -pub fn validator_controllers() -> Vec { +pub(crate) fn validator_controllers() -> Vec { Session::validators() .into_iter() .map(|s| Staking::bonded(&s).expect("no controller for validator")) .collect() } -pub fn on_offence_in_era( +pub(crate) fn on_offence_in_era( offenders: &[OffenceDetails< AccountId, pallet_session::historical::IdentificationTuple, @@ -669,7 +755,7 @@ pub fn on_offence_in_era( } } -pub fn on_offence_now( +pub(crate) fn on_offence_now( offenders: &[OffenceDetails>], slash_fraction: &[Perbill], ) { @@ -679,7 +765,7 @@ pub fn on_offence_now( // winners will be chosen by simply their unweighted total backing stake. Nominator stake is // distributed evenly. -pub fn horrible_phragmen_with_post_processing( +pub(crate) fn horrible_phragmen_with_post_processing( do_reduce: bool, ) -> (CompactAssignments, Vec, PhragmenScore) { let mut backing_stake_of: BTreeMap = BTreeMap::new(); @@ -787,7 +873,7 @@ pub fn horrible_phragmen_with_post_processing( // Note: this should always logically reproduce [`offchain_election::prepare_submission`], yet we // cannot do it since we want to have `tweak` injected into the process. -pub fn prepare_submission_with( +pub(crate) fn prepare_submission_with( do_reduce: bool, tweak: impl FnOnce(&mut Vec>), ) -> (CompactAssignments, Vec, PhragmenScore) { @@ -798,10 +884,10 @@ pub fn prepare_submission_with( } = Staking::do_phragmen::().unwrap(); let winners = winners.into_iter().map(|(w, _)| w).collect::>(); - let stake_of = |who: &AccountId| -> ExtendedBalance { - >::convert( + let stake_of = |who: &AccountId| -> VoteWeight { + >::convert( Staking::slashable_balance_of(&who) - ) as ExtendedBalance + ) }; let mut staked = sp_phragmen::assignment_ratio_to_staked(assignments, stake_of); @@ -840,7 +926,7 @@ pub fn prepare_submission_with( let score = { let staked = sp_phragmen::assignment_ratio_to_staked( assignments_reduced.clone(), - Staking::slashable_balance_of_extended, + Staking::slashable_balance_of_vote_weight, ); let (support_map, _) = build_support_map::( @@ -863,7 +949,7 @@ pub fn prepare_submission_with( } /// Make all validator and nominator request their payment -pub fn make_all_reward_payment_before_migration(era: EraIndex) { +pub(crate) fn make_all_reward_payment_before_migration(era: EraIndex) { let validators_with_reward = ErasRewardPoints::::get(era).individual.keys() .cloned() .collect::>(); @@ -895,7 +981,7 @@ pub fn make_all_reward_payment_before_migration(era: EraIndex) { } /// Make all validator and nominator request their payment -pub fn make_all_reward_payment(era: EraIndex) { +pub(crate) fn make_all_reward_payment(era: EraIndex) { let validators_with_reward = ErasRewardPoints::::get(era).individual.keys() .cloned() .collect::>(); @@ -927,3 +1013,13 @@ macro_rules! assert_session_era { ); }; } + +pub(crate) fn staking_events() -> Vec> { + System::events().into_iter().map(|r| r.event).filter_map(|e| { + if let MetaEvent::staking(inner) = e { + Some(inner) + } else { + None + } + }).collect() +} diff --git a/frame/staking/src/offchain_election.rs b/frame/staking/src/offchain_election.rs index 0d4cf49f103b9fd634ad178235159244f4f743b2..572703f895ec70a3a25f8192f0e0406d58d43975 100644 --- a/frame/staking/src/offchain_election.rs +++ b/frame/staking/src/offchain_election.rs @@ -16,17 +16,18 @@ //! Helpers for offchain worker election. +use codec::Decode; use crate::{ Call, CompactAssignments, Module, NominatorIndex, OffchainAccuracy, Trait, ValidatorIndex, }; -use frame_system::offchain::SubmitUnsignedTransaction; +use frame_system::offchain::SubmitTransaction; use sp_phragmen::{ build_support_map, evaluate_support, reduce, Assignment, ExtendedBalance, PhragmenResult, - PhragmenScore, + PhragmenScore, equalize, }; use sp_runtime::offchain::storage::StorageValueRef; -use sp_runtime::PerThing; -use sp_runtime::RuntimeDebug; +use sp_runtime::{PerThing, RuntimeDebug, traits::{TrailingZeroInput, Zero}}; +use frame_support::{debug, traits::Get}; use sp_std::{convert::TryInto, prelude::*}; /// Error types related to the offchain election machinery. @@ -113,18 +114,18 @@ pub(crate) fn compute_offchain_election() -> Result<(), OffchainElecti // process and prepare it for submission. let (winners, compact, score) = prepare_submission::(assignments, winners, true)?; - // defensive-only: active era can never be none except genesis. - let era = >::active_era().map(|e| e.index).unwrap_or_default(); + // defensive-only: current era can never be none except genesis. + let current_era = >::current_era().unwrap_or_default(); // send it. - let call: ::Call = Call::submit_election_solution_unsigned( + let call = Call::submit_election_solution_unsigned( winners, compact, score, - era, + current_era, ).into(); - T::SubmitTransaction::submit_unsigned(call) + SubmitTransaction::>::submit_unsigned_transaction(call) .map_err(|_| OffchainElectionError::PoolSubmissionFailed) } @@ -159,21 +160,41 @@ pub fn prepare_submission( }; // Clean winners. - let winners = winners - .into_iter() - .map(|(w, _)| w) - .collect::>(); + let winners = sp_phragmen::to_without_backing(winners); // convert into absolute value and to obtain the reduced version. let mut staked = sp_phragmen::assignment_ratio_to_staked( assignments, - >::slashable_balance_of_extended, + >::slashable_balance_of_vote_weight, ); + // reduce if do_reduce { reduce(&mut staked); } + let (mut support_map, _) = build_support_map::(&winners, &staked); + + // equalize a random number of times. + let iterations_executed = match T::MaxIterations::get() { + 0 => { + // Don't run equalize at all + 0 + } + iterations @ _ => { + let seed = sp_io::offchain::random_seed(); + let iterations = ::decode(&mut TrailingZeroInput::new(seed.as_ref())) + .expect("input is padded with zeroes; qed") % iterations.saturating_add(1); + equalize( + &mut staked, + &mut support_map, + Zero::zero(), + iterations as usize, + ) + } + }; + + // Convert back to ratio assignment. This takes less space. let low_accuracy_assignment = sp_phragmen::assignment_staked_to_ratio(staked); @@ -188,7 +209,7 @@ pub fn prepare_submission( let score = { let staked = sp_phragmen::assignment_ratio_to_staked( low_accuracy_assignment.clone(), - >::slashable_balance_of_extended, + >::slashable_balance_of_vote_weight, ); let (support_map, _) = build_support_map::(&winners, &staked); @@ -215,5 +236,12 @@ pub fn prepare_submission( } } + debug::native::debug!( + target: "staking", + "prepared solution after {} equalization iterations with score {:?}", + iterations_executed, + score, + ); + Ok((winners_indexed, compact, score)) } diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index 29a395b89de4dda8aa2c56755ed941a52ff99411..229b1e2c969950667fd5c097845ed2bf3873c738 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -18,7 +18,7 @@ //! //! Note that these helpers should NOT be used with the actual crate tests, but are rather designed //! for when the module is being externally tested (i.e. fuzzing, benchmarking, e2e tests). Enabling -//! this feature in the current crate's Cargo.toml will leak the all of this into a normal release +//! this feature in the current crate's Cargo.toml will leak all of this into a normal release //! build. Just don't do it. use crate::*; @@ -153,13 +153,13 @@ pub fn get_weak_solution( let mut backing_stake_of: BTreeMap> = BTreeMap::new(); // self stake - >::enumerate().for_each(|(who, _p)| { + >::iter().for_each(|(who, _p)| { *backing_stake_of.entry(who.clone()).or_insert(Zero::zero()) += >::slashable_balance_of(&who) }); // add nominator stuff - >::enumerate().for_each(|(who, nomination)| { + >::iter().for_each(|(who, nomination)| { nomination.targets.into_iter().for_each(|v| { *backing_stake_of.entry(v).or_insert(Zero::zero()) += >::slashable_balance_of(&who) @@ -176,7 +176,7 @@ pub fn get_weak_solution( .collect(); let mut staked_assignments: Vec> = Vec::new(); - >::enumerate().for_each(|(who, nomination)| { + >::iter().for_each(|(who, nomination)| { let mut dist: Vec<(T::AccountId, ExtendedBalance)> = Vec::new(); nomination.targets.into_iter().for_each(|v| { if winners.iter().find(|&w| *w == v).is_some() { @@ -325,16 +325,29 @@ pub fn clean(era: EraIndex) ::AccountId: codec::EncodeLike, u32: codec::EncodeLike, { - >::enumerate().for_each(|(k, _)| { + >::iter().for_each(|(k, _)| { let ctrl = >::bonded(&k).unwrap(); >::remove(&k); >::remove(&k); >::remove(&ctrl); >::remove(k, era); }); - >::enumerate().for_each(|(k, _)| >::remove(k)); + >::iter().for_each(|(k, _)| >::remove(k)); >::remove_all(); >::remove_all(); >::kill(); QueuedScore::kill(); } + +/// get the active era. +pub fn active_era() -> EraIndex { + >::active_era().unwrap().index +} + +/// initialize the first era. +pub fn init_active_era() { + ActiveEra::put(ActiveEraInfo { + index: 1, + start: None, + }) +} diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 17770817696c55c940534ffa8577a359e4cf5664..84c937d324cab2f2f47feb979c88d955dba72aad 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -33,7 +33,7 @@ use crate::Store; #[test] fn force_unstake_works() { // Verifies initial conditions of mock - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { // Account 11 is stashed and locked, and account 10 is the controller assert_eq!(Staking::bonded(&11), Some(10)); // Cant transfer @@ -55,7 +55,7 @@ fn force_unstake_works() { #[test] fn basic_setup_works() { // Verifies initial conditions of mock - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { // Account 11 is stashed and locked, and account 10 is the controller assert_eq!(Staking::bonded(&11), Some(10)); // Account 21 is stashed and locked, and account 20 is the controller @@ -77,7 +77,7 @@ fn basic_setup_works() { assert_eq!(Staking::ledger(&1), None); // ValidatorPrefs are default - assert_eq!(>::iter().collect::>(), vec![ + assert_eq_uvec!(>::iter().collect::>(), vec![ (31, ValidatorPrefs::default()), (21, ValidatorPrefs::default()), (11, ValidatorPrefs::default()) @@ -105,7 +105,8 @@ fn basic_setup_works() { others: vec![ IndividualExposure { who: 101, value: 375 }] }, ); - // initial slot_stake + + // initial total stake = 1125 + 1375 assert_eq!(Staking::eras_total_stake(Staking::active_era().unwrap().index), 2500); @@ -121,16 +122,12 @@ fn basic_setup_works() { // New era is not being forced assert_eq!(Staking::force_era(), Forcing::NotForcing); - - // All exposures must be correct. - check_exposure_all(Staking::active_era().unwrap().index); - check_nominator_all(Staking::active_era().unwrap().index); }); } #[test] fn change_controller_works() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_eq!(Staking::bonded(&11), Some(10)); assert!(Session::validators().contains(&11)); @@ -155,8 +152,9 @@ fn rewards_should_work() { // should check that: // * rewards get recorded per session // * rewards get paid per Era + // * `RewardRemainder::on_unbalanced` is called // * Check that nominators are also rewarded - ExtBuilder::default().nominate(true).build().execute_with(|| { + ExtBuilder::default().nominate(true).build_and_execute(|| { let init_balance_10 = Balances::total_balance(&10); let init_balance_11 = Balances::total_balance(&11); let init_balance_20 = Balances::total_balance(&20); @@ -200,6 +198,8 @@ fn rewards_should_work() { start_session(3); assert_eq!(Staking::active_era().unwrap().index, 1); + assert_eq!(mock::REWARD_REMAINDER_UNBALANCED.with(|v| *v.borrow()), 7050); + assert_eq!(*mock::staking_events().last().unwrap(), RawEvent::EraPayout(0, 2350, 7050)); mock::make_all_reward_payment(0); assert_eq_error_rate!(Balances::total_balance(&10), init_balance_10 + part_for_10 * total_payout_0*2/3, 2); @@ -223,6 +223,8 @@ fn rewards_should_work() { assert!(total_payout_1 > 10); // Test is meaningful if reward something mock::start_era(2); + assert_eq!(mock::REWARD_REMAINDER_UNBALANCED.with(|v| *v.borrow()), 7050*2); + assert_eq!(*mock::staking_events().last().unwrap(), RawEvent::EraPayout(1, 2350, 7050)); mock::make_all_reward_payment(1); assert_eq_error_rate!(Balances::total_balance(&10), init_balance_10 + part_for_10 * (total_payout_0 * 2/3 + total_payout_1), 2); @@ -343,11 +345,9 @@ fn less_than_needed_candidates_works() { // But the exposure is updated in a simple way. No external votes exists. // This is purely self-vote. assert!( - ErasStakers::::iter_prefix(Staking::active_era().unwrap().index) + ErasStakers::::iter_prefix_values(Staking::active_era().unwrap().index) .all(|exposure| exposure.others.is_empty()) ); - check_exposure_all(Staking::active_era().unwrap().index); - check_nominator_all(Staking::active_era().unwrap().index); }); } @@ -466,7 +466,7 @@ fn nominating_and_rewards_should_work() { // ------ check the staked value of all parties. // 30 and 40 are not chosen anymore - assert_eq!(ErasStakers::::iter_prefix(Staking::active_era().unwrap().index).count(), 2); + assert_eq!(ErasStakers::::iter_prefix_values(Staking::active_era().unwrap().index).count(), 2); assert_eq!( Staking::eras_stakers(Staking::active_era().unwrap().index, 11), Exposure { @@ -529,9 +529,6 @@ fn nominating_and_rewards_should_work() { initial_balance + 5 * payout_for_20 / 11, 1, ); - - check_exposure_all(Staking::active_era().unwrap().index); - check_nominator_all(Staking::active_era().unwrap().index); }); } @@ -542,7 +539,7 @@ fn nominators_also_get_slashed() { // 10 - is the controller of 11 // 11 - is the stash. // 2 - is the nominator of 20, 10 - ExtBuilder::default().nominate(false).build().execute_with(|| { + ExtBuilder::default().nominate(false).build_and_execute(|| { assert_eq!(Staking::validator_count(), 2); // Set payee to controller @@ -589,8 +586,7 @@ fn nominators_also_get_slashed() { // initial + first era reward + slash assert_eq!(Balances::total_balance(&11), initial_balance - validator_slash); assert_eq!(Balances::total_balance(&2), initial_balance - nominator_slash); - check_exposure_all(Staking::active_era().unwrap().index); - check_nominator_all(Staking::active_era().unwrap().index); + // Because slashing happened. assert!(is_disabled(10)); }); @@ -602,7 +598,7 @@ fn double_staking_should_fail() { // * an account already bonded as stash cannot be be stashed again. // * an account already bonded as stash cannot nominate. // * an account already bonded as controller can nominate. - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { let arbitrary_value = 5; // 2 = controller, 1 stashed => ok assert_ok!( @@ -625,7 +621,7 @@ fn double_staking_should_fail() { fn double_controlling_should_fail() { // should test (in the same order): // * an account already bonded as controller CANNOT be reused as the controller of another account. - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { let arbitrary_value = 5; // 2 = controller, 1 stashed => ok assert_ok!(Staking::bond( @@ -644,7 +640,7 @@ fn double_controlling_should_fail() { #[test] fn session_and_eras_work() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_eq!(Staking::active_era().unwrap().index, 0); assert_eq!(Session::current_index(), 0); @@ -682,7 +678,7 @@ fn session_and_eras_work() { #[test] fn forcing_new_era_works() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { // normal flow of session. assert_eq!(Staking::active_era().unwrap().index, 0); start_session(0); @@ -738,7 +734,7 @@ fn forcing_new_era_works() { #[test] fn cannot_transfer_staked_balance() { // Tests that a stash account cannot transfer funds - ExtBuilder::default().nominate(false).build().execute_with(|| { + ExtBuilder::default().nominate(false).build_and_execute(|| { // Confirm account 11 is stashed assert_eq!(Staking::bonded(&11), Some(10)); // Confirm account 11 has some free balance @@ -763,7 +759,7 @@ fn cannot_transfer_staked_balance_2() { // Tests that a stash account cannot transfer funds // Same test as above but with 20, and more accurate. // 21 has 2000 free balance but 1000 at stake - ExtBuilder::default().nominate(false).fair(true).build().execute_with(|| { + ExtBuilder::default().nominate(false).fair(true).build_and_execute(|| { // Confirm account 21 is stashed assert_eq!(Staking::bonded(&21), Some(20)); // Confirm account 21 has some free balance @@ -782,7 +778,7 @@ fn cannot_transfer_staked_balance_2() { #[test] fn cannot_reserve_staked_balance() { // Checks that a bonded account cannot reserve balance from free balance - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { // Confirm account 11 is stashed assert_eq!(Staking::bonded(&11), Some(10)); // Confirm account 11 has some free balance @@ -805,7 +801,7 @@ fn cannot_reserve_staked_balance() { #[test] fn reward_destination_works() { // Rewards go to the correct destination as determined in Payee - ExtBuilder::default().nominate(false).build().execute_with(|| { + ExtBuilder::default().nominate(false).build_and_execute(|| { // Check that account 11 is a validator assert!(Session::validators().contains(&11)); // Check the balance of the validator account @@ -904,7 +900,7 @@ fn validator_payment_prefs_work() { // Test that validator preferences are correctly honored // Note: unstake threshold is being directly tested in slashing tests. // This test will focus on validator payment. - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { let commission = Perbill::from_percent(40); >::insert(&11, ValidatorPrefs { commission: commission.clone(), @@ -935,9 +931,6 @@ fn validator_payment_prefs_work() { let reward_of_100 = shared_cut * exposure_1.others[0].value / exposure_1.total; assert_eq_error_rate!(Balances::total_balance(&10), balance_era_1_10 + reward_of_10, 2); assert_eq_error_rate!(Balances::total_balance(&100), balance_era_1_100 + reward_of_100, 2); - - check_exposure_all(Staking::active_era().unwrap().index); - check_nominator_all(Staking::active_era().unwrap().index); }); } @@ -947,7 +940,7 @@ fn bond_extra_works() { // Tests that extra `free_balance` in the stash can be added to stake // NOTE: this tests only verifies `StakingLedger` for correct updates // See `bond_extra_and_withdraw_unbonded_works` for more details and updates on `Exposure`. - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { // Check that account 10 is a validator assert!(>::contains_key(11)); // Check that account 10 is bonded to account 11 @@ -976,7 +969,7 @@ fn bond_extra_works() { })); // Call the bond_extra function with a large number, should handle it - assert_ok!(Staking::bond_extra(Origin::signed(11), u64::max_value())); + assert_ok!(Staking::bond_extra(Origin::signed(11), Balance::max_value())); // The full amount of the funds should now be in the total and active assert_eq!(Staking::ledger(&10), Some(StakingLedger { stash: 11, @@ -995,7 +988,7 @@ fn bond_extra_and_withdraw_unbonded_works() { // * It can add extra funds to the bonded account. // * it can unbond a portion of its funds from the stash account. // * Once the unbonding period is done, it can actually take the funds out of the stash. - ExtBuilder::default().nominate(false).build().execute_with(|| { + ExtBuilder::default().nominate(false).build_and_execute(|| { // Set payee to controller. avoids confusion assert_ok!(Staking::set_payee(Origin::signed(10), RewardDestination::Controller)); @@ -1081,7 +1074,7 @@ fn bond_extra_and_withdraw_unbonded_works() { #[test] fn too_many_unbond_calls_should_not_work() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { // locked at era 0 until 3 for _ in 0..MAX_UNLOCKING_CHUNKS-1 { assert_ok!(Staking::unbond(Origin::signed(10), 1)); @@ -1359,7 +1352,7 @@ fn rebond_is_fifo() { #[test] fn reward_to_stake_works() { - ExtBuilder::default().nominate(false).fair(false).build().execute_with(|| { + ExtBuilder::default().nominate(false).fair(false).build_and_execute(|| { // Confirm validator count is 2 assert_eq!(Staking::validator_count(), 2); // Confirm account 10 and 20 are validators @@ -1401,9 +1394,6 @@ fn reward_to_stake_works() { // -- new infos assert_eq!(Staking::eras_stakers(Staking::active_era().unwrap().index, 11).total, 1000 + total_payout_0 / 2); assert_eq!(Staking::eras_stakers(Staking::active_era().unwrap().index, 21).total, 69 + total_payout_0 / 2); - - check_exposure_all(Staking::active_era().unwrap().index); - check_nominator_all(Staking::active_era().unwrap().index); }); } @@ -1411,7 +1401,7 @@ fn reward_to_stake_works() { fn on_free_balance_zero_stash_removes_validator() { // Tests that validator storage items are cleaned up when stash is empty // Tests that storage items are untouched when controller is empty - ExtBuilder::default().existential_deposit(10).build().execute_with(|| { + ExtBuilder::default().existential_deposit(10).build_and_execute(|| { // Check the balance of the validator account assert_eq!(Balances::free_balance(10), 256); // Check the balance of the stash account @@ -1430,7 +1420,7 @@ fn on_free_balance_zero_stash_removes_validator() { assert!(>::contains_key(&11)); // Reduce free_balance of controller to 0 - let _ = Balances::slash(&10, u64::max_value()); + let _ = Balances::slash(&10, Balance::max_value()); // Check the balance of the stash account has not been touched assert_eq!(Balances::free_balance(11), 256000); @@ -1444,7 +1434,7 @@ fn on_free_balance_zero_stash_removes_validator() { assert!(>::contains_key(&11)); // Reduce free_balance of stash to 0 - let _ = Balances::slash(&11, u64::max_value()); + let _ = Balances::slash(&11, Balance::max_value()); // Check total balance of stash assert_eq!(Balances::total_balance(&11), 0); @@ -1464,7 +1454,7 @@ fn on_free_balance_zero_stash_removes_validator() { fn on_free_balance_zero_stash_removes_nominator() { // Tests that nominator storage items are cleaned up when stash is empty // Tests that storage items are untouched when controller is empty - ExtBuilder::default().existential_deposit(10).build().execute_with(|| { + ExtBuilder::default().existential_deposit(10).build_and_execute(|| { // Make 10 a nominator assert_ok!(Staking::nominate(Origin::signed(10), vec![20])); // Check that account 10 is a nominator @@ -1484,7 +1474,7 @@ fn on_free_balance_zero_stash_removes_nominator() { assert!(>::contains_key(&11)); // Reduce free_balance of controller to 0 - let _ = Balances::slash(&10, u64::max_value()); + let _ = Balances::slash(&10, Balance::max_value()); // Check total balance of account 10 assert_eq!(Balances::total_balance(&10), 0); @@ -1500,7 +1490,7 @@ fn on_free_balance_zero_stash_removes_nominator() { assert!(>::contains_key(&11)); // Reduce free_balance of stash to 0 - let _ = Balances::slash(&11, u64::max_value()); + let _ = Balances::slash(&11, Balance::max_value()); // Check total balance of stash assert_eq!(Balances::total_balance(&11), 0); @@ -1520,7 +1510,7 @@ fn on_free_balance_zero_stash_removes_nominator() { #[test] fn switching_roles() { // Test that it should be possible to switch between roles (nominator, validator, idle) with minimal overhead. - ExtBuilder::default().nominate(false).build().execute_with(|| { + ExtBuilder::default().nominate(false).build_and_execute(|| { // Reset reward destination for i in &[10, 20] { assert_ok!(Staking::set_payee(Origin::signed(*i), RewardDestination::Controller)); } @@ -1557,15 +1547,12 @@ fn switching_roles() { mock::start_era(2); assert_eq_uvec!(validator_controllers(), vec![2, 20]); - - check_exposure_all(Staking::active_era().unwrap().index); - check_nominator_all(Staking::active_era().unwrap().index); }); } #[test] fn wrong_vote_is_null() { - ExtBuilder::default().nominate(false).validator_pool(true).build().execute_with(|| { + ExtBuilder::default().nominate(false).validator_pool(true).build_and_execute(|| { assert_eq_uvec!(validator_controllers(), vec![40, 30]); // put some money in account that we'll use. @@ -1687,8 +1674,6 @@ fn bond_with_little_staked_value_bounded() { Balances::free_balance(&10), init_balance_10 + total_payout_0 / 3 + total_payout_1 / 3, ); - check_exposure_all(Staking::active_era().unwrap().index); - check_nominator_all(Staking::active_era().unwrap().index); }); } @@ -1707,80 +1692,39 @@ fn new_era_elects_correct_number_of_validators() { Session::on_initialize(System::block_number()); assert_eq!(validator_controllers().len(), 1); - check_exposure_all(Staking::active_era().unwrap().index); - check_nominator_all(Staking::active_era().unwrap().index); }) } #[test] -fn phragmen_should_not_overflow_validators() { - ExtBuilder::default().nominate(false).build().execute_with(|| { - let _ = Staking::chill(Origin::signed(10)); - let _ = Staking::chill(Origin::signed(20)); +fn phragmen_should_not_overflow() { + ExtBuilder::default().nominate(false).build_and_execute(|| { + // This is the maximum value that we can have as the outcome of CurrencyToVote. + type Votes = u64; - bond_validator(3, 2, u64::max_value()); - bond_validator(5, 4, u64::max_value()); - - bond_nominator(7, 6, u64::max_value() / 2, vec![3, 5]); - bond_nominator(9, 8, u64::max_value() / 2, vec![3, 5]); - - mock::start_era(1); - - assert_eq_uvec!(validator_controllers(), vec![4, 2]); - - // This test will fail this. Will saturate. - // check_exposure_all(); - assert_eq!(Staking::eras_stakers(Staking::active_era().unwrap().index, 3).total, u64::max_value()); - assert_eq!(Staking::eras_stakers(Staking::active_era().unwrap().index, 5).total, u64::max_value()); - }) -} - -#[test] -fn phragmen_should_not_overflow_nominators() { - ExtBuilder::default().nominate(false).build().execute_with(|| { let _ = Staking::chill(Origin::signed(10)); let _ = Staking::chill(Origin::signed(20)); - bond_validator(3, 2, u64::max_value() / 2); - bond_validator(5, 4, u64::max_value() / 2); - - bond_nominator(7, 6, u64::max_value(), vec![3, 5]); - bond_nominator(9, 8, u64::max_value(), vec![3, 5]); - - mock::start_era(1); - - assert_eq_uvec!(validator_controllers(), vec![4, 2]); - - // Saturate. - assert_eq!(Staking::eras_stakers(Staking::active_era().unwrap().index, 3).total, u64::max_value()); - assert_eq!(Staking::eras_stakers(Staking::active_era().unwrap().index, 5).total, u64::max_value()); - }) -} - -#[test] -fn phragmen_should_not_overflow_ultimate() { - ExtBuilder::default().nominate(false).build().execute_with(|| { - bond_validator(3, 2, u64::max_value()); - bond_validator(5, 4, u64::max_value()); + bond_validator(3, 2, Votes::max_value() as Balance); + bond_validator(5, 4, Votes::max_value() as Balance); - bond_nominator(7, 6, u64::max_value(), vec![3, 5]); - bond_nominator(9, 8, u64::max_value(), vec![3, 5]); + bond_nominator(7, 6, Votes::max_value() as Balance, vec![3, 5]); + bond_nominator(9, 8, Votes::max_value() as Balance, vec![3, 5]); mock::start_era(1); assert_eq_uvec!(validator_controllers(), vec![4, 2]); - // Saturate. - assert_eq!(Staking::eras_stakers(Staking::active_era().unwrap().index, 3).total, u64::max_value()); - assert_eq!(Staking::eras_stakers(Staking::active_era().unwrap().index, 5).total, u64::max_value()); + // We can safely convert back to values within [u64, u128]. + assert!(Staking::eras_stakers(active_era(), 3).total > Votes::max_value() as Balance); + assert!(Staking::eras_stakers(active_era(), 5).total > Votes::max_value() as Balance); }) } #[test] -fn reward_validator_slashing_validator_doesnt_overflow() { - ExtBuilder::default().build().execute_with(|| { - let stake = u32::max_value() as u64 * 2; - let reward_slash = u32::max_value() as u64 * 2; +fn reward_validator_slashing_validator_does_not_overflow() { + ExtBuilder::default().build_and_execute(|| { + let stake = u64::max_value() as Balance * 2; + let reward_slash = u64::max_value() as Balance * 2; // Assert multiplication overflows in balance arithmetic. assert!(stake.checked_mul(reward_slash).is_none()); @@ -1834,7 +1778,7 @@ fn reward_validator_slashing_validator_doesnt_overflow() { #[test] fn reward_from_authorship_event_handler_works() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { use pallet_authorship::EventHandler; assert_eq!(>::author(), 11); @@ -1861,7 +1805,7 @@ fn reward_from_authorship_event_handler_works() { #[test] fn add_reward_points_fns_works() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { // Not mandatory but must be coherent with rewards assert_eq!(Session::validators(), vec![21, 11]); @@ -1889,7 +1833,7 @@ fn add_reward_points_fns_works() { #[test] fn unbonded_balance_is_not_slashable() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { // total amount staked is slashable. assert_eq!(Staking::slashable_balance_of(&11), 1000); @@ -1904,30 +1848,30 @@ fn unbonded_balance_is_not_slashable() { fn era_is_always_same_length() { // This ensures that the sessions is always of the same length if there is no forcing no // session changes. - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { let session_per_era = >::get(); mock::start_era(1); - assert_eq!(Staking::eras_start_session_index(active_era()).unwrap(), session_per_era); + assert_eq!(Staking::eras_start_session_index(current_era()).unwrap(), session_per_era); mock::start_era(2); - assert_eq!(Staking::eras_start_session_index(active_era()).unwrap(), session_per_era * 2u32); + assert_eq!(Staking::eras_start_session_index(current_era()).unwrap(), session_per_era * 2u32); let session = Session::current_index(); ForceEra::put(Forcing::ForceNew); advance_session(); advance_session(); - assert_eq!(Staking::active_era().unwrap().index, 3); - assert_eq!(Staking::eras_start_session_index(active_era()).unwrap(), session + 2); + assert_eq!(current_era(), 3); + assert_eq!(Staking::eras_start_session_index(current_era()).unwrap(), session + 2); mock::start_era(4); - assert_eq!(Staking::eras_start_session_index(active_era()).unwrap(), session + 2u32 + session_per_era); + assert_eq!(Staking::eras_start_session_index(current_era()).unwrap(), session + 2u32 + session_per_era); }); } #[test] fn offence_forces_new_era() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { on_offence_now( &[OffenceDetails { offender: ( @@ -1945,7 +1889,7 @@ fn offence_forces_new_era() { #[test] fn offence_ensures_new_era_without_clobbering() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_ok!(Staking::force_new_era_always(Origin::ROOT)); assert_eq!(Staking::force_era(), Forcing::ForceAlways); @@ -1966,7 +1910,7 @@ fn offence_ensures_new_era_without_clobbering() { #[test] fn offence_deselects_validator_even_when_slash_is_zero() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert!(Session::validators().contains(&11)); assert!(>::contains_key(11)); @@ -1995,7 +1939,7 @@ fn offence_deselects_validator_even_when_slash_is_zero() { fn slashing_performed_according_exposure() { // This test checks that slashing is performed according the exposure (or more precisely, // historical exposure), not the current balance. - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_eq!(Staking::eras_stakers(Staking::active_era().unwrap().index, 11).own, 1000); // Handle an offence with a historical exposure. @@ -2021,7 +1965,7 @@ fn slashing_performed_according_exposure() { #[test] fn slash_in_old_span_does_not_deselect() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { mock::start_era(1); assert!(>::contains_key(11)); @@ -2087,7 +2031,6 @@ fn slash_in_old_span_does_not_deselect() { assert_eq!(Staking::force_era(), Forcing::NotForcing); assert!(>::contains_key(11)); assert!(Session::validators().contains(&11)); - assert_ledger_consistent(11); }); } @@ -2095,7 +2038,7 @@ fn slash_in_old_span_does_not_deselect() { fn reporters_receive_their_slice() { // This test verifies that the reporters of the offence receive their slice from the slashed // amount. - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { // The reporters' reward is calculated from the total exposure. let initial_balance = 1125; @@ -2118,7 +2061,6 @@ fn reporters_receive_their_slice() { let reward_each = reward / 2; // split into two pieces. assert_eq!(Balances::free_balance(1), 10 + reward_each); assert_eq!(Balances::free_balance(2), 20 + reward_each); - assert_ledger_consistent(11); }); } @@ -2126,7 +2068,7 @@ fn reporters_receive_their_slice() { fn subsequent_reports_in_same_span_pay_out_less() { // This test verifies that the reporters of the offence receive their slice from the slashed // amount, but less and less if they submit multiple reports in one span. - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { // The reporters' reward is calculated from the total exposure. let initial_balance = 1125; @@ -2165,14 +2107,13 @@ fn subsequent_reports_in_same_span_pay_out_less() { // 50% * (10% * (initial_balance / 2) - prior_payout) let reward = ((initial_balance / 20) - prior_payout) / 2; assert_eq!(Balances::free_balance(1), 10 + prior_payout + reward); - assert_ledger_consistent(11); }); } #[test] fn invulnerables_are_not_slashed() { // For invulnerable validators no slashing is performed. - ExtBuilder::default().invulnerables(vec![11]).build().execute_with(|| { + ExtBuilder::default().invulnerables(vec![11]).build_and_execute(|| { assert_eq!(Balances::free_balance(11), 1000); assert_eq!(Balances::free_balance(21), 2000); @@ -2208,15 +2149,13 @@ fn invulnerables_are_not_slashed() { initial_balance - (2 * other.value / 10), ); } - assert_ledger_consistent(11); - assert_ledger_consistent(21); }); } #[test] fn dont_slash_if_fraction_is_zero() { // Don't slash if the fraction is zero. - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_eq!(Balances::free_balance(11), 1000); on_offence_now( @@ -2233,8 +2172,6 @@ fn dont_slash_if_fraction_is_zero() { // The validator hasn't been slashed. The new era is not forced. assert_eq!(Balances::free_balance(11), 1000); assert_eq!(Staking::force_era(), Forcing::ForceNew); - - assert_ledger_consistent(11); }); } @@ -2242,7 +2179,7 @@ fn dont_slash_if_fraction_is_zero() { fn only_slash_for_max_in_era() { // multiple slashes within one era are only applied if it is more than any previous slash in the // same era. - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { assert_eq!(Balances::free_balance(11), 1000); on_offence_now( @@ -2284,14 +2221,13 @@ fn only_slash_for_max_in_era() { // The validator got slashed 10% more. assert_eq!(Balances::free_balance(11), 400); - assert_ledger_consistent(11); }) } #[test] fn garbage_collection_after_slashing() { // ensures that `SlashingSpans` and `SpanSlash` of an account is removed after reaping. - ExtBuilder::default().existential_deposit(2).build().execute_with(|| { + ExtBuilder::default().existential_deposit(2).build_and_execute(|| { assert_eq!(Balances::free_balance(11), 256_000); on_offence_now( @@ -2335,7 +2271,7 @@ fn garbage_collection_after_slashing() { fn garbage_collection_on_window_pruning() { // ensures that `ValidatorSlashInEra` and `NominatorSlashInEra` are cleared after // `BondingDuration`. - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { mock::start_era(1); assert_eq!(Balances::free_balance(11), 1000); @@ -2376,7 +2312,7 @@ fn garbage_collection_on_window_pruning() { #[test] fn slashing_nominators_by_span_max() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { mock::start_era(1); mock::start_era(2); mock::start_era(3); @@ -2474,7 +2410,7 @@ fn slashing_nominators_by_span_max() { #[test] fn slashes_are_summed_across_spans() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { mock::start_era(1); mock::start_era(2); mock::start_era(3); @@ -2532,7 +2468,7 @@ fn slashes_are_summed_across_spans() { #[test] fn deferred_slashes_are_deferred() { - ExtBuilder::default().slash_defer_duration(2).build().execute_with(|| { + ExtBuilder::default().slash_defer_duration(2).build_and_execute(|| { mock::start_era(1); assert_eq!(Balances::free_balance(11), 1000); @@ -2575,7 +2511,7 @@ fn deferred_slashes_are_deferred() { #[test] fn remove_deferred() { - ExtBuilder::default().slash_defer_duration(2).build().execute_with(|| { + ExtBuilder::default().slash_defer_duration(2).build_and_execute(|| { mock::start_era(1); assert_eq!(Balances::free_balance(11), 1000); @@ -2651,7 +2587,7 @@ fn remove_deferred() { #[test] fn remove_multi_deferred() { - ExtBuilder::default().slash_defer_duration(2).build().execute_with(|| { + ExtBuilder::default().slash_defer_duration(2).build_and_execute(|| { mock::start_era(1); assert_eq!(Balances::free_balance(11), 1000); @@ -2738,6 +2674,7 @@ fn remove_multi_deferred() { mod offchain_phragmen { use crate::*; + use codec::Encode; use frame_support::{assert_noop, assert_ok}; use sp_runtime::transaction_validity::TransactionSource; use mock::*; @@ -2778,14 +2715,19 @@ mod offchain_phragmen { bond_nominator(voter, 1000 + voter, 100, vec![21, 31, 41]); } - fn offchainify(ext: &mut TestExternalities) -> Arc> { - let (offchain, _state) = TestOffchainExt::new(); - let (pool, state) = TestTransactionPoolExt::new(); + /// convert an externalities to one that can handle offchain worker tests. + fn offchainify(ext: &mut TestExternalities, iterations: u32) -> Arc> { + let (offchain, offchain_state) = TestOffchainExt::new(); + let (pool, pool_state) = TestTransactionPoolExt::new(); + + let mut seed = [0_u8; 32]; + seed[0..4].copy_from_slice(&iterations.to_le_bytes()); + offchain_state.write().seed = seed; ext.register_extension(OffchainExt::new(offchain)); ext.register_extension(TransactionPoolExt::new(pool)); - state + pool_state } #[test] @@ -2872,9 +2814,30 @@ mod offchain_phragmen { }) } + #[test] + fn offchain_election_flag_is_triggered_when_forcing() { + ExtBuilder::default() + .session_per_era(5) + .session_length(10) + .election_lookahead(3) + .build() + .execute_with(|| { + run_to_block(7); + assert_session_era!(0, 0); + + run_to_block(12); + ForceEra::put(Forcing::ForceNew); + run_to_block(13); + assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); + + run_to_block(17); // instead of 47 + assert_eq!(Staking::era_election_status(), ElectionStatus::Open(17)); + }) + } + #[test] fn election_on_chain_fallback_works() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { start_session(1); start_session(2); assert_eq!(Staking::era_election_status(), ElectionStatus::Closed); @@ -2902,7 +2865,6 @@ mod offchain_phragmen { fn offchain_wont_work_if_snapshot_fails() { ExtBuilder::default() .offchain_phragmen_ext() - .election_lookahead(3) .build() .execute_with(|| { run_to_block(12); @@ -2932,23 +2894,21 @@ mod offchain_phragmen { .execute_with(|| { run_to_block(12); assert!(Staking::snapshot_validators().is_some()); + // given assert_eq!(Staking::era_election_status(), ElectionStatus::Open(12)); - let call = crate::Call::bond(999, 998, Default::default()); - let outer: mock::Call = call.into(); - - let lock_staking: LockStakingStatus = Default::default(); - assert_eq!( - lock_staking.validate(&10, &outer, Default::default(), Default::default(),), - TransactionValidity::Err(InvalidTransaction::Stale.into()), - ) + // chill et. al. are now not allowed. + assert_noop!( + Staking::chill(Origin::signed(10)), + Error::::CallNotAllowed, + ); }) } #[test] fn signed_result_can_be_submitted() { - // should check that we have a new validator set normally, - // event says that it comes from offchain. + // should check that we have a new validator set normally, event says that it comes from + // offchain. ExtBuilder::default() .offchain_phragmen_ext() .build() @@ -2963,7 +2923,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), )); let queued_result = Staking::queued_elected().unwrap(); @@ -3006,7 +2966,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), )); let queued_result = Staking::queued_elected().unwrap(); @@ -3035,8 +2995,8 @@ mod offchain_phragmen { #[test] fn early_solution_submission_is_rejected() { - // should check that we have a new validator set normally, - // event says that it comes from offchain. + // should check that we have a new validator set normally, event says that it comes from + // offchain. ExtBuilder::default() .offchain_phragmen_ext() .build() @@ -3056,7 +3016,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ), Error::::PhragmenEarlySubmission, ); @@ -3082,7 +3042,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), )); // a bad solution @@ -3093,7 +3053,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ), Error::::PhragmenWeakSubmission, ); @@ -3119,7 +3079,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), )); // a better solution @@ -3129,7 +3089,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), )); }) } @@ -3141,7 +3101,7 @@ mod offchain_phragmen { .offchain_phragmen_ext() .validator_count(2) .build(); - let state = offchainify(&mut ext); + let state = offchainify(&mut ext, 0); ext.execute_with(|| { run_to_block(12); @@ -3165,7 +3125,51 @@ mod offchain_phragmen { &inner, ), TransactionValidity::Ok(ValidTransaction { - priority: 1125, // the proposed slot stake. + priority: UnsignedPriority::get() + 1125, // the proposed slot stake. + requires: vec![], + provides: vec![("StakingOffchain", current_era()).encode()], + longevity: 3, + propagate: false, + }) + ) + }) + } + + #[test] + fn offchain_worker_runs_with_equalise() { + // Offchain worker equalises based on the number provided by randomness. See the difference + // in the priority, which comes from the computed score. + let mut ext = ExtBuilder::default() + .offchain_phragmen_ext() + .validator_count(2) + .max_offchain_iterations(2) + .build(); + let state = offchainify(&mut ext, 2); + ext.execute_with(|| { + run_to_block(12); + + // local key 11 is in the elected set. + assert_eq_uvec!(Session::validators(), vec![11, 21]); + assert_eq!(state.read().transactions.len(), 0); + Staking::offchain_worker(12); + assert_eq!(state.read().transactions.len(), 1); + + let encoded = state.read().transactions[0].clone(); + let extrinsic: Extrinsic = Decode::decode(&mut &*encoded).unwrap(); + + let call = extrinsic.call; + let inner = match call { + mock::Call::Staking(inner) => inner, + }; + + assert_eq!( + ::validate_unsigned( + TransactionSource::Local, + &inner, + ), + TransactionValidity::Ok(ValidTransaction { + // the proposed slot stake, with equalize. + priority: UnsignedPriority::get() + 1250, requires: vec![], provides: vec![("StakingOffchain", active_era()).encode()], longevity: 3, @@ -3181,7 +3185,7 @@ mod offchain_phragmen { .offchain_phragmen_ext() .validator_count(4) .build(); - let state = offchainify(&mut ext); + let state = offchainify(&mut ext, 0); ext.execute_with(|| { run_to_block(12); // put a good solution on-chain @@ -3191,7 +3195,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ),); // now run the offchain worker in the same chain state. @@ -3242,7 +3246,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ), Error::::PhragmenBogusWinnerCount, ); @@ -3273,7 +3277,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ), Error::::PhragmenBogusWinnerCount, ); @@ -3302,7 +3306,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ),); }) } @@ -3333,7 +3337,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ), Error::::PhragmenBogusCompact, ); @@ -3366,7 +3370,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ), Error::::PhragmenBogusCompact, ); @@ -3398,7 +3402,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ), Error::::PhragmenBogusWinner, ); @@ -3434,7 +3438,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ), Error::::PhragmenBogusEdge, ); @@ -3470,7 +3474,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ), Error::::PhragmenBogusSelfVote, ); @@ -3506,7 +3510,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ), Error::::PhragmenBogusSelfVote, ); @@ -3541,7 +3545,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ), Error::::PhragmenBogusCompact, ); @@ -3583,7 +3587,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ), Error::::PhragmenBogusNomination, ); @@ -3611,7 +3615,7 @@ mod offchain_phragmen { run_to_block(20); // slash 10. This must happen outside of the election window. - let offender_expo = Staking::eras_stakers(active_era(), 11); + let offender_expo = Staking::eras_stakers(Staking::active_era().unwrap().index, 11); on_offence_now( &[OffenceDetails { offender: (11, offender_expo.clone()), @@ -3646,7 +3650,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), )); // a wrong solution. @@ -3665,7 +3669,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ), Error::::PhragmenSlashedNomination, ); @@ -3693,7 +3697,7 @@ mod offchain_phragmen { winners, compact, score, - active_era(), + current_era(), ), Error::::PhragmenBogusScore, ); @@ -3706,7 +3710,7 @@ mod offchain_phragmen { .offchain_phragmen_ext() .validator_count(4) .build(); - let state = offchainify(&mut ext); + let state = offchainify(&mut ext, 0); ext.execute_with(|| { use offchain_election::OFFCHAIN_HEAD_DB; @@ -3730,7 +3734,7 @@ mod offchain_phragmen { .offchain_phragmen_ext() .validator_count(4) .build(); - let _ = offchainify(&mut ext); + let _ = offchainify(&mut ext, 0); ext.execute_with(|| { use offchain_election::OFFCHAIN_HEAD_DB; @@ -3780,7 +3784,7 @@ mod offchain_phragmen { run_to_block(12); assert_eq!(Staking::era_election_status(), ElectionStatus::Open(12)); - let offender_expo = Staking::eras_stakers(active_era(), 10); + let offender_expo = Staking::eras_stakers(Staking::active_era().unwrap().index, 10); // panic from the impl in mock on_offence_now( @@ -3796,7 +3800,7 @@ mod offchain_phragmen { #[test] fn slash_kicks_validators_not_nominators_and_disables_nominator_for_kicked_validator() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { mock::start_era(1); assert_eq_uvec!(Session::validators(), vec![11, 21]); @@ -3805,8 +3809,8 @@ fn slash_kicks_validators_not_nominators_and_disables_nominator_for_kicked_valid assert_eq!(Balances::free_balance(101), 2000); // 11 and 21 both have the support of 100 - let exposure_11 = Staking::eras_stakers(active_era(), &11); - let exposure_21 = Staking::eras_stakers(active_era(), &21); + let exposure_11 = Staking::eras_stakers(Staking::active_era().unwrap().index, &11); + let exposure_21 = Staking::eras_stakers(Staking::active_era().unwrap().index, &21); assert_eq!(exposure_11.total, 1000 + 125); assert_eq!(exposure_21.total, 1000 + 375); @@ -3846,8 +3850,8 @@ fn slash_kicks_validators_not_nominators_and_disables_nominator_for_kicked_valid assert_ok!(Staking::validate(Origin::signed(10), Default::default())); mock::start_era(2); - let exposure_11 = Staking::eras_stakers(active_era(), &11); - let exposure_21 = Staking::eras_stakers(active_era(), &21); + let exposure_11 = Staking::eras_stakers(Staking::active_era().unwrap().index, &11); + let exposure_21 = Staking::eras_stakers(Staking::active_era().unwrap().index, &21); // 10 is re-elected, but without the support of 100 assert_eq!(exposure_11.total, 900); @@ -3863,7 +3867,7 @@ fn claim_reward_at_the_last_era_and_no_double_claim_and_invalid_claim() { // * rewards get paid until history_depth for both validators and nominators // * an invalid era to claim doesn't update last_reward // * double claim of one era fails - ExtBuilder::default().nominate(true).build().execute_with(|| { + ExtBuilder::default().nominate(true).build_and_execute(|| { let init_balance_10 = Balances::total_balance(&10); let init_balance_100 = Balances::total_balance(&100); @@ -3943,12 +3947,12 @@ fn claim_reward_at_the_last_era_and_no_double_claim_and_invalid_claim() { #[test] fn zero_slash_keeps_nominators() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { mock::start_era(1); assert_eq!(Balances::free_balance(11), 1000); - let exposure = Staking::eras_stakers(active_era(), 11); + let exposure = Staking::eras_stakers(Staking::active_era().unwrap().index, 11); assert_eq!(Balances::free_balance(101), 2000); on_offence_now( @@ -3981,7 +3985,7 @@ fn zero_slash_keeps_nominators() { #[test] fn six_session_delay() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { use pallet_session::SessionManager; let val_set = Session::validators(); @@ -4035,11 +4039,11 @@ fn test_max_nominator_rewarded_per_validator_and_cant_steal_someone_else_reward( // * If nominator nomination is below the $MaxNominatorRewardedPerValidator other nominator // then the nominator can't claim its reward // * A nominator can't claim another nominator reward - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { for i in 0..=::MaxNominatorRewardedPerValidator::get() { - let stash = 10_000 + i as u64; - let controller = 20_000 + i as u64; - let balance = 10_000 + i as u64; + let stash = 10_000 + i as AccountId; + let controller = 20_000 + i as AccountId; + let balance = 10_000 + i as Balance; Balances::make_free_balance_be(&stash, balance); assert_ok!( Staking::bond( @@ -4063,8 +4067,8 @@ fn test_max_nominator_rewarded_per_validator_and_cant_steal_someone_else_reward( // Assert only nominators from 1 to Max are rewarded for i in 0..=::MaxNominatorRewardedPerValidator::get() { - let stash = 10_000 + i as u64; - let balance = 10_000 + i as u64; + let stash = 10_000 + i as AccountId; + let balance = 10_000 + i as Balance; if stash == 10_000 { assert!(Balances::free_balance(&stash) == balance); } else { @@ -4076,7 +4080,7 @@ fn test_max_nominator_rewarded_per_validator_and_cant_steal_someone_else_reward( #[test] fn set_history_depth_works() { - ExtBuilder::default().build().execute_with(|| { + ExtBuilder::default().build_and_execute(|| { mock::start_era(10); Staking::set_history_depth(Origin::ROOT, 20).unwrap(); assert!(::ErasTotalStake::contains_key(10 - 4)); @@ -4097,14 +4101,14 @@ fn set_history_depth_works() { fn test_payout_stakers() { // Here we will test validator can set `max_nominators_payout` and it works. // We also test that `payout_extra_nominators` works. - ExtBuilder::default().has_stakers(false).build().execute_with(|| { + ExtBuilder::default().has_stakers(false).build_and_execute(|| { let balance = 1000; // Create three validators: bond_validator(11, 10, balance); // Default(64) // Create nominators, targeting stash of validators for i in 0..100 { - bond_nominator(1000 + i, 100 + i, balance + i, vec![11]); + bond_nominator(1000 + i, 100 + i, balance + i as Balance, vec![11]); } mock::start_era(1); @@ -4119,11 +4123,11 @@ fn test_payout_stakers() { // Validator payout goes to controller. assert!(Balances::free_balance(&10) > balance); for i in 36..100 { - assert!(Balances::free_balance(&(100 + i)) > balance + i); + assert!(Balances::free_balance(&(100 + i)) > balance + i as Balance); } // The bottom 36 do not for i in 0..36 { - assert_eq!(Balances::free_balance(&(100 + i)), balance + i); + assert_eq!(Balances::free_balance(&(100 + i)), balance + i as Balance); } // We track rewards in `claimed_rewards` vec @@ -4177,14 +4181,14 @@ fn test_payout_stakers() { #[test] fn payout_stakers_handles_basic_errors() { // Here we will test payouts handle all errors. - ExtBuilder::default().has_stakers(false).build().execute_with(|| { + ExtBuilder::default().has_stakers(false).build_and_execute(|| { // Same setup as the test above let balance = 1000; bond_validator(11, 10, balance); // Default(64) // Create nominators, targeting stash for i in 0..100 { - bond_nominator(1000 + i, 100 + i, balance + i, vec![11]); + bond_nominator(1000 + i, 100 + i, balance + i as Balance, vec![11]); } mock::start_era(1); @@ -4221,7 +4225,7 @@ fn payout_stakers_handles_basic_errors() { #[test] fn bond_during_era_correctly_populates_claimed_rewards() { - ExtBuilder::default().has_stakers(false).build().execute_with(|| { + ExtBuilder::default().has_stakers(false).build_and_execute(|| { // Era = None bond_validator(9, 8, 1000); assert_eq!( @@ -4343,7 +4347,7 @@ fn test_last_reward_migration() { sp_io::TestExternalities::new(s).execute_with(|| { HistoryDepth::put(84); CurrentEra::put(99); - let nominations = Nominations:: { + let nominations = Nominations:: { targets: vec![], submitted_in: 0, suppressed: false @@ -4404,7 +4408,7 @@ fn rewards_should_work_before_migration() { // * rewards get recorded per session // * rewards get paid per Era // * Check that nominators are also rewarded - ExtBuilder::default().nominate(true).build().execute_with(|| { + ExtBuilder::default().nominate(true).build_and_execute(|| { MigrateEra::put(10); let init_balance_10 = Balances::total_balance(&10); let init_balance_11 = Balances::total_balance(&11); @@ -4495,7 +4499,7 @@ fn migrate_era_should_work() { // * rewards get recorded per session // * rewards get paid per Era // * Check that nominators are also rewarded - ExtBuilder::default().nominate(true).build().execute_with(|| { + ExtBuilder::default().nominate(true).build_and_execute(|| { MigrateEra::put(1); let init_balance_10 = Balances::total_balance(&10); let init_balance_11 = Balances::total_balance(&11); @@ -4583,7 +4587,7 @@ fn migrate_era_should_work() { #[test] #[should_panic] fn migrate_era_should_handle_error() { - ExtBuilder::default().nominate(true).build().execute_with(|| { + ExtBuilder::default().nominate(true).build_and_execute(|| { MigrateEra::put(1); let init_balance_10 = Balances::total_balance(&10); let init_balance_11 = Balances::total_balance(&11); @@ -4635,7 +4639,7 @@ fn migrate_era_should_handle_errors_2() { // * rewards get recorded per session // * rewards get paid per Era // * Check that nominators are also rewarded - ExtBuilder::default().nominate(true).build().execute_with(|| { + ExtBuilder::default().nominate(true).build_and_execute(|| { MigrateEra::put(1); let init_balance_10 = Balances::total_balance(&10); let init_balance_11 = Balances::total_balance(&11); diff --git a/frame/sudo/Cargo.toml b/frame/sudo/Cargo.toml index 4216b94ec12f12ccf8514698a78a25e3e9752ac6..25988b1fe30c945ba8b53ef48944057108d2c16d 100644 --- a/frame/sudo/Cargo.toml +++ b/frame/sudo/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-sudo" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,17 +8,20 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for sudo" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } [features] default = ["std"] @@ -31,6 +34,3 @@ std = [ "frame-support/std", "frame-system/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/sudo/src/lib.rs b/frame/sudo/src/lib.rs index 3f2eacdbf8139a83530fcf7536b6ad981e6af58f..d19c92358c287e74a4d87de3e1dcf3d0dd2a9ae6 100644 --- a/frame/sudo/src/lib.rs +++ b/frame/sudo/src/lib.rs @@ -58,7 +58,7 @@ //! //! decl_module! { //! pub struct Module for enum Call where origin: T::Origin { -//! #[weight = frame_support::weights::SimpleDispatchInfo::default()] +//! #[weight = 0] //! pub fn privileged_function(origin) -> dispatch::DispatchResult { //! ensure_root(origin)?; //! @@ -87,12 +87,12 @@ #![cfg_attr(not(feature = "std"), no_std)] use sp_std::prelude::*; -use sp_runtime::traits::{StaticLookup, Dispatchable}; +use sp_runtime::{DispatchResult, traits::{StaticLookup, Dispatchable}}; use frame_support::{ Parameter, decl_module, decl_event, decl_storage, decl_error, ensure, }; -use frame_support::weights::{GetDispatchInfo, FunctionOf}; +use frame_support::weights::{GetDispatchInfo, FunctionOf, Pays}; use frame_system::{self as system, ensure_signed}; pub trait Trait: frame_system::Trait { @@ -123,22 +123,15 @@ decl_module! { #[weight = FunctionOf( |args: (&Box<::Call>,)| args.0.get_dispatch_info().weight + 10_000, |args: (&Box<::Call>,)| args.0.get_dispatch_info().class, - true + Pays::Yes, )] fn sudo(origin, call: Box<::Call>) { // This is a public call, so we ensure that the origin is some signed account. let sender = ensure_signed(origin)?; ensure!(sender == Self::key(), Error::::RequireSudo); - let res = match call.dispatch(frame_system::RawOrigin::Root.into()) { - Ok(_) => true, - Err(e) => { - sp_runtime::print(e); - false - } - }; - - Self::deposit_event(RawEvent::Sudid(res)); + let res = call.dispatch(frame_system::RawOrigin::Root.into()); + Self::deposit_event(RawEvent::Sudid(res.map(|_| ()).map_err(|e| e.error))); } /// Authenticates the current sudo key and sets the given AccountId (`new`) as the new sudo key. @@ -150,7 +143,7 @@ decl_module! { /// - Limited storage reads. /// - One DB change. /// # - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn set_key(origin, new: ::Source) { // This is a public call, so we ensure that the origin is some signed account. let sender = ensure_signed(origin)?; @@ -179,7 +172,7 @@ decl_module! { |args: (&::Source, &Box<::Call>,)| { args.1.get_dispatch_info().class }, - true + Pays::Yes, )] fn sudo_as(origin, who: ::Source, call: Box<::Call>) { // This is a public call, so we ensure that the origin is some signed account. @@ -204,7 +197,7 @@ decl_module! { decl_event!( pub enum Event where AccountId = ::AccountId { /// A sudo just took place. - Sudid(bool), + Sudid(DispatchResult), /// The sudoer just switched identity; the old key is supplied. KeyChanged(AccountId), /// A sudo just took place. diff --git a/frame/support/Cargo.toml b/frame/support/Cargo.toml index 3bad72a115716b6f5c1f6297bf62ef40d24674a2..0168705da7e343f682dd38209a15bf93efbc126c 100644 --- a/frame/support/Cargo.toml +++ b/frame/support/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-support" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,33 +8,35 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Support code for the runtime." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] log = "0.4" serde = { version = "1.0.101", optional = true, features = ["derive"] } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -frame-metadata = { version = "11.0.0-alpha.5", default-features = false, path = "../metadata" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -sp-arithmetic = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/arithmetic" } -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" } -frame-support-procedural = { version = "2.0.0-alpha.5", path = "./procedural" } +frame-metadata = { version = "11.0.0-dev", default-features = false, path = "../metadata" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-tracing = { version = "2.0.0-dev", default-features = false, path = "../../primitives/tracing" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-arithmetic = { version = "2.0.0-dev", default-features = false, path = "../../primitives/arithmetic" } +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../../primitives/inherents" } +frame-support-procedural = { version = "2.0.0-dev", path = "./procedural" } paste = "0.1.6" once_cell = { version = "1", default-features = false, optional = true } -sp-state-machine = { version = "0.8.0-alpha.5", optional = true, path = "../../primitives/state-machine" } +sp-state-machine = { version = "0.8.0-dev", optional = true, path = "../../primitives/state-machine" } bitmask = { version = "0.5.0", default-features = false } impl-trait-for-tuples = "0.1.3" -tracing = { version = "0.1.10", optional = true } [dev-dependencies] pretty_assertions = "0.6.1" -frame-system = { version = "2.0.0-alpha.5", path = "../system" } +frame-system = { version = "2.0.0-dev", path = "../system" } [features] default = ["std"] std = [ - "tracing", "once_cell", "bitmask/std", "serde", @@ -42,6 +44,7 @@ std = [ "codec/std", "sp-std/std", "sp-runtime/std", + "sp-tracing/std", "sp-arithmetic/std", "frame-metadata/std", "sp-inherents/std", @@ -50,6 +53,3 @@ std = [ nightly = [] strict = [] runtime-benchmarks = [] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/support/procedural/Cargo.toml b/frame/support/procedural/Cargo.toml index 2f7450e4b86d3ea960a9b9b2e1e3b2c32b1af306..55e551343214d0711945d6c3d3008d03086836dd 100644 --- a/frame/support/procedural/Cargo.toml +++ b/frame/support/procedural/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-support-procedural" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,14 +8,14 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Proc macro of Support code for the runtime." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [lib] proc-macro = true [dependencies] -frame-support-procedural-tools = { version = "2.0.0-alpha.5", path = "./tools" } +frame-support-procedural-tools = { version = "2.0.0-dev", path = "./tools" } proc-macro2 = "1.0.6" quote = "1.0.3" syn = { version = "1.0.7", features = ["full"] } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/support/procedural/src/storage/parse.rs b/frame/support/procedural/src/storage/parse.rs index af568c78cc6b72f3f71a9827ef4c260a11851425..beb9beff707b3c8254cb2ea1c5ff6418cc77e0b6 100644 --- a/frame/support/procedural/src/storage/parse.rs +++ b/frame/support/procedural/src/storage/parse.rs @@ -166,7 +166,7 @@ struct DeclStorageLine { #[derive(Parse, ToTokens, Debug)] struct DeclStorageGetterBody { - fn_keyword: Option, + fn_keyword: Token![fn], ident: Ident, } diff --git a/frame/support/procedural/tools/Cargo.toml b/frame/support/procedural/tools/Cargo.toml index f199f1245db0499aa90425ac6da3966fa2132201..f64ad9b1e66edcd87640a73ce257ab0a74730713 100644 --- a/frame/support/procedural/tools/Cargo.toml +++ b/frame/support/procedural/tools/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-support-procedural-tools" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,12 +8,12 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Proc macro helpers for procedural macros" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -frame-support-procedural-tools-derive = { version = "2.0.0-alpha.5", path = "./derive" } +frame-support-procedural-tools-derive = { version = "2.0.0-dev", path = "./derive" } proc-macro2 = "1.0.6" quote = "1.0.3" syn = { version = "1.0.7", features = ["full", "visit"] } proc-macro-crate = "0.1.4" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/support/procedural/tools/derive/Cargo.toml b/frame/support/procedural/tools/derive/Cargo.toml index bf6346ab1b67a63c57fb021550dec9a8fda0420a..75721508e8f0f138d7d4c64da1c1a4822530b00c 100644 --- a/frame/support/procedural/tools/derive/Cargo.toml +++ b/frame/support/procedural/tools/derive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-support-procedural-tools-derive" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,6 +8,9 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Use to derive parsing for parsing struct." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [lib] proc-macro = true @@ -15,6 +18,3 @@ proc-macro = true proc-macro2 = "1.0.6" quote = { version = "1.0.3", features = ["proc-macro"] } syn = { version = "1.0.7", features = ["proc-macro" ,"full", "extra-traits", "parsing"] } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/support/src/dispatch.rs b/frame/support/src/dispatch.rs index 026b30d0f43d0cb358173819019e4de9def0e2e0..081574726a62cbd85eafb266eb22e34b909fdafe 100644 --- a/frame/support/src/dispatch.rs +++ b/frame/support/src/dispatch.rs @@ -24,8 +24,8 @@ pub use frame_metadata::{ ModuleConstantMetadata, DefaultByte, DefaultByteGetter, ModuleErrorMetadata, ErrorMetadata }; pub use crate::weights::{ - SimpleDispatchInfo, GetDispatchInfo, DispatchInfo, WeighData, ClassifyDispatch, - TransactionPriority, Weight, PaysFee, PostDispatchInfo, WithPostDispatchInfo, + GetDispatchInfo, DispatchInfo, WeighData, ClassifyDispatch, TransactionPriority, Weight, + PaysFee, PostDispatchInfo, WithPostDispatchInfo, }; pub use sp_runtime::{traits::Dispatchable, DispatchError}; pub use crate::traits::{CallMetadata, GetCallMetadata, GetCallName}; @@ -46,14 +46,10 @@ pub type DispatchResult = Result<(), sp_runtime::DispatchError>; pub type DispatchErrorWithPostInfo = sp_runtime::DispatchErrorWithPostInfo; - -/// A type that cannot be instantiated. -pub enum Never {} - /// Serializable version of Dispatchable. /// This value can be used as a "function" in an extrinsic. pub trait Callable { - type Call: Dispatchable + Codec + Clone + PartialEq + Eq; + type Call: Dispatchable + Codec + Clone + PartialEq + Eq; } // dirty hack to work around serde_derive issue @@ -74,14 +70,13 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// # #[macro_use] /// # extern crate frame_support; /// # use frame_support::dispatch; -/// # use frame_support::weights::SimpleDispatchInfo; /// # use frame_system::{self as system, Trait, ensure_signed}; /// decl_module! { /// pub struct Module for enum Call where origin: T::Origin { /// /// // Private functions are dispatchable, but not available to other /// // FRAME pallets. -/// #[weight = SimpleDispatchInfo::default()] +/// #[weight = 0] /// fn my_function(origin, var: u64) -> dispatch::DispatchResult { /// // Your implementation /// Ok(()) @@ -89,7 +84,7 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// /// // Public functions are both dispatchable and available to other /// // FRAME pallets. -/// #[weight = SimpleDispatchInfo::default()] +/// #[weight = 0] /// pub fn my_public_function(origin) -> dispatch::DispatchResult { /// // Your implementation /// Ok(()) @@ -117,17 +112,16 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// # #[macro_use] /// # extern crate frame_support; /// # use frame_support::dispatch; -/// # use frame_support::weights::SimpleDispatchInfo; /// # use frame_system::{self as system, Trait, ensure_signed}; /// decl_module! { /// pub struct Module for enum Call where origin: T::Origin { -/// #[weight = SimpleDispatchInfo::default()] +/// #[weight = 0] /// fn my_long_function(origin) -> dispatch::DispatchResult { /// // Your implementation /// Ok(()) /// } /// -/// #[weight = SimpleDispatchInfo::default()] +/// #[weight = 0] /// fn my_short_function(origin) { /// // Your implementation /// } @@ -153,11 +147,10 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// # #[macro_use] /// # extern crate frame_support; /// # use frame_support::dispatch::{DispatchResultWithPostInfo, WithPostDispatchInfo}; -/// # use frame_support::weights::SimpleDispatchInfo; /// # use frame_system::{self as system, Trait, ensure_signed}; /// decl_module! { /// pub struct Module for enum Call where origin: T::Origin { -/// #[weight = SimpleDispatchInfo::FixedNormal(1_000_000)] +/// #[weight = 1_000_000] /// fn my_long_function(origin, do_expensive_calc: bool) -> DispatchResultWithPostInfo { /// ensure_signed(origin).map_err(|e| e.with_weight(100_000))?; /// if do_expensive_calc { @@ -182,11 +175,10 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug {} /// # #[macro_use] /// # extern crate frame_support; /// # use frame_support::dispatch; -/// # use frame_support::weights::SimpleDispatchInfo; /// # use frame_system::{self as system, Trait, ensure_signed, ensure_root}; /// decl_module! { /// pub struct Module for enum Call where origin: T::Origin { -/// #[weight = SimpleDispatchInfo::default()] +/// #[weight = 0] /// fn my_privileged_function(origin) -> dispatch::DispatchResult { /// ensure_root(origin)?; /// // Your implementation @@ -1030,12 +1022,7 @@ macro_rules! decl_module { for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* { fn on_initialize(_block_number_not_used: $trait_instance::BlockNumber) -> $return { - use $crate::sp_std::if_std; - if_std! { - use $crate::tracing; - let span = tracing::span!(tracing::Level::DEBUG, "on_initialize"); - let _enter = span.enter(); - } + $crate::sp_tracing::enter_span!("on_initialize"); { $( $impl )* } } } @@ -1051,12 +1038,7 @@ macro_rules! decl_module { for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* { fn on_initialize($param: $param_ty) -> $return { - use $crate::sp_std::if_std; - if_std! { - use $crate::tracing; - let span = tracing::span!(tracing::Level::DEBUG, "on_initialize"); - let _enter = span.enter(); - } + $crate::sp_tracing::enter_span!("on_initialize"); { $( $impl )* } } } @@ -1082,12 +1064,7 @@ macro_rules! decl_module { for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* { fn on_runtime_upgrade() -> $return { - use $crate::sp_std::if_std; - if_std! { - use $crate::tracing; - let span = tracing::span!(tracing::Level::DEBUG, "on_runtime_upgrade"); - let _enter = span.enter(); - } + $crate::sp_tracing::enter_span!("on_runtime_upgrade"); { $( $impl )* } } } @@ -1114,12 +1091,7 @@ macro_rules! decl_module { for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* { fn on_finalize(_block_number_not_used: $trait_instance::BlockNumber) { - use $crate::sp_std::if_std; - if_std! { - use $crate::tracing; - let span = tracing::span!(tracing::Level::DEBUG, "on_finalize"); - let _enter = span.enter(); - } + $crate::sp_tracing::enter_span!("on_finalize"); { $( $impl )* } } } @@ -1135,12 +1107,7 @@ macro_rules! decl_module { for $module<$trait_instance$(, $instance)?> where $( $other_where_bounds )* { fn on_finalize($param: $param_ty) { - use $crate::sp_std::if_std; - if_std! { - use $crate::tracing; - let span = tracing::span!(tracing::Level::DEBUG, "on_finalize"); - let _enter = span.enter(); - } + $crate::sp_tracing::enter_span!("on_finalize"); { $( $impl )* } } } @@ -1209,15 +1176,9 @@ macro_rules! decl_module { $vis fn $name( $origin: $origin_ty $(, $param: $param_ty )* ) -> $crate::dispatch::DispatchResult { - $crate::sp_std::if_std! { - use $crate::tracing; - let span = tracing::span!(tracing::Level::DEBUG, stringify!($name)); - let _enter = span.enter(); - } - { - { $( $impl )* } - Ok(()) - } + $crate::sp_tracing::enter_span!(stringify!($name)); + { $( $impl )* } + Ok(()) } }; @@ -1234,13 +1195,8 @@ macro_rules! decl_module { ) => { $(#[doc = $doc_attr])* $vis fn $name($origin: $origin_ty $(, $param: $param_ty )* ) -> $result { - use $crate::sp_std::if_std; - if_std! { - use $crate::tracing; - let span = tracing::span!(tracing::Level::DEBUG, stringify!($name)); - let _enter = span.enter(); - } - { $( $impl )* } + $crate::sp_tracing::enter_span!(stringify!($name)); + $( $impl )* } }; @@ -1352,7 +1308,7 @@ macro_rules! decl_module { { #[doc(hidden)] #[codec(skip)] - __PhantomItem($crate::sp_std::marker::PhantomData<($trait_instance, $($instance)?)>, $crate::dispatch::Never), + __PhantomItem($crate::sp_std::marker::PhantomData<($trait_instance, $($instance)?)>, $crate::Never), $( $generated_variants )* } }; @@ -1477,16 +1433,17 @@ macro_rules! decl_module { match *self { $( $call_type::$fn_name( $( ref $param_name ),* ) => { + let base_weight = $weight; let weight = >::weigh_data( - &$weight, + &base_weight, ($( $param_name, )*) ); let class = >::classify_dispatch( - &$weight, + &base_weight, ($( $param_name, )*) ); let pays_fee = >::pays_fee( - &$weight, + &base_weight, ($( $param_name, )*) ); $crate::dispatch::DispatchInfo { @@ -1593,6 +1550,7 @@ macro_rules! decl_module { { type Trait = $trait_instance; type Origin = $origin_type; + type Info = $crate::weights::DispatchInfo; type PostInfo = $crate::weights::PostDispatchInfo; fn dispatch(self, _origin: Self::Origin) -> $crate::dispatch::DispatchResultWithPostInfo { match self { @@ -1720,6 +1678,7 @@ macro_rules! impl_outer_dispatch { impl $crate::dispatch::Dispatchable for $call_type { type Origin = $origin; type Trait = $call_type; + type Info = $crate::weights::DispatchInfo; type PostInfo = $crate::weights::PostDispatchInfo; fn dispatch( self, @@ -2084,7 +2043,7 @@ macro_rules! __check_reserved_fn_name { #[allow(dead_code)] mod tests { use super::*; - use crate::weights::{DispatchInfo, DispatchClass}; + use crate::weights::{DispatchInfo, DispatchClass, Pays}; use crate::traits::{ CallMetadata, GetCallMetadata, GetCallName, OnInitialize, OnFinalize, OnRuntimeUpgrade }; @@ -2110,25 +2069,25 @@ mod tests { decl_module! { pub struct Module for enum Call where origin: T::Origin, T::AccountId: From { /// Hi, this is a comment. - #[weight = SimpleDispatchInfo::default()] + #[weight = 0] fn aux_0(_origin) -> DispatchResult { unreachable!() } - #[weight = SimpleDispatchInfo::default()] + #[weight = 0] fn aux_1(_origin, #[compact] _data: u32,) -> DispatchResult { unreachable!() } - #[weight = SimpleDispatchInfo::default()] + #[weight = 0] fn aux_2(_origin, _data: i32, _data2: String) -> DispatchResult { unreachable!() } - #[weight = SimpleDispatchInfo::FixedNormal(3)] + #[weight = 3] fn aux_3(_origin) -> DispatchResult { unreachable!() } - #[weight = SimpleDispatchInfo::default()] + #[weight = 0] fn aux_4(_origin, _data: i32) -> DispatchResult { unreachable!() } - #[weight = SimpleDispatchInfo::default()] + #[weight = 0] fn aux_5(_origin, _data: i32, #[compact] _data2: u32,) -> DispatchResult { unreachable!() } - #[weight = SimpleDispatchInfo::FixedOperational(5)] + #[weight = (5, DispatchClass::Operational)] fn operational(_origin) { unreachable!() } fn on_initialize(n: T::BlockNumber,) -> Weight { if n.into() == 42 { panic!("on_initialize") } 7 } @@ -2287,17 +2246,12 @@ mod tests { // operational. assert_eq!( Call::::operational().get_dispatch_info(), - DispatchInfo { weight: 5, class: DispatchClass::Operational, pays_fee: true }, - ); - // default weight. - assert_eq!( - Call::::aux_0().get_dispatch_info(), - DispatchInfo { weight: 10_000, class: DispatchClass::Normal, pays_fee: true }, + DispatchInfo { weight: 5, class: DispatchClass::Operational, pays_fee: Pays::Yes }, ); // custom basic assert_eq!( Call::::aux_3().get_dispatch_info(), - DispatchInfo { weight: 3, class: DispatchClass::Normal, pays_fee: true }, + DispatchInfo { weight: 3, class: DispatchClass::Normal, pays_fee: Pays::Yes }, ); } diff --git a/frame/support/src/error.rs b/frame/support/src/error.rs index f619250726deda5caea1f76894d1a75793458f01..115920f39a933440d61825661830cbc391588a9f 100644 --- a/frame/support/src/error.rs +++ b/frame/support/src/error.rs @@ -35,7 +35,7 @@ pub use frame_metadata::{ModuleErrorMetadata, ErrorMetadata, DecodeDifferent}; /// /// ``` /// # use frame_support::{decl_error, decl_module}; -/// # use frame_support::weights::SimpleDispatchInfo; +/// # /// decl_error! { /// /// Errors that can occur in my module. /// pub enum MyError for Module { @@ -55,7 +55,7 @@ pub use frame_metadata::{ModuleErrorMetadata, ErrorMetadata, DecodeDifferent}; /// pub struct Module for enum Call where origin: T::Origin { /// type Error = MyError; /// -/// #[weight = SimpleDispatchInfo::default()] +/// #[weight = 0] /// fn do_something(origin) -> frame_support::dispatch::DispatchResult { /// Err(MyError::::YouAreNotCoolEnough.into()) /// } @@ -89,7 +89,7 @@ macro_rules! decl_error { #[doc(hidden)] __Ignore( $crate::sp_std::marker::PhantomData<($generic, $( $inst_generic)?)>, - $crate::dispatch::Never, + $crate::Never, ), $( $( #[doc = $doc_attr] )* diff --git a/frame/support/src/hash.rs b/frame/support/src/hash.rs index 693e929a309e366b0605805d1721429904d04806..40cb1f612f7bf61e4a499e08feba56c1bfe6ea41 100644 --- a/frame/support/src/hash.rs +++ b/frame/support/src/hash.rs @@ -60,7 +60,12 @@ pub trait StorageHasher: 'static { } /// Hasher to use to hash keys to insert to storage. +/// +/// Reversible hasher store the encoded key after the hash part. pub trait ReversibleStorageHasher: StorageHasher { + /// Split the hash part out of the input. + /// + /// I.e. for input `&[hash ++ key ++ some]` returns `&[key ++ some]` fn reverse(x: &[u8]) -> &[u8]; } diff --git a/frame/support/src/inherent.rs b/frame/support/src/inherent.rs index a21bd361b618dc515755a1cb444b4ccb22193283..4e09bf9dd817398864bd8e71c86dd0e82befb4fd 100644 --- a/frame/support/src/inherent.rs +++ b/frame/support/src/inherent.rs @@ -77,7 +77,7 @@ macro_rules! impl_outer_inherent { let mut result = $crate::inherent::CheckInherentsResult::new(); for xt in block.extrinsics() { if $crate::inherent::Extrinsic::is_signed(xt).unwrap_or(false) { - break; + break } $( @@ -88,7 +88,7 @@ macro_rules! impl_outer_inherent { $module::INHERENT_IDENTIFIER, &e ).expect("There is only one fatal error; qed"); if e.is_fatal_error() { - return result; + return result } } } @@ -97,6 +97,41 @@ macro_rules! impl_outer_inherent { )* } + $( + match $module::is_inherent_required(self) { + Ok(Some(e)) => { + let found = block.extrinsics().iter().any(|xt| { + if $crate::inherent::Extrinsic::is_signed(xt).unwrap_or(false) { + return false + } + + match xt.function { + Call::$call(_) => true, + _ => false, + } + }); + + if !found { + result.put_error( + $module::INHERENT_IDENTIFIER, &e + ).expect("There is only one fatal error; qed"); + if e.is_fatal_error() { + return result + } + } + }, + Ok(None) => (), + Err(e) => { + result.put_error( + $module::INHERENT_IDENTIFIER, &e + ).expect("There is only one fatal error; qed"); + if e.is_fatal_error() { + return result + } + }, + } + )* + result } } diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 81438ea1bdeb5d0d733bd15a18f45db6d3c1cc75..b0e7395e9ff2f36ea03f32b3fdb27f233fb031e4 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -23,8 +23,9 @@ extern crate self as frame_support; #[macro_use] extern crate bitmask; -#[cfg(feature = "std")] -pub extern crate tracing; + +#[doc(hidden)] +pub use sp_tracing; #[cfg(feature = "std")] pub use serde; @@ -68,7 +69,7 @@ pub mod weights; pub use self::hash::{ Twox256, Twox128, Blake2_256, Blake2_128, Identity, Twox64Concat, Blake2_128Concat, Hashable, - StorageHasher + StorageHasher, ReversibleStorageHasher }; pub use self::storage::{ StorageValue, StorageMap, StorageDoubleMap, StoragePrefixedMap, IterableStorageMap, @@ -77,6 +78,10 @@ pub use self::storage::{ pub use self::dispatch::{Parameter, Callable, IsSubType}; pub use sp_runtime::{self, ConsensusEngineId, print, traits::Printable}; +/// A type that cannot be instantiated. +#[derive(Debug)] +pub enum Never {} + /// Macro for easily creating a new implementation of the `Get` trait. Use similarly to /// how you would declare a `const`: /// @@ -140,6 +145,8 @@ macro_rules! ord_parameter_types { fn contains(t: &$type) -> bool { &$value == t } fn sorted_members() -> $crate::sp_std::prelude::Vec<$type> { vec![$value] } fn count() -> usize { 1 } + #[cfg(feature = "runtime-benchmarks")] + fn add(_: &$type) {} } } } @@ -186,10 +193,6 @@ macro_rules! assert_noop { } } -/// Panic if an expression doesn't evaluate to an `Err`. -/// -/// Used as `assert_err!(expression_to_assert, expected_err_expression)`. - /// Assert an expression returns an error specified. /// /// Used as `assert_err!(expression_to_assert, expected_error_expression)` @@ -201,6 +204,18 @@ macro_rules! assert_err { } } +/// Assert an expression returns an error specified. +/// +/// This can be used on`DispatchResultWithPostInfo` when the post info should +/// be ignored. +#[macro_export] +#[cfg(feature = "std")] +macro_rules! assert_err_ignore_postinfo { + ( $x:expr , $y:expr $(,)? ) => { + assert_err!($x.map(|_| ()).map_err(|e| e.error), $y); + } +} + /// Panic if an expression doesn't evaluate to `Ok`. /// /// Used as `assert_ok!(expression_to_assert, expected_ok_expression)`, @@ -264,8 +279,6 @@ mod tests { map hasher(identity) T::BlockNumber => T::BlockNumber; pub GenericData2 get(fn generic_data2): map hasher(blake2_128_concat) T::BlockNumber => Option; - pub GetterNoFnKeyword get(no_fn): Option; - pub DataDM config(test_config) build(|_| vec![(15u32, 16u32, 42u64)]): double_map hasher(twox_64_concat) u32, hasher(blake2_128_concat) u32 => u64; pub GenericDataDM: @@ -551,15 +564,6 @@ mod tests { ), documentation: DecodeDifferent::Encode(&[]), }, - StorageEntryMetadata { - name: DecodeDifferent::Encode("GetterNoFnKeyword"), - modifier: StorageEntryModifier::Optional, - ty: StorageEntryType::Plain(DecodeDifferent::Encode("u32")), - default: DecodeDifferent::Encode( - DefaultByteGetter(&__GetByteStructGetterNoFnKeyword(PhantomData::)) - ), - documentation: DecodeDifferent::Encode(&[]), - }, StorageEntryMetadata { name: DecodeDifferent::Encode("DataDM"), modifier: StorageEntryModifier::Default, diff --git a/frame/support/src/metadata.rs b/frame/support/src/metadata.rs index d9c8136d3c442a99a57e74fe6851193818ac0bb5..081f392b9ecf16d2bedbdcc90046a04ef4d7983b 100644 --- a/frame/support/src/metadata.rs +++ b/frame/support/src/metadata.rs @@ -256,9 +256,8 @@ mod tests { struct TestExtension; impl sp_runtime::traits::SignedExtension for TestExtension { type AccountId = u32; - type Call = u32; + type Call = (); type AdditionalSigned = u32; - type DispatchInfo = (); type Pre = (); const IDENTIFIER: &'static str = "testextension"; fn additional_signed(&self) -> Result { @@ -270,9 +269,8 @@ mod tests { struct TestExtension2; impl sp_runtime::traits::SignedExtension for TestExtension2 { type AccountId = u32; - type Call = u32; + type Call = (); type AdditionalSigned = u32; - type DispatchInfo = (); type Pre = (); const IDENTIFIER: &'static str = "testextension2"; fn additional_signed(&self) -> Result { @@ -336,7 +334,6 @@ mod tests { mod event_module { use crate::dispatch::DispatchResult; - use crate::weights::SimpleDispatchInfo; pub trait Trait: super::system::Trait { type Balance; @@ -354,7 +351,7 @@ mod tests { pub struct Module for enum Call where origin: T::Origin { type Error = Error; - #[weight = SimpleDispatchInfo::default()] + #[weight = 0] fn aux_0(_origin) -> DispatchResult { unreachable!() } } } diff --git a/frame/support/src/storage/child.rs b/frame/support/src/storage/child.rs index d4d046a9d4245837702917e4cb5214d5af858d26..658908d258a2ffee230b555a38aa18865e74c276 100644 --- a/frame/support/src/storage/child.rs +++ b/frame/support/src/storage/child.rs @@ -16,100 +16,90 @@ //! Operation on runtime child storages. //! -//! This module is a currently only a variant of unhashed with additional `storage_key`. -//! Note that `storage_key` must be unique and strong (strong in the sense of being long enough to -//! avoid collision from a resistant hash function (which unique implies)). -//! -//! A **key collision free** unique id is required as parameter to avoid key collision -//! between child tries. -//! This unique id management and generation responsibility is delegated to pallet module. -// NOTE: could replace unhashed by having only one kind of storage (root being null storage key (storage_key can become Option<&[u8]>). +//! This module is a currently only a variant of unhashed with additional `child_info`. +// NOTE: could replace unhashed by having only one kind of storage (top trie being the child info +// of null length parent storage key). use crate::sp_std::prelude::*; use codec::{Codec, Encode, Decode}; -pub use sp_core::storage::ChildInfo; +pub use sp_core::storage::{ChildInfo, ChildType}; /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry. pub fn get( - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Option { - let (data, child_type) = child_info.info(); - sp_io::storage::child_get( - storage_key, - data, - child_type, - key, - ).and_then(|v| { - Decode::decode(&mut &v[..]).map(Some).unwrap_or_else(|_| { - // TODO #3700: error should be handleable. - runtime_print!("ERROR: Corrupted state in child trie at {:?}/{:?}", storage_key, key); - None - }) - }) + match child_info.child_type() { + ChildType::ParentKeyId => { + let storage_key = child_info.storage_key(); + sp_io::default_child_storage::get( + storage_key, + key, + ).and_then(|v| { + Decode::decode(&mut &v[..]).map(Some).unwrap_or_else(|_| { + // TODO #3700: error should be handleable. + runtime_print!("ERROR: Corrupted state in child trie at {:?}/{:?}", storage_key, key); + None + }) + }) + }, + } } /// Return the value of the item in storage under `key`, or the type's default if there is no /// explicit entry. pub fn get_or_default( - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> T { - get(storage_key, child_info, key).unwrap_or_else(Default::default) + get(child_info, key).unwrap_or_else(Default::default) } /// Return the value of the item in storage under `key`, or `default_value` if there is no /// explicit entry. pub fn get_or( - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], default_value: T, ) -> T { - get(storage_key, child_info, key).unwrap_or(default_value) + get(child_info, key).unwrap_or(default_value) } /// Return the value of the item in storage under `key`, or `default_value()` if there is no /// explicit entry. pub fn get_or_else T>( - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], default_value: F, ) -> T { - get(storage_key, child_info, key).unwrap_or_else(default_value) + get(child_info, key).unwrap_or_else(default_value) } /// Put `value` in storage under `key`. pub fn put( - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], value: &T, ) { - let (data, child_type) = child_info.info(); - value.using_encoded(|slice| - sp_io::storage::child_set( - storage_key, - data, - child_type, - key, - slice, - ) - ); + match child_info.child_type() { + ChildType::ParentKeyId => value.using_encoded(|slice| + sp_io::default_child_storage::set( + child_info.storage_key(), + key, + slice, + ) + ), + } } /// Remove `key` from storage, returning its value if it had an explicit entry or `None` otherwise. pub fn take( - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Option { - let r = get(storage_key, child_info, key); + let r = get(child_info, key); if r.is_some() { - kill(storage_key, child_info, key); + kill(child_info, key); } r } @@ -117,113 +107,106 @@ pub fn take( /// Remove `key` from storage, returning its value, or, if there was no explicit entry in storage, /// the default for its type. pub fn take_or_default( - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> T { - take(storage_key, child_info, key).unwrap_or_else(Default::default) + take(child_info, key).unwrap_or_else(Default::default) } /// Return the value of the item in storage under `key`, or `default_value` if there is no /// explicit entry. Ensure there is no explicit entry on return. pub fn take_or( - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], default_value: T, ) -> T { - take(storage_key, child_info, key).unwrap_or(default_value) + take(child_info, key).unwrap_or(default_value) } /// Return the value of the item in storage under `key`, or `default_value()` if there is no /// explicit entry. Ensure there is no explicit entry on return. pub fn take_or_else T>( - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], default_value: F, ) -> T { - take(storage_key, child_info, key).unwrap_or_else(default_value) + take(child_info, key).unwrap_or_else(default_value) } /// Check to see if `key` has an explicit entry in storage. pub fn exists( - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> bool { - let (data, child_type) = child_info.info(); - sp_io::storage::child_read( - storage_key, data, child_type, - key, &mut [0;0][..], 0, - ).is_some() + match child_info.child_type() { + ChildType::ParentKeyId => sp_io::default_child_storage::read( + child_info.storage_key(), + key, &mut [0;0][..], 0, + ).is_some(), + } } /// Remove all `storage_key` key/values pub fn kill_storage( - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, ) { - let (data, child_type) = child_info.info(); - sp_io::storage::child_storage_kill( - storage_key, - data, - child_type, - ) + match child_info.child_type() { + ChildType::ParentKeyId => sp_io::default_child_storage::storage_kill( + child_info.storage_key(), + ), + } } /// Ensure `key` has no explicit entry in storage. pub fn kill( - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) { - let (data, child_type) = child_info.info(); - sp_io::storage::child_clear( - storage_key, - data, - child_type, - key, - ); + match child_info.child_type() { + ChildType::ParentKeyId => { + sp_io::default_child_storage::clear( + child_info.storage_key(), + key, + ); + }, + } } /// Get a Vec of bytes from storage. pub fn get_raw( - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Option> { - let (data, child_type) = child_info.info(); - sp_io::storage::child_get( - storage_key, - data, - child_type, - key, - ) + match child_info.child_type() { + ChildType::ParentKeyId => sp_io::default_child_storage::get( + child_info.storage_key(), + key, + ), + } } /// Put a raw byte slice into storage. pub fn put_raw( - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], value: &[u8], ) { - let (data, child_type) = child_info.info(); - sp_io::storage::child_set( - storage_key, - data, - child_type, - key, - value, - ) + match child_info.child_type() { + ChildType::ParentKeyId => sp_io::default_child_storage::set( + child_info.storage_key(), + key, + value, + ), + } } /// Calculate current child root value. -pub fn child_root( - storage_key: &[u8], +pub fn root( + child_info: &ChildInfo, ) -> Vec { - sp_io::storage::child_root( - storage_key, - ) + match child_info.child_type() { + ChildType::ParentKeyId => sp_io::default_child_storage::root( + child_info.storage_key(), + ), + } } diff --git a/frame/support/src/storage/generator/double_map.rs b/frame/support/src/storage/generator/double_map.rs index 9d05ff0b2d6d94ac1db87a730fc39015720b9938..c7e4c10017e706275727f687228516bb6e55aca0 100644 --- a/frame/support/src/storage/generator/double_map.rs +++ b/frame/support/src/storage/generator/double_map.rs @@ -17,7 +17,7 @@ use sp_std::prelude::*; use sp_std::borrow::Borrow; use codec::{Ref, FullCodec, FullEncode, Decode, Encode, EncodeLike, EncodeAppend}; -use crate::{storage::{self, unhashed}, traits::Len}; +use crate::{storage::{self, unhashed}, traits::Len, Never}; use crate::hash::{StorageHasher, Twox128, ReversibleStorageHasher}; /// Generator for `StorageDoubleMap` used by `decl_storage`. @@ -208,7 +208,7 @@ impl storage::StorageDoubleMap for G where unhashed::kill_prefix(Self::storage_double_map_final_key1(k1).as_ref()) } - fn iter_prefix(k1: KArg1) -> storage::PrefixIterator where + fn iter_prefix_values(k1: KArg1) -> storage::PrefixIterator where KArg1: ?Sized + EncodeLike { let prefix = Self::storage_double_map_final_key1(k1); @@ -223,14 +223,24 @@ impl storage::StorageDoubleMap for G where KArg1: EncodeLike, KArg2: EncodeLike, F: FnOnce(&mut Self::Query) -> R, + { + Self::try_mutate(k1, k2, |v| Ok::(f(v))).expect("`Never` can not be constructed; qed") + } + + fn try_mutate(k1: KArg1, k2: KArg2, f: F) -> Result where + KArg1: EncodeLike, + KArg2: EncodeLike, + F: FnOnce(&mut Self::Query) -> Result, { let final_key = Self::storage_double_map_final_key(k1, k2); let mut val = G::from_optional_value_to_query(unhashed::get(final_key.as_ref())); let ret = f(&mut val); - match G::from_query_to_optional_value(val) { - Some(ref val) => unhashed::put(final_key.as_ref(), val), - None => unhashed::kill(final_key.as_ref()), + if ret.is_ok() { + match G::from_query_to_optional_value(val) { + Some(ref val) => unhashed::put(final_key.as_ref(), val), + None => unhashed::kill(final_key.as_ref()), + } } ret } @@ -334,41 +344,47 @@ impl storage::StorageDoubleMap for G where } } -/// Utility to iterate through items in a storage map. -pub struct MapIterator { +/// Iterate over a prefix and decode raw_key and raw_value into `T`. +pub struct MapIterator { prefix: Vec, previous_key: Vec, + /// If true then value are removed while iterating drain: bool, - _phantom: ::sp_std::marker::PhantomData<(K, V, Hasher)>, + /// Function that take `(raw_key_without_prefix, raw_value)` and decode `T`. + /// `raw_key_without_prefix` is the raw storage key without the prefix iterated on. + closure: fn(&[u8], &[u8]) -> Result, } -impl< - K: Decode + Sized, - V: Decode + Sized, - Hasher: ReversibleStorageHasher -> Iterator for MapIterator { - type Item = (K, V); +impl Iterator for MapIterator { + type Item = T; - fn next(&mut self) -> Option<(K, V)> { + fn next(&mut self) -> Option { loop { let maybe_next = sp_io::storage::next_key(&self.previous_key) .filter(|n| n.starts_with(&self.prefix)); break match maybe_next { Some(next) => { self.previous_key = next; - match unhashed::get::(&self.previous_key) { - Some(value) => { - if self.drain { - unhashed::kill(&self.previous_key) - } - let mut key_material = Hasher::reverse(&self.previous_key[self.prefix.len()..]); - match K::decode(&mut key_material) { - Ok(key) => Some((key, value)), - Err(_) => continue, - } + let raw_value = match unhashed::get_raw(&self.previous_key) { + Some(raw_value) => raw_value, + None => { + frame_support::print("ERROR: next_key returned a key with no value in MapIterator"); + continue } - None => continue, + }; + if self.drain { + unhashed::kill(&self.previous_key) } + let raw_key_without_prefix = &self.previous_key[self.prefix.len()..]; + let item = match (self.closure)(raw_key_without_prefix, &raw_value[..]) { + Ok(item) => item, + Err(_e) => { + frame_support::print("ERROR: (key, value) failed to decode in MapIterator"); + continue + } + }; + + Some(item) } None => None, } @@ -385,30 +401,50 @@ impl< G::Hasher1: ReversibleStorageHasher, G::Hasher2: ReversibleStorageHasher { - type Iterator = MapIterator; + type PrefixIterator = MapIterator<(K2, V)>; + type Iterator = MapIterator<(K1, K2, V)>; - /// Enumerate all elements in the map. - fn iter(k1: impl EncodeLike) -> Self::Iterator { + fn iter_prefix(k1: impl EncodeLike) -> Self::PrefixIterator { let prefix = G::storage_double_map_final_key1(k1); - Self::Iterator { + Self::PrefixIterator { prefix: prefix.clone(), previous_key: prefix, drain: false, - _phantom: Default::default(), + closure: |raw_key_without_prefix, mut raw_value| { + let mut key_material = G::Hasher2::reverse(raw_key_without_prefix); + Ok((K2::decode(&mut key_material)?, V::decode(&mut raw_value)?)) + }, } } - /// Enumerate all elements in the map. - fn drain(k1: impl EncodeLike) -> Self::Iterator { - let prefix = G::storage_double_map_final_key1(k1); + fn drain_prefix(k1: impl EncodeLike) -> Self::PrefixIterator { + let mut iterator = Self::iter_prefix(k1); + iterator.drain = true; + iterator + } + + fn iter() -> Self::Iterator { + let prefix = G::prefix_hash(); Self::Iterator { prefix: prefix.clone(), previous_key: prefix, - drain: true, - _phantom: Default::default(), + drain: false, + closure: |raw_key_without_prefix, mut raw_value| { + let mut k1_k2_material = G::Hasher1::reverse(raw_key_without_prefix); + let k1 = K1::decode(&mut k1_k2_material)?; + let mut k2_material = G::Hasher2::reverse(k1_k2_material); + let k2 = K2::decode(&mut k2_material)?; + Ok((k1, k2, V::decode(&mut raw_value)?)) + }, } } + fn drain() -> Self::Iterator { + let mut iterator = Self::iter(); + iterator.drain = true; + iterator + } + fn translate Option>(f: F) { let prefix = G::prefix_hash(); let mut previous_key = prefix.clone(); @@ -431,33 +467,111 @@ impl< } } +/// Test iterators for StorageDoubleMap #[cfg(test)] -mod test { - use sp_io::TestExternalities; - use crate::storage::{self, StorageDoubleMap}; - use crate::hash::Twox128; +#[allow(dead_code)] +mod test_iterators { + use codec::{Encode, Decode}; + use crate::storage::{generator::StorageDoubleMap, IterableStorageDoubleMap, unhashed}; + + pub trait Trait { + type Origin; + type BlockNumber; + } + + crate::decl_module! { + pub struct Module for enum Call where origin: T::Origin {} + } + + #[derive(PartialEq, Eq, Clone, Encode, Decode)] + struct NoDef(u32); + + crate::decl_storage! { + trait Store for Module as Test { + DoubleMap: double_map hasher(blake2_128_concat) u16, hasher(blake2_128_concat) u32 => u64; + } + } + + fn key_before_prefix(mut prefix: Vec) -> Vec { + let last = prefix.iter_mut().last().unwrap(); + assert!(*last != 0, "mock function not implemented for this prefix"); + *last -= 1; + prefix + } + + fn key_after_prefix(mut prefix: Vec) -> Vec { + let last = prefix.iter_mut().last().unwrap(); + assert!(*last != 255, "mock function not implemented for this prefix"); + *last += 1; + prefix + } + + fn key_in_prefix(mut prefix: Vec) -> Vec { + prefix.push(0); + prefix + } #[test] - fn iter_prefix_works() { - TestExternalities::default().execute_with(|| { - struct MyStorage; - impl storage::generator::StorageDoubleMap for MyStorage { - type Query = Option; - fn module_prefix() -> &'static [u8] { b"MyModule" } - fn storage_prefix() -> &'static [u8] { b"MyStorage" } - type Hasher1 = Twox128; - type Hasher2 = Twox128; - fn from_optional_value_to_query(v: Option) -> Self::Query { v } - fn from_query_to_optional_value(v: Self::Query) -> Option { v } + fn double_map_reversible_reversible_iteration() { + sp_io::TestExternalities::default().execute_with(|| { + // All map iterator + let prefix = DoubleMap::prefix_hash(); + + unhashed::put(&key_before_prefix(prefix.clone()), &1u64); + unhashed::put(&key_after_prefix(prefix.clone()), &1u64); + + for i in 0..4 { + DoubleMap::insert(i as u16, i as u32, i as u64); } - MyStorage::insert(1, 3, 7); - MyStorage::insert(1, 4, 8); - MyStorage::insert(2, 5, 9); - MyStorage::insert(2, 6, 10); + assert_eq!( + DoubleMap::iter().collect::>(), + vec![(3, 3, 3), (0, 0, 0), (2, 2, 2), (1, 1, 1)], + ); + + assert_eq!( + DoubleMap::iter_values().collect::>(), + vec![3, 0, 2, 1], + ); + + assert_eq!( + DoubleMap::drain().collect::>(), + vec![(3, 3, 3), (0, 0, 0), (2, 2, 2), (1, 1, 1)], + ); - assert_eq!(MyStorage::iter_prefix(1).collect::>(), vec![7, 8]); - assert_eq!(MyStorage::iter_prefix(2).collect::>(), vec![10, 9]); - }); + assert_eq!(DoubleMap::iter().collect::>(), vec![]); + assert_eq!(unhashed::get(&key_before_prefix(prefix.clone())), Some(1u64)); + assert_eq!(unhashed::get(&key_after_prefix(prefix.clone())), Some(1u64)); + + // Prefix iterator + let k1 = 3 << 8; + let prefix = DoubleMap::storage_double_map_final_key1(k1); + + unhashed::put(&key_before_prefix(prefix.clone()), &1u64); + unhashed::put(&key_after_prefix(prefix.clone()), &1u64); + + for i in 0..4 { + DoubleMap::insert(k1, i as u32, i as u64); + } + + assert_eq!( + DoubleMap::iter_prefix(k1).collect::>(), + vec![(0, 0), (2, 2), (1, 1), (3, 3)], + ); + + assert_eq!( + DoubleMap::iter_prefix_values(k1).collect::>(), + vec![0, 2, 1, 3], + ); + + assert_eq!( + DoubleMap::drain_prefix(k1).collect::>(), + vec![(0, 0), (2, 2), (1, 1), (3, 3)], + ); + + assert_eq!(DoubleMap::iter_prefix(k1).collect::>(), vec![]); + assert_eq!(unhashed::get(&key_before_prefix(prefix.clone())), Some(1u64)); + assert_eq!(unhashed::get(&key_after_prefix(prefix.clone())), Some(1u64)); + }) } } diff --git a/frame/support/src/storage/generator/map.rs b/frame/support/src/storage/generator/map.rs index c29a9a223aacf578d162864ef9a119aa015e78bc..cc871072f5f40f550464c33400978b53358970a8 100644 --- a/frame/support/src/storage/generator/map.rs +++ b/frame/support/src/storage/generator/map.rs @@ -18,7 +18,7 @@ use sp_std::prelude::*; use sp_std::borrow::Borrow; use codec::{FullCodec, FullEncode, Decode, Encode, EncodeLike, Ref, EncodeAppend}; -use crate::{storage::{self, unhashed}, traits::Len}; +use crate::{storage::{self, unhashed}, traits::Len, Never}; use crate::hash::{StorageHasher, Twox128, ReversibleStorageHasher}; /// Generator for `StorageMap` used by `decl_storage`. @@ -229,27 +229,11 @@ impl> storage::StorageMap } fn mutate, R, F: FnOnce(&mut Self::Query) -> R>(key: KeyArg, f: F) -> R { - let final_key = Self::storage_map_final_key(key); - let mut val = G::from_optional_value_to_query(unhashed::get(final_key.as_ref())); - - let ret = f(&mut val); - match G::from_query_to_optional_value(val) { - Some(ref val) => unhashed::put(final_key.as_ref(), &val), - None => unhashed::kill(final_key.as_ref()), - } - ret + Self::try_mutate(key, |v| Ok::(f(v))).expect("`Never` can not be constructed; qed") } fn mutate_exists, R, F: FnOnce(&mut Option) -> R>(key: KeyArg, f: F) -> R { - let final_key = Self::storage_map_final_key(key); - let mut val = unhashed::get(final_key.as_ref()); - - let ret = f(&mut val); - match val { - Some(ref val) => unhashed::put(final_key.as_ref(), &val), - None => unhashed::kill(final_key.as_ref()), - } - ret + Self::try_mutate_exists(key, |v| Ok::(f(v))).expect("`Never` can not be constructed; qed") } fn try_mutate, R, E, F: FnOnce(&mut Self::Query) -> Result>( diff --git a/frame/support/src/storage/generator/mod.rs b/frame/support/src/storage/generator/mod.rs index 687d8a3c9361ba1ca217ee22947a349ae9095955..07e75055f356a87435fc0ec700e5afdb0795c877 100644 --- a/frame/support/src/storage/generator/mod.rs +++ b/frame/support/src/storage/generator/mod.rs @@ -37,6 +37,7 @@ mod tests { use sp_io::TestExternalities; use codec::Encode; use crate::storage::{unhashed, generator::StorageValue, IterableStorageMap}; + use crate::{assert_noop, assert_ok}; struct Runtime {} pub trait Trait { @@ -57,6 +58,7 @@ mod tests { trait Store for Module as Runtime { Value get(fn value) config(): (u64, u64); NumberMap: map hasher(identity) u32 => u64; + DoubleMap: double_map hasher(identity) u32, hasher(identity) u32 => u64; } } @@ -102,4 +104,54 @@ mod tests { ); }) } + + #[test] + fn try_mutate_works() { + let t = GenesisConfig::default().build_storage().unwrap(); + TestExternalities::new(t).execute_with(|| { + assert_eq!(Value::get(), (0, 0)); + assert_eq!(NumberMap::get(0), 0); + assert_eq!(DoubleMap::get(0, 0), 0); + + // `assert_noop` ensures that the state does not change + assert_noop!(Value::try_mutate(|value| -> Result<(), &'static str> { + *value = (2, 2); + Err("don't change value") + }), "don't change value"); + + assert_noop!(NumberMap::try_mutate(0, |value| -> Result<(), &'static str> { + *value = 4; + Err("don't change value") + }), "don't change value"); + + assert_noop!(DoubleMap::try_mutate(0, 0, |value| -> Result<(), &'static str> { + *value = 6; + Err("don't change value") + }), "don't change value"); + + // Showing this explicitly for clarity + assert_eq!(Value::get(), (0, 0)); + assert_eq!(NumberMap::get(0), 0); + assert_eq!(DoubleMap::get(0, 0), 0); + + assert_ok!(Value::try_mutate(|value| -> Result<(), &'static str> { + *value = (2, 2); + Ok(()) + })); + + assert_ok!(NumberMap::try_mutate(0, |value| -> Result<(), &'static str> { + *value = 4; + Ok(()) + })); + + assert_ok!(DoubleMap::try_mutate(0, 0, |value| -> Result<(), &'static str> { + *value = 6; + Ok(()) + })); + + assert_eq!(Value::get(), (2, 2)); + assert_eq!(NumberMap::get(0), 4); + assert_eq!(DoubleMap::get(0, 0), 6); + }); + } } diff --git a/frame/support/src/storage/generator/value.rs b/frame/support/src/storage/generator/value.rs index 9e26131f48949637be4a8779169e97bddc9396a0..dd9bbded4b8ebb10f8553a24db37099935a15290 100644 --- a/frame/support/src/storage/generator/value.rs +++ b/frame/support/src/storage/generator/value.rs @@ -17,7 +17,12 @@ #[cfg(not(feature = "std"))] use sp_std::prelude::*; use codec::{FullCodec, Encode, EncodeAppend, EncodeLike, Decode}; -use crate::{storage::{self, unhashed}, hash::{Twox128, StorageHasher}, traits::Len}; +use crate::{ + Never, + storage::{self, unhashed}, + hash::{Twox128, StorageHasher}, + traits::Len +}; /// Generator for `StorageValue` used by `decl_storage`. /// @@ -104,12 +109,18 @@ impl> storage::StorageValue for G { } fn mutate R>(f: F) -> R { + Self::try_mutate(|v| Ok::(f(v))).expect("`Never` can not be constructed; qed") + } + + fn try_mutate Result>(f: F) -> Result { let mut val = G::get(); let ret = f(&mut val); - match G::from_query_to_optional_value(val) { - Some(ref val) => G::put(val), - None => G::kill(), + if ret.is_ok() { + match G::from_query_to_optional_value(val) { + Some(ref val) => G::put(val), + None => G::kill(), + } } ret } diff --git a/frame/support/src/storage/migration.rs b/frame/support/src/storage/migration.rs index 8e6beefa8886e702fbd0536bfaa75d86d5f505b9..264c3c644e144ee6d042e6323ee915a53f61acb9 100644 --- a/frame/support/src/storage/migration.rs +++ b/frame/support/src/storage/migration.rs @@ -159,7 +159,7 @@ pub fn get_storage_value(module: &[u8], item: &[u8], hash: &[ frame_support::storage::unhashed::get::(&key) } -/// Get a particular value in storage by the `module`, the map's `item` name and the key `hash`. +/// Take a particular value in storage by the `module`, the map's `item` name and the key `hash`. pub fn take_storage_value(module: &[u8], item: &[u8], hash: &[u8]) -> Option { let mut key = vec![0u8; 32 + hash.len()]; key[0..16].copy_from_slice(&Twox128::hash(module)); diff --git a/frame/support/src/storage/mod.rs b/frame/support/src/storage/mod.rs index efec36b540abcc4ec8ab695e233919eeaf2de5b2..3a811c20e364433f9ea6bd542b95a9acddef2b01 100644 --- a/frame/support/src/storage/mod.rs +++ b/frame/support/src/storage/mod.rs @@ -80,6 +80,9 @@ pub trait StorageValue { /// Mutate the value fn mutate R>(f: F) -> R; + /// Mutate the value if closure returns `Ok` + fn try_mutate Result>(f: F) -> Result; + /// Clear the storage value. fn kill(); @@ -240,18 +243,29 @@ pub trait IterableStorageDoubleMap< K2: FullCodec, V: FullCodec >: StorageDoubleMap { - /// The type that iterates over all `(key, value)`. - type Iterator: Iterator; + /// The type that iterates over all `(key2, value)`. + type PrefixIterator: Iterator; + + /// The type that iterates over all `(key1, key2, value)`. + type Iterator: Iterator; /// Enumerate all elements in the map with first key `k1` in no particular order. If you add or /// remove values whose first key is `k1` to the map while doing this, you'll get undefined /// results. - fn iter(k1: impl EncodeLike) -> Self::Iterator; + fn iter_prefix(k1: impl EncodeLike) -> Self::PrefixIterator; /// Remove all elements from the map with first key `k1` and iterate through them in no /// particular order. If you add elements with first key `k1` to the map while doing this, /// you'll get undefined results. - fn drain(k1: impl EncodeLike) -> Self::Iterator; + fn drain_prefix(k1: impl EncodeLike) -> Self::PrefixIterator; + + /// Enumerate all elements in the map in no particular order. If you add or remove values to + /// the map while doing this, you'll get undefined results. + fn iter() -> Self::Iterator; + + /// Remove all elements from the map and iterate through them in no particular order. If you + /// add elements to the map while doing this, you'll get undefined results. + fn drain() -> Self::Iterator; /// Translate the values of all elements by a function `f`, in the map in no particular order. /// By returning `None` from `f` for an element, you'll remove it from the map. @@ -269,21 +283,25 @@ pub trait StorageDoubleMap { /// The type that get/take returns. type Query; + /// Get the storage key used to fetch a value corresponding to a specific key. fn hashed_key_for(k1: KArg1, k2: KArg2) -> Vec where KArg1: EncodeLike, KArg2: EncodeLike; + /// Does the value (explicitly) exist in storage? fn contains_key(k1: KArg1, k2: KArg2) -> bool where KArg1: EncodeLike, KArg2: EncodeLike; + /// Load the value associated with the given key from the double map. fn get(k1: KArg1, k2: KArg2) -> Self::Query where KArg1: EncodeLike, KArg2: EncodeLike; + /// Take a value from storage, removing it afterwards. fn take(k1: KArg1, k2: KArg2) -> Self::Query where KArg1: EncodeLike, @@ -297,28 +315,43 @@ pub trait StorageDoubleMap { YKArg1: EncodeLike, YKArg2: EncodeLike; + /// Store a value to be associated with the given keys from the double map. fn insert(k1: KArg1, k2: KArg2, val: VArg) where KArg1: EncodeLike, KArg2: EncodeLike, VArg: EncodeLike; + /// Remove the value under the given keys. fn remove(k1: KArg1, k2: KArg2) where KArg1: EncodeLike, KArg2: EncodeLike; + /// Remove all values under the first key. fn remove_prefix(k1: KArg1) where KArg1: ?Sized + EncodeLike; - fn iter_prefix(k1: KArg1) -> PrefixIterator + /// Iterate over values that share the first key. + fn iter_prefix_values(k1: KArg1) -> PrefixIterator where KArg1: ?Sized + EncodeLike; + /// Mutate the value under the given keys. fn mutate(k1: KArg1, k2: KArg2, f: F) -> R where KArg1: EncodeLike, KArg2: EncodeLike, F: FnOnce(&mut Self::Query) -> R; + /// Mutate the value under the given keys when the closure returns `Ok`. + fn try_mutate(k1: KArg1, k2: KArg2, f: F) -> Result + where + KArg1: EncodeLike, + KArg2: EncodeLike, + F: FnOnce(&mut Self::Query) -> Result; + + /// Append the given item to the value in the storage. + /// + /// `V` is required to implement `codec::EncodeAppend`. fn append( k1: KArg1, k2: KArg2, @@ -333,6 +366,10 @@ pub trait StorageDoubleMap { Items: IntoIterator, Items::IntoIter: ExactSizeIterator; + /// Safely append the given items to the value in the storage. If a codec error occurs, then the + /// old (presumably corrupt) value is replaced with the given `items`. + /// + /// `V` is required to implement `codec::EncodeAppend`. fn append_or_insert( k1: KArg1, k2: KArg2, diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs index 7e2040ee234a75fbcd2b488b7ebb40c64bfbefd7..3d5a3403a9724aff6a0167c6da6131b78154352a 100644 --- a/frame/support/src/traits.rs +++ b/frame/support/src/traits.rs @@ -190,9 +190,7 @@ impl Get for () { fn get() -> T { T::default() } } -/// A trait for querying whether a type can be said to statically "contain" a value. Similar -/// in nature to `Get`, except it is designed to be lazy rather than active (you can't ask it to -/// enumerate all values that it contains) and work for multiple values rather than just one. +/// A trait for querying whether a type can be said to "contain" a value. pub trait Contains { /// Return `true` if this "contains" the given value `t`. fn contains(t: &T) -> bool { Self::sorted_members().binary_search(t).is_ok() } @@ -208,7 +206,15 @@ pub trait Contains { /// /// **Should be used for benchmarking only!!!** #[cfg(feature = "runtime-benchmarks")] - fn add(t: &T); + fn add(_t: &T) { unimplemented!() } +} + +/// A trait for querying bound for the length of an implementation of `Contains` +pub trait ContainsLengthBound { + /// Minimum number of elements contained + fn min_len() -> usize; + /// Maximum number of elements contained + fn max_len() -> usize; } /// Determiner to say whether a given account is unused. diff --git a/frame/support/src/weights.rs b/frame/support/src/weights.rs index ea3368550f301b96f3dd82bc15daf2d000a5b60f..9a8faf0c9145fc1c9e4eb5108a42b21056650414 100644 --- a/frame/support/src/weights.rs +++ b/frame/support/src/weights.rs @@ -16,29 +16,118 @@ //! # Primitives for transaction weighting. //! -//! All dispatchable functions defined in `decl_module!` must provide two trait implementations: -//! - [`WeightData`]: To determine the weight of the dispatch. -//! - [`ClassifyDispatch`]: To determine the class of the dispatch. See the enum definition for -//! more information on dispatch classes. +//! Every dispatchable function is responsible for providing `#[weight = $x]` attribute. In this +//! snipped, `$x` can be any user provided struct that implements the following traits: //! -//! Every dispatchable function is responsible for providing this data via an optional `#[weight = -//! $x]` attribute. In this snipped, `$x` can be any user provided struct that implements the -//! two aforementioned traits. +//! - [`WeighData`]: the weight amount. +//! - [`ClassifyDispatch`]: class of the dispatch. +//! - [`PaysFee`]: weather this weight should be translated to fee and deducted upon dispatch. //! //! Substrate then bundles then output information of the two traits into [`DispatchInfo`] struct -//! and provides it by implementing the [`GetDispatchInfo`] for all `Call` variants, and opaque -//! extrinsic types. +//! and provides it by implementing the [`GetDispatchInfo`] for all `Call` both inner and outer call +//! types. //! -//! If no `#[weight]` is defined, the macro automatically injects the `Default` implementation of -//! the [`SimpleDispatchInfo`]. +//! Substrate provides two pre-defined ways to annotate weight: //! -//! Note that the decl_module macro _cannot_ enforce this and will simply fail if an invalid struct -//! (something that does not implement `Weighable`) is passed in. +//! ### 1. Fixed values +//! +//! This can only be used when all 3 traits can be resolved statically. You have 3 degrees of +//! configuration: +//! +//! 1. Define only weight, **in which case `ClassifyDispatch` will be `Normal` and `PaysFee` will be +//! `Yes`**. +//! +//! ``` +//! # use frame_system::{self as system, Trait}; +//! frame_support::decl_module! { +//! pub struct Module for enum Call where origin: T::Origin { +//! #[weight = 1000] +//! fn dispatching(origin) { unimplemented!() } +//! } +//! } +//! # fn main() {} +//! ``` +//! +//! 2.1 Define weight and class, **in which case `PaysFee` would be `Yes`**. +//! +//! ``` +//! # use frame_system::{self as system, Trait}; +//! # use frame_support::weights::DispatchClass; +//! frame_support::decl_module! { +//! pub struct Module for enum Call where origin: T::Origin { +//! #[weight = (1000, DispatchClass::Operational)] +//! fn dispatching(origin) { unimplemented!() } +//! } +//! } +//! # fn main() {} +//! ``` +//! +//! 2.2 Define weight and `PaysFee`, **in which case `ClassifyDispatch` would be `Normal`**. +//! +//! ``` +//! # use frame_system::{self as system, Trait}; +//! # use frame_support::weights::Pays; +//! frame_support::decl_module! { +//! pub struct Module for enum Call where origin: T::Origin { +//! #[weight = (1000, Pays::No)] +//! fn dispatching(origin) { unimplemented!() } +//! } +//! } +//! # fn main() {} +//! ``` +//! +//! 3. Define all 3 parameters. +//! +//! ``` +//! # use frame_system::{self as system, Trait}; +//! # use frame_support::weights::{DispatchClass, Pays}; +//! frame_support::decl_module! { +//! pub struct Module for enum Call where origin: T::Origin { +//! #[weight = (1000, DispatchClass::Operational, Pays::No)] +//! fn dispatching(origin) { unimplemented!() } +//! } +//! } +//! # fn main() {} +//! ``` +//! +//! ### 2. Define weights as a function of input arguments using `FunctionOf` tuple struct. This struct works +//! in a similar manner as above. 3 items must be provided and each can be either a fixed value or a +//! function/closure with the same parameters list as the dispatchable function itself, wrapper in a +//! tuple. +//! +//! Using this only makes sense if you want to use a function for at least one of the elements. If +//! all 3 are static values, providing a raw tuple is easier. +//! +//! ``` +//! # use frame_system::{self as system, Trait}; +//! # use frame_support::weights::{DispatchClass, FunctionOf, Pays}; +//! frame_support::decl_module! { +//! pub struct Module for enum Call where origin: T::Origin { +//! #[weight = FunctionOf( +//! // weight, function. +//! |args: (&u32, &u64)| *args.0 as u64 + args.1, +//! // class, fixed. +//! DispatchClass::Operational, +//! // pays fee, function. +//! |args: (&u32, &u64)| if *args.0 > 1000 { Pays::Yes } else { Pays::No }, +//! )] +//! fn dispatching(origin, a: u32, b: u64) { unimplemented!() } +//! } +//! } +//! # fn main() {} +//! ``` +//! FRAME assumes a weight of `1_000_000_000_000` equals 1 second of compute on a standard machine. +//! +//! Latest machine specification used to benchmark are: +//! - Digital Ocean: ubuntu-s-2vcpu-4gb-ams3-01 +//! - 2x Intel(R) Xeon(R) CPU E5-2650 v4 @ 2.20GHz +//! - 4GB RAM +//! - Ubuntu 19.10 (GNU/Linux 5.3.0-18-generic x86_64) +//! - rustc 1.42.0 (b8cedc004 2020-03-09) #[cfg(feature = "std")] use serde::{Serialize, Deserialize}; use codec::{Encode, Decode}; -use sp_arithmetic::traits::Bounded; use sp_runtime::{ RuntimeDebug, traits::SignedExtension, @@ -50,7 +139,7 @@ use crate::dispatch::{DispatchErrorWithPostInfo, DispatchError}; pub use sp_runtime::transaction_validity::TransactionPriority; /// Numeric range of a transaction weight. -pub type Weight = u32; +pub type Weight = u64; /// Means of weighing some particular kind of data (`T`). pub trait WeighData { @@ -68,15 +157,27 @@ pub trait ClassifyDispatch { } /// Indicates if dispatch function should pay fees or not. -/// If set to false, the block resource limits are applied, yet no fee is deducted. +/// If set to `Pays::No`, the block resource limits are applied, yet no fee is deducted. pub trait PaysFee { - fn pays_fee(&self, _target: T) -> bool { - true + fn pays_fee(&self, _target: T) -> Pays; +} + +/// Explicit enum to denote if a transaction pays fee or not. +#[derive(Clone, Copy, Eq, PartialEq, RuntimeDebug, Encode, Decode)] +pub enum Pays { + /// Transactor will pay related fees. + Yes, + /// Transactor will NOT pay related fees. + No, +} + +impl Default for Pays { + fn default() -> Self { + Self::Yes } } -/// A generalized group of dispatch types. This is only distinguishing normal, user-triggered transactions -/// (`Normal`) and anything beyond which serves a higher purpose to the system (`Operational`). +/// A generalized group of dispatch types. #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] #[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] #[derive(PartialEq, Eq, Clone, Copy, Encode, Decode, RuntimeDebug)] @@ -102,22 +203,7 @@ pub enum DispatchClass { impl Default for DispatchClass { fn default() -> Self { - DispatchClass::Normal - } -} - -impl From for DispatchClass { - fn from(tx: SimpleDispatchInfo) -> Self { - match tx { - SimpleDispatchInfo::FixedOperational(_) => DispatchClass::Operational, - SimpleDispatchInfo::MaxOperational => DispatchClass::Operational, - - SimpleDispatchInfo::FixedNormal(_) => DispatchClass::Normal, - SimpleDispatchInfo::MaxNormal => DispatchClass::Normal, - SimpleDispatchInfo::InsecureFreeNormal => DispatchClass::Normal, - - SimpleDispatchInfo::FixedMandatory(_) => DispatchClass::Mandatory, - } + Self::Normal } } @@ -129,7 +215,16 @@ pub struct DispatchInfo { /// Class of this transaction. pub class: DispatchClass, /// Does this transaction pay fees. - pub pays_fee: bool, + pub pays_fee: Pays, +} + +/// A `Dispatchable` function (aka transaction) that can carry some static information along with +/// it, using the `#[weight]` attribute. +pub trait GetDispatchInfo { + /// Return a `DispatchInfo`, containing relevant information of this dispatch. + /// + /// This is done independently of its encoded size. + fn get_dispatch_info(&self) -> DispatchInfo; } /// Weight information that is only available post dispatch. @@ -139,6 +234,21 @@ pub struct PostDispatchInfo { pub actual_weight: Option, } +impl PostDispatchInfo { + /// Calculate how much (if any) weight was not used by the `Dispatchable`. + pub fn calc_unspent(&self, info: &DispatchInfo) -> Weight { + if let Some(actual_weight) = self.actual_weight { + if actual_weight >= info.weight { + 0 + } else { + info.weight - actual_weight + } + } else { + 0 + } + } +} + impl From> for PostDispatchInfo { fn from(actual_weight: Option) -> Self { Self { @@ -166,7 +276,7 @@ impl sp_runtime::traits::Printable for PostDispatchInfo { } /// Allows easy conversion from `DispatchError` to `DispatchErrorWithPostInfo` for dispatchables -/// that want to return a custom a posteriori weight on error. +/// that want to return a custom a posterior weight on error. pub trait WithPostDispatchInfo { /// Call this on your modules custom errors type in order to return a custom weight on error. /// @@ -190,93 +300,75 @@ impl WithPostDispatchInfo for T where } } -/// A `Dispatchable` function (aka transaction) that can carry some static information along with -/// it, using the `#[weight]` attribute. -pub trait GetDispatchInfo { - /// Return a `DispatchInfo`, containing relevant information of this dispatch. - /// - /// This is done independently of its encoded size. - fn get_dispatch_info(&self) -> DispatchInfo; +impl WeighData for Weight { + fn weigh_data(&self, _: T) -> Weight { + return *self + } } -/// Default type used with the `#[weight = x]` attribute in a substrate chain. -/// -/// A user may pass in any other type that implements the correct traits. If not, the `Default` -/// implementation of [`SimpleDispatchInfo`] is used. -/// -/// For each generalized group (`Normal` and `Operation`): -/// - A `Fixed` variant means weight fee is charged normally and the weight is the number -/// specified in the inner value of the variant. -/// - A `Free` variant is equal to `::Fixed(0)`. Note that this does not guarantee inclusion. -/// - A `Max` variant is equal to `::Fixed(Weight::max_value())`. -/// -/// As for the generalized groups themselves: -/// - `Normal` variants will be assigned a priority proportional to their weight. They can only -/// consume a portion (defined in the system module) of the maximum block resource limits. -/// - `Operational` variants will be assigned the maximum priority. They can potentially consume -/// the entire block resource limit. -#[derive(Clone, Copy)] -pub enum SimpleDispatchInfo { - /// A normal dispatch with fixed weight. - FixedNormal(Weight), - /// A normal dispatch with the maximum weight. - MaxNormal, - /// A normal dispatch with no weight. Base and bytes fees still need to be paid. - InsecureFreeNormal, - /// An operational dispatch with fixed weight. - FixedOperational(Weight), - /// An operational dispatch with the maximum weight. - MaxOperational, - /// A mandatory dispatch with fixed weight. - /// - /// NOTE: Signed transactions may not (directly) dispatch this kind of a call, so the other - /// attributes concerning transactability (e.g. priority, fee paying) are moot. - FixedMandatory(Weight), +impl ClassifyDispatch for Weight { + fn classify_dispatch(&self, _: T) -> DispatchClass { + DispatchClass::Normal + } +} + +impl PaysFee for Weight { + fn pays_fee(&self, _: T) -> Pays { + Pays::Yes + } } -impl WeighData for SimpleDispatchInfo { +impl WeighData for (Weight, DispatchClass, Pays) { fn weigh_data(&self, _: T) -> Weight { - match self { - SimpleDispatchInfo::FixedNormal(w) => *w, - SimpleDispatchInfo::MaxNormal => Bounded::max_value(), - SimpleDispatchInfo::InsecureFreeNormal => Bounded::min_value(), - SimpleDispatchInfo::FixedOperational(w) => *w, - SimpleDispatchInfo::MaxOperational => Bounded::max_value(), - SimpleDispatchInfo::FixedMandatory(w) => *w, - } + return self.0 } } -impl ClassifyDispatch for SimpleDispatchInfo { +impl ClassifyDispatch for (Weight, DispatchClass, Pays) { fn classify_dispatch(&self, _: T) -> DispatchClass { - DispatchClass::from(*self) + self.1 } } -impl PaysFee for SimpleDispatchInfo { - fn pays_fee(&self, _: T) -> bool { - match self { - SimpleDispatchInfo::FixedNormal(_) => true, - SimpleDispatchInfo::MaxNormal => true, - SimpleDispatchInfo::InsecureFreeNormal => true, - SimpleDispatchInfo::FixedOperational(_) => true, - SimpleDispatchInfo::MaxOperational => true, - SimpleDispatchInfo::FixedMandatory(_) => true, - } +impl PaysFee for (Weight, DispatchClass, Pays) { + fn pays_fee(&self, _: T) -> Pays { + self.2 } } -impl Default for SimpleDispatchInfo { - fn default() -> Self { - // Default weight of all transactions. - SimpleDispatchInfo::FixedNormal(10_000) +impl WeighData for (Weight, DispatchClass) { + fn weigh_data(&self, _: T) -> Weight { + return self.0 + } +} + +impl ClassifyDispatch for (Weight, DispatchClass) { + fn classify_dispatch(&self, _: T) -> DispatchClass { + self.1 + } +} + +impl PaysFee for (Weight, DispatchClass) { + fn pays_fee(&self, _: T) -> Pays { + Pays::Yes + } +} + +impl WeighData for (Weight, Pays) { + fn weigh_data(&self, _: T) -> Weight { + return self.0 } } -impl SimpleDispatchInfo { - /// An _additive zero_ variant of SimpleDispatchInfo. - pub fn zero() -> Self { - Self::FixedNormal(0) +impl ClassifyDispatch for (Weight, Pays) { + fn classify_dispatch(&self, _: T) -> DispatchClass { + DispatchClass::Normal + } +} + +impl PaysFee for (Weight, Pays) { + fn pays_fee(&self, _: T) -> Pays { + self.1 } } @@ -287,8 +379,8 @@ impl SimpleDispatchInfo { /// argument list as the dispatched, wrapped in a tuple. /// - `CD`: a raw `DispatchClass` value or a closure that returns a `DispatchClass` /// with the same argument list as the dispatched, wrapped in a tuple. -/// - `PF`: a `bool` for whether this dispatch pays fee or not or a closure that -/// returns a bool with the same argument list as the dispatched, wrapped in a tuple. +/// - `PF`: a `Pays` variant for whether this dispatch pays fee or not or a closure that +/// returns a `Pays` variant with the same argument list as the dispatched, wrapped in a tuple. pub struct FunctionOf(pub WD, pub CD, pub PF); // `WeighData` as a raw value @@ -324,17 +416,17 @@ impl ClassifyDispatch for FunctionOf where } // `PaysFee` as a raw value -impl PaysFee for FunctionOf { - fn pays_fee(&self, _: Args) -> bool { +impl PaysFee for FunctionOf { + fn pays_fee(&self, _: Args) -> Pays { self.2 } } // `PaysFee` as a closure impl PaysFee for FunctionOf where - PF : Fn(Args) -> bool + PF : Fn(Args) -> Pays { - fn pays_fee(&self, args: Args) -> bool { + fn pays_fee(&self, args: Args) -> Pays { (self.2)(args) } } @@ -369,51 +461,127 @@ impl GetDispatchInfo for sp_runtime::testing::TestX // for testing: weight == size. DispatchInfo { weight: self.encode().len() as _, - pays_fee: true, + pays_fee: Pays::Yes, ..Default::default() } } } +/// The weight of database operations that the runtime can invoke. +#[derive(Clone, Copy, Eq, PartialEq, Default, RuntimeDebug, Encode, Decode)] +pub struct RuntimeDbWeight { + pub read: Weight, + pub write: Weight, +} + +impl RuntimeDbWeight { + pub fn reads(self, r: Weight) -> Weight { + self.read.saturating_mul(r) + } + + pub fn writes(self, w: Weight) -> Weight { + self.write.saturating_mul(w) + } + + pub fn reads_writes(self, r: Weight, w: Weight) -> Weight { + let read_weight = self.read.saturating_mul(r); + let write_weight = self.write.saturating_mul(w); + read_weight.saturating_add(write_weight) + } +} + #[cfg(test)] #[allow(dead_code)] mod tests { - use crate::decl_module; + use crate::{decl_module, parameter_types, traits::Get}; use super::*; pub trait Trait { type Origin; type Balance; type BlockNumber; + type DbWeight: Get; } pub struct TraitImpl {} + parameter_types! { + pub const DbWeight: RuntimeDbWeight = RuntimeDbWeight { + read: 100, + write: 1000, + }; + } + impl Trait for TraitImpl { type Origin = u32; type BlockNumber = u32; type Balance = u32; + type DbWeight = DbWeight; } decl_module! { pub struct Module for enum Call where origin: T::Origin { // no arguments, fixed weight - #[weight = SimpleDispatchInfo::FixedNormal(1000)] - fn f0(_origin) { unimplemented!(); } + #[weight = 1000] + fn f00(_origin) { unimplemented!(); } + + #[weight = (1000, DispatchClass::Mandatory)] + fn f01(_origin) { unimplemented!(); } + + #[weight = (1000, Pays::No)] + fn f02(_origin) { unimplemented!(); } + + #[weight = (1000, DispatchClass::Operational, Pays::No)] + fn f03(_origin) { unimplemented!(); } // weight = a x 10 + b - #[weight = FunctionOf(|args: (&u32, &u32)| args.0 * 10 + args.1, DispatchClass::Normal, true)] + #[weight = FunctionOf(|args: (&u32, &u32)| (args.0 * 10 + args.1) as Weight, DispatchClass::Normal, Pays::Yes)] fn f11(_origin, _a: u32, _eb: u32) { unimplemented!(); } - #[weight = FunctionOf(|_: (&u32, &u32)| 0, DispatchClass::Operational, true)] + #[weight = FunctionOf(|_: (&u32, &u32)| 0, DispatchClass::Operational, Pays::Yes)] fn f12(_origin, _a: u32, _eb: u32) { unimplemented!(); } + + #[weight = T::DbWeight::get().reads(3) + T::DbWeight::get().writes(2) + 10_000] + fn f2(_origin) { unimplemented!(); } + + #[weight = T::DbWeight::get().reads_writes(6, 5) + 40_000] + fn f21(_origin) { unimplemented!(); } + } } #[test] fn weights_are_correct() { + // #[weight = 1000] + let info = Call::::f00().get_dispatch_info(); + assert_eq!(info.weight, 1000); + assert_eq!(info.class, DispatchClass::Normal); + assert_eq!(info.pays_fee, Pays::Yes); + + // #[weight = (1000, DispatchClass::Mandatory)] + let info = Call::::f01().get_dispatch_info(); + assert_eq!(info.weight, 1000); + assert_eq!(info.class, DispatchClass::Mandatory); + assert_eq!(info.pays_fee, Pays::Yes); + + // #[weight = (1000, Pays::No)] + let info = Call::::f02().get_dispatch_info(); + assert_eq!(info.weight, 1000); + assert_eq!(info.class, DispatchClass::Normal); + assert_eq!(info.pays_fee, Pays::No); + + // #[weight = (1000, DispatchClass::Operational, Pays::No)] + let info = Call::::f03().get_dispatch_info(); + assert_eq!(info.weight, 1000); + assert_eq!(info.class, DispatchClass::Operational); + assert_eq!(info.pays_fee, Pays::No); + assert_eq!(Call::::f11(10, 20).get_dispatch_info().weight, 120); assert_eq!(Call::::f11(10, 20).get_dispatch_info().class, DispatchClass::Normal); - assert_eq!(Call::::f0().get_dispatch_info().weight, 1000); + assert_eq!(Call::::f12(10, 20).get_dispatch_info().weight, 0); + assert_eq!(Call::::f12(10, 20).get_dispatch_info().class, DispatchClass::Operational); + assert_eq!(Call::::f2().get_dispatch_info().weight, 12300); + assert_eq!(Call::::f21().get_dispatch_info().weight, 45600); + assert_eq!(Call::::f2().get_dispatch_info().class, DispatchClass::Normal); } } diff --git a/frame/support/test/Cargo.toml b/frame/support/test/Cargo.toml index 773523579b0b4ce8d580e5e80aad796591947fdf..77899788d2d96e6f876e97d11a9c4f31408146a4 100644 --- a/frame/support/test/Cargo.toml +++ b/frame/support/test/Cargo.toml @@ -8,17 +8,21 @@ publish = false homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", default-features = false, features = ["derive"] } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-io ={ path = "../../../primitives/io", default-features = false , version = "2.0.0-alpha.5"} -sp-state-machine = { version = "0.8.0-alpha.5", optional = true, path = "../../../primitives/state-machine" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../" } -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/inherents" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/runtime" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../../primitives/core" } +sp-io ={ path = "../../../primitives/io", default-features = false , version = "2.0.0-dev"} +sp-state-machine = { version = "0.8.0-dev", optional = true, path = "../../../primitives/state-machine" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../" } +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/inherents" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/runtime" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../../primitives/core" } trybuild = "1.0.17" pretty_assertions = "0.6.1" +rustversion = "1.0.0" [features] default = ["std"] @@ -32,6 +36,3 @@ std = [ "sp-runtime/std", "sp-state-machine", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/support/test/tests/construct_runtime_ui.rs b/frame/support/test/tests/construct_runtime_ui.rs index acddf01f037a665fd1c03a4e1643ebf1298168ac..d094d73ba54e9a7de4dded597c5dd9f4aedfa97d 100644 --- a/frame/support/test/tests/construct_runtime_ui.rs +++ b/frame/support/test/tests/construct_runtime_ui.rs @@ -1,5 +1,22 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + use std::env; +#[rustversion::attr(not(stable), ignore)] #[test] fn ui() { // As trybuild is using `cargo check`, we don't need the real WASM binaries. diff --git a/frame/support/test/tests/decl_error.rs b/frame/support/test/tests/decl_error.rs index 4191e79f2417ca59d118e0265094fa4e9a5b8782..eb14ae7502fe3bbcd773575e0f776dea26e41448 100644 --- a/frame/support/test/tests/decl_error.rs +++ b/frame/support/test/tests/decl_error.rs @@ -19,6 +19,7 @@ use sp_runtime::{generic, traits::{BlakeTwo256, Block as _, Verify}, DispatchError}; use sp_core::{H256, sr25519}; + mod system; pub trait Currency {} @@ -32,7 +33,7 @@ mod module1 { pub struct Module, I: Instance = DefaultInstance> for enum Call where origin: ::Origin { - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn fail(_origin) -> frame_support::dispatch::DispatchResult { Err(Error::::Something.into()) } @@ -59,7 +60,7 @@ mod module2 { pub struct Module for enum Call where origin: ::Origin { - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] pub fn fail(_origin) -> frame_support::dispatch::DispatchResult { Err(Error::::Something.into()) } diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index ea5d32fea3b222b754331aff047d65782b671911..5cb7fa1972a05f86ff13b3776f080368153431d4 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -55,7 +55,7 @@ mod module1 { fn deposit_event() = default; - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn one(origin) { system::ensure_root(origin)?; Self::deposit_event(RawEvent::AnotherVariant(3)); @@ -300,7 +300,7 @@ fn new_test_ext() -> sp_io::TestExternalities { fn storage_instance_independence() { let mut storage = sp_core::storage::Storage { top: std::collections::BTreeMap::new(), - children: std::collections::HashMap::new() + children_default: std::collections::HashMap::new() }; sp_state_machine::BasicExternalities::execute_with_storage(&mut storage, || { module2::Value::::put(0); diff --git a/frame/support/test/tests/reserved_keyword.rs b/frame/support/test/tests/reserved_keyword.rs index d6cc4bba3b0b02e50d3a1c81e9c12a4c605043d0..6f2bbb47474152b451d7cfe7667a099a02c50e65 100644 --- a/frame/support/test/tests/reserved_keyword.rs +++ b/frame/support/test/tests/reserved_keyword.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +#[rustversion::attr(not(stable), ignore)] #[test] fn reserved_keyword() { // As trybuild is using `cargo check`, we don't need the real WASM binaries. diff --git a/frame/support/test/tests/reserved_keyword/on_initialize.rs b/frame/support/test/tests/reserved_keyword/on_initialize.rs index 8eacc836c48685c1516ce129780c71f0ee636ea1..0751c600cccb2f67ceead54353818a8dd6aba514 100644 --- a/frame/support/test/tests/reserved_keyword/on_initialize.rs +++ b/frame/support/test/tests/reserved_keyword/on_initialize.rs @@ -19,7 +19,7 @@ macro_rules! reserved { frame_support::decl_module! { pub struct Module for enum Call where origin: T::Origin { - #[weight = frame_support::weights::SimpleDispatchInfo::default()] + #[weight = 0] fn $reserved(_origin) -> dispatch::DispatchResult { unreachable!() } } } diff --git a/frame/system/Cargo.toml b/frame/system/Cargo.toml index 78288cff917e233306fe0bee01f5a04d96263f16..210d3664ff6a1371222bddfae4406cec4ab37747 100644 --- a/frame/system/Cargo.toml +++ b/frame/system/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-system" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,20 +8,23 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME system module" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { path = "../../primitives/io", default-features = false, version = "2.0.0-alpha.5"} -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-version = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/version" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { path = "../../primitives/io", default-features = false, version = "2.0.0-dev"} +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-version = { version = "2.0.0-dev", default-features = false, path = "../../primitives/version" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } impl-trait-for-tuples = "0.1.3" [dev-dependencies] criterion = "0.2.11" -sp-externalities = { version = "0.8.0-alpha.5", path = "../../primitives/externalities" } +sp-externalities = { version = "0.8.0-dev", path = "../../primitives/externalities" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../test-utils/runtime/client" } [features] @@ -41,6 +44,3 @@ runtime-benchmarks = ["sp-runtime/runtime-benchmarks"] [[bench]] name = "bench" harness = false - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/system/benches/bench.rs b/frame/system/benches/bench.rs index 90a4ad1d34de80c7991a7a87fb3fb480e1945cd1..6594862e5c6d56647620b14218cc4e05d9850764 100644 --- a/frame/system/benches/bench.rs +++ b/frame/system/benches/bench.rs @@ -72,6 +72,9 @@ impl system::Trait for Runtime { type Event = Event; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/frame/system/rpc/runtime-api/Cargo.toml b/frame/system/rpc/runtime-api/Cargo.toml index 9519427297425504a8fd1b9c769c5c2a6edd899e..2097e112663f51190b85210df9a29c9d3730f536 100644 --- a/frame/system/rpc/runtime-api/Cargo.toml +++ b/frame/system/rpc/runtime-api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-system-rpc-runtime-api" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,8 +8,11 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Runtime API definition required by System RPC extensions." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../../../primitives/api" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../../../../primitives/api" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } [features] @@ -18,6 +21,3 @@ std = [ "sp-api/std", "codec/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 050ab2654b51fd443552cc22b4c135fec2c08ea7..234e427d3a3a7d569ad3933b582d837ccf3c28f4 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -68,14 +68,14 @@ //! ### Example - Get extrinsic count and parent hash for the current block //! //! ``` -//! use frame_support::{decl_module, dispatch, weights::SimpleDispatchInfo}; +//! use frame_support::{decl_module, dispatch}; //! use frame_system::{self as system, ensure_signed}; //! //! pub trait Trait: system::Trait {} //! //! decl_module! { //! pub struct Module for enum Call where origin: T::Origin { -//! #[weight = SimpleDispatchInfo::default()] +//! #[weight = 0] //! pub fn system_module_example(origin) -> dispatch::DispatchResult { //! let _sender = ensure_signed(origin)?; //! let _extrinsic_count = >::extrinsic_count(); @@ -109,17 +109,22 @@ use sp_runtime::{ self, CheckEqual, AtLeast32Bit, Zero, SignedExtension, Lookup, LookupError, SimpleBitOps, Hash, Member, MaybeDisplay, BadOrigin, SaturatedConversion, MaybeSerialize, MaybeSerializeDeserialize, MaybeMallocSizeOf, StaticLookup, One, Bounded, + Dispatchable, DispatchInfoOf, PostDispatchInfoOf, }, }; use sp_core::{ChangesTrieConfiguration, storage::well_known_keys}; use frame_support::{ - decl_module, decl_event, decl_storage, decl_error, storage, Parameter, ensure, debug, + decl_module, decl_event, decl_storage, decl_error, Parameter, ensure, debug, + storage::{self, generator::StorageValue}, traits::{ Contains, Get, ModuleToIndex, OnNewAccount, OnKilledAccount, IsDeadAccount, Happened, StoredMap, EnsureOrigin, }, - weights::{Weight, DispatchInfo, DispatchClass, SimpleDispatchInfo, FunctionOf} + weights::{ + Weight, RuntimeDbWeight, DispatchInfo, PostDispatchInfo, DispatchClass, + FunctionOf, Pays, + } }; use codec::{Encode, Decode, FullCodec, EncodeLike}; @@ -146,7 +151,7 @@ pub trait Trait: 'static + Eq + Clone { + Clone; /// The aggregated `Call` type. - type Call: Debug; + type Call: Dispatchable + Debug; /// Account index (aka nonce) type. This stores the number of previous transactions associated /// with a sender account. @@ -194,6 +199,15 @@ pub trait Trait: 'static + Eq + Clone { /// The maximum weight of a block. type MaximumBlockWeight: Get; + /// The weight of runtime database operations the runtime can invoke. + type DbWeight: Get; + + /// The base weight of executing a block, independent of the transactions in the block. + type BlockExecutionWeight: Get; + + /// The base weight of an Extrinsic in the block, independent of the of extrinsic being executed. + type ExtrinsicBaseWeight: Get; + /// The maximum length of a block (in bytes). type MaximumBlockLength: Get; @@ -468,33 +482,64 @@ decl_module! { pub struct Module for enum Call where origin: T::Origin { type Error = Error; + /// The maximum weight of a block. + const MaximumBlockWeight: Weight = T::MaximumBlockWeight::get(); + + /// The weight of runtime database operations the runtime can invoke. + const DbWeight: RuntimeDbWeight = T::DbWeight::get(); + + /// The base weight of executing a block, independent of the transactions in the block. + const BlockExecutionWeight: Weight = T::BlockExecutionWeight::get(); + + /// The base weight of an Extrinsic in the block, independent of the of extrinsic being executed. + const ExtrinsicBaseWeight: Weight = T::ExtrinsicBaseWeight::get(); + + /// The maximum length of a block (in bytes). + const MaximumBlockLength: u32 = T::MaximumBlockLength::get(); + /// A dispatch that will fill the block weight up to the given ratio. // TODO: This should only be available for testing, rather than in general usage, but // that's not possible at present (since it's within the decl_module macro). #[weight = FunctionOf( |(ratio,): (&Perbill,)| *ratio * T::MaximumBlockWeight::get(), DispatchClass::Operational, - true, + Pays::Yes, )] fn fill_block(origin, _ratio: Perbill) { ensure_root(origin)?; } /// Make some on-chain remark. - #[weight = SimpleDispatchInfo::FixedNormal(10_000)] + /// + /// # + /// - `O(1)` + /// # + #[weight = 0] fn remark(origin, _remark: Vec) { ensure_signed(origin)?; } /// Set the number of pages in the WebAssembly environment's heap. - #[weight = SimpleDispatchInfo::FixedOperational(10_000)] + /// + /// # + /// - `O(1)` + /// - 1 storage write. + /// # + #[weight = (0, DispatchClass::Operational)] fn set_heap_pages(origin, pages: u64) { ensure_root(origin)?; storage::unhashed::put_raw(well_known_keys::HEAP_PAGES, &pages.encode()); } /// Set the new runtime code. - #[weight = SimpleDispatchInfo::FixedOperational(200_000)] + /// + /// # + /// - `O(C + S)` where `C` length of `code` and `S` complexity of `can_set_code` + /// - 1 storage write (codec `O(C)`). + /// - 1 call to `can_set_code`: `O(S)` (calls `sp_io::misc::runtime_version` which is expensive). + /// - 1 event. + /// # + #[weight = (200_000_000, DispatchClass::Operational)] pub fn set_code(origin, code: Vec) { Self::can_set_code(origin, &code)?; @@ -503,7 +548,13 @@ decl_module! { } /// Set the new runtime code without doing any checks of the given `code`. - #[weight = SimpleDispatchInfo::FixedOperational(200_000)] + /// + /// # + /// - `O(C)` where `C` length of `code` + /// - 1 storage write (codec `O(C)`). + /// - 1 event. + /// # + #[weight = (200_000_000, DispatchClass::Operational)] pub fn set_code_without_checks(origin, code: Vec) { ensure_root(origin)?; storage::unhashed::put_raw(well_known_keys::CODE, &code); @@ -511,7 +562,13 @@ decl_module! { } /// Set the new changes trie configuration. - #[weight = SimpleDispatchInfo::FixedOperational(20_000)] + /// + /// # + /// - `O(D)` where `D` length of `Digest` + /// - 1 storage write or delete (codec `O(1)`). + /// - 1 call to `deposit_log`: `O(D)` (which depends on the length of `Digest`) + /// # + #[weight = (20_000_000, DispatchClass::Operational)] pub fn set_changes_trie_config(origin, changes_trie_config: Option) { ensure_root(origin)?; match changes_trie_config.clone() { @@ -529,7 +586,12 @@ decl_module! { } /// Set some items of storage. - #[weight = SimpleDispatchInfo::FixedOperational(10_000)] + /// + /// # + /// - `O(I)` where `I` length of `items` + /// - `I` storage writes (`O(1)`). + /// # + #[weight = (0, DispatchClass::Operational)] fn set_storage(origin, items: Vec) { ensure_root(origin)?; for i in &items { @@ -538,7 +600,12 @@ decl_module! { } /// Kill some items from storage. - #[weight = SimpleDispatchInfo::FixedOperational(10_000)] + /// + /// # + /// - `O(VK)` where `V` length of `keys` and `K` length of one key + /// - `V` storage deletions. + /// # + #[weight = (0, DispatchClass::Operational)] fn kill_storage(origin, keys: Vec) { ensure_root(origin)?; for key in &keys { @@ -547,7 +614,12 @@ decl_module! { } /// Kill all storage items with a key that starts with the given prefix. - #[weight = SimpleDispatchInfo::FixedOperational(10_000)] + /// + /// # + /// - `O(P)` where `P` amount of keys with prefix `prefix` + /// - `P` storage deletions. + /// # + #[weight = (0, DispatchClass::Operational)] fn kill_prefix(origin, prefix: Key) { ensure_root(origin)?; storage::unhashed::kill_prefix(&prefix); @@ -555,7 +627,14 @@ decl_module! { /// Kill the sending account, assuming there are no references outstanding and the composite /// data is equal to its default value. - #[weight = SimpleDispatchInfo::FixedOperational(25_000)] + /// + /// # + /// - `O(K)` with `K` being complexity of `on_killed_account` + /// - 1 storage read and deletion. + /// - 1 call to `on_killed_account` callback with unknown complexity `K` + /// - 1 event. + /// # + #[weight = (25_000_000, DispatchClass::Operational)] fn suicide(origin) { let who = ensure_signed(origin)?; let account = Account::::get(&who); @@ -620,9 +699,12 @@ impl< #[cfg(feature = "runtime-benchmarks")] fn successful_origin() -> O { - let caller: AccountId = Default::default(); - // Who::add(&caller); - O::from(RawOrigin::Signed(caller)) + let members = Who::sorted_members(); + let first_member = match members.get(0) { + Some(account) => account.clone(), + None => Default::default(), + }; + O::from(RawOrigin::Signed(first_member.clone())) } } @@ -773,17 +855,10 @@ impl Module { old_event_count }; - // Appending can only fail if `Events` can not be decoded or - // when we try to insert more than `u32::max_value()` events. - // - // We perform early return if we've reached the maximum capacity of the event list, - // so `Events` seems to be corrupted. Also, this has happened after the start of execution - // (since the event list is cleared at the block initialization). - if >::append([event].iter()).is_err() { - // The most sensible thing to do here is to just ignore this event and wait until the - // new block. - return; - } + // We use append api here to avoid bringing all events in the runtime when we push a + // new one in the list. + let encoded_event = event.encode(); + sp_io::storage::append(&Events::::storage_value_final_key()[..], encoded_event); for topic in topics { // The same applies here. @@ -808,6 +883,23 @@ impl Module { AllExtrinsicsWeight::get().unwrap_or_default() } + /// Get the quota ratio of each dispatch class type. This indicates that all operational and mandatory + /// dispatches can use the full capacity of any resource, while user-triggered ones can consume + /// a portion. + pub fn get_dispatch_limit_ratio(class: DispatchClass) -> Perbill { + match class { + DispatchClass::Operational | DispatchClass::Mandatory + => ::one(), + DispatchClass::Normal => T::AvailableBlockRatio::get(), + } + } + + /// The maximum weight of an allowable extrinsic. Only one of these could exist in a block. + pub fn max_extrinsic_weight(class: DispatchClass) -> Weight { + let limit = Self::get_dispatch_limit_ratio(class) * T::MaximumBlockWeight::get(); + limit - (T::BlockExecutionWeight::get() + T::ExtrinsicBaseWeight::get()) + } + pub fn all_extrinsics_len() -> u32 { AllExtrinsicsLen::get().unwrap_or_default() } @@ -831,7 +923,7 @@ impl Module { /// If no previous weight exists, the function initializes the weight to zero. pub fn register_extra_weight_unchecked(weight: Weight) { let current_weight = AllExtrinsicsWeight::get().unwrap_or_default(); - let next_weight = current_weight.saturating_add(weight).min(T::MaximumBlockWeight::get()); + let next_weight = current_weight.saturating_add(weight); AllExtrinsicsWeight::put(next_weight); } @@ -908,6 +1000,11 @@ impl Module { } /// Deposits a log and ensures it matches the block's log data. + /// + /// # + /// - `O(D)` where `D` length of `Digest` + /// - 1 storage mutation (codec `O(D)`). + /// # pub fn deposit_log(item: DigestItemOf) { let mut l = >::get(); l.push(item); @@ -923,7 +1020,7 @@ impl Module { >::hashed_key().to_vec() => T::BlockNumber::one().encode(), >::hashed_key().to_vec() => [69u8; 32].encode() ], - children: map![], + children_default: map![], }) } @@ -1164,33 +1261,40 @@ pub fn split_inner(option: Option, splitter: impl FnOnce(T) -> (R, S #[derive(Encode, Decode, Clone, Eq, PartialEq)] pub struct CheckWeight(PhantomData); -impl CheckWeight { - /// Get the quota ratio of each dispatch class type. This indicates that all operational +impl CheckWeight where + T::Call: Dispatchable +{ + /// Get the quota ratio of each dispatch class type. This indicates that all operational and mandatory /// dispatches can use the full capacity of any resource, while user-triggered ones can consume /// a portion. fn get_dispatch_limit_ratio(class: DispatchClass) -> Perbill { - match class { - DispatchClass::Operational | DispatchClass::Mandatory - => ::one(), - DispatchClass::Normal => T::AvailableBlockRatio::get(), - } + Module::::get_dispatch_limit_ratio(class) } /// Checks if the current extrinsic can fit into the block with respect to block weight limits. /// /// Upon successes, it returns the new block weight as a `Result`. fn check_weight( - info: ::DispatchInfo, + info: &DispatchInfoOf, ) -> Result { let current_weight = Module::::all_extrinsics_weight(); let maximum_weight = T::MaximumBlockWeight::get(); let limit = Self::get_dispatch_limit_ratio(info.class) * maximum_weight; - let added_weight = info.weight.min(limit); - let next_weight = current_weight.saturating_add(added_weight); - if next_weight > limit && info.class != DispatchClass::Mandatory { - Err(InvalidTransaction::ExhaustsResources.into()) - } else { + if info.class == DispatchClass::Mandatory { + // If we have a dispatch that must be included in the block, it ignores all the limits. + let extrinsic_weight = info.weight.saturating_add(T::ExtrinsicBaseWeight::get()); + let next_weight = current_weight.saturating_add(extrinsic_weight); Ok(next_weight) + } else { + let extrinsic_weight = info.weight.checked_add(T::ExtrinsicBaseWeight::get()) + .ok_or(InvalidTransaction::ExhaustsResources)?; + let next_weight = current_weight.checked_add(extrinsic_weight) + .ok_or(InvalidTransaction::ExhaustsResources)?; + if next_weight > limit { + Err(InvalidTransaction::ExhaustsResources.into()) + } else { + Ok(next_weight) + } } } @@ -1198,7 +1302,7 @@ impl CheckWeight { /// /// Upon successes, it returns the new block length as a `Result`. fn check_block_length( - info: ::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> Result { let current_len = Module::::all_extrinsics_len(); @@ -1214,7 +1318,7 @@ impl CheckWeight { } /// get the priority of an extrinsic denoted by `info`. - fn get_priority(info: ::DispatchInfo) -> TransactionPriority { + fn get_priority(info: &DispatchInfoOf) -> TransactionPriority { match info.class { DispatchClass::Normal => info.weight.into(), DispatchClass::Operational => Bounded::max_value(), @@ -1232,7 +1336,7 @@ impl CheckWeight { /// /// It checks and notes the new weight and length. fn do_pre_dispatch( - info: ::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> Result<(), TransactionValidityError> { let next_len = Self::check_block_length(info, len)?; @@ -1246,7 +1350,7 @@ impl CheckWeight { /// /// It only checks that the block weight and length limit will not exceed. fn do_validate( - info: ::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { // ignore the next weight and length. If they return `Ok`, then it is below the limit. @@ -1257,11 +1361,12 @@ impl CheckWeight { } } -impl SignedExtension for CheckWeight { +impl SignedExtension for CheckWeight where + T::Call: Dispatchable +{ type AccountId = T::AccountId; type Call = T::Call; type AdditionalSigned = (); - type DispatchInfo = DispatchInfo; type Pre = (); const IDENTIFIER: &'static str = "CheckWeight"; @@ -1271,7 +1376,7 @@ impl SignedExtension for CheckWeight { self, _who: &Self::AccountId, _call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> Result<(), TransactionValidityError> { if info.class == DispatchClass::Mandatory { @@ -1284,7 +1389,7 @@ impl SignedExtension for CheckWeight { &self, _who: &Self::AccountId, _call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { if info.class == DispatchClass::Mandatory { @@ -1295,7 +1400,7 @@ impl SignedExtension for CheckWeight { fn pre_dispatch_unsigned( _call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> Result<(), TransactionValidityError> { Self::do_pre_dispatch(info, len) @@ -1303,7 +1408,7 @@ impl SignedExtension for CheckWeight { fn validate_unsigned( _call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { Self::do_validate(info, len) @@ -1311,7 +1416,8 @@ impl SignedExtension for CheckWeight { fn post_dispatch( _pre: Self::Pre, - info: Self::DispatchInfo, + info: &DispatchInfoOf, + post_info: &PostDispatchInfoOf, _len: usize, result: &DispatchResult, ) -> Result<(), TransactionValidityError> { @@ -1321,6 +1427,14 @@ impl SignedExtension for CheckWeight { if info.class == DispatchClass::Mandatory && result.is_err() { Err(InvalidTransaction::BadMandatory)? } + + let unspent = post_info.calc_unspent(info); + if unspent > 0 { + AllExtrinsicsWeight::mutate(|weight| { + *weight = weight.map(|w| w.saturating_sub(unspent)); + }) + } + Ok(()) } } @@ -1360,11 +1474,12 @@ impl Debug for CheckNonce { } } -impl SignedExtension for CheckNonce { +impl SignedExtension for CheckNonce where + T::Call: Dispatchable +{ type AccountId = T::AccountId; type Call = T::Call; type AdditionalSigned = (); - type DispatchInfo = DispatchInfo; type Pre = (); const IDENTIFIER: &'static str = "CheckNonce"; @@ -1374,7 +1489,7 @@ impl SignedExtension for CheckNonce { self, who: &Self::AccountId, _call: &Self::Call, - _info: Self::DispatchInfo, + _info: &DispatchInfoOf, _len: usize, ) -> Result<(), TransactionValidityError> { let mut account = Account::::get(who); @@ -1396,7 +1511,7 @@ impl SignedExtension for CheckNonce { &self, who: &Self::AccountId, _call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, _len: usize, ) -> TransactionValidity { // check index @@ -1455,7 +1570,6 @@ impl SignedExtension for CheckEra { type AccountId = T::AccountId; type Call = T::Call; type AdditionalSigned = T::Hash; - type DispatchInfo = DispatchInfo; type Pre = (); const IDENTIFIER: &'static str = "CheckEra"; @@ -1463,7 +1577,7 @@ impl SignedExtension for CheckEra { &self, _who: &Self::AccountId, _call: &Self::Call, - _info: Self::DispatchInfo, + _info: &DispatchInfoOf, _len: usize, ) -> TransactionValidity { let current_u64 = >::block_number().saturated_into::(); @@ -1512,7 +1626,6 @@ impl SignedExtension for CheckGenesis { type AccountId = T::AccountId; type Call = ::Call; type AdditionalSigned = T::Hash; - type DispatchInfo = DispatchInfo; type Pre = (); const IDENTIFIER: &'static str = "CheckGenesis"; @@ -1548,7 +1661,6 @@ impl SignedExtension for CheckVersion { type AccountId = T::AccountId; type Call = ::Call; type AdditionalSigned = u32; - type DispatchInfo = DispatchInfo; type Pre = (); const IDENTIFIER: &'static str = "CheckVersion"; @@ -1574,18 +1686,18 @@ impl Lookup for ChainContext { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use sp_std::cell::RefCell; use sp_core::H256; - use sp_runtime::{traits::{BlakeTwo256, IdentityLookup}, testing::Header, DispatchError}; - use frame_support::{impl_outer_origin, parameter_types}; + use sp_runtime::{traits::{BlakeTwo256, IdentityLookup, SignedExtension}, testing::Header, DispatchError}; + use frame_support::{impl_outer_origin, parameter_types, assert_ok, assert_err}; impl_outer_origin! { pub enum Origin for Test where system = super {} } - #[derive(Clone, Eq, PartialEq)] + #[derive(Clone, Eq, PartialEq, Debug)] pub struct Test; parameter_types! { @@ -1600,6 +1712,13 @@ mod tests { spec_version: 1, impl_version: 1, apis: sp_version::create_apis_vec!([]), + transaction_version: 1, + }; + pub const BlockExecutionWeight: Weight = 10; + pub const ExtrinsicBaseWeight: Weight = 5; + pub const DbWeight: RuntimeDbWeight = RuntimeDbWeight { + read: 10, + write: 100, }; } @@ -1612,9 +1731,23 @@ mod tests { fn on_killed_account(who: &u64) { KILLED.with(|r| r.borrow_mut().push(*who)) } } + #[derive(Debug, codec::Encode, codec::Decode)] + pub struct Call; + + impl Dispatchable for Call { + type Origin = (); + type Trait = (); + type Info = DispatchInfo; + type PostInfo = PostDispatchInfo; + fn dispatch(self, _origin: Self::Origin) + -> sp_runtime::DispatchResultWithInfo { + panic!("Do not use dummy implementation for dispatch."); + } + } + impl Trait for Test { type Origin = Origin; - type Call = (); + type Call = Call; type Index = u64; type BlockNumber = u64; type Hash = H256; @@ -1625,6 +1758,9 @@ mod tests { type Event = u16; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = DbWeight; + type BlockExecutionWeight = BlockExecutionWeight; + type ExtrinsicBaseWeight = ExtrinsicBaseWeight; type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = Version; @@ -1647,10 +1783,13 @@ mod tests { type System = Module; - const CALL: &::Call = &(); + const CALL: &::Call = &Call; fn new_test_ext() -> sp_io::TestExternalities { - GenesisConfig::default().build_storage::().unwrap().into() + let mut ext: sp_io::TestExternalities = GenesisConfig::default().build_storage::().unwrap().into(); + // Add to each test the initial weight of a block + ext.execute_with(|| System::register_extra_weight_unchecked(::BlockExecutionWeight::get())); + ext } fn normal_weight_limit() -> Weight { @@ -1848,14 +1987,14 @@ mod tests { let info = DispatchInfo::default(); let len = 0_usize; // stale - assert!(CheckNonce::(0).validate(&1, CALL, info, len).is_err()); - assert!(CheckNonce::(0).pre_dispatch(&1, CALL, info, len).is_err()); + assert!(CheckNonce::(0).validate(&1, CALL, &info, len).is_err()); + assert!(CheckNonce::(0).pre_dispatch(&1, CALL, &info, len).is_err()); // correct - assert!(CheckNonce::(1).validate(&1, CALL, info, len).is_ok()); - assert!(CheckNonce::(1).pre_dispatch(&1, CALL, info, len).is_ok()); + assert!(CheckNonce::(1).validate(&1, CALL, &info, len).is_ok()); + assert!(CheckNonce::(1).pre_dispatch(&1, CALL, &info, len).is_ok()); // future - assert!(CheckNonce::(5).validate(&1, CALL, info, len).is_ok()); - assert!(CheckNonce::(5).pre_dispatch(&1, CALL, info, len).is_err()); + assert!(CheckNonce::(5).validate(&1, CALL, &info, len).is_ok()); + assert!(CheckNonce::(5).pre_dispatch(&1, CALL, &info, len).is_err()); }) } @@ -1865,11 +2004,11 @@ mod tests { let normal_limit = normal_weight_limit(); let small = DispatchInfo { weight: 100, ..Default::default() }; let medium = DispatchInfo { - weight: normal_limit - 1, + weight: normal_limit - ::ExtrinsicBaseWeight::get(), ..Default::default() }; let big = DispatchInfo { - weight: normal_limit + 1, + weight: normal_limit - ::ExtrinsicBaseWeight::get() + 1, ..Default::default() }; let len = 0_usize; @@ -1880,77 +2019,208 @@ mod tests { if f { assert!(r.is_err()) } else { assert!(r.is_ok()) } }; - reset_check_weight(small, false, 0); - reset_check_weight(medium, false, 0); - reset_check_weight(big, true, 1); + reset_check_weight(&small, false, 0); + reset_check_weight(&medium, false, 0); + reset_check_weight(&big, true, 1); }) } #[test] - fn signed_ext_check_weight_fee_works() { + fn signed_ext_check_weight_refund_works() { new_test_ext().execute_with(|| { - let free = DispatchInfo { weight: 0, ..Default::default() }; + // This is half of the max block weight + let info = DispatchInfo { weight: 512, ..Default::default() }; + let post_info = PostDispatchInfo { actual_weight: Some(128), }; let len = 0_usize; - assert_eq!(System::all_extrinsics_weight(), 0); - let r = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, free, len); - assert!(r.is_ok()); - assert_eq!(System::all_extrinsics_weight(), 0); + // We allow 75% for normal transaction, so we put 25% - extrinsic base weight + AllExtrinsicsWeight::put(256 - ::ExtrinsicBaseWeight::get()); + + let pre = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &info, len).unwrap(); + assert_eq!(AllExtrinsicsWeight::get().unwrap(), info.weight + 256); + + assert!( + CheckWeight::::post_dispatch(pre, &info, &post_info, len, &Ok(())) + .is_ok() + ); + assert_eq!( + AllExtrinsicsWeight::get().unwrap(), + post_info.actual_weight.unwrap() + 256, + ); }) } #[test] - fn signed_ext_check_weight_max_works() { + fn signed_ext_check_weight_actual_weight_higher_than_max_is_capped() { new_test_ext().execute_with(|| { - let max = DispatchInfo { weight: Weight::max_value(), ..Default::default() }; + let info = DispatchInfo { weight: 512, ..Default::default() }; + let post_info = PostDispatchInfo { actual_weight: Some(700), }; let len = 0_usize; - let normal_limit = normal_weight_limit(); - assert_eq!(System::all_extrinsics_weight(), 0); - let r = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, max, len); + AllExtrinsicsWeight::put(128); + + let pre = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &info, len).unwrap(); + assert_eq!( + AllExtrinsicsWeight::get().unwrap(), + info.weight + 128 + ::ExtrinsicBaseWeight::get(), + ); + + assert!( + CheckWeight::::post_dispatch(pre, &info, &post_info, len, &Ok(())) + .is_ok() + ); + assert_eq!( + AllExtrinsicsWeight::get().unwrap(), + info.weight + 128 + ::ExtrinsicBaseWeight::get(), + ); + }) + } + + #[test] + fn zero_weight_extrinsic_still_has_base_weight() { + new_test_ext().execute_with(|| { + let free = DispatchInfo { weight: 0, ..Default::default() }; + let len = 0_usize; + + // Initial weight from `BlockExecutionWeight` + assert_eq!(System::all_extrinsics_weight(), ::BlockExecutionWeight::get()); + let r = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &free, len); assert!(r.is_ok()); - assert_eq!(System::all_extrinsics_weight(), normal_limit); + assert_eq!( + System::all_extrinsics_weight(), + ::ExtrinsicBaseWeight::get() + ::BlockExecutionWeight::get() + ); }) } + #[test] + fn max_extrinsic_weight_is_allowed_but_nothing_more() { + // Dispatch Class Normal + new_test_ext().execute_with(|| { + let one = DispatchInfo { weight: 1, ..Default::default() }; + let max = DispatchInfo { weight: System::max_extrinsic_weight(DispatchClass::Normal), ..Default::default() }; + let len = 0_usize; + + assert_ok!(CheckWeight::::do_pre_dispatch(&max, len)); + assert_err!( + CheckWeight::::do_pre_dispatch(&one, len), + InvalidTransaction::ExhaustsResources, + ); + // Weight should be 75% of 1024 (max block weight) + assert_eq!(System::all_extrinsics_weight(), 768); + }); + + // Dispatch Class Operational + new_test_ext().execute_with(|| { + let one = DispatchInfo { + weight: 1, + class: DispatchClass::Operational, + ..Default::default() + }; + let max = DispatchInfo { + weight: System::max_extrinsic_weight(DispatchClass::Operational), + class: DispatchClass::Operational, + ..Default::default() + }; + let len = 0_usize; + + assert_ok!(CheckWeight::::do_pre_dispatch(&max, len)); + assert_err!( + CheckWeight::::do_pre_dispatch(&one, len), + InvalidTransaction::ExhaustsResources, + ); + // Weight should be 100% of max block weight + assert_eq!(System::all_extrinsics_weight(), ::MaximumBlockWeight::get()); + }); + } + + #[test] + fn mandatory_extrinsic_doesnt_care_about_limits() { + new_test_ext().execute_with(|| { + let max = DispatchInfo { + weight: Weight::max_value(), + class: DispatchClass::Mandatory, + ..Default::default() + }; + let len = 0_usize; + + assert_ok!(CheckWeight::::do_pre_dispatch(&max, len)); + assert_eq!(System::all_extrinsics_weight(), Weight::max_value()); + assert!(System::all_extrinsics_weight() > ::MaximumBlockWeight::get()); + }); + } + + #[test] + fn register_extra_weight_unchecked_doesnt_care_about_limits() { + new_test_ext().execute_with(|| { + System::register_extra_weight_unchecked(Weight::max_value()); + assert_eq!(System::all_extrinsics_weight(), Weight::max_value()); + assert!(System::all_extrinsics_weight() > ::MaximumBlockWeight::get()); + }); + } + + #[test] + fn full_block_with_normal_and_operational() { + new_test_ext().execute_with(|| { + // Max block is 1024 + // Max normal is 768 (75%) + // 10 is taken for block execution weight + // So normal extrinsic can be 758 weight (-5 for base extrinsic weight) + // And Operational can be 256 to produce a full block (-5 for base) + + assert_eq!(System::max_extrinsic_weight(DispatchClass::Normal), 753); + + let max_normal = DispatchInfo { weight: 753, ..Default::default() }; + let rest_operational = DispatchInfo { weight: 251, class: DispatchClass::Operational, ..Default::default() }; + + let len = 0_usize; + + assert_ok!(CheckWeight::::do_pre_dispatch(&max_normal, len)); + assert_eq!(System::all_extrinsics_weight(), 768); + assert_ok!(CheckWeight::::do_pre_dispatch(&rest_operational, len)); + assert_eq!(::MaximumBlockWeight::get(), 1024); + assert_eq!(System::all_extrinsics_weight(), ::MaximumBlockWeight::get()); + }); + } + #[test] fn signed_ext_check_weight_works_operational_tx() { new_test_ext().execute_with(|| { let normal = DispatchInfo { weight: 100, ..Default::default() }; - let op = DispatchInfo { weight: 100, class: DispatchClass::Operational, pays_fee: true }; + let op = DispatchInfo { weight: 100, class: DispatchClass::Operational, pays_fee: Pays::Yes }; let len = 0_usize; let normal_limit = normal_weight_limit(); // given almost full block AllExtrinsicsWeight::put(normal_limit); // will not fit. - assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, normal, len).is_err()); + assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &normal, len).is_err()); // will fit. - assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, op, len).is_ok()); + assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &op, len).is_ok()); // likewise for length limit. let len = 100_usize; AllExtrinsicsLen::put(normal_length_limit()); - assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, normal, len).is_err()); - assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, op, len).is_ok()); + assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &normal, len).is_err()); + assert!(CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &op, len).is_ok()); }) } #[test] fn signed_ext_check_weight_priority_works() { new_test_ext().execute_with(|| { - let normal = DispatchInfo { weight: 100, class: DispatchClass::Normal, pays_fee: true }; - let op = DispatchInfo { weight: 100, class: DispatchClass::Operational, pays_fee: true }; + let normal = DispatchInfo { weight: 100, class: DispatchClass::Normal, pays_fee: Pays::Yes }; + let op = DispatchInfo { weight: 100, class: DispatchClass::Operational, pays_fee: Pays::Yes }; let len = 0_usize; let priority = CheckWeight::(PhantomData) - .validate(&1, CALL, normal, len) + .validate(&1, CALL, &normal, len) .unwrap() .priority; assert_eq!(priority, 100); let priority = CheckWeight::(PhantomData) - .validate(&1, CALL, op, len) + .validate(&1, CALL, &op, len) .unwrap() .priority; assert_eq!(priority, u64::max_value()); @@ -1968,16 +2238,16 @@ mod tests { if f { assert!(r.is_err()) } else { assert!(r.is_ok()) } }; - reset_check_weight(normal, normal_limit - 1, false); - reset_check_weight(normal, normal_limit, false); - reset_check_weight(normal, normal_limit + 1, true); + reset_check_weight(&normal, normal_limit - 1, false); + reset_check_weight(&normal, normal_limit, false); + reset_check_weight(&normal, normal_limit + 1, true); // Operational ones don't have this limit. - let op = DispatchInfo { weight: 0, class: DispatchClass::Operational, pays_fee: true }; - reset_check_weight(op, normal_limit, false); - reset_check_weight(op, normal_limit + 100, false); - reset_check_weight(op, 1024, false); - reset_check_weight(op, 1025, true); + let op = DispatchInfo { weight: 0, class: DispatchClass::Operational, pays_fee: Pays::Yes }; + reset_check_weight(&op, normal_limit, false); + reset_check_weight(&op, normal_limit + 100, false); + reset_check_weight(&op, 1024, false); + reset_check_weight(&op, 1025, true); }) } @@ -2000,7 +2270,7 @@ mod tests { #[test] fn signed_ext_check_era_should_change_longevity() { new_test_ext().execute_with(|| { - let normal = DispatchInfo { weight: 100, class: DispatchClass::Normal, pays_fee: true }; + let normal = DispatchInfo { weight: 100, class: DispatchClass::Normal, pays_fee: Pays::Yes }; let len = 0_usize; let ext = ( CheckWeight::(PhantomData), @@ -2009,7 +2279,7 @@ mod tests { System::set_block_number(17); >::insert(16, H256::repeat_byte(1)); - assert_eq!(ext.validate(&1, CALL, normal, len).unwrap().longevity, 15); + assert_eq!(ext.validate(&1, CALL, &normal, len).unwrap().longevity, 15); }) } @@ -2026,6 +2296,7 @@ mod tests { _: &str, _: &[u8], _: &mut dyn sp_externalities::Externalities, + _: sp_core::traits::MissingHostFunctions, ) -> Result, String> { Ok(self.0.clone()) } diff --git a/frame/system/src/offchain.rs b/frame/system/src/offchain.rs index a3fe3e00ca4be879ed6074cc6b13f75e0fbae83f..04cd3001e43a3899cdbdafaa7fb3a1eeb9bed9db 100644 --- a/frame/system/src/offchain.rs +++ b/frame/system/src/offchain.rs @@ -15,353 +15,832 @@ // along with Substrate. If not, see . //! Module helpers for off-chain calls. +//! +//! ## Overview +//! +//! This module provides transaction related helpers to: +//! - Submit a raw unsigned transaction +//! - Submit an unsigned transaction with a signed payload +//! - Submit a signed transction. +//! +//! ## Usage +//! +//! Please refer to [`example-offchain-worker`](../../pallet_example_offchain_worker/index.html) for +//! a concrete example usage of this crate. +//! +//! ### Submit a raw unsigned transaction +//! +//! To submit a raw unsigned transaction, [`SubmitTransaction`](./struct.SubmitTransaction.html) +//! can be used. +//! +//! ### Signing transactions +//! +//! To be able to use signing, the following trait should be implemented: +//! +//! - [`AppCrypto`](./trait.AppCrypto.html): where an application-specific key +//! is defined and can be used by this module's helpers for signing. +//! - [`CreateSignedTransaction`](./trait.CreateSignedTransaction.html): where +//! the manner in which the transaction is constructed is defined. +//! +//! #### Submit an unsigned transaction with a signed payload +//! +//! Initially, a payload instance that implements the `SignedPayload` trait should be defined. +//! See [`PricePayload`](../../pallet_example_offchain_worker/struct.PricePayload.html) +//! +//! The payload type that is defined defined can then be signed and submitted onchain. +//! +//! #### Submit a signed transaction +//! +//! [`Signer`](./struct.Signer.html) can be used to sign/verify payloads +//! + +#![warn(missing_docs)] use codec::Encode; -use sp_std::convert::TryInto; -use sp_std::prelude::Vec; -use sp_runtime::app_crypto::{RuntimeAppPublic, AppPublic, AppSignature}; +use sp_std::collections::btree_set::BTreeSet; +use sp_std::convert::{TryInto, TryFrom}; +use sp_std::prelude::{Box, Vec}; +use sp_runtime::app_crypto::RuntimeAppPublic; use sp_runtime::traits::{Extrinsic as ExtrinsicT, IdentifyAccount, One}; -use frame_support::{debug, storage::StorageMap}; +use frame_support::{debug, storage::StorageMap, RuntimeDebug}; -/// Creates runtime-specific signed transaction. +/// Marker struct used to flag using all supported keys to sign a payload. +pub struct ForAll {} +/// Marker struct used to flag using any of the supported keys to sign a payload. +pub struct ForAny {} + +/// Provides the ability to directly submit signed and unsigned +/// transaction onchain. /// -/// This trait should be implemented by your `Runtime` to be able -/// to submit `SignedTransaction`s` to the pool from off-chain code. -pub trait CreateTransaction { - /// A `Public` key representing a particular `AccountId`. - type Public: IdentifyAccount + Clone; - /// A `Signature` generated by the `Signer`. - type Signature; +/// For submitting unsigned transactions, `submit_unsigned_transaction` +/// utility function can be used. However, this struct is used by `Signer` +/// to submit a signed transactions providing the signature along with the call. +pub struct SubmitTransaction, OverarchingCall> { + _phantom: sp_std::marker::PhantomData<(T, OverarchingCall)> +} - /// Attempt to create signed extrinsic data that encodes call from given account. - /// - /// Runtime implementation is free to construct the payload to sign and the signature - /// in any way it wants. - /// Returns `None` if signed extrinsic could not be created (either because signing failed - /// or because of any other runtime-specific reason). - fn create_transaction>( - call: Extrinsic::Call, - public: Self::Public, - account: T::AccountId, - nonce: T::Index, - ) -> Option<(Extrinsic::Call, Extrinsic::SignaturePayload)>; +impl SubmitTransaction +where + T: SendTransactionTypes, +{ + /// Submit transaction onchain by providing the call and an optional signature + pub fn submit_transaction( + call: >::OverarchingCall, + signature: Option<::SignaturePayload>, + ) -> Result<(), ()> { + let xt = T::Extrinsic::new(call.into(), signature).ok_or(())?; + sp_io::offchain::submit_transaction(xt.encode()) + } + + /// A convenience method to submit an unsigned transaction onchain. + pub fn submit_unsigned_transaction( + call: >::OverarchingCall, + ) -> Result<(), ()> { + SubmitTransaction::::submit_transaction(call, None) + } } -/// A trait responsible for signing a payload using given account. +/// Provides an implementation for signing transaction payloads. /// -/// This trait is usually going to represent a local public key -/// that has ability to sign arbitrary `Payloads`. +/// Keys used for signing are defined when instantiating the signer object. +/// Signing can be done using: /// -/// NOTE: Most likely you don't need to implement this trait manually. -/// It has a blanket implementation for all `RuntimeAppPublic` types, -/// so it's enough to pass an application-specific crypto type. +/// - All supported keys in the keystore +/// - Any of the supported keys in the keystore +/// - An intersection of in-keystore keys and the list of provided keys /// -/// To easily create `SignedTransaction`s have a look at the -/// [`TransactionSubmitter`] type. -pub trait Signer { - /// Sign any encodable payload with given account and produce a signature. +/// The signer is then able to: +/// - Submit a unsigned transaction with a signed payload +/// - Submit a signed transaction +#[derive(RuntimeDebug)] +pub struct Signer, X = ForAny> { + accounts: Option>, + _phantom: sp_std::marker::PhantomData<(X, C)>, +} + +impl, X> Default for Signer { + fn default() -> Self { + Self { + accounts: Default::default(), + _phantom: Default::default(), + } + } +} + +impl, X> Signer { + /// Use all available keys for signing. + pub fn all_accounts() -> Signer { + Default::default() + } + + /// Use any of the available keys for signing. + pub fn any_account() -> Signer { + Default::default() + } + + /// Use provided `accounts` for signing. /// - /// Returns `Some` if signing succeeded and `None` in case the `account` couldn't - /// be used (for instance we couldn't convert it to required application specific crypto). - fn sign(public: Public, payload: &Payload) -> Option; + /// Note that not all keys will be necessarily used. The provided + /// vector of accounts will be intersected with the supported keys + /// in the keystore and the resulting list will be used for signing. + pub fn with_filter(mut self, accounts: Vec) -> Self { + self.accounts = Some(accounts); + self + } + + /// Check if there are any keys that could be used for signing. + pub fn can_sign(&self) -> bool { + self.accounts_from_keys().count() > 0 + } + + /// Return a vector of the intersection between + /// all available accounts and the provided accounts + /// in `with_filter`. If no accounts are provided, + /// use all accounts by default. + fn accounts_from_keys<'a>(&'a self) -> Box> + 'a> { + let keystore_accounts = self.keystore_accounts(); + match self.accounts { + None => Box::new(keystore_accounts), + Some(ref keys) => { + let keystore_lookup: BTreeSet<::Public> = keystore_accounts + .map(|account| account.public).collect(); + + Box::new(keys.into_iter() + .enumerate() + .map(|(index, key)| { + let account_id = key.clone().into_account(); + Account::new(index, account_id, key.clone()) + }) + .filter(move |account| keystore_lookup.contains(&account.public))) + } + } + } + + fn keystore_accounts(&self) -> impl Iterator> { + C::RuntimeAppPublic::all() + .into_iter() + .enumerate() + .map(|(index, key)| { + let generic_public = C::GenericPublic::from(key); + let public = generic_public.into(); + let account_id = public.clone().into_account(); + Account::new(index, account_id, public.clone()) + }) + } } -/// A `Signer` implementation for any `AppPublic` type. -/// -/// This implementation additionally supports conversion to/from multi-signature/multi-signer -/// wrappers. -/// If the wrapped crypto doesn't match `AppPublic`s crypto `None` is returned. -impl Signer for TAnyAppPublic where - TAnyAppPublic: RuntimeAppPublic - + AppPublic - + From<::Generic>, - ::Signature: AppSignature, - Signature: From< - <::Signature as AppSignature>::Generic - >, - Public: TryInto<::Generic> -{ - fn sign(public: Public, raw_payload: &Payload) -> Option { - raw_payload.using_encoded(|payload| { - let public = public.try_into().ok()?; - TAnyAppPublic::from(public).sign(&payload) - .map( - <::Signature as AppSignature> - ::Generic::from - ) - .map(Signature::from) + +impl> Signer { + fn for_all(&self, f: F) -> Vec<(Account, R)> where + F: Fn(&Account) -> Option, + { + let accounts = self.accounts_from_keys(); + accounts + .into_iter() + .filter_map(|account| { + f(&account).map(|res| (account, res)) + }) + .collect() + } +} + +impl> Signer { + fn for_any(&self, f: F) -> Option<(Account, R)> where + F: Fn(&Account) -> Option, + { + let accounts = self.accounts_from_keys(); + for account in accounts.into_iter() { + let res = f(&account); + if let Some(res) = res { + return Some((account, res)); + } + } + None + } +} + +impl> SignMessage for Signer { + type SignatureData = Vec<(Account, T::Signature)>; + + fn sign_message(&self, message: &[u8]) -> Self::SignatureData { + self.for_all(|account| C::sign(message, account.public.clone())) + } + + fn sign(&self, f: F) -> Self::SignatureData where + F: Fn(&Account) -> TPayload, + TPayload: SignedPayload, + { + self.for_all(|account| f(account).sign::()) + } +} + +impl> SignMessage for Signer { + type SignatureData = Option<(Account, T::Signature)>; + + fn sign_message(&self, message: &[u8]) -> Self::SignatureData { + self.for_any(|account| C::sign(message, account.public.clone())) + } + + fn sign(&self, f: F) -> Self::SignatureData where + F: Fn(&Account) -> TPayload, + TPayload: SignedPayload, + { + self.for_any(|account| f(account).sign::()) + } +} + +impl< + T: CreateSignedTransaction + SigningTypes, + C: AppCrypto, + LocalCall, +> SendSignedTransaction for Signer { + type Result = Option<(Account, Result<(), ()>)>; + + fn send_signed_transaction( + &self, + f: impl Fn(&Account) -> LocalCall, + ) -> Self::Result { + self.for_any(|account| { + let call = f(account); + self.send_single_signed_transaction(account, call) }) } } -/// Retrieves a public key type for given `SignAndSubmitTransaction`. -pub type PublicOf = -< - >::CreateTransaction - as - CreateTransaction>::Extrinsic> ->::Public; +impl< + T: SigningTypes + CreateSignedTransaction, + C: AppCrypto, + LocalCall, +> SendSignedTransaction for Signer { + type Result = Vec<(Account, Result<(), ()>)>; + + fn send_signed_transaction( + &self, + f: impl Fn(&Account) -> LocalCall, + ) -> Self::Result { + self.for_all(|account| { + let call = f(account); + self.send_single_signed_transaction(account, call) + }) + } +} + +impl< + T: SigningTypes + SendTransactionTypes, + C: AppCrypto, + LocalCall, +> SendUnsignedTransaction for Signer { + type Result = Option<(Account, Result<(), ()>)>; + + fn send_unsigned_transaction( + &self, + f: F, + f2: impl Fn(TPayload, T::Signature) -> LocalCall, + ) -> Self::Result + where + F: Fn(&Account) -> TPayload, + TPayload: SignedPayload, + { + self.for_any(|account| { + let payload = f(account); + let signature= payload.sign::()?; + let call = f2(payload, signature); + self.submit_unsigned_transaction(call) + }) + } +} + +impl< + T: SigningTypes + SendTransactionTypes, + C: AppCrypto, + LocalCall, +> SendUnsignedTransaction for Signer { + type Result = Vec<(Account, Result<(), ()>)>; + + fn send_unsigned_transaction( + &self, + f: F, + f2: impl Fn(TPayload, T::Signature) -> LocalCall, + ) -> Self::Result + where + F: Fn(&Account) -> TPayload, + TPayload: SignedPayload { + self.for_all(|account| { + let payload = f(account); + let signature = payload.sign::()?; + let call = f2(payload, signature); + self.submit_unsigned_transaction(call) + }) + } +} + +/// Details of an account for which a private key is contained in the keystore. +#[derive(RuntimeDebug, PartialEq)] +pub struct Account { + /// Index on the provided list of accounts or list of all accounts. + pub index: usize, + /// Runtime-specific `AccountId`. + pub id: T::AccountId, + /// A runtime-specific `Public` key for that key pair. + pub public: T::Public, +} + +impl Account { + /// Create a new Account instance + pub fn new(index: usize, id: T::AccountId, public: T::Public) -> Self { + Self { index, id, public } + } +} + +impl Clone for Account where + T::AccountId: Clone, + T::Public: Clone, +{ + fn clone(&self) -> Self { + Self { + index: self.index, + id: self.id.clone(), + public: self.public.clone(), + } + } +} -/// A trait to sign and submit transactions in off-chain calls. +/// A type binding runtime-level `Public/Signature` pair with crypto wrapped by `RuntimeAppPublic`. /// -/// NOTE: Most likely you should not implement this trait yourself. -/// There is an implementation for -/// [`TransactionSubmitter`] type, which -/// you should use. -pub trait SignAndSubmitTransaction { - /// Unchecked extrinsic type. - type Extrinsic: ExtrinsicT + Encode; - - /// A runtime-specific type to produce signed data for the extrinsic. - type CreateTransaction: CreateTransaction; - - /// A type used to sign transactions created using `CreateTransaction`. - type Signer: Signer< - PublicOf, - >::Signature, - >; - - /// Sign given call and submit it to the transaction pool. - /// - /// Returns `Ok` if the transaction was submitted correctly - /// and `Err` if the key for given `id` was not found or the - /// transaction was rejected from the pool. - fn sign_and_submit(call: impl Into, public: PublicOf) -> Result<(), ()> { - let call = call.into(); - let id = public.clone().into_account(); - let mut account = super::Account::::get(&id); - debug::native::debug!( - target: "offchain", - "Creating signed transaction from account: {:?} (nonce: {:?})", - id, - account.nonce, - ); - let (call, signature_data) = Self::CreateTransaction - ::create_transaction::(call, public, id.clone(), account.nonce) - .ok_or(())?; - // increment the nonce. This is fine, since the code should always - // be running in off-chain context, so we NEVER persists data. - account.nonce += One::one(); - super::Account::::insert(&id, account); - - let xt = Self::Extrinsic::new(call, Some(signature_data)).ok_or(())?; - sp_io::offchain::submit_transaction(xt.encode()) +/// Implementations of this trait should specify the app-specific public/signature types. +/// This is merely a wrapper around an existing `RuntimeAppPublic` type, but with +/// extra non-application-specific crypto type that is being wrapped (e.g. `sr25519`, `ed25519`). +/// This is needed to later on convert into runtime-specific `Public` key, which might support +/// multiple different crypto. +/// The point of this trait is to be able to easily convert between `RuntimeAppPublic`, the wrapped +/// (generic = non application-specific) crypto types and the `Public` type required by the runtime. +/// +/// TODO [#5662] Potentially use `IsWrappedBy` types, or find some other way to make it easy to +/// obtain unwrapped crypto (and wrap it back). +/// +/// Example (pseudo-)implementation: +/// ```ignore +/// // im-online specific crypto +/// type RuntimeAppPublic = ImOnline(sr25519::Public); +/// // wrapped "raw" crypto +/// type GenericPublic = sr25519::Public; +/// type GenericSignature = sr25519::Signature; +/// +/// // runtime-specific public key +/// type Public = MultiSigner: From; +/// type Signature = MulitSignature: From; +/// ``` +pub trait AppCrypto { + /// A application-specific crypto. + type RuntimeAppPublic: RuntimeAppPublic; + + /// A raw crypto public key wrapped by `RuntimeAppPublic`. + type GenericPublic: + From + + Into + + TryFrom + + Into; + + /// A matching raw crypto `Signature` type. + type GenericSignature: + From<::Signature> + + Into<::Signature> + + TryFrom + + Into; + + /// Sign payload with the private key to maps to the provided public key. + fn sign(payload: &[u8], public: Public) -> Option { + let p: Self::GenericPublic = public.try_into().ok()?; + let x = Into::::into(p); + x.sign(&payload) + .map(|x| { + let sig: Self::GenericSignature = x.into(); + sig + }) + .map(Into::into) + } + + /// Verify signature against the provided public key. + fn verify(payload: &[u8], public: Public, signature: Signature) -> bool { + let p: Self::GenericPublic = match public.try_into() { + Ok(a) => a, + _ => return false + }; + let x = Into::::into(p); + let signature: Self::GenericSignature = match signature.try_into() { + Ok(a) => a, + _ => return false + }; + let signature = Into::<< + Self::RuntimeAppPublic as RuntimeAppPublic + >::Signature>::into(signature); + + x.verify(&payload, &signature) } } -/// A trait to submit unsigned transactions in off-chain calls. +/// A wrapper around the types which are used for signing. +/// +/// This trait adds extra bounds to `Public` and `Signature` types of the runtime +/// that are necessary to use these types for signing. /// -/// NOTE: Most likely you should not implement this trait yourself. -/// There is an implementation for -/// [`TransactionSubmitter`] type, which -/// you should use. -pub trait SubmitUnsignedTransaction { - /// Unchecked extrinsic type. - type Extrinsic: ExtrinsicT + Encode; - - /// Submit given call to the transaction pool as unsigned transaction. +/// TODO [#5663] Could this be just `T::Signature as traits::Verify>::Signer`? +/// Seems that this may cause issues with bounds resolution. +pub trait SigningTypes: crate::Trait { + /// A public key that is capable of identifing `AccountId`s. /// - /// Returns `Ok` if the transaction was submitted correctly - /// and `Err` if transaction was rejected from the pool. - fn submit_unsigned(call: impl Into) -> Result<(), ()> { - let xt = Self::Extrinsic::new(call.into(), None).ok_or(())?; - let encoded_xt = xt.encode(); - sp_io::offchain::submit_transaction(encoded_xt) - } + /// Usually that's either a raw crypto public key (e.g. `sr25519::Public`) or + /// an aggregate type for multiple crypto public keys, like `MulitSigner`. + type Public: Clone + + PartialEq + + IdentifyAccount + + core::fmt::Debug + + codec::Codec + + Ord; + + /// A matching `Signature` type. + type Signature: Clone + + PartialEq + + core::fmt::Debug + + codec::Codec; } -/// A utility trait to easily create signed transactions -/// from accounts in node's local keystore. +/// A definition of types required to submit transactions from within the runtime. +pub trait SendTransactionTypes { + /// The extrinsic type expected by the runtime. + type Extrinsic: ExtrinsicT + codec::Encode; + /// The runtime's call type. + /// + /// This has additional bound to be able to be created from pallet-local `Call` types. + type OverarchingCall: From; +} + +/// Create signed transaction. /// -/// NOTE: Most likely you should not implement this trait yourself. -/// There is an implementation for -/// [`TransactionSubmitter`] type, which -/// you should use. -pub trait SubmitSignedTransaction { - /// A `SignAndSubmitTransaction` implementation. - type SignAndSubmit: SignAndSubmitTransaction; - - /// Find local keys that match given list of accounts. +/// This trait is meant to be implemented by the runtime and is responsible for constructing +/// a payload to be signed and contained within the extrinsic. +/// This will most likely include creation of `SignedExtra` (a set of `SignedExtensions`). +/// Note that the result can be altered by inspecting the `Call` (for instance adjusting +/// fees, or mortality depending on the `pallet` being called). +pub trait CreateSignedTransaction: SendTransactionTypes + SigningTypes { + /// Attempt to create signed extrinsic data that encodes call from given account. /// - /// Technically it finds an intersection between given list of `AccountId`s - /// and accounts that are represented by public keys in local keystore. - /// If `None` is passed it returns all accounts in the keystore. + /// Runtime implementation is free to construct the payload to sign and the signature + /// in any way it wants. + /// Returns `None` if signed extrinsic could not be created (either because signing failed + /// or because of any other runtime-specific reason). + fn create_transaction>( + call: Self::OverarchingCall, + public: Self::Public, + account: Self::AccountId, + nonce: Self::Index, + ) -> Option<(Self::OverarchingCall, ::SignaturePayload)>; +} + +/// A message signer. +pub trait SignMessage { + /// A signature data. /// - /// Returns both public keys and `AccountId`s of accounts that are available. - /// Such accounts can later be used to sign a payload or send signed transactions. - fn find_local_keys(accounts: Option>) -> Vec<( - T::AccountId, - PublicOf, - )>; - - /// Find all available local keys. + /// May contain account used for signing and the `Signature` itself. + type SignatureData; + + /// Sign a message. /// - /// This is equivalent of calling `find_local_keys(None)`. - fn find_all_local_keys() -> Vec<(T::AccountId, PublicOf)> { - Self::find_local_keys(None as Option>) - } + /// Implementation of this method should return + /// a result containing the signature. + fn sign_message(&self, message: &[u8]) -> Self::SignatureData; - /// Check if there are keys for any of given accounts that could be used to send a transaction. + /// Construct and sign given payload. /// - /// This check can be used as an early-exit condition to avoid doing too - /// much work, before we actually realise that there are no accounts that you - /// we could use for signing. - fn can_sign_with(accounts: Option>) -> bool { - !Self::find_local_keys(accounts).is_empty() - } + /// This method expects `f` to return a `SignedPayload` + /// object which is then used for signing. + fn sign(&self, f: F) -> Self::SignatureData where + F: Fn(&Account) -> TPayload, + TPayload: SignedPayload, + ; +} - /// Check if there are any keys that could be used for signing. +/// Submit a signed transaction to the transaction pool. +pub trait SendSignedTransaction< + T: SigningTypes + CreateSignedTransaction, + C: AppCrypto, + LocalCall +> { + /// A submission result. /// - /// This is equivalent of calling `can_sign_with(None)`. - fn can_sign() -> bool { - Self::can_sign_with(None as Option>) + /// This should contain an indication of success and the account that was used for signing. + type Result; + + /// Submit a signed transaction to the local pool. + /// + /// Given `f` closure will be called for every requested account and expects a `Call` object + /// to be returned. + /// The call is then wrapped into a transaction (see `#CreateSignedTransaction`), signed and + /// submitted to the pool. + fn send_signed_transaction( + &self, + f: impl Fn(&Account) -> LocalCall, + ) -> Self::Result; + + /// Wraps the call into transaction, signs using given account and submits to the pool. + fn send_single_signed_transaction( + &self, + account: &Account, + call: LocalCall, + ) -> Option> { + let mut account_data = crate::Account::::get(&account.id); + debug::native::debug!( + target: "offchain", + "Creating signed transaction from account: {:?} (nonce: {:?})", + account.id, + account_data.nonce, + ); + let (call, signature) = T::create_transaction::( + call.into(), + account.public.clone(), + account.id.clone(), + account_data.nonce + )?; + let res = SubmitTransaction:: + ::submit_transaction(call, Some(signature)); + + if res.is_ok() { + // increment the nonce. This is fine, since the code should always + // be running in off-chain context, so we NEVER persists data. + account_data.nonce += One::one(); + crate::Account::::insert(&account.id, account_data); + } + + Some(res) } +} - /// Create and submit signed transactions from supported accounts. +/// Submit an unsigned transaction onchain with a signed payload +pub trait SendUnsignedTransaction< + T: SigningTypes + SendTransactionTypes, + LocalCall, +> { + /// A submission result. /// - /// This method should intersect given list of accounts with the ones - /// supported locally and submit signed transaction containing given `Call` - /// with every of them. + /// Should contain the submission result and the account(s) that signed the payload. + type Result; + + /// Send an unsigned transaction with a signed payload. /// - /// Returns a vector of results and account ids that were supported. - #[must_use] - fn submit_signed_from( - call: impl Into + Clone, - accounts: impl IntoIterator, - ) -> Vec<(T::AccountId, Result<(), ()>)> { - let keys = Self::find_local_keys(Some(accounts)); - keys.into_iter().map(|(account, pub_key)| { - let call = call.clone().into(); - ( - account, - Self::SignAndSubmit::sign_and_submit(call, pub_key) - ) - }).collect() - } - - /// Create and submit signed transactions from all local accounts. + /// This method takes `f` and `f2` where: + /// - `f` is called for every account and is expected to return a `SignedPayload` object. + /// - `f2` is then called with the `SignedPayload` returned by `f` and the signature and is + /// expected to return a `Call` object to be embedded into transaction. + fn send_unsigned_transaction( + &self, + f: F, + f2: impl Fn(TPayload, T::Signature) -> LocalCall, + ) -> Self::Result + where + F: Fn(&Account) -> TPayload, + TPayload: SignedPayload; + + /// Submits an unsigned call to the transaction pool. + fn submit_unsigned_transaction( + &self, + call: LocalCall + ) -> Option> { + Some(SubmitTransaction:: + ::submit_unsigned_transaction(call.into())) + } +} + +/// Utility trait to be implemented on payloads that can be signed. +pub trait SignedPayload: Encode { + /// Return a public key that is expected to have a matching key in the keystore, + /// which should be used to sign the payload. + fn public(&self) -> T::Public; + + /// Sign the payload using the implementor's provided public key. /// - /// This method submits a signed transaction from all local accounts - /// for given application crypto. + /// Returns `Some(signature)` if public key is supported. + fn sign>(&self) -> Option { + self.using_encoded(|payload| C::sign(payload, self.public())) + } + + /// Verify signature against payload. /// - /// Returns a vector of results and account ids that were supported. - #[must_use] - fn submit_signed( - call: impl Into + Clone, - ) -> Vec<(T::AccountId, Result<(), ()>)> { - let keys = Self::find_all_local_keys(); - keys.into_iter().map(|(account, pub_key)| { - let call = call.clone().into(); - ( - account, - Self::SignAndSubmit::sign_and_submit(call, pub_key) - ) - }).collect() + /// Returns a bool indicating whether the signature is valid or not. + fn verify>(&self, signature: T::Signature) -> bool { + self.using_encoded(|payload| C::verify(payload, self.public(), signature)) } } -/// A default type used to submit transactions to the pool. -/// -/// This is passed into each runtime as an opaque associated type that can have either of: -/// - [`SignAndSubmitTransaction`] -/// - [`SubmitUnsignedTransaction`] -/// - [`SubmitSignedTransaction`] -/// and used accordingly. -/// -/// This struct should be constructed by providing the following generic parameters: -/// * `Signer` - Usually the application specific key type (see `app_crypto`). -/// * `CreateTransaction` - A type that is able to produce signed transactions, -/// usually it's going to be the entire `Runtime` object. -/// * `Extrinsic` - A runtime-specific type for in-block extrinsics. -/// -/// If you only need the ability to submit unsigned transactions, -/// you may substitute both `Signer` and `CreateTransaction` with any type. -pub struct TransactionSubmitter { - _signer: sp_std::marker::PhantomData<(Signer, CreateTransaction, Extrinsic)>, -} -impl Default for TransactionSubmitter { - fn default() -> Self { - Self { - _signer: Default::default(), - } +#[cfg(test)] +mod tests { + use super::*; + use codec::Decode; + use crate::tests::{Test as TestRuntime, Call}; + use sp_core::offchain::{testing, TransactionPoolExt}; + use sp_runtime::testing::{UintAuthorityId, TestSignature, TestXt}; + + impl SigningTypes for TestRuntime { + type Public = UintAuthorityId; + type Signature = TestSignature; } -} -/// A blanket implementation to simplify creation of transaction signer & submitter in the runtime. -impl SignAndSubmitTransaction for TransactionSubmitter where - T: crate::Trait, - C: CreateTransaction, - S: Signer<>::Public, >::Signature>, - E: ExtrinsicT + Encode, -{ - type Extrinsic = E; - type CreateTransaction = C; - type Signer = S; -} + type Extrinsic = TestXt; -/// A blanket implementation to use the same submitter for unsigned transactions as well. -impl SubmitUnsignedTransaction for TransactionSubmitter where - T: crate::Trait, - E: ExtrinsicT + Encode, -{ - type Extrinsic = E; -} + impl SendTransactionTypes for TestRuntime { + type Extrinsic = Extrinsic; + type OverarchingCall = Call; + } -/// A blanket implementation to support local keystore of application-crypto types. -impl SubmitSignedTransaction for TransactionSubmitter where - T: crate::Trait, - C: CreateTransaction, - E: ExtrinsicT + Encode, - S: Signer<>::Public, >::Signature>, - // Make sure we can unwrap the app crypto key. - S: RuntimeAppPublic + AppPublic + Into<::Generic>, - // Make sure we can convert from wrapped crypto to public key (e.g. `MultiSigner`) - S::Generic: Into>, - // For simplicity we require the same trait to implement `SignAndSubmitTransaction` too. - Self: SignAndSubmitTransaction, -{ - type SignAndSubmit = Self; - - fn find_local_keys(accounts: Option>) -> Vec<( - T::AccountId, - PublicOf, - )> { - // Convert app-specific keys into generic ones. - let local_accounts_and_keys = S::all() - .into_iter() - .map(|app_key| { - // unwrap app-crypto - let generic_pub_key: ::Generic = app_key.into(); - // convert to expected public key type (might be MultiSigner) - let signer_pub_key: PublicOf = generic_pub_key.into(); - // lookup accountid for that pubkey - let account = signer_pub_key.clone().into_account(); - (account, signer_pub_key) - }).collect::>(); - - if let Some(accounts) = accounts { - let mut local_accounts_and_keys = local_accounts_and_keys; - // sort by accountId to allow bin-search. - local_accounts_and_keys.sort_by(|a, b| a.0.cmp(&b.0)); - - // get all the matching accounts - accounts.into_iter().filter_map(|acc| { - let idx = local_accounts_and_keys.binary_search_by(|a| a.0.cmp(&acc)).ok()?; - local_accounts_and_keys.get(idx).cloned() - }).collect() - } else { - // just return all account ids and keys - local_accounts_and_keys - } + #[derive(codec::Encode, codec::Decode)] + struct SimplePayload { + pub public: UintAuthorityId, + pub data: Vec, } - fn can_sign_with(accounts: Option>) -> bool { - // early exit if we care about any account. - if accounts.is_none() { - !S::all().is_empty() - } else { - !Self::find_local_keys(accounts).is_empty() + impl SignedPayload for SimplePayload { + fn public(&self) -> UintAuthorityId { + self.public.clone() } } + + struct DummyAppCrypto; + // Bind together the `SigningTypes` with app-crypto and the wrapper types. + // here the implementation is pretty dummy, because we use the same type for + // both application-specific crypto and the runtime crypto, but in real-life + // runtimes it's going to use different types everywhere. + impl AppCrypto for DummyAppCrypto { + type RuntimeAppPublic = UintAuthorityId; + type GenericPublic = UintAuthorityId; + type GenericSignature = TestSignature; + } + + fn assert_account( + next: Option<(Account, Result<(), ()>)>, + index: usize, + id: u64, + ) { + assert_eq!(next, Some((Account { + index, + id, + public: id.into(), + }, Ok(())))); + } + + #[test] + fn should_send_unsigned_with_signed_payload_with_all_accounts() { + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + + let mut t = sp_io::TestExternalities::default(); + t.register_extension(TransactionPoolExt::new(pool)); + + // given + UintAuthorityId::set_all_keys(vec![0xf0, 0xf1, 0xf2]); + + t.execute_with(|| { + // when + let result = Signer:: + ::all_accounts() + .send_unsigned_transaction( + |account| SimplePayload { + data: vec![1, 2, 3], + public: account.public.clone() + }, + |_payload, _signature| { + Call + } + ); + + // then + let mut res = result.into_iter(); + assert_account(res.next(), 0, 0xf0); + assert_account(res.next(), 1, 0xf1); + assert_account(res.next(), 2, 0xf2); + assert_eq!(res.next(), None); + + // check the transaction pool content: + let tx1 = pool_state.write().transactions.pop().unwrap(); + let _tx2 = pool_state.write().transactions.pop().unwrap(); + let _tx3 = pool_state.write().transactions.pop().unwrap(); + assert!(pool_state.read().transactions.is_empty()); + let tx1 = Extrinsic::decode(&mut &*tx1).unwrap(); + assert_eq!(tx1.signature, None); + }); + } + + #[test] + fn should_send_unsigned_with_signed_payload_with_any_account() { + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + + let mut t = sp_io::TestExternalities::default(); + t.register_extension(TransactionPoolExt::new(pool)); + + // given + UintAuthorityId::set_all_keys(vec![0xf0, 0xf1, 0xf2]); + + t.execute_with(|| { + // when + let result = Signer:: + ::any_account() + .send_unsigned_transaction( + |account| SimplePayload { + data: vec![1, 2, 3], + public: account.public.clone() + }, + |_payload, _signature| { + Call + } + ); + + // then + let mut res = result.into_iter(); + assert_account(res.next(), 0, 0xf0); + assert_eq!(res.next(), None); + + // check the transaction pool content: + let tx1 = pool_state.write().transactions.pop().unwrap(); + assert!(pool_state.read().transactions.is_empty()); + let tx1 = Extrinsic::decode(&mut &*tx1).unwrap(); + assert_eq!(tx1.signature, None); + }); + } + + #[test] + fn should_send_unsigned_with_signed_payload_with_all_account_and_filter() { + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + + let mut t = sp_io::TestExternalities::default(); + t.register_extension(TransactionPoolExt::new(pool)); + + // given + UintAuthorityId::set_all_keys(vec![0xf0, 0xf1, 0xf2]); + + t.execute_with(|| { + // when + let result = Signer:: + ::all_accounts() + .with_filter(vec![0xf2.into(), 0xf1.into()]) + .send_unsigned_transaction( + |account| SimplePayload { + data: vec![1, 2, 3], + public: account.public.clone() + }, + |_payload, _signature| { + Call + } + ); + + // then + let mut res = result.into_iter(); + assert_account(res.next(), 0, 0xf2); + assert_account(res.next(), 1, 0xf1); + assert_eq!(res.next(), None); + + // check the transaction pool content: + let tx1 = pool_state.write().transactions.pop().unwrap(); + let _tx2 = pool_state.write().transactions.pop().unwrap(); + assert!(pool_state.read().transactions.is_empty()); + let tx1 = Extrinsic::decode(&mut &*tx1).unwrap(); + assert_eq!(tx1.signature, None); + }); + } + + #[test] + fn should_send_unsigned_with_signed_payload_with_any_account_and_filter() { + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + + let mut t = sp_io::TestExternalities::default(); + t.register_extension(TransactionPoolExt::new(pool)); + + // given + UintAuthorityId::set_all_keys(vec![0xf0, 0xf1, 0xf2]); + + t.execute_with(|| { + // when + let result = Signer:: + ::any_account() + .with_filter(vec![0xf2.into(), 0xf1.into()]) + .send_unsigned_transaction( + |account| SimplePayload { + data: vec![1, 2, 3], + public: account.public.clone() + }, + |_payload, _signature| { + Call + } + ); + + // then + let mut res = result.into_iter(); + assert_account(res.next(), 0, 0xf2); + assert_eq!(res.next(), None); + + // check the transaction pool content: + let tx1 = pool_state.write().transactions.pop().unwrap(); + assert!(pool_state.read().transactions.is_empty()); + let tx1 = Extrinsic::decode(&mut &*tx1).unwrap(); + assert_eq!(tx1.signature, None); + }); + } + } diff --git a/frame/timestamp/Cargo.toml b/frame/timestamp/Cargo.toml index eb7358e197a189dd33b55e985c1b9a65b86a72d8..7691421bbfed16b4726d1eb1f3752fe8868026d8 100644 --- a/frame/timestamp/Cargo.toml +++ b/frame/timestamp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-timestamp" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,23 +9,26 @@ repository = "https://github.com/paritytech/substrate/" description = "FRAME Timestamp Module" documentation = "https://docs.rs/pallet-timestamp" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io", optional = true } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" } -frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -sp-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/timestamp" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io", optional = true } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../../primitives/inherents" } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../benchmarking", optional = true } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +sp-timestamp = { version = "2.0.0-dev", default-features = false, path = "../../primitives/timestamp" } impl-trait-for-tuples = "0.1.3" [dev-dependencies] -sp-io ={ version = "2.0.0-alpha.5", path = "../../primitives/io" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } +sp-io ={ version = "2.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } [features] default = ["std"] @@ -41,6 +44,3 @@ std = [ "sp-timestamp/std" ] runtime-benchmarks = ["frame-benchmarking", "sp-io"] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/timestamp/src/benchmarking.rs b/frame/timestamp/src/benchmarking.rs index 65b4dbf2b079ea23d4b2622b61847b99e41ce518..c468bf82fba6f9570c471ce7aa615f2658db3389 100644 --- a/frame/timestamp/src/benchmarking.rs +++ b/frame/timestamp/src/benchmarking.rs @@ -21,16 +21,44 @@ use super::*; use sp_std::prelude::*; use frame_system::RawOrigin; +use frame_support::{ensure, traits::OnFinalize}; use frame_benchmarking::benchmarks; +use crate::Module as Timestamp; + const MAX_TIME: u32 = 100; benchmarks! { - _ { - let n in 1 .. MAX_TIME => (); - } + _ { } set { - let n in ...; - }: _(RawOrigin::None, n.into()) + let t in 1 .. MAX_TIME; + }: _(RawOrigin::None, t.into()) + verify { + ensure!(Timestamp::::now() == t.into(), "Time was not set."); + } + + on_finalize { + let t in 1 .. MAX_TIME; + Timestamp::::set(RawOrigin::None.into(), t.into())?; + ensure!(DidUpdate::exists(), "Time was not set."); + }: { Timestamp::::on_finalize(t.into()); } + verify { + ensure!(!DidUpdate::exists(), "Time was not removed."); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_set::()); + assert_ok!(test_benchmark_on_finalize::()); + }); + } } diff --git a/frame/timestamp/src/lib.rs b/frame/timestamp/src/lib.rs index 8ba756d68329141b7bd7ad9512bb279d475cdd69..a2a6762464eb75f9224d5714ccc6705165c387ab 100644 --- a/frame/timestamp/src/lib.rs +++ b/frame/timestamp/src/lib.rs @@ -69,7 +69,7 @@ //! //! decl_module! { //! pub struct Module for enum Call where origin: T::Origin { -//! #[weight = frame_support::weights::SimpleDispatchInfo::default()] +//! #[weight = 0] //! pub fn get_time(origin) -> dispatch::DispatchResult { //! let _sender = ensure_signed(origin)?; //! let _now = >::get(); @@ -100,7 +100,7 @@ use frame_support::debug; use frame_support::{ Parameter, decl_storage, decl_module, traits::{Time, UnixTime, Get}, - weights::SimpleDispatchInfo, + weights::{DispatchClass}, }; use sp_runtime::{ RuntimeString, @@ -147,7 +147,13 @@ decl_module! { /// `MinimumPeriod`. /// /// The dispatch origin for this call must be `Inherent`. - #[weight = SimpleDispatchInfo::FixedMandatory(10_000)] + /// + /// # + /// - `O(T)` where `T` complexity of `on_timestamp_set` + /// - 2 storage mutations (codec `O(1)`). + /// - 1 event handler `on_timestamp_set` `O(T)`. + /// # + #[weight = (0, DispatchClass::Mandatory)] fn set(origin, #[compact] now: T::Moment) { ensure_none(origin)?; assert!(!::DidUpdate::exists(), "Timestamp must be updated only once in the block"); @@ -161,6 +167,10 @@ decl_module! { >::on_timestamp_set(now); } + /// # + /// - `O(1)` + /// - 1 storage deletion (codec `O(1)`). + /// # fn on_finalize() { assert!(::DidUpdate::take(), "Timestamp must be updated once in the block"); } @@ -271,6 +281,11 @@ mod tests { use sp_core::H256; use sp_runtime::{Perbill, traits::{BlakeTwo256, IdentityLookup}, testing::Header}; + pub fn new_test_ext() -> TestExternalities { + let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + TestExternalities::new(t) + } + impl_outer_origin! { pub enum Origin for Test where system = frame_system {} } @@ -296,6 +311,9 @@ mod tests { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); @@ -316,8 +334,7 @@ mod tests { #[test] fn timestamp_works() { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - TestExternalities::new(t).execute_with(|| { + new_test_ext().execute_with(|| { Timestamp::set_timestamp(42); assert_ok!(Timestamp::dispatch(Call::set(69), Origin::NONE)); assert_eq!(Timestamp::now(), 69); @@ -327,8 +344,7 @@ mod tests { #[test] #[should_panic(expected = "Timestamp must be updated only once in the block")] fn double_timestamp_should_fail() { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - TestExternalities::new(t).execute_with(|| { + new_test_ext().execute_with(|| { Timestamp::set_timestamp(42); assert_ok!(Timestamp::dispatch(Call::set(69), Origin::NONE)); let _ = Timestamp::dispatch(Call::set(70), Origin::NONE); @@ -338,8 +354,7 @@ mod tests { #[test] #[should_panic(expected = "Timestamp must increment by at least between sequential blocks")] fn block_period_minimum_enforced() { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - TestExternalities::new(t).execute_with(|| { + new_test_ext().execute_with(|| { Timestamp::set_timestamp(42); let _ = Timestamp::dispatch(Call::set(46), Origin::NONE); }); diff --git a/frame/transaction-payment/Cargo.toml b/frame/transaction-payment/Cargo.toml index 811c2885b1bdaa674247c7e9510531efa8fe0e95..a969f3008688435c0401c0d91eb0988919d90a03 100644 --- a/frame/transaction-payment/Cargo.toml +++ b/frame/transaction-payment/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-transaction-payment" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,18 +8,22 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet to manage transaction payments" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-alpha.5", default-features = false, path = "./rpc/runtime-api" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-dev", default-features = false, path = "./rpc/runtime-api" } [dev-dependencies] -sp-io = { version = "2.0.0-alpha.5", path = "../../primitives/io" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } +sp-io = { version = "2.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } +sp-storage = { version = "2.0.0-dev", path = "../../primitives/storage" } [features] default = ["std"] @@ -31,6 +35,3 @@ std = [ "frame-system/std", "pallet-transaction-payment-rpc-runtime-api/std" ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/transaction-payment/rpc/Cargo.toml b/frame/transaction-payment/rpc/Cargo.toml index cd74829b29b049fb18b29a661b00bd8763c3c725..35f421915af3975a6ad820523a45207a564af327 100644 --- a/frame/transaction-payment/rpc/Cargo.toml +++ b/frame/transaction-payment/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-transaction-payment-rpc" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,18 +8,18 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "RPC interface for the transaction payment module." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0" } jsonrpc-core = "14.0.3" jsonrpc-core-client = "14.0.5" jsonrpc-derive = "14.0.3" -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sp-rpc = { version = "2.0.0-alpha.5", path = "../../../primitives/rpc" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sp-rpc = { version = "2.0.0-dev", path = "../../../primitives/rpc" } serde = { version = "1.0.101", features = ["derive"] } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.5", path = "../../../primitives/api" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } -pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-alpha.5", path = "./runtime-api" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-dev", path = "../../../primitives/api" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } +pallet-transaction-payment-rpc-runtime-api = { version = "2.0.0-dev", path = "./runtime-api" } diff --git a/frame/transaction-payment/rpc/runtime-api/Cargo.toml b/frame/transaction-payment/rpc/runtime-api/Cargo.toml index 447590111c7ee65ee098a2a508647c5fe7cefc57..1170e043eea84dccf37aef9ba1bd9bb35ad7bb08 100644 --- a/frame/transaction-payment/rpc/runtime-api/Cargo.toml +++ b/frame/transaction-payment/rpc/runtime-api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-transaction-payment-rpc-runtime-api" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,13 +8,16 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "RPC runtime API for transaction payment FRAME pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../../../primitives/api" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../../../../primitives/api" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../../../support" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../../../primitives/std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../../../primitives/runtime" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../../../support" } [dev-dependencies] serde_json = "1.0.41" @@ -29,6 +32,3 @@ std = [ "sp-runtime/std", "frame-support/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/transaction-payment/src/lib.rs b/frame/transaction-payment/src/lib.rs index 1078927747b228cd7b7787aa2d911ab41818ef7d..c5992ab1298b8b293eb9c149121afbcbf76a45f8 100644 --- a/frame/transaction-payment/src/lib.rs +++ b/frame/transaction-payment/src/lib.rs @@ -36,19 +36,23 @@ use codec::{Encode, Decode}; use frame_support::{ decl_storage, decl_module, traits::{Currency, Get, OnUnbalanced, ExistenceRequirement, WithdrawReason, Imbalance}, - weights::{Weight, DispatchInfo, GetDispatchInfo}, + weights::{Weight, DispatchInfo, PostDispatchInfo, GetDispatchInfo, Pays}, + dispatch::DispatchResult, }; use sp_runtime::{ - Fixed64, + Fixed128, transaction_validity::{ TransactionPriority, ValidTransaction, InvalidTransaction, TransactionValidityError, TransactionValidity, }, - traits::{Zero, Saturating, SignedExtension, SaturatedConversion, Convert}, + traits::{ + Zero, Saturating, SignedExtension, SaturatedConversion, Convert, Dispatchable, + DispatchInfoOf, PostDispatchInfoOf, UniqueSaturatedFrom, UniqueSaturatedInto, + }, }; use pallet_transaction_payment_rpc_runtime_api::RuntimeDispatchInfo; -type Multiplier = Fixed64; +type Multiplier = Fixed128; type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; type NegativeImbalanceOf = @@ -63,9 +67,6 @@ pub trait Trait: frame_system::Trait { /// if any. type OnTransactionPayment: OnUnbalanced>; - /// The fee to be paid for making a transaction; the base. - type TransactionBaseFee: Get>; - /// The fee to be paid for making a transaction; the per-byte portion. type TransactionByteFee: Get>; @@ -84,9 +85,6 @@ decl_storage! { decl_module! { pub struct Module for enum Call where origin: T::Origin { - /// The fee to be paid for making a transaction; the base. - const TransactionBaseFee: BalanceOf = T::TransactionBaseFee::get(); - /// The fee to be paid for making a transaction; the per-byte portion. const TransactionByteFee: BalanceOf = T::TransactionByteFee::get(); @@ -95,6 +93,20 @@ decl_module! { *fm = T::FeeMultiplierUpdate::convert(*fm) }); } + + fn on_runtime_upgrade() -> Weight { + // TODO: Remove this code after on-chain upgrade from u32 to u64 weights + use sp_runtime::Fixed64; + use frame_support::migration::take_storage_value; + if let Some(old_next_fee_multiplier) = take_storage_value::(b"TransactionPayment", b"NextFeeMultiplier", &[]) { + let raw_multiplier = old_next_fee_multiplier.into_inner() as i128; + // Fixed64 used 10^9 precision, where Fixed128 uses 10^18, so we need to add 9 zeros. + let new_raw_multiplier: i128 = raw_multiplier.saturating_mul(1_000_000_000); + let new_next_fee_multiplier: Fixed128 = Fixed128::from_parts(new_raw_multiplier); + NextFeeMultiplier::put(new_next_fee_multiplier); + } + 0 + } } } @@ -113,6 +125,7 @@ impl Module { where T: Send + Sync, BalanceOf: Send + Sync, + T::Call: Dispatchable, { // NOTE: we can actually make it understand `ChargeTransactionPayment`, but would be some // hassle for sure. We have to make it aware of the index of `ChargeTransactionPayment` in @@ -121,24 +134,11 @@ impl Module { // a very very little potential gain in the future. let dispatch_info = ::get_dispatch_info(&unchecked_extrinsic); - let partial_fee = - >::compute_fee(len, dispatch_info, 0u32.into()); + let partial_fee = Self::compute_fee(len, &dispatch_info, 0u32.into()); let DispatchInfo { weight, class, .. } = dispatch_info; RuntimeDispatchInfo { weight, class, partial_fee } } -} - -/// Require the transactor pay for themselves and maybe include a tip to gain additional priority -/// in the queue. -#[derive(Encode, Decode, Clone, Eq, PartialEq)] -pub struct ChargeTransactionPayment(#[codec(compact)] BalanceOf); - -impl ChargeTransactionPayment { - /// utility constructor. Used only in client/factory code. - pub fn from(fee: BalanceOf) -> Self { - Self(fee) - } /// Compute the final fee value for a particular transaction. /// @@ -156,37 +156,92 @@ impl ChargeTransactionPayment { /// final_fee = base_fee + targeted_fee_adjustment(len_fee + weight_fee) + tip; pub fn compute_fee( len: u32, - info: ::DispatchInfo, + info: &DispatchInfoOf, tip: BalanceOf, - ) -> BalanceOf - where - BalanceOf: Sync + Send, + ) -> BalanceOf where + T::Call: Dispatchable, { - if info.pays_fee { + if info.pays_fee == Pays::Yes { let len = >::from(len); let per_byte = T::TransactionByteFee::get(); let len_fee = per_byte.saturating_mul(len); - - let weight_fee = { - // cap the weight to the maximum defined in runtime, otherwise it will be the - // `Bounded` maximum of its data type, which is not desired. - let capped_weight = info.weight - .min(::MaximumBlockWeight::get()); - T::WeightToFee::convert(capped_weight) - }; + let unadjusted_weight_fee = Self::weight_to_fee(info.weight); // the adjustable part of the fee - let adjustable_fee = len_fee.saturating_add(weight_fee); + let adjustable_fee = len_fee.saturating_add(unadjusted_weight_fee); let targeted_fee_adjustment = NextFeeMultiplier::get(); - // adjusted_fee = adjustable_fee + (adjustable_fee * targeted_fee_adjustment) - let adjusted_fee = targeted_fee_adjustment.saturated_multiply_accumulate(adjustable_fee); + let adjusted_fee = targeted_fee_adjustment.saturated_multiply_accumulate(adjustable_fee.saturated_into()); - let base_fee = T::TransactionBaseFee::get(); - base_fee.saturating_add(adjusted_fee).saturating_add(tip) + let base_fee = Self::weight_to_fee(T::ExtrinsicBaseWeight::get()); + base_fee.saturating_add(adjusted_fee.saturated_into()).saturating_add(tip) } else { tip } } + + /// Compute the fee for the specified weight. + /// + /// This fee is already adjusted by the per block fee adjustment factor and is therefore + /// the share that the weight contributes to the overall fee of a transaction. + pub fn weight_to_fee_with_adjustment(weight: Weight) -> Balance where + Balance: UniqueSaturatedFrom + { + let fee = UniqueSaturatedInto::::unique_saturated_into(Self::weight_to_fee(weight)); + UniqueSaturatedFrom::unique_saturated_from( + NextFeeMultiplier::get().saturated_multiply_accumulate(fee) + ) + } + + fn weight_to_fee(weight: Weight) -> BalanceOf { + // cap the weight to the maximum defined in runtime, otherwise it will be the + // `Bounded` maximum of its data type, which is not desired. + let capped_weight = weight.min(::MaximumBlockWeight::get()); + T::WeightToFee::convert(capped_weight) + } +} + +/// Require the transactor pay for themselves and maybe include a tip to gain additional priority +/// in the queue. +#[derive(Encode, Decode, Clone, Eq, PartialEq)] +pub struct ChargeTransactionPayment(#[codec(compact)] BalanceOf); + +impl ChargeTransactionPayment where + T::Call: Dispatchable, + BalanceOf: Send + Sync, +{ + /// utility constructor. Used only in client/factory code. + pub fn from(fee: BalanceOf) -> Self { + Self(fee) + } + + fn withdraw_fee( + &self, + who: &T::AccountId, + info: &DispatchInfoOf, + len: usize, + ) -> Result<(BalanceOf, Option>), TransactionValidityError> { + let tip = self.0; + let fee = Module::::compute_fee(len as u32, info, tip); + + // Only mess with balances if fee is not zero. + if fee.is_zero() { + return Ok((fee, None)); + } + + match T::Currency::withdraw( + who, + fee, + if tip.is_zero() { + WithdrawReason::TransactionPayment.into() + } else { + WithdrawReason::TransactionPayment | WithdrawReason::Tip + }, + ExistenceRequirement::KeepAlive, + ) { + Ok(imbalance) => Ok((fee, Some(imbalance))), + Err(_) => Err(InvalidTransaction::Payment.into()), + } + } } impl sp_std::fmt::Debug for ChargeTransactionPayment { @@ -200,46 +255,25 @@ impl sp_std::fmt::Debug for ChargeTransactionPayment } } -impl SignedExtension for ChargeTransactionPayment - where BalanceOf: Send + Sync +impl SignedExtension for ChargeTransactionPayment where + BalanceOf: Send + Sync + From, + T::Call: Dispatchable, { const IDENTIFIER: &'static str = "ChargeTransactionPayment"; type AccountId = T::AccountId; type Call = T::Call; type AdditionalSigned = (); - type DispatchInfo = DispatchInfo; - type Pre = (); + type Pre = (BalanceOf, Self::AccountId, Option>); fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } fn validate( &self, who: &Self::AccountId, _call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { - // pay any fees. - let tip = self.0; - let fee = Self::compute_fee(len as u32, info, tip); - // Only mess with balances if fee is not zero. - if !fee.is_zero() { - let imbalance = match T::Currency::withdraw( - who, - fee, - if tip.is_zero() { - WithdrawReason::TransactionPayment.into() - } else { - WithdrawReason::TransactionPayment | WithdrawReason::Tip - }, - ExistenceRequirement::KeepAlive, - ) { - Ok(imbalance) => imbalance, - Err(_) => return InvalidTransaction::Payment.into(), - }; - let imbalances = imbalance.split(tip); - T::OnTransactionPayment::on_unbalanceds(Some(imbalances.0).into_iter() - .chain(Some(imbalances.1))); - } + let (fee, _) = self.withdraw_fee(who, info, len)?; let mut r = ValidTransaction::default(); // NOTE: we probably want to maximize the _fee (of any type) per weight unit_ here, which @@ -247,15 +281,57 @@ impl SignedExtension for ChargeTransactionPayment r.priority = fee.saturated_into::(); Ok(r) } + + fn pre_dispatch( + self, + who: &Self::AccountId, + _call: &Self::Call, + info: &DispatchInfoOf, + len: usize + ) -> Result { + let (_, imbalance) = self.withdraw_fee(who, info, len)?; + Ok((self.0, who.clone(), imbalance)) + } + + fn post_dispatch( + pre: Self::Pre, + info: &DispatchInfoOf, + post_info: &PostDispatchInfoOf, + _len: usize, + _result: &DispatchResult, + ) -> Result<(), TransactionValidityError> { + let (tip, who, imbalance) = pre; + if let Some(payed) = imbalance { + let refund = Module::::weight_to_fee_with_adjustment(post_info.calc_unspent(info)); + let actual_payment = match T::Currency::deposit_into_existing(&who, refund) { + Ok(refund_imbalance) => { + // The refund cannot be larger than the up front payed max weight. + // `PostDispatchInfo::calc_unspent` guards against such a case. + match payed.offset(refund_imbalance) { + Ok(actual_payment) => actual_payment, + Err(_) => return Err(InvalidTransaction::Payment.into()), + } + } + // We do not recreate the account using the refund. The up front payment + // is gone in that case. + Err(_) => payed, + }; + let imbalances = actual_payment.split(tip); + T::OnTransactionPayment::on_unbalanceds(Some(imbalances.0).into_iter() + .chain(Some(imbalances.1))); + } + Ok(()) + } } #[cfg(test)] mod tests { use super::*; + use core::num::NonZeroI128; use codec::Encode; use frame_support::{ impl_outer_dispatch, impl_outer_origin, parameter_types, - weights::{DispatchClass, DispatchInfo, GetDispatchInfo, Weight}, + weights::{DispatchClass, DispatchInfo, PostDispatchInfo, GetDispatchInfo, Weight}, }; use pallet_balances::Call as BalancesCall; use pallet_transaction_payment_rpc_runtime_api::RuntimeDispatchInfo; @@ -285,6 +361,15 @@ mod tests { pub enum Origin for Runtime {} } + thread_local! { + static EXTRINSIC_BASE_WEIGHT: RefCell = RefCell::new(0); + } + + pub struct ExtrinsicBaseWeight; + impl Get for ExtrinsicBaseWeight { + fn get() -> u64 { EXTRINSIC_BASE_WEIGHT.with(|v| *v.borrow()) } + } + parameter_types! { pub const BlockHashCount: u64 = 250; pub const MaximumBlockWeight: Weight = 1024; @@ -305,6 +390,9 @@ mod tests { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = ExtrinsicBaseWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -326,16 +414,10 @@ mod tests { type AccountStore = System; } thread_local! { - static TRANSACTION_BASE_FEE: RefCell = RefCell::new(0); static TRANSACTION_BYTE_FEE: RefCell = RefCell::new(1); static WEIGHT_TO_FEE: RefCell = RefCell::new(1); } - pub struct TransactionBaseFee; - impl Get for TransactionBaseFee { - fn get() -> u64 { TRANSACTION_BASE_FEE.with(|v| *v.borrow()) } - } - pub struct TransactionByteFee; impl Get for TransactionByteFee { fn get() -> u64 { TRANSACTION_BYTE_FEE.with(|v| *v.borrow()) } @@ -351,7 +433,6 @@ mod tests { impl Trait for Runtime { type Currency = pallet_balances::Module; type OnTransactionPayment = (); - type TransactionBaseFee = TransactionBaseFee; type TransactionByteFee = TransactionByteFee; type WeightToFee = WeightToFee; type FeeMultiplierUpdate = (); @@ -363,7 +444,7 @@ mod tests { pub struct ExtBuilder { balance_factor: u64, - base_fee: u64, + base_weight: u64, byte_fee: u64, weight_to_fee: u64 } @@ -372,7 +453,7 @@ mod tests { fn default() -> Self { Self { balance_factor: 1, - base_fee: 0, + base_weight: 0, byte_fee: 1, weight_to_fee: 1, } @@ -380,8 +461,8 @@ mod tests { } impl ExtBuilder { - pub fn base_fee(mut self, base_fee: u64) -> Self { - self.base_fee = base_fee; + pub fn base_weight(mut self, base_weight: u64) -> Self { + self.base_weight = base_weight; self } pub fn byte_fee(mut self, byte_fee: u64) -> Self { @@ -397,7 +478,7 @@ mod tests { self } fn set_constants(&self) { - TRANSACTION_BASE_FEE.with(|v| *v.borrow_mut() = self.base_fee); + EXTRINSIC_BASE_WEIGHT.with(|v| *v.borrow_mut() = self.base_weight); TRANSACTION_BYTE_FEE.with(|v| *v.borrow_mut() = self.byte_fee); WEIGHT_TO_FEE.with(|v| *v.borrow_mut() = self.weight_to_fee); } @@ -424,31 +505,77 @@ mod tests { /// create a transaction info struct from weight. Handy to avoid building the whole struct. pub fn info_from_weight(w: Weight) -> DispatchInfo { - DispatchInfo { weight: w, pays_fee: true, ..Default::default() } + // pays: yes -- class: normal + DispatchInfo { weight: w, ..Default::default() } + } + + fn post_info_from_weight(w: Weight) -> PostDispatchInfo { + PostDispatchInfo { actual_weight: Some(w), } + } + + fn default_post_info() -> PostDispatchInfo { + PostDispatchInfo { actual_weight: None, } } #[test] fn signed_extension_transaction_payment_work() { ExtBuilder::default() .balance_factor(10) - .base_fee(5) + .base_weight(5) .build() .execute_with(|| { let len = 10; + let pre = ChargeTransactionPayment::::from(0) + .pre_dispatch(&1, CALL, &info_from_weight(5), len) + .unwrap(); + assert_eq!(Balances::free_balance(1), 100 - 5 - 5 - 10); + assert!( - ChargeTransactionPayment::::from(0) - .pre_dispatch(&1, CALL, info_from_weight(5), len) + ChargeTransactionPayment:: + ::post_dispatch(pre, &info_from_weight(5), &default_post_info(), len, &Ok(())) .is_ok() ); assert_eq!(Balances::free_balance(1), 100 - 5 - 5 - 10); + let pre = ChargeTransactionPayment::::from(5 /* tipped */) + .pre_dispatch(&2, CALL, &info_from_weight(100), len) + .unwrap(); + assert_eq!(Balances::free_balance(2), 200 - 5 - 10 - 100 - 5); + assert!( - ChargeTransactionPayment::::from(5 /* tipped */) - .pre_dispatch(&2, CALL, info_from_weight(3), len) + ChargeTransactionPayment:: + ::post_dispatch(pre, &info_from_weight(100), &post_info_from_weight(50), len, &Ok(())) .is_ok() ); - assert_eq!(Balances::free_balance(2), 200 - 5 - 10 - 3 - 5); + assert_eq!(Balances::free_balance(2), 200 - 5 - 10 - 50 - 5); + }); + } + + #[test] + fn signed_extension_transaction_payment_multiplied_refund_works() { + ExtBuilder::default() + .balance_factor(10) + .base_weight(5) + .build() + .execute_with(|| + { + let len = 10; + NextFeeMultiplier::put(Fixed128::from_rational(1, NonZeroI128::new(2).unwrap())); + + let pre = ChargeTransactionPayment::::from(5 /* tipped */) + .pre_dispatch(&2, CALL, &info_from_weight(100), len) + .unwrap(); + // 5 base fee, 3/2 * 10 byte fee, 3/2 * 100 weight fee, 5 tip + assert_eq!(Balances::free_balance(2), 200 - 5 - 15 - 150 - 5); + + assert!( + ChargeTransactionPayment:: + ::post_dispatch(pre, &info_from_weight(100), &post_info_from_weight(50), len, &Ok(())) + .is_ok() + ); + // 75 (3/2 of the returned 50 units of weight ) is refunded + assert_eq!(Balances::free_balance(2), 200 - 5 - 15 - 75 - 5); }); } @@ -463,7 +590,7 @@ mod tests { // maximum weight possible assert!( ChargeTransactionPayment::::from(0) - .pre_dispatch(&1, CALL, info_from_weight(Weight::max_value()), 10) + .pre_dispatch(&1, CALL, &info_from_weight(Weight::max_value()), 10) .is_ok() ); // fee will be proportional to what is the actual maximum weight in the runtime. @@ -477,7 +604,7 @@ mod tests { #[test] fn signed_extension_allows_free_transactions() { ExtBuilder::default() - .base_fee(100) + .base_weight(100) .balance_factor(0) .build() .execute_with(|| @@ -491,11 +618,11 @@ mod tests { let operational_transaction = DispatchInfo { weight: 0, class: DispatchClass::Operational, - pays_fee: false, + pays_fee: Pays::No, }; assert!( ChargeTransactionPayment::::from(0) - .validate(&1, CALL, operational_transaction , len) + .validate(&1, CALL, &operational_transaction , len) .is_ok() ); @@ -503,11 +630,11 @@ mod tests { let free_transaction = DispatchInfo { weight: 0, class: DispatchClass::Normal, - pays_fee: true, + pays_fee: Pays::Yes, }; assert!( ChargeTransactionPayment::::from(0) - .validate(&1, CALL, free_transaction , len) + .validate(&1, CALL, &free_transaction , len) .is_err() ); }); @@ -516,18 +643,18 @@ mod tests { #[test] fn signed_ext_length_fee_is_also_updated_per_congestion() { ExtBuilder::default() - .base_fee(5) + .base_weight(5) .balance_factor(10) .build() .execute_with(|| { // all fees should be x1.5 - NextFeeMultiplier::put(Fixed64::from_rational(1, 2)); + NextFeeMultiplier::put(Fixed128::from_rational(1, NonZeroI128::new(2).unwrap())); let len = 10; assert!( ChargeTransactionPayment::::from(10) // tipped - .pre_dispatch(&1, CALL, info_from_weight(3), len) + .pre_dispatch(&1, CALL, &info_from_weight(3), len) .is_ok() ); assert_eq!(Balances::free_balance(1), 100 - 10 - 5 - (10 + 3) * 3 / 2); @@ -544,13 +671,13 @@ mod tests { let ext = xt.encode(); let len = ext.len() as u32; ExtBuilder::default() - .base_fee(5) + .base_weight(5) .weight_fee(2) .build() .execute_with(|| { // all fees should be x1.5 - NextFeeMultiplier::put(Fixed64::from_rational(1, 2)); + NextFeeMultiplier::put(Fixed128::from_rational(1, NonZeroI128::new(2).unwrap())); assert_eq!( TransactionPayment::query_info(xt, len), @@ -558,7 +685,7 @@ mod tests { weight: info.weight, class: info.class, partial_fee: - 5 /* base */ + 5 * 2 /* base * weight_fee */ + ( len as u64 /* len * 1 */ + info.weight.min(MaximumBlockWeight::get()) as u64 * 2 /* weight * weight_to_fee */ @@ -572,80 +699,80 @@ mod tests { #[test] fn compute_fee_works_without_multiplier() { ExtBuilder::default() - .base_fee(100) + .base_weight(100) .byte_fee(10) .balance_factor(0) .build() .execute_with(|| { // Next fee multiplier is zero - assert_eq!(NextFeeMultiplier::get(), Fixed64::from_natural(0)); + assert_eq!(NextFeeMultiplier::get(), Fixed128::from_natural(0)); // Tip only, no fees works let dispatch_info = DispatchInfo { weight: 0, class: DispatchClass::Operational, - pays_fee: false, + pays_fee: Pays::No, }; - assert_eq!(ChargeTransactionPayment::::compute_fee(0, dispatch_info, 10), 10); + assert_eq!(Module::::compute_fee(0, &dispatch_info, 10), 10); // No tip, only base fee works let dispatch_info = DispatchInfo { weight: 0, class: DispatchClass::Operational, - pays_fee: true, + pays_fee: Pays::Yes, }; - assert_eq!(ChargeTransactionPayment::::compute_fee(0, dispatch_info, 0), 100); + assert_eq!(Module::::compute_fee(0, &dispatch_info, 0), 100); // Tip + base fee works - assert_eq!(ChargeTransactionPayment::::compute_fee(0, dispatch_info, 69), 169); + assert_eq!(Module::::compute_fee(0, &dispatch_info, 69), 169); // Len (byte fee) + base fee works - assert_eq!(ChargeTransactionPayment::::compute_fee(42, dispatch_info, 0), 520); + assert_eq!(Module::::compute_fee(42, &dispatch_info, 0), 520); // Weight fee + base fee works let dispatch_info = DispatchInfo { weight: 1000, class: DispatchClass::Operational, - pays_fee: true, + pays_fee: Pays::Yes, }; - assert_eq!(ChargeTransactionPayment::::compute_fee(0, dispatch_info, 0), 1100); + assert_eq!(Module::::compute_fee(0, &dispatch_info, 0), 1100); }); } #[test] fn compute_fee_works_with_multiplier() { ExtBuilder::default() - .base_fee(100) + .base_weight(100) .byte_fee(10) .balance_factor(0) .build() .execute_with(|| { // Add a next fee multiplier - NextFeeMultiplier::put(Fixed64::from_rational(1, 2)); // = 1/2 = .5 + NextFeeMultiplier::put(Fixed128::from_rational(1, NonZeroI128::new(2).unwrap())); // = 1/2 = .5 // Base fee is unaffected by multiplier let dispatch_info = DispatchInfo { weight: 0, class: DispatchClass::Operational, - pays_fee: true, + pays_fee: Pays::Yes, }; - assert_eq!(ChargeTransactionPayment::::compute_fee(0, dispatch_info, 0), 100); + assert_eq!(Module::::compute_fee(0, &dispatch_info, 0), 100); // Everything works together :) let dispatch_info = DispatchInfo { weight: 123, class: DispatchClass::Operational, - pays_fee: true, + pays_fee: Pays::Yes, }; // 123 weight, 456 length, 100 base // adjustable fee = (123 * 1) + (456 * 10) = 4683 // adjusted fee = (4683 * .5) + 4683 = 7024.5 -> 7024 // final fee = 100 + 7024 + 789 tip = 7913 - assert_eq!(ChargeTransactionPayment::::compute_fee(456, dispatch_info, 789), 7913); + assert_eq!(Module::::compute_fee(456, &dispatch_info, 789), 7913); }); } #[test] fn compute_fee_does_not_overflow() { ExtBuilder::default() - .base_fee(100) + .base_weight(100) .byte_fee(10) .balance_factor(0) .build() @@ -653,18 +780,102 @@ mod tests { { // Overflow is handled let dispatch_info = DispatchInfo { - weight: ::max_value(), + weight: Weight::max_value(), class: DispatchClass::Operational, - pays_fee: true, + pays_fee: Pays::Yes, }; assert_eq!( - ChargeTransactionPayment::::compute_fee( + Module::::compute_fee( ::max_value(), - dispatch_info, + &dispatch_info, ::max_value() ), ::max_value() ); }); } + + #[test] + fn refund_does_not_recreate_account() { + ExtBuilder::default() + .balance_factor(10) + .base_weight(5) + .build() + .execute_with(|| + { + let len = 10; + let pre = ChargeTransactionPayment::::from(5 /* tipped */) + .pre_dispatch(&2, CALL, &info_from_weight(100), len) + .unwrap(); + assert_eq!(Balances::free_balance(2), 200 - 5 - 10 - 100 - 5); + + // kill the account between pre and post dispatch + assert!(Balances::transfer(Some(2).into(), 3, Balances::free_balance(2)).is_ok()); + assert_eq!(Balances::free_balance(2), 0); + + assert!( + ChargeTransactionPayment:: + ::post_dispatch(pre, &info_from_weight(100), &post_info_from_weight(50), len, &Ok(())) + .is_ok() + ); + assert_eq!(Balances::free_balance(2), 0); + }); + } + + #[test] + fn actual_weight_higher_than_max_refunds_nothing() { + ExtBuilder::default() + .balance_factor(10) + .base_weight(5) + .build() + .execute_with(|| + { + let len = 10; + let pre = ChargeTransactionPayment::::from(5 /* tipped */) + .pre_dispatch(&2, CALL, &info_from_weight(100), len) + .unwrap(); + assert_eq!(Balances::free_balance(2), 200 - 5 - 10 - 100 - 5); + + assert!( + ChargeTransactionPayment:: + ::post_dispatch(pre, &info_from_weight(100), &post_info_from_weight(101), len, &Ok(())) + .is_ok() + ); + assert_eq!(Balances::free_balance(2), 200 - 5 - 10 - 100 - 5); + }); + } + + // TODO Remove after u32 to u64 weights upgrade + #[test] + fn upgrade_to_fixed128_works() { + // TODO You can remove this from dev-dependencies after removing this test + use sp_storage::Storage; + use sp_runtime::Fixed64; + use frame_support::storage::generator::StorageValue; + use frame_support::traits::OnRuntimeUpgrade; + use core::num::NonZeroI128; + + let mut s = Storage::default(); + + let original_multiplier = Fixed64::from_rational(1, 2); + + let data = vec![ + ( + NextFeeMultiplier::storage_value_final_key().to_vec(), + original_multiplier.encode().to_vec() + ), + ]; + + s.top = data.into_iter().collect(); + + sp_io::TestExternalities::new(s).execute_with(|| { + let old_value = NextFeeMultiplier::get(); + assert!(old_value != Fixed128::from_rational(1, NonZeroI128::new(2).unwrap())); + + // Convert Fixed64(.5) to Fixed128(.5) + TransactionPayment::on_runtime_upgrade(); + let new_value = NextFeeMultiplier::get(); + assert_eq!(new_value, Fixed128::from_rational(1, NonZeroI128::new(2).unwrap())); + }); + } } diff --git a/frame/treasury/Cargo.toml b/frame/treasury/Cargo.toml index 6951c0eba9ae7bd566681e385ab8bf7ca9400718..c00ae225c1eaeda5180e8d6005f451305a92b439 100644 --- a/frame/treasury/Cargo.toml +++ b/frame/treasury/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-treasury" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,20 +8,23 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet to manage treasury" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -pallet-balances = { version = "2.0.0-alpha.5", default-features = false, path = "../balances" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +pallet-balances = { version = "2.0.0-dev", default-features = false, path = "../balances" } -frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-io ={ version = "2.0.0-alpha.5", path = "../../primitives/io" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } +sp-io ={ version = "2.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } [features] default = ["std"] @@ -38,6 +41,3 @@ runtime-benchmarks = [ "frame-benchmarking", "frame-support/runtime-benchmarks", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/treasury/src/benchmarking.rs b/frame/treasury/src/benchmarking.rs index 0f9582ebc42d1b20905e399f085dfe74a65d2cfe..f901576c95d4b0c8d384737502654672ccd8a795 100644 --- a/frame/treasury/src/benchmarking.rs +++ b/frame/treasury/src/benchmarking.rs @@ -217,3 +217,25 @@ benchmarks! { Treasury::::on_initialize(T::BlockNumber::zero()); } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_propose_spend::()); + assert_ok!(test_benchmark_reject_proposal::()); + assert_ok!(test_benchmark_approve_proposal::()); + assert_ok!(test_benchmark_report_awesome::()); + assert_ok!(test_benchmark_retract_tip::()); + assert_ok!(test_benchmark_tip_new::()); + assert_ok!(test_benchmark_tip::()); + assert_ok!(test_benchmark_close_tip::()); + assert_ok!(test_benchmark_on_initialize::()); + }); + } +} diff --git a/frame/treasury/src/lib.rs b/frame/treasury/src/lib.rs index f07b9b511e1682edfd3c450239ed95eef2f546a2..270a710a2c29e31cd15435e3a242988c931b1dc7 100644 --- a/frame/treasury/src/lib.rs +++ b/frame/treasury/src/lib.rs @@ -98,8 +98,8 @@ use frame_support::traits::{ use sp_runtime::{Permill, ModuleId, Percent, RuntimeDebug, traits::{ Zero, StaticLookup, AccountIdConversion, Saturating, Hash, BadOrigin }}; -use frame_support::weights::{Weight, WeighData, SimpleDispatchInfo}; -use frame_support::traits::{Contains, EnsureOrigin}; +use frame_support::weights::{Weight, DispatchClass}; +use frame_support::traits::{Contains, ContainsLengthBound, EnsureOrigin}; use codec::{Encode, Decode}; use frame_system::{self as system, ensure_signed, ensure_root}; @@ -110,10 +110,10 @@ type BalanceOf = <::Currency as Currency< = <::Currency as Currency<::AccountId>>::PositiveImbalance; type NegativeImbalanceOf = <::Currency as Currency<::AccountId>>::NegativeImbalance; -/// The treasury's module id, used for deriving its sovereign account ID. -const MODULE_ID: ModuleId = ModuleId(*b"py/trsry"); - pub trait Trait: frame_system::Trait { + /// The treasury's module id, used for deriving its sovereign account ID. + type ModuleId: Get; + /// The staking balance. type Currency: Currency + ReservableCurrency; @@ -124,7 +124,9 @@ pub trait Trait: frame_system::Trait { type RejectOrigin: EnsureOrigin; /// Origin from which tippers must come. - type Tippers: Contains; + /// + /// `ContainsLengthBound::max_len` must be cost free (i.e. no storage read or heavy operation). + type Tippers: Contains + ContainsLengthBound; /// The period for which a tip remains open after is has achieved threshold tippers. type TipCountdown: Get; @@ -313,6 +315,9 @@ decl_module! { /// The amount held on deposit per byte within the tip report reason. const TipReportDepositPerByte: BalanceOf = T::TipReportDepositPerByte::get(); + + /// The treasury's module id, used for deriving its sovereign account ID. + const ModuleId: ModuleId = T::ModuleId::get(); type Error = Error; @@ -323,11 +328,11 @@ decl_module! { /// proposal is awarded. /// /// # - /// - O(1). - /// - Limited storage reads. - /// - One DB change, one extra DB entry. + /// - Complexity: O(1) + /// - DbReads: `ProposalCount`, `origin account` + /// - DbWrites: `ProposalCount`, `Proposals`, `origin account` /// # - #[weight = SimpleDispatchInfo::FixedNormal(500_000)] + #[weight = 120_000_000 + T::DbWeight::get().reads_writes(1, 2)] fn propose_spend( origin, #[compact] value: BalanceOf, @@ -350,11 +355,11 @@ decl_module! { /// Reject a proposed spend. The original deposit will be slashed. /// /// # - /// - O(1). - /// - Limited storage reads. - /// - One DB clear. + /// - Complexity: O(1) + /// - DbReads: `Proposals`, `rejected proposer account` + /// - DbWrites: `Proposals`, `rejected proposer account` /// # - #[weight = SimpleDispatchInfo::FixedOperational(100_000)] + #[weight = (130_000_000 + T::DbWeight::get().reads_writes(2, 2), DispatchClass::Operational)] fn reject_proposal(origin, #[compact] proposal_id: ProposalIndex) { T::RejectOrigin::try_origin(origin) .map(|_| ()) @@ -372,11 +377,11 @@ decl_module! { /// and the original deposit will be returned. /// /// # - /// - O(1). - /// - Limited storage reads. - /// - One DB change. + /// - Complexity: O(1). + /// - DbReads: `Proposals`, `Approvals` + /// - DbWrite: `Approvals` /// # - #[weight = SimpleDispatchInfo::FixedOperational(100_000)] + #[weight = (34_000_000 + T::DbWeight::get().reads_writes(2, 1), DispatchClass::Operational)] fn approve_proposal(origin, #[compact] proposal_id: ProposalIndex) { T::ApproveOrigin::try_origin(origin) .map(|_| ()) @@ -400,12 +405,12 @@ decl_module! { /// Emits `NewTip` if successful. /// /// # - /// - `O(R)` where `R` length of `reason`. - /// - One balance operation. - /// - One storage mutation (codec `O(R)`). - /// - One event. + /// - Complexity: `O(R)` where `R` length of `reason`. + /// - encoding and hashing of 'reason' + /// - DbReads: `Reasons`, `Tips`, `who account data` + /// - DbWrites: `Tips`, `who account data` /// # - #[weight = SimpleDispatchInfo::FixedNormal(100_000)] + #[weight = 140_000_000 + 4_000 * reason.len() as Weight + T::DbWeight::get().reads_writes(3, 2)] fn report_awesome(origin, reason: Vec, who: T::AccountId) { let finder = ensure_signed(origin)?; @@ -442,12 +447,12 @@ decl_module! { /// Emits `TipRetracted` if successful. /// /// # - /// - `O(T)` - /// - One balance operation. - /// - Two storage removals (one read, codec `O(T)`). - /// - One event. + /// - Complexity: `O(1)` + /// - Depends on the length of `T::Hash` which is fixed. + /// - DbReads: `Tips`, `origin account` + /// - DbWrites: `Reasons`, `Tips`, `origin account` /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = 120_000_000 + T::DbWeight::get().reads_writes(1, 2)] fn retract_tip(origin, hash: T::Hash) { let who = ensure_signed(origin)?; let tip = Tips::::get(&hash).ok_or(Error::::UnknownTip)?; @@ -474,12 +479,18 @@ decl_module! { /// Emits `NewTip` if successful. /// /// # - /// - `O(R + T)` where `R` length of `reason`, `T` is the number of tippers. `T` is - /// naturally capped as a membership set, `R` is limited through transaction-size. - /// - Two storage insertions (codecs `O(R)`, `O(T)`), one read `O(1)`. - /// - One event. + /// - Complexity: `O(R + T)` where `R` length of `reason`, `T` is the number of tippers. + /// - `O(T)`: decoding `Tipper` vec of length `T` + /// `T` is charged as upper bound given by `ContainsLengthBound`. + /// The actual cost depends on the implementation of `T::Tippers`. + /// - `O(R)`: hashing and encoding of reason of length `R` + /// - DbReads: `Tippers`, `Reasons` + /// - DbWrites: `Reasons`, `Tips` /// # - #[weight = SimpleDispatchInfo::FixedNormal(150_000)] + #[weight = 110_000_000 + + 4_000 * reason.len() as Weight + + 480_000 * T::Tippers::max_len() as Weight + + T::DbWeight::get().reads_writes(2, 2)] fn tip_new(origin, reason: Vec, who: T::AccountId, tip_value: BalanceOf) { let tipper = ensure_signed(origin)?; ensure!(T::Tippers::contains(&tipper), BadOrigin); @@ -509,11 +520,18 @@ decl_module! { /// has started. /// /// # - /// - `O(T)` - /// - One storage mutation (codec `O(T)`), one storage read `O(1)`. - /// - Up to one event. + /// - Complexity: `O(T)` where `T` is the number of tippers. + /// decoding `Tipper` vec of length `T`, insert tip and check closing, + /// `T` is charged as upper bound given by `ContainsLengthBound`. + /// The actual cost depends on the implementation of `T::Tippers`. + /// + /// Actually weight could be lower as it depends on how many tips are in `OpenTip` but it + /// is weighted as if almost full i.e of length `T-1`. + /// - DbReads: `Tippers`, `Tips` + /// - DbWrites: `Tips` /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = 68_000_000 + 2_000_000 * T::Tippers::max_len() as Weight + + T::DbWeight::get().reads_writes(2, 1)] fn tip(origin, hash: T::Hash, tip_value: BalanceOf) { let tipper = ensure_signed(origin)?; ensure!(T::Tippers::contains(&tipper), BadOrigin); @@ -535,11 +553,15 @@ decl_module! { /// as the hash of the tuple of the original tip `reason` and the beneficiary account ID. /// /// # - /// - `O(T)` - /// - One storage retrieval (codec `O(T)`) and two removals. - /// - Up to three balance operations. + /// - Complexity: `O(T)` where `T` is the number of tippers. + /// decoding `Tipper` vec of length `T`. + /// `T` is charged as upper bound given by `ContainsLengthBound`. + /// The actual cost depends on the implementation of `T::Tippers`. + /// - DbReads: `Tips`, `Tippers`, `tip finder` + /// - DbWrites: `Reasons`, `Tips`, `Tippers`, `tip finder` /// # - #[weight = SimpleDispatchInfo::FixedNormal(50_000)] + #[weight = 220_000_000 + 1_100_000 * T::Tippers::max_len() as Weight + + T::DbWeight::get().reads_writes(3, 3)] fn close_tip(origin, hash: T::Hash) { ensure_signed(origin)?; @@ -549,16 +571,26 @@ decl_module! { // closed. Reasons::::remove(&tip.reason); Tips::::remove(hash); - Self::payout_tip(tip); + Self::payout_tip(hash, tip); } + /// # + /// - Complexity: `O(A)` where `A` is the number of approvals + /// - Db reads and writes: `Approvals`, `pot account data` + /// - Db reads and writes per approval: + /// `Proposals`, `proposer account data`, `beneficiary account data` + /// - The weight is overestimated if some approvals got missed. + /// # fn on_initialize(n: T::BlockNumber) -> Weight { // Check to see if we should spend some funds! if (n % T::SpendPeriod::get()).is_zero() { - Self::spend_funds(); - } + let approvals_len = Self::spend_funds(); - SimpleDispatchInfo::default().weigh_data(()) + 270_000_000 * approvals_len + + T::DbWeight::get().reads_writes(2 + approvals_len * 3, 2 + approvals_len * 3) + } else { + 0 + } } } } @@ -571,7 +603,7 @@ impl Module { /// This actually does computation. If you need to keep using it, then make sure you cache the /// value and only call this once. pub fn account_id() -> T::AccountId { - MODULE_ID.into_account() + T::ModuleId::get().into_account() } /// The needed bond for a proposal whose spend is `value`. @@ -627,7 +659,7 @@ impl Module { /// /// Up to three balance operations. /// Plus `O(T)` (`T` is Tippers length). - fn payout_tip(tip: OpenTip, T::BlockNumber, T::Hash>) { + fn payout_tip(hash: T::Hash, tip: OpenTip, T::BlockNumber, T::Hash>) { let mut tips = tip.tips; Self::retain_active_tips(&mut tips); tips.sort_by_key(|i| i.1); @@ -647,16 +679,18 @@ impl Module { } // same as above: best-effort only. let _ = T::Currency::transfer(&treasury, &tip.who, payout, KeepAlive); + Self::deposit_event(RawEvent::TipClosed(hash, tip.who, payout)); } - // Spend some money! - fn spend_funds() { + /// Spend some money! returns number of approvals before spend. + fn spend_funds() -> u64 { let mut budget_remaining = Self::pot(); Self::deposit_event(RawEvent::Spending(budget_remaining)); let mut missed_any = false; let mut imbalance = >::zero(); - Approvals::mutate(|v| { + let prior_approvals_len = Approvals::mutate(|v| { + let prior_approvals_len = v.len() as u64; v.retain(|&index| { // Should always be true, but shouldn't panic if false or we're screwed. if let Some(p) = Self::proposals(index) { @@ -680,6 +714,7 @@ impl Module { false } }); + prior_approvals_len }); if !missed_any { @@ -706,6 +741,8 @@ impl Module { } Self::deposit_event(RawEvent::Rollover(budget_remaining)); + + prior_approvals_len } /// Return the amount of money in the pot. diff --git a/frame/treasury/src/tests.rs b/frame/treasury/src/tests.rs index 1f6dbecef516e6fd9c9d62729a0245026017db35..18b1d1c50eebd9c283e98861b736bd87f0fdd83a 100644 --- a/frame/treasury/src/tests.rs +++ b/frame/treasury/src/tests.rs @@ -19,14 +19,14 @@ #![cfg(test)] use super::*; - +use std::cell::RefCell; use frame_support::{ - assert_noop, assert_ok, impl_outer_origin, parameter_types, weights::Weight, + assert_noop, assert_ok, impl_outer_origin, impl_outer_event, parameter_types, weights::Weight, traits::{Contains, OnInitialize} }; use sp_core::H256; use sp_runtime::{ - Perbill, + Perbill, ModuleId, testing::Header, traits::{BlakeTwo256, IdentityLookup, BadOrigin}, }; @@ -35,6 +35,21 @@ impl_outer_origin! { pub enum Origin for Test where system = frame_system {} } + +mod treasury { + // Re-export needed for `impl_outer_event!`. + pub use super::super::*; +} + +impl_outer_event! { + pub enum Event for Test { + system, + pallet_balances, + treasury, + } +} + + #[derive(Clone, Eq, PartialEq)] pub struct Test; parameter_types! { @@ -53,9 +68,12 @@ impl frame_system::Trait for Test { type AccountId = u64; type Lookup = IdentityLookup; type Header = Header; - type Event = (); + type Event = Event; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type AvailableBlockRatio = AvailableBlockRatio; type MaximumBlockLength = MaximumBlockLength; type Version = (); @@ -69,21 +87,35 @@ parameter_types! { } impl pallet_balances::Trait for Test { type Balance = u64; - type Event = (); + type Event = Event; type DustRemoval = (); type ExistentialDeposit = ExistentialDeposit; type AccountStore = System; } +thread_local! { + static TEN_TO_FOURTEEN: RefCell> = RefCell::new(vec![10,11,12,13,14]); +} pub struct TenToFourteen; impl Contains for TenToFourteen { - fn contains(n: &u64) -> bool { - *n >= 10 && *n <= 14 - } fn sorted_members() -> Vec { - vec![10, 11, 12, 13, 14] + TEN_TO_FOURTEEN.with(|v| { + v.borrow().clone() + }) } #[cfg(feature = "runtime-benchmarks")] - fn add(_: &u64) { unimplemented!() } + fn add(new: &u64) { + TEN_TO_FOURTEEN.with(|v| { + let mut members = v.borrow_mut(); + members.push(*new); + members.sort(); + }) + } +} +impl ContainsLengthBound for TenToFourteen { + fn max_len() -> usize { + TEN_TO_FOURTEEN.with(|v| v.borrow().len()) + } + fn min_len() -> usize { 0 } } parameter_types! { pub const ProposalBond: Permill = Permill::from_percent(5); @@ -94,8 +126,10 @@ parameter_types! { pub const TipFindersFee: Percent = Percent::from_percent(20); pub const TipReportDepositBase: u64 = 1; pub const TipReportDepositPerByte: u64 = 1; + pub const TreasuryModuleId: ModuleId = ModuleId(*b"py/trsry"); } impl Trait for Test { + type ModuleId = TreasuryModuleId; type Currency = pallet_balances::Module; type ApproveOrigin = frame_system::EnsureRoot; type RejectOrigin = frame_system::EnsureRoot; @@ -104,7 +138,7 @@ impl Trait for Test { type TipFindersFee = TipFindersFee; type TipReportDepositBase = TipReportDepositBase; type TipReportDepositPerByte = TipReportDepositPerByte; - type Event = (); + type Event = Event; type ProposalRejection = (); type ProposalBond = ProposalBond; type ProposalBondMinimum = ProposalBondMinimum; @@ -115,7 +149,7 @@ type System = frame_system::Module; type Balances = pallet_balances::Module; type Treasury = Module; -fn new_test_ext() -> sp_io::TestExternalities { +pub fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig::{ // Total issuance will be 200 with treasury account initialized at ED. @@ -197,15 +231,41 @@ fn report_awesome_from_beneficiary_and_tip_works() { #[test] fn close_tip_works() { new_test_ext().execute_with(|| { + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), 101); assert_eq!(Treasury::pot(), 100); assert_ok!(Treasury::tip_new(Origin::signed(10), b"awesome.dot".to_vec(), 3, 10)); + let h = tip_hash(); + + assert_eq!( + System::events().into_iter().map(|r| r.event) + .filter_map(|e| { + if let Event::treasury(inner) = e { Some(inner) } else { None } + }) + .last() + .unwrap(), + RawEvent::NewTip(h), + ); + assert_ok!(Treasury::tip(Origin::signed(11), h.clone(), 10)); + assert_noop!(Treasury::close_tip(Origin::signed(0), h.into()), Error::::StillOpen); assert_ok!(Treasury::tip(Origin::signed(12), h.clone(), 10)); + + assert_eq!( + System::events().into_iter().map(|r| r.event) + .filter_map(|e| { + if let Event::treasury(inner) = e { Some(inner) } else { None } + }) + .last() + .unwrap(), + RawEvent::TipClosing(h), + ); + assert_noop!(Treasury::close_tip(Origin::signed(0), h.into()), Error::::Premature); System::set_block_number(2); @@ -213,6 +273,16 @@ fn close_tip_works() { assert_ok!(Treasury::close_tip(Origin::signed(0), h.into())); assert_eq!(Balances::free_balance(3), 10); + assert_eq!( + System::events().into_iter().map(|r| r.event) + .filter_map(|e| { + if let Event::treasury(inner) = e { Some(inner) } else { None } + }) + .last() + .unwrap(), + RawEvent::TipClosed(h, 3, 10), + ); + assert_noop!(Treasury::close_tip(Origin::signed(100), h.into()), Error::::UnknownTip); }); } diff --git a/frame/utility/Cargo.toml b/frame/utility/Cargo.toml index cf1042d8521da3e0c8c538eaef51c48de03b5439..a830f8ab5b57f405ac8ec94d1e3d63ee09e55c12 100644 --- a/frame/utility/Cargo.toml +++ b/frame/utility/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-utility" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,21 +8,24 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME utilities pallet" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } -frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } [features] default = ["std"] @@ -39,6 +42,3 @@ runtime-benchmarks = [ "frame-benchmarking", "frame-support/runtime-benchmarks", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/utility/src/benchmarking.rs b/frame/utility/src/benchmarking.rs index f16754fad591e09c8e7e06abfb394a4ddc4c3c28..fc8783b49aa87aaf0813bcc467defae82abc7bf3 100644 --- a/frame/utility/src/benchmarking.rs +++ b/frame/utility/src/benchmarking.rs @@ -146,3 +146,24 @@ benchmarks! { Utility::::as_multi(RawOrigin::Signed(caller.clone()).into(), s as u16, signatories.clone(), None, call.clone())?; }: _(RawOrigin::Signed(caller), s as u16, signatories, timepoint, call_hash) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{new_test_ext, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + new_test_ext().execute_with(|| { + assert_ok!(test_benchmark_batch::()); + assert_ok!(test_benchmark_as_sub::()); + assert_ok!(test_benchmark_as_multi_create::()); + assert_ok!(test_benchmark_as_multi_approve::()); + assert_ok!(test_benchmark_as_multi_complete::()); + assert_ok!(test_benchmark_approve_as_multi_create::()); + assert_ok!(test_benchmark_approve_as_multi_approve::()); + assert_ok!(test_benchmark_cancel_as_multi::()); + }); + } +} diff --git a/frame/utility/src/lib.rs b/frame/utility/src/lib.rs index 49aea15a0f595a7633c2f7ce6fc8d6dfafe953f0..39109a6f6aebd22d516a7ae96c430528cceed8bb 100644 --- a/frame/utility/src/lib.rs +++ b/frame/utility/src/lib.rs @@ -67,8 +67,8 @@ use sp_core::TypeId; use sp_io::hashing::blake2_256; use frame_support::{decl_module, decl_event, decl_error, decl_storage, Parameter, ensure, RuntimeDebug}; use frame_support::{traits::{Get, ReservableCurrency, Currency}, - weights::{GetDispatchInfo, DispatchClass,FunctionOf}, - dispatch::PostDispatchInfo, + weights::{Weight, GetDispatchInfo, DispatchClass, FunctionOf, Pays}, + dispatch::{DispatchResultWithPostInfo, DispatchErrorWithPostInfo, PostDispatchInfo}, }; use frame_system::{self as system, ensure_signed}; use sp_runtime::{DispatchError, DispatchResult, traits::Dispatchable}; @@ -170,7 +170,8 @@ decl_event! { /// Events type. pub enum Event where AccountId = ::AccountId, - BlockNumber = ::BlockNumber + BlockNumber = ::BlockNumber, + CallHash = [u8; 32] { /// Batch of dispatches did not complete fully. Index of first failing dispatch given, as /// well as the error. @@ -178,17 +179,17 @@ decl_event! { /// Batch of dispatches completed fully with no error. BatchCompleted, /// A new multisig operation has begun. First param is the account that is approving, - /// second is the multisig account. - NewMultisig(AccountId, AccountId), + /// second is the multisig account, third is hash of the call. + NewMultisig(AccountId, AccountId, CallHash), /// A multisig operation has been approved by someone. First param is the account that is - /// approving, third is the multisig account. - MultisigApproval(AccountId, Timepoint, AccountId), + /// approving, third is the multisig account, fourth is hash of the call. + MultisigApproval(AccountId, Timepoint, AccountId, CallHash), /// A multisig operation has been executed. First param is the account that is - /// approving, third is the multisig account. - MultisigExecuted(AccountId, Timepoint, AccountId, DispatchResult), + /// approving, third is the multisig account, fourth is hash of the call to be executed. + MultisigExecuted(AccountId, Timepoint, AccountId, CallHash, DispatchResult), /// A multisig operation has been cancelled. First param is the account that is - /// cancelling, third is the multisig account. - MultisigCancelled(AccountId, Timepoint, AccountId), + /// cancelling, third is the multisig account, fourth is hash of the call. + MultisigCancelled(AccountId, Timepoint, AccountId, CallHash), } } @@ -200,6 +201,25 @@ impl TypeId for IndexedUtilityModuleId { const TYPE_ID: [u8; 4] = *b"suba"; } +mod weight_of { + use super::*; + + /// - Base Weight: + /// - Create: 137.5 + 0.274 * S µs + /// - Approve: 103.8 + .266 * S µs + /// - Complete: 116.2 + .754 * S µs + /// - DB Weight: + /// - Reads: Multisig Storage, [Caller Account] + /// - Writes: Multisig Storage, [Caller Account] + /// - Plus Call Weight + pub fn as_multi(other_sig_len: usize, call_weight: Weight) -> Weight { + call_weight + .saturating_add(150_000_000) + .saturating_add((other_sig_len as Weight).saturating_mul(750_000)) + .saturating_add(T::DbWeight::get().reads_writes(1, 1)) + } +} + decl_module! { pub struct Module for enum Call where origin: T::Origin { type Error = Error; @@ -216,8 +236,9 @@ decl_module! { /// - `calls`: The calls to be dispatched from the same origin. /// /// # - /// - The sum of the weights of the `calls`. - /// - One event. + /// - Base weight: 63.78 µs + /// - Plus the sum of the weights of the `calls`. + /// - Plus one additional event. (repeat read/write) /// # /// /// This will return `Ok` in all circumstances. To determine the success of the batch, an @@ -229,7 +250,7 @@ decl_module! { |args: (&Vec<::Call>,)| { args.0.iter() .map(|call| call.get_dispatch_info().weight) - .fold(10_000, |a, n| a + n) + .fold(65_000_000, |a: Weight, n| a.saturating_add(n)) }, |args: (&Vec<::Call>,)| { let all_operational = args.0.iter() @@ -241,7 +262,7 @@ decl_module! { DispatchClass::Normal } }, - true + Pays::Yes, )] fn batch(origin, calls: Vec<::Call>) { for (index, call) in calls.into_iter().enumerate() { @@ -259,12 +280,15 @@ decl_module! { /// The dispatch origin for this call must be _Signed_. /// /// # - /// - The weight of the `call` + 10,000. + /// - Base weight: 5.1 µs + /// - Plus the weight of the `call` /// # #[weight = FunctionOf( - |args: (&u16, &Box<::Call>)| args.1.get_dispatch_info().weight + 10_000, + |args: (&u16, &Box<::Call>)| { + args.1.get_dispatch_info().weight.saturating_add(5_000_000) + }, |args: (&u16, &Box<::Call>)| args.1.get_dispatch_info().class, - true + Pays::Yes, )] fn as_sub(origin, index: u16, call: Box<::Call>) -> DispatchResult { let who = ensure_signed(origin)?; @@ -313,27 +337,37 @@ decl_module! { /// - Storage: inserts one item, value size bounded by `MaxSignatories`, with a /// deposit taken for its lifetime of /// `MultisigDepositBase + threshold * MultisigDepositFactor`. + /// ------------------------------- + /// - Base Weight: + /// - Create: 137.5 + 0.274 * S µs + /// - Approve: 103.8 + .266 * S µs + /// - Complete: 116.2 + .754 * S µs + /// - DB Weight: + /// - Reads: Multisig Storage, [Caller Account] + /// - Writes: Multisig Storage, [Caller Account] + /// - Plus Call Weight /// # #[weight = FunctionOf( |args: (&u16, &Vec, &Option>, &Box<::Call>)| { - args.3.get_dispatch_info().weight + 10_000 * (args.1.len() as u32 + 1) + weight_of::as_multi::(args.1.len(),args.3.get_dispatch_info().weight) }, |args: (&u16, &Vec, &Option>, &Box<::Call>)| { args.3.get_dispatch_info().class }, - true + Pays::Yes, )] fn as_multi(origin, threshold: u16, other_signatories: Vec, maybe_timepoint: Option>, call: Box<::Call>, - ) -> DispatchResult { + ) -> DispatchResultWithPostInfo { let who = ensure_signed(origin)?; ensure!(threshold >= 1, Error::::ZeroThreshold); let max_sigs = T::MaxSignatories::get() as usize; ensure!(!other_signatories.is_empty(), Error::::TooFewSignatories); - ensure!(other_signatories.len() < max_sigs, Error::::TooManySignatories); + let other_signatories_len = other_signatories.len(); + ensure!(other_signatories_len < max_sigs, Error::::TooManySignatories); let signatories = Self::ensure_sorted_and_insert(other_signatories, who.clone())?; let id = Self::multi_account_id(&signatories, threshold); @@ -347,8 +381,9 @@ decl_module! { if (m.approvals.len() as u16) < threshold - 1 { m.approvals.insert(pos, who.clone()); >::insert(&id, call_hash, m); - Self::deposit_event(RawEvent::MultisigApproval(who, timepoint, id)); - return Ok(()) + Self::deposit_event(RawEvent::MultisigApproval(who, timepoint, id, call_hash)); + // Call is not made, so the actual weight does not include call + return Ok(Some(weight_of::as_multi::(other_signatories_len, 0)).into()) } } else { if (m.approvals.len() as u16) < threshold { @@ -360,8 +395,9 @@ decl_module! { let _ = T::Currency::unreserve(&m.depositor, m.deposit); >::remove(&id, call_hash); Self::deposit_event(RawEvent::MultisigExecuted( - who, timepoint, id, result.map(|_| ()).map_err(|e| e.error) + who, timepoint, id, call_hash, result.map(|_| ()).map_err(|e| e.error) )); + return Ok(None.into()) } else { ensure!(maybe_timepoint.is_none(), Error::::UnexpectedTimepoint); if threshold > 1 { @@ -374,13 +410,32 @@ decl_module! { depositor: who.clone(), approvals: vec![who.clone()], }); - Self::deposit_event(RawEvent::NewMultisig(who, id)); + Self::deposit_event(RawEvent::NewMultisig(who, id, call_hash)); + // Call is not made, so we can return that weight + return Ok(Some(weight_of::as_multi::(other_signatories_len, 0)).into()) } else { - return call.dispatch(frame_system::RawOrigin::Signed(id).into()) - .map(|_| ()).map_err(|e| e.error) + let result = call.dispatch(frame_system::RawOrigin::Signed(id).into()); + match result { + Ok(post_dispatch_info) => { + match post_dispatch_info.actual_weight { + Some(actual_weight) => return Ok(Some(weight_of::as_multi::(other_signatories_len, actual_weight)).into()), + None => return Ok(None.into()), + } + }, + Err(err) => { + match err.post_info.actual_weight { + Some(actual_weight) => { + let weight_used = weight_of::as_multi::(other_signatories_len, actual_weight); + return Err(DispatchErrorWithPostInfo { post_info: Some(weight_used).into(), error: err.error.into() }) + }, + None => { + return Err(err) + } + } + } + } } } - Ok(()) } /// Register approval for a dispatch to be made from a deterministic composite account if @@ -414,13 +469,22 @@ decl_module! { /// - Storage: inserts one item, value size bounded by `MaxSignatories`, with a /// deposit taken for its lifetime of /// `MultisigDepositBase + threshold * MultisigDepositFactor`. + /// ---------------------------------- + /// - Base Weight: + /// - Create: 139.1 + 0.202 * S + /// - Approve: 96.6 + 0.328 * S + /// - DB Weight: + /// - Read: Multisig Storage, [Caller Account] + /// - Write: Multisig Storage, [Caller Account] /// # #[weight = FunctionOf( |args: (&u16, &Vec, &Option>, &[u8; 32])| { - 10_000 * (args.1.len() as u32 + 1) + T::DbWeight::get().reads_writes(1, 1) + .saturating_add(140_000_000) + .saturating_add((args.1.len() as Weight).saturating_mul(350_000)) }, DispatchClass::Normal, - true + Pays::Yes, )] fn approve_as_multi(origin, threshold: u16, @@ -444,7 +508,7 @@ decl_module! { if let Err(pos) = m.approvals.binary_search(&who) { m.approvals.insert(pos, who.clone()); >::insert(&id, call_hash, m); - Self::deposit_event(RawEvent::MultisigApproval(who, timepoint, id)); + Self::deposit_event(RawEvent::MultisigApproval(who, timepoint, id, call_hash)); } else { Err(Error::::AlreadyApproved)? } @@ -460,7 +524,7 @@ decl_module! { depositor: who.clone(), approvals: vec![who.clone()], }); - Self::deposit_event(RawEvent::NewMultisig(who, id)); + Self::deposit_event(RawEvent::NewMultisig(who, id, call_hash)); } else { Err(Error::::NoApprovalsNeeded)? } @@ -489,13 +553,20 @@ decl_module! { /// - One event. /// - I/O: 1 read `O(S)`, one remove. /// - Storage: removes one item. + /// ---------------------------------- + /// - Base Weight: 126.6 + 0.126 * S + /// - DB Weight: + /// - Read: Multisig Storage, [Caller Account] + /// - Write: Multisig Storage, [Caller Account] /// # #[weight = FunctionOf( |args: (&u16, &Vec, &Timepoint, &[u8; 32])| { - 10_000 * (args.1.len() as u32 + 1) + T::DbWeight::get().reads_writes(1, 1) + .saturating_add(130_000_000) + .saturating_add((args.1.len() as Weight).saturating_mul(130_000)) }, DispatchClass::Normal, - true + Pays::Yes, )] fn cancel_as_multi(origin, threshold: u16, @@ -520,7 +591,7 @@ decl_module! { let _ = T::Currency::unreserve(&m.depositor, m.deposit); >::remove(&id, call_hash); - Self::deposit_event(RawEvent::MultisigCancelled(who, timepoint, id)); + Self::deposit_event(RawEvent::MultisigCancelled(who, timepoint, id, call_hash)); Ok(()) } } diff --git a/frame/utility/src/tests.rs b/frame/utility/src/tests.rs index 9fcd525020a8096df1e6692678b9474b9bb89225..360ff78d308c9235d85c04a1829eb01f3748b9b2 100644 --- a/frame/utility/src/tests.rs +++ b/frame/utility/src/tests.rs @@ -71,6 +71,9 @@ impl frame_system::Trait for Test { type Event = TestEvent; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -109,7 +112,7 @@ type Utility = Module; use pallet_balances::Call as BalancesCall; use pallet_balances::Error as BalancesError; -fn new_test_ext() -> sp_io::TestExternalities { +pub fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 10)], @@ -303,10 +306,10 @@ fn multisig_2_of_3_cannot_reissue_same_call() { assert_eq!(Balances::free_balance(multi), 5); assert_ok!(Utility::as_multi(Origin::signed(1), 2, vec![2, 3], None, call.clone())); - assert_ok!(Utility::as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), call)); + assert_ok!(Utility::as_multi(Origin::signed(3), 2, vec![1, 2], Some(now()), call.clone())); let err = DispatchError::from(BalancesError::::InsufficientBalance).stripped(); - expect_event(RawEvent::MultisigExecuted(3, now(), multi, Err(err))); + expect_event(RawEvent::MultisigExecuted(3, now(), multi, call.using_encoded(blake2_256), Err(err))); }); } diff --git a/frame/vesting/Cargo.toml b/frame/vesting/Cargo.toml index e40062706ff8559101e0390300b2a8af58a4174c..79430990c32474410bcb6ebfd1dae8cf7a34bec2 100644 --- a/frame/vesting/Cargo.toml +++ b/frame/vesting/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-vesting" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,21 +8,24 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for manage vesting" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } enumflags2 = { version = "0.6.2" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../support" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" } -frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../system" } +frame-benchmarking = { version = "2.0.0-dev", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -pallet-balances = { version = "2.0.0-alpha.5", path = "../balances" } -sp-storage = { version = "2.0.0-alpha.5", path = "../../primitives/storage" } +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +pallet-balances = { version = "2.0.0-dev", path = "../balances" } +sp-storage = { version = "2.0.0-dev", path = "../../primitives/storage" } hex-literal = "0.2.1" [features] @@ -37,6 +40,3 @@ std = [ "frame-system/std", ] runtime-benchmarks = ["frame-benchmarking"] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/frame/vesting/src/benchmarking.rs b/frame/vesting/src/benchmarking.rs index 2ef8ed9ef8dbb867136b37ceb43b85cdb3b21fc7..10f19af65ee186ab1be9b5b0c6b5a194de1b3524 100644 --- a/frame/vesting/src/benchmarking.rs +++ b/frame/vesting/src/benchmarking.rs @@ -21,106 +21,193 @@ use super::*; use frame_system::{RawOrigin, Module as System}; -use sp_io::hashing::blake2_256; use frame_benchmarking::{benchmarks, account}; +use sp_runtime::traits::Bounded; use crate::Module as Vesting; const SEED: u32 = 0; const MAX_LOCKS: u32 = 20; -fn add_locks(l: u32) { - for id in 0..l { - let lock_id = <[u8; 8]>::decode(&mut &id.using_encoded(blake2_256)[..]) - .unwrap_or_default(); - let locker = account("locker", 0, SEED); - let locked = 1; +type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; + +fn add_locks(who: &T::AccountId, n: u8) { + for id in 0..n { + let lock_id = [id; 8]; + let locked = 100; let reasons = WithdrawReason::Transfer | WithdrawReason::Reserve; - T::Currency::set_lock(lock_id, &locker, locked.into(), reasons); + T::Currency::set_lock(lock_id, who, locked.into(), reasons); } } -fn setup(b: u32) -> T::AccountId { - let locked = 1; - let per_block = 1; - let starting_block = 0; - - let caller = account("caller", 0, SEED); - System::::set_block_number(0.into()); - - // Add schedule to avoid `NotVesting` error. - let _ = Vesting::::add_vesting_schedule( - &caller, - locked.into(), - per_block.into(), - starting_block.into(), - ); - - // Set lock and block number to take different code paths. - let reasons = WithdrawReason::Transfer | WithdrawReason::Reserve; - T::Currency::set_lock(VESTING_ID, &caller, locked.into(), reasons); - System::::set_block_number(b.into()); - - caller +fn add_vesting_schedule(who: &T::AccountId) -> Result<(), &'static str> { + let locked = 100; + let per_block = 10; + let starting_block = 1; + + System::::set_block_number(0.into()); + + // Add schedule to avoid `NotVesting` error. + Vesting::::add_vesting_schedule( + &who, + locked.into(), + per_block.into(), + starting_block.into(), + )?; + Ok(()) } benchmarks! { - _ { - // Number of previous locks. - // It doesn't seems to influence the timings for lower values. - let l in 0 .. MAX_LOCKS => add_locks::(l); - } + _ { } vest_locked { - let l in ...; - - let caller = setup::(0u32); + let l in 0 .. MAX_LOCKS; - }: vest(RawOrigin::Signed(caller)) - - vest_not_locked { - let l in ...; + let caller = account("caller", 0, SEED); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + add_locks::(&caller, l as u8); + add_vesting_schedule::(&caller)?; + // At block zero, everything is vested. + System::::set_block_number(T::BlockNumber::zero()); + assert_eq!( + Vesting::::vesting_balance(&caller), + Some(100.into()), + "Vesting schedule not added", + ); + }: vest(RawOrigin::Signed(caller.clone())) + verify { + // Nothing happened since everything is still vested. + assert_eq!( + Vesting::::vesting_balance(&caller), + Some(100.into()), + "Vesting schedule was removed", + ); + } - let caller = setup::(1u32); + vest_unlocked { + let l in 0 .. MAX_LOCKS; - }: vest(RawOrigin::Signed(caller)) + let caller = account("caller", 0, SEED); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + add_locks::(&caller, l as u8); + add_vesting_schedule::(&caller)?; + // At block 20, everything is unvested. + System::::set_block_number(20.into()); + assert_eq!( + Vesting::::vesting_balance(&caller), + Some(BalanceOf::::zero()), + "Vesting schedule still active", + ); + }: vest(RawOrigin::Signed(caller.clone())) + verify { + // Vesting schedule is removed! + assert_eq!( + Vesting::::vesting_balance(&caller), + None, + "Vesting schedule was not removed", + ); + } vest_other_locked { - let l in ...; + let l in 0 .. MAX_LOCKS; - let other: T::AccountId = setup::(0u32); + let other: T::AccountId = account("other", 0, SEED); let other_lookup: ::Source = T::Lookup::unlookup(other.clone()); + T::Currency::make_free_balance_be(&other, BalanceOf::::max_value()); + add_locks::(&other, l as u8); + add_vesting_schedule::(&other)?; + // At block zero, everything is vested. + System::::set_block_number(T::BlockNumber::zero()); + assert_eq!( + Vesting::::vesting_balance(&other), + Some(100.into()), + "Vesting schedule not added", + ); - let caller = account("caller", 0, SEED); - - }: vest_other(RawOrigin::Signed(caller), other_lookup) + let caller: T::AccountId = account("caller", 0, SEED); + }: vest_other(RawOrigin::Signed(caller.clone()), other_lookup) + verify { + // Nothing happened since everything is still vested. + assert_eq!( + Vesting::::vesting_balance(&other), + Some(100.into()), + "Vesting schedule was removed", + ); + } - vest_other_not_locked { - let l in ...; + vest_other_unlocked { + let l in 0 .. MAX_LOCKS; - let other: T::AccountId = setup::(1u32); + let other: T::AccountId = account("other", 0, SEED); let other_lookup: ::Source = T::Lookup::unlookup(other.clone()); + T::Currency::make_free_balance_be(&other, BalanceOf::::max_value()); + add_locks::(&other, l as u8); + add_vesting_schedule::(&other)?; + // At block 20, everything is unvested. + System::::set_block_number(20.into()); + assert_eq!( + Vesting::::vesting_balance(&other), + Some(BalanceOf::::zero()), + "Vesting schedule still active", + ); - let caller = account("caller", 0, SEED); - - }: vest_other(RawOrigin::Signed(caller), other_lookup) + let caller: T::AccountId = account("caller", 0, SEED); + }: vest_other(RawOrigin::Signed(caller.clone()), other_lookup) + verify { + // Vesting schedule is removed! + assert_eq!( + Vesting::::vesting_balance(&other), + None, + "Vesting schedule was not removed", + ); + } vested_transfer { - let u in 0 .. 1000; + let l in 0 .. MAX_LOCKS; - let from = account("from", u, SEED); - let to = account("to", u, SEED); - let to_lookup: ::Source = T::Lookup::unlookup(to); + let caller: T::AccountId = account("caller", 0, SEED); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + let target: T::AccountId = account("target", 0, SEED); + let target_lookup: ::Source = T::Lookup::unlookup(target.clone()); + // Give target existing locks + add_locks::(&target, l as u8); let transfer_amount = T::MinVestedTransfer::get(); let vesting_schedule = VestingInfo { locked: transfer_amount, - per_block: 1.into(), - starting_block: 0.into(), + per_block: 10.into(), + starting_block: 1.into(), }; + }: _(RawOrigin::Signed(caller), target_lookup, vesting_schedule) + verify { + assert_eq!( + T::MinVestedTransfer::get(), + T::Currency::free_balance(&target), + "Transfer didn't happen", + ); + assert_eq!( + Vesting::::vesting_balance(&target), + Some(T::MinVestedTransfer::get()), + "Lock not created", + ); + } +} - let _ = T::Currency::make_free_balance_be(&from, transfer_amount * 10.into()); - - }: _(RawOrigin::Signed(from), to_lookup, vesting_schedule) +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{ExtBuilder, Test}; + use frame_support::assert_ok; + + #[test] + fn test_benchmarks() { + ExtBuilder::default().existential_deposit(256).build().execute_with(|| { + assert_ok!(test_benchmark_vest_locked::()); + assert_ok!(test_benchmark_vest_unlocked::()); + assert_ok!(test_benchmark_vest_other_locked::()); + assert_ok!(test_benchmark_vest_other_unlocked::()); + assert_ok!(test_benchmark_vested_transfer::()); + }); + } } diff --git a/frame/vesting/src/lib.rs b/frame/vesting/src/lib.rs index b0c98e78bd6f7243e4c017f133726fccf16407a9..d8e937d550f247570f186517d27c9b3a2eac359e 100644 --- a/frame/vesting/src/lib.rs +++ b/frame/vesting/src/lib.rs @@ -57,7 +57,7 @@ use frame_support::traits::{ Currency, LockableCurrency, VestingSchedule, WithdrawReason, LockIdentifier, ExistenceRequirement, Get }; -use frame_support::weights::SimpleDispatchInfo; + use frame_system::{self as system, ensure_signed}; mod benchmarking; @@ -190,11 +190,13 @@ decl_module! { /// /// # /// - `O(1)`. - /// - One balance-lock operation. - /// - One storage read (codec `O(1)`) and up to one removal. - /// - One event. + /// - DbWeight: 2 Reads, 2 Writes + /// - Reads: Vesting Storage, Balances Locks, [Sender Account] + /// - Writes: Vesting Storage, Balances Locks, [Sender Account] + /// - Benchmark: 147.5 µs (min square analysis) + /// - Assuming less than 50 locks on any user, else we may want factor in number of locks. /// # - #[weight = SimpleDispatchInfo::default()] + #[weight = 150_000_000 + T::DbWeight::get().reads_writes(2, 2)] fn vest(origin) -> DispatchResult { let who = ensure_signed(origin)?; Self::update_lock(who) @@ -211,12 +213,13 @@ decl_module! { /// /// # /// - `O(1)`. - /// - Up to one account lookup. - /// - One balance-lock operation. - /// - One storage read (codec `O(1)`) and up to one removal. - /// - One event. + /// - DbWeight: 3 Reads, 3 Writes + /// - Reads: Vesting Storage, Balances Locks, Target Account + /// - Writes: Vesting Storage, Balances Locks, Target Account + /// - Benchmark: 150.4 µs (min square analysis) + /// - Assuming less than 50 locks on any user, else we may want factor in number of locks. /// # - #[weight = SimpleDispatchInfo::default()] + #[weight = 150_000_000 + T::DbWeight::get().reads_writes(3, 3)] fn vest_other(origin, target: ::Source) -> DispatchResult { ensure_signed(origin)?; Self::update_lock(T::Lookup::lookup(target)?) @@ -233,10 +236,14 @@ decl_module! { /// Emits `VestingCreated`. /// /// # - /// - Creates a new storage entry, but is protected by a minimum transfer - /// amount needed to succeed. + /// - `O(1)`. + /// - DbWeight: 3 Reads, 3 Writes + /// - Reads: Vesting Storage, Balances Locks, Target Account, [Sender Account] + /// - Writes: Vesting Storage, Balances Locks, Target Account, [Sender Account] + /// - Benchmark: 263 µs (min square analysis) + /// - Assuming less than 50 locks on any user, else we may want factor in number of locks. /// # - #[weight = SimpleDispatchInfo::FixedNormal(1_000_000)] + #[weight = 300_000_000 + T::DbWeight::get().reads_writes(3, 3)] pub fn vested_transfer( origin, target: ::Source, @@ -381,6 +388,9 @@ mod tests { type Event = (); type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); diff --git a/primitives/allocator/Cargo.toml b/primitives/allocator/Cargo.toml index 635a3c9128febcd6fd46690ea3e91043d935e70f..8530e0df0e7d22246eb4effa84ab35bd74fb7bab 100644 --- a/primitives/allocator/Cargo.toml +++ b/primitives/allocator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-allocator" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,10 +9,13 @@ repository = "https://github.com/paritytech/substrate/" description = "Collection of allocator implementations." documentation = "https://docs.rs/sp-allocator" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-std = { version = "2.0.0-alpha.5", path = "../std", default-features = false } -sp-core = { version = "2.0.0-alpha.5", path = "../core", default-features = false } -sp-wasm-interface = { version = "2.0.0-alpha.5", path = "../wasm-interface", default-features = false } +sp-std = { version = "2.0.0-dev", path = "../std", default-features = false } +sp-core = { version = "2.0.0-dev", path = "../core", default-features = false } +sp-wasm-interface = { version = "2.0.0-dev", path = "../wasm-interface", default-features = false } log = { version = "0.4.8", optional = true } derive_more = { version = "0.99.2", optional = true } @@ -25,6 +28,3 @@ std = [ "log", "derive_more", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/api/Cargo.toml b/primitives/api/Cargo.toml index fedeeceb3faadfb8e3ec6ff19b7204aa97e6904b..43decc4f6e60b62e2057661469158e72d78a592d 100644 --- a/primitives/api/Cargo.toml +++ b/primitives/api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-api" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,14 +8,17 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Substrate runtime api primitives" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-api-proc-macro = { version = "2.0.0-alpha.5", path = "proc-macro" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } -sp-version = { version = "2.0.0-alpha.5", default-features = false, path = "../version" } -sp-state-machine = { version = "0.8.0-alpha.5", optional = true, path = "../../primitives/state-machine" } +sp-api-proc-macro = { version = "2.0.0-dev", path = "proc-macro" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../core" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../runtime" } +sp-version = { version = "2.0.0-dev", default-features = false, path = "../version" } +sp-state-machine = { version = "0.8.0-dev", optional = true, path = "../../primitives/state-machine" } hash-db = { version = "0.15.2", optional = true } [dev-dependencies] @@ -32,6 +35,3 @@ std = [ "sp-version/std", "hash-db", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/api/proc-macro/Cargo.toml b/primitives/api/proc-macro/Cargo.toml index 25c5ae13435a5a107cee146acc2a0661afea8c9f..a970b3e750a87941b9cb4dd05f7411664e3a378c 100644 --- a/primitives/api/proc-macro/Cargo.toml +++ b/primitives/api/proc-macro/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-api-proc-macro" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,6 +9,9 @@ repository = "https://github.com/paritytech/substrate/" description = "Macros for declaring and implementing runtime apis." documentation = "https://docs.rs/sp-api-proc-macro" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [lib] proc-macro = true @@ -24,6 +27,3 @@ proc-macro-crate = "0.1.4" [features] default = [ "std" ] std = [] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/api/proc-macro/src/decl_runtime_apis.rs b/primitives/api/proc-macro/src/decl_runtime_apis.rs index e9f3087912e26f322214fa22cdbca397a4a2c332..440176bc468585e76cde32d3e07627e550d59a42 100644 --- a/primitives/api/proc-macro/src/decl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/decl_runtime_apis.rs @@ -407,6 +407,7 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result { at: &#crate_::BlockId, args: Vec, changes: &std::cell::RefCell<#crate_::OverlayedChanges>, + offchain_changes: &std::cell::RefCell<#crate_::OffchainOverlayedChanges>, storage_transaction_cache: &std::cell::RefCell< #crate_::StorageTransactionCache >, @@ -436,6 +437,7 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result { native_call: None, arguments: args, overlayed_changes: changes, + offchain_changes, storage_transaction_cache, initialize_block, context, @@ -456,6 +458,7 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result { native_call, arguments: args, overlayed_changes: changes, + offchain_changes, storage_transaction_cache, initialize_block, context, @@ -876,6 +879,53 @@ struct CheckTraitDecl { errors: Vec, } +impl CheckTraitDecl { + /// Check the given trait. + /// + /// All errors will be collected in `self.errors`. + fn check(&mut self, trait_: &ItemTrait) { + self.check_method_declarations(trait_.items.iter().filter_map(|i| match i { + TraitItem::Method(method) => Some(method), + _ => None, + })); + + visit::visit_item_trait(self, trait_); + } + + /// Check that the given method declarations are correct. + /// + /// Any error is stored in `self.errors`. + fn check_method_declarations<'a>(&mut self, methods: impl Iterator) { + let mut method_to_signature_changed = HashMap::>>::new(); + + methods.into_iter().for_each(|method| { + let attributes = remove_supported_attributes(&mut method.attrs.clone()); + + let changed_in = match get_changed_in(&attributes) { + Ok(r) => r, + Err(e) => { self.errors.push(e); return; }, + }; + + method_to_signature_changed + .entry(method.sig.ident.clone()) + .or_default() + .push(changed_in); + }); + + method_to_signature_changed.into_iter().for_each(|(f, changed)| { + // If `changed_in` is `None`, it means it is the current "default" method that calls + // into the latest implementation. + if changed.iter().filter(|c| c.is_none()).count() == 0 { + self.errors.push(Error::new( + f.span(), + "There is no 'default' method with this name (without `changed_in` attribute).\n\ + The 'default' method is used to call into the latest implementation.", + )); + } + }); + } +} + impl<'ast> Visit<'ast> for CheckTraitDecl { fn visit_fn_arg(&mut self, input: &'ast FnArg) { if let FnArg::Receiver(_) = input { @@ -923,7 +973,7 @@ impl<'ast> Visit<'ast> for CheckTraitDecl { /// Check that the trait declarations are in the format we expect. fn check_trait_decls(decls: &[ItemTrait]) -> Result<()> { let mut checker = CheckTraitDecl { errors: Vec::new() }; - decls.iter().for_each(|decl| visit::visit_item_trait(&mut checker, &decl)); + decls.iter().for_each(|decl| checker.check(decl)); if let Some(err) = checker.errors.pop() { Err(checker.errors.into_iter().fold(err, |mut err, other| { diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index 7def6aa0fb73af3cca94b33ab1bb9ed7d6a8f70e..7423dbec559c886629de6fdb8c9e841cb89b96a2 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -207,6 +207,7 @@ fn generate_runtime_api_base_structures() -> Result { commit_on_success: std::cell::RefCell, initialized_block: std::cell::RefCell>>, changes: std::cell::RefCell<#crate_::OverlayedChanges>, + offchain_changes: std::cell::RefCell<#crate_::OffchainOverlayedChanges>, storage_transaction_cache: std::cell::RefCell< #crate_::StorageTransactionCache >, @@ -335,6 +336,7 @@ fn generate_runtime_api_base_structures() -> Result { commit_on_success: true.into(), initialized_block: None.into(), changes: Default::default(), + offchain_changes: Default::default(), recorder: Default::default(), storage_transaction_cache: Default::default(), }.into() @@ -353,6 +355,7 @@ fn generate_runtime_api_base_structures() -> Result { &C, &Self, &std::cell::RefCell<#crate_::OverlayedChanges>, + &std::cell::RefCell<#crate_::OffchainOverlayedChanges>, &std::cell::RefCell<#crate_::StorageTransactionCache>, &std::cell::RefCell>>, &Option<#crate_::ProofRecorder>, @@ -366,6 +369,7 @@ fn generate_runtime_api_base_structures() -> Result { &self.call, self, &self.changes, + &self.offchain_changes, &self.storage_transaction_cache, &self.initialized_block, &self.recorder, @@ -517,6 +521,7 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> { call_runtime_at, core_api, changes, + offchain_changes, storage_transaction_cache, initialized_block, recorder @@ -527,6 +532,7 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> { at, params_encoded, changes, + offchain_changes, storage_transaction_cache, initialized_block, params.map(|p| { diff --git a/primitives/api/src/lib.rs b/primitives/api/src/lib.rs index 74bcf19a9949e43316001d34b7ccfe3bda2b73cb..f8a22c45665ffd899f19b4e7200bd31a2a7c4bd4 100644 --- a/primitives/api/src/lib.rs +++ b/primitives/api/src/lib.rs @@ -27,6 +27,8 @@ //! //! Besides the macros and the [`Core`] runtime api, this crates provides the [`Metadata`] runtime //! api, the [`ApiExt`] trait, the [`CallApiAt`] trait and the [`ConstructRuntimeApi`] trait. +//! +//! On a meta level this implies, the client calls the generated API from the client perspective. #![cfg_attr(not(feature = "std"), no_std)] @@ -44,6 +46,8 @@ pub use sp_core::NativeOrEncoded; #[doc(hidden)] #[cfg(feature = "std")] pub use hash_db::Hasher; +#[cfg(feature = "std")] +pub use sp_core::offchain::storage::OffchainOverlayedChanges; #[doc(hidden)] #[cfg(not(feature = "std"))] pub use sp_core::to_substrate_wasm_fn_return_value; @@ -53,7 +57,7 @@ pub use sp_runtime::{ Block as BlockT, GetNodeBlockType, GetRuntimeBlockType, HashFor, NumberFor, Header as HeaderT, Hash as HashT, }, - generic::BlockId, transaction_validity::TransactionValidity, + generic::BlockId, transaction_validity::TransactionValidity, RuntimeString, }; #[doc(hidden)] pub use sp_core::{offchain, ExecutionContext}; @@ -113,7 +117,9 @@ use std::{panic::UnwindSafe, cell::RefCell}; /// change is highlighted with the `#[changed_in(2)]` attribute above a method. A method that is /// tagged with this attribute is callable by the name `METHOD_before_version_VERSION`. This /// method will only support calling into wasm, trying to call into native will fail (change the -/// spec version!). Such a method also does not need to be implemented in the runtime. +/// spec version!). Such a method also does not need to be implemented in the runtime. It is +/// required that there exist the "default" of the method without the `#[changed_in(_)]` attribute, +/// this method will be used to call the current default implementation. /// /// ```rust /// sp_api::decl_runtime_apis! { @@ -222,6 +228,7 @@ pub use sp_api_proc_macro::decl_runtime_apis; /// impl_version: 0, /// // Here we are exposing the runtime api versions. /// apis: RUNTIME_API_VERSIONS, +/// transaction_version: 1, /// }; /// /// # fn main() {} @@ -428,6 +435,8 @@ pub struct CallApiAtParams<'a, Block: BlockT, C, NC, Backend: StateBackend, /// The overlayed changes that are on top of the state. pub overlayed_changes: &'a RefCell, + /// The overlayed changes to be applied to the offchain worker database. + pub offchain_changes: &'a RefCell, /// The cache for storage transactions. pub storage_transaction_cache: &'a RefCell>, /// Determines if the function requires that `initialize_block` should be called before calling @@ -518,13 +527,53 @@ pub trait RuntimeApiInfo { #[cfg(feature = "std")] pub type ApiErrorFor = <>::Api as ApiErrorExt>::Error; +#[derive(codec::Encode, codec::Decode)] +pub struct OldRuntimeVersion { + pub spec_name: RuntimeString, + pub impl_name: RuntimeString, + pub authoring_version: u32, + pub spec_version: u32, + pub impl_version: u32, + pub apis: ApisVec, +} + +impl From for RuntimeVersion { + fn from(x: OldRuntimeVersion) -> Self { + Self { + spec_name: x.spec_name, + impl_name: x.impl_name, + authoring_version: x.authoring_version, + spec_version: x.spec_version, + impl_version: x.impl_version, + apis: x.apis, + transaction_version: 1, + } + } +} + +impl From for OldRuntimeVersion { + fn from(x: RuntimeVersion) -> Self { + Self { + spec_name: x.spec_name, + impl_name: x.impl_name, + authoring_version: x.authoring_version, + spec_version: x.spec_version, + impl_version: x.impl_version, + apis: x.apis, + } + } +} + decl_runtime_apis! { /// The `Core` runtime api that every Substrate runtime needs to implement. #[core_trait] - #[api_version(2)] + #[api_version(3)] pub trait Core { /// Returns the version of the runtime. fn version() -> RuntimeVersion; + /// Returns the version of the runtime. + #[changed_in(3)] + fn version() -> OldRuntimeVersion; /// Execute the given block. #[skip_initialize_block] fn execute_block(block: Block); diff --git a/primitives/api/test/Cargo.toml b/primitives/api/test/Cargo.toml index f2e66afc24abbac9f4c5604701d6b6f558bf3b4d..a945399f1b66ea830721a630c546888ece019830 100644 --- a/primitives/api/test/Cargo.toml +++ b/primitives/api/test/Cargo.toml @@ -8,23 +8,26 @@ publish = false homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-api = { version = "2.0.0-alpha.5", path = "../" } +sp-api = { version = "2.0.0-dev", path = "../" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } -sp-version = { version = "2.0.0-alpha.5", path = "../../version" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../runtime" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../blockchain" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../../primitives/consensus/common" } -sc-block-builder = { version = "0.8.0-alpha.5", path = "../../../client/block-builder" } +sp-version = { version = "2.0.0-dev", path = "../../version" } +sp-runtime = { version = "2.0.0-dev", path = "../../runtime" } +sp-blockchain = { version = "2.0.0-dev", path = "../../blockchain" } +sp-consensus = { version = "0.8.0-dev", path = "../../../primitives/consensus/common" } +sc-block-builder = { version = "0.8.0-dev", path = "../../../client/block-builder" } codec = { package = "parity-scale-codec", version = "1.3.0" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../../primitives/state-machine" } +sp-state-machine = { version = "0.8.0-dev", path = "../../../primitives/state-machine" } trybuild = "1.0.17" rustversion = "1.0.0" [dev-dependencies] criterion = "0.3.0" substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } -sp-core = { version = "2.0.0-alpha.5", path = "../../core" } +sp-core = { version = "2.0.0-dev", path = "../../core" } [[bench]] name = "bench" @@ -34,6 +37,3 @@ harness = false [features] default = [ "std" ] std = [] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/api/test/tests/decl_and_impl.rs b/primitives/api/test/tests/decl_and_impl.rs index a09bd0412c8afe9f15deb078a3f338dd0937ac59..4a8c8cd6628c895dec8bbf5a920b15684471380b 100644 --- a/primitives/api/test/tests/decl_and_impl.rs +++ b/primitives/api/test/tests/decl_and_impl.rs @@ -112,7 +112,7 @@ mock_impl_runtime_apis! { } } -type TestClient = substrate_test_runtime_client::sc_client::Client< +type TestClient = substrate_test_runtime_client::client::Client< substrate_test_runtime_client::Backend, substrate_test_runtime_client::Executor, Block, diff --git a/primitives/api/test/tests/ui/changed_in_no_default_method.rs b/primitives/api/test/tests/ui/changed_in_no_default_method.rs new file mode 100644 index 0000000000000000000000000000000000000000..6af183a4cde910248575e93ca57e437a24950fbf --- /dev/null +++ b/primitives/api/test/tests/ui/changed_in_no_default_method.rs @@ -0,0 +1,19 @@ +use sp_runtime::traits::GetNodeBlockType; +use substrate_test_runtime_client::runtime::Block; + +/// The declaration of the `Runtime` type and the implementation of the `GetNodeBlockType` +/// trait are done by the `construct_runtime!` macro in a real runtime. +struct Runtime {} +impl GetNodeBlockType for Runtime { + type NodeBlock = Block; +} + +sp_api::decl_runtime_apis! { + #[api_version(2)] + pub trait Api { + #[changed_in(2)] + fn test(data: u64); + } +} + +fn main() {} diff --git a/primitives/api/test/tests/ui/changed_in_no_default_method.stderr b/primitives/api/test/tests/ui/changed_in_no_default_method.stderr new file mode 100644 index 0000000000000000000000000000000000000000..ed4c0f9088573e78211f6e9560fc781e24936c02 --- /dev/null +++ b/primitives/api/test/tests/ui/changed_in_no_default_method.stderr @@ -0,0 +1,6 @@ +error: There is no 'default' method with this name (without `changed_in` attribute). +The 'default' method is used to call into the latest implementation. + --> $DIR/changed_in_no_default_method.rs:15:6 + | +15 | fn test(data: u64); + | ^^^^ diff --git a/primitives/api/test/tests/ui/mock_only_one_error_type.stderr b/primitives/api/test/tests/ui/mock_only_one_error_type.stderr index 8f026838c96b8db0e503179c745b3a76f4a2c181..7cec5246ca825a742bfad7a52ba446bd2be4f4c7 100644 --- a/primitives/api/test/tests/ui/mock_only_one_error_type.stderr +++ b/primitives/api/test/tests/ui/mock_only_one_error_type.stderr @@ -23,5 +23,5 @@ error[E0277]: the trait bound `u32: std::convert::From` is > > > - > - and 16 others + > + and 18 others diff --git a/primitives/application-crypto/Cargo.toml b/primitives/application-crypto/Cargo.toml index 7c3fb5357997ffa2e7470c5aaf94cca168594fc8..be9e5e5a1116ddcf883c5cbd40ee5f026855ab38 100644 --- a/primitives/application-crypto/Cargo.toml +++ b/primitives/application-crypto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-application-crypto" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" description = "Provides facilities for generating application specific crypto wrapper types." @@ -9,13 +9,16 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sp-application-crypto" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../core" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } [features] default = [ "std" ] @@ -31,6 +34,3 @@ full_crypto = [ "sp-io/disable_panic_handler", "sp-io/disable_oom", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/application-crypto/src/lib.rs b/primitives/application-crypto/src/lib.rs index 07e2b45106ee9690c841a5d842eebdc4325afd59..8bad474ede81e1281648953b035be1dbd1cc63ee 100644 --- a/primitives/application-crypto/src/lib.rs +++ b/primitives/application-crypto/src/lib.rs @@ -25,7 +25,7 @@ pub use sp_core::{self, crypto::{CryptoType, CryptoTypePublicPair, Public, Deriv #[doc(hidden)] #[cfg(feature = "full_crypto")] pub use sp_core::crypto::{SecretStringError, DeriveJunction, Ss58Codec, Pair}; -pub use sp_core::crypto::{CryptoTypeId, KeyTypeId, key_types}; +pub use sp_core::crypto::{KeyTypeId, key_types}; #[doc(hidden)] pub use codec; @@ -103,17 +103,8 @@ macro_rules! app_crypto_pair { type Signature = Signature; type DeriveError = <$pair as $crate::Pair>::DeriveError; - #[cfg(feature = "std")] - fn generate_with_phrase(password: Option<&str>) -> (Self, String, Self::Seed) { - let r = <$pair>::generate_with_phrase(password); - (Self(r.0), r.1, r.2) - } - #[cfg(feature = "std")] - fn from_phrase(phrase: &str, password: Option<&str>) - -> Result<(Self, Self::Seed), $crate::SecretStringError> - { - <$pair>::from_phrase(phrase, password).map(|r| (Self(r.0), r.1)) - } + $crate::app_crypto_pair_functions_if_std!($pair); + fn derive< Iter: Iterator >(&self, path: Iter, seed: Option) -> Result<(Self, Option), Self::DeriveError> { @@ -158,10 +149,38 @@ macro_rules! app_crypto_pair { }; } +/// Implements functions for the `Pair` trait when `feature = "std"` is enabled. +#[doc(hidden)] +#[cfg(feature = "std")] +#[macro_export] +macro_rules! app_crypto_pair_functions_if_std { + ($pair:ty) => { + fn generate_with_phrase(password: Option<&str>) -> (Self, String, Self::Seed) { + let r = <$pair>::generate_with_phrase(password); + (Self(r.0), r.1, r.2) + } + + fn from_phrase(phrase: &str, password: Option<&str>) + -> Result<(Self, Self::Seed), $crate::SecretStringError> + { + <$pair>::from_phrase(phrase, password).map(|r| (Self(r.0), r.1)) + } + } +} + +#[doc(hidden)] +#[cfg(not(feature = "std"))] +#[macro_export] +macro_rules! app_crypto_pair_functions_if_std { + ($pair:ty) => {} +} + + /// Declares Public type which is functionally equivalent to `$public`, but is new /// Application-specific type whose identifier is `$key_type`. /// can only be used together with `full_crypto` feature /// For full functionality, app_crypto_public_common! must be called too. +#[doc(hidden)] #[macro_export] macro_rules! app_crypto_public_full_crypto { ($public:ty, $key_type:expr) => { @@ -195,6 +214,7 @@ macro_rules! app_crypto_public_full_crypto { /// Application-specific type whose identifier is `$key_type`. /// can only be used without `full_crypto` feature /// For full functionality, app_crypto_public_common! must be called too. +#[doc(hidden)] #[macro_export] macro_rules! app_crypto_public_not_full_crypto { ($public:ty, $key_type:expr) => { @@ -223,44 +243,11 @@ macro_rules! app_crypto_public_not_full_crypto { /// Declares Public type which is functionally equivalent to `$public`, but is new /// Application-specific type whose identifier is `$key_type`. /// For full functionality, app_crypto_public_(not)_full_crypto! must be called too. +#[doc(hidden)] #[macro_export] macro_rules! app_crypto_public_common { ($public:ty, $sig:ty, $key_type:expr) => { - impl $crate::Derive for Public { - #[cfg(feature = "std")] - fn derive>(&self, - path: Iter - ) -> Option { - self.0.derive(path).map(Self) - } - } - - #[cfg(feature = "std")] - impl std::fmt::Display for Public { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - use $crate::Ss58Codec; - write!(f, "{}", self.0.to_ss58check()) - } - } - #[cfg(feature = "std")] - impl $crate::serde::Serialize for Public { - fn serialize(&self, serializer: S) -> std::result::Result where - S: $crate::serde::Serializer - { - use $crate::Ss58Codec; - serializer.serialize_str(&self.to_ss58check()) - } - } - #[cfg(feature = "std")] - impl<'de> $crate::serde::Deserialize<'de> for Public { - fn deserialize(deserializer: D) -> std::result::Result where - D: $crate::serde::Deserializer<'de> - { - use $crate::Ss58Codec; - Public::from_ss58check(&String::deserialize(deserializer)?) - .map_err(|e| $crate::serde::de::Error::custom(format!("{:?}", e))) - } - } + $crate::app_crypto_public_common_if_std!(); impl AsRef<[u8]> for Public { fn as_ref(&self) -> &[u8] { self.0.as_ref() } @@ -309,10 +296,63 @@ macro_rules! app_crypto_public_common { } } +/// Implements traits for the public key type if `feature = "std"` is enabled. +#[cfg(feature = "std")] +#[doc(hidden)] +#[macro_export] +macro_rules! app_crypto_public_common_if_std { + () => { + impl $crate::Derive for Public { + fn derive>(&self, + path: Iter + ) -> Option { + self.0.derive(path).map(Self) + } + } + + impl std::fmt::Display for Public { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + use $crate::Ss58Codec; + write!(f, "{}", self.0.to_ss58check()) + } + } + + impl $crate::serde::Serialize for Public { + fn serialize(&self, serializer: S) -> std::result::Result where + S: $crate::serde::Serializer + { + use $crate::Ss58Codec; + serializer.serialize_str(&self.to_ss58check()) + } + } + + impl<'de> $crate::serde::Deserialize<'de> for Public { + fn deserialize(deserializer: D) -> std::result::Result where + D: $crate::serde::Deserializer<'de> + { + use $crate::Ss58Codec; + Public::from_ss58check(&String::deserialize(deserializer)?) + .map_err(|e| $crate::serde::de::Error::custom(format!("{:?}", e))) + } + } + } +} + +#[cfg(not(feature = "std"))] +#[doc(hidden)] +#[macro_export] +macro_rules! app_crypto_public_common_if_std { + () => { + impl $crate::Derive for Public {} + } +} + + /// Declares Signature type which is functionally equivalent to `$sig`, but is new /// Application-specific type whose identifier is `$key_type`. /// can only be used together with `full_crypto` feature /// For full functionality, app_crypto_public_common! must be called too. +#[doc(hidden)] #[macro_export] macro_rules! app_crypto_signature_full_crypto { ($sig:ty, $key_type:expr) => { @@ -345,6 +385,7 @@ macro_rules! app_crypto_signature_full_crypto { /// Application-specific type whose identifier is `$key_type`. /// can only be used without `full_crypto` feature /// For full functionality, app_crypto_public_common! must be called too. +#[doc(hidden)] #[macro_export] macro_rules! app_crypto_signature_not_full_crypto { ($sig:ty, $key_type:expr) => { @@ -372,6 +413,7 @@ macro_rules! app_crypto_signature_not_full_crypto { /// Declares Signature type which is functionally equivalent to `$sig`, but is new /// Application-specific type whose identifier is `$key_type`. /// For full functionality, app_crypto_public_(not)_full_crypto! must be called too. +#[doc(hidden)] #[macro_export] macro_rules! app_crypto_signature_common { ($sig:ty, $key_type:expr) => { diff --git a/primitives/application-crypto/test/Cargo.toml b/primitives/application-crypto/test/Cargo.toml index d34840b4eb5206429fbf829f60c2e11187b838f4..47b477ddd37df6e11207dd86e68d670d5a8e9225 100644 --- a/primitives/application-crypto/test/Cargo.toml +++ b/primitives/application-crypto/test/Cargo.toml @@ -9,12 +9,12 @@ publish = false homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" -[dependencies] -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../core" } -substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../runtime" } -sp-api = { version = "2.0.0-alpha.5", path = "../../api" } -sp-application-crypto = { version = "2.0.0-alpha.5", path = "../" } - [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../core" } +substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } +sp-runtime = { version = "2.0.0-dev", path = "../../runtime" } +sp-api = { version = "2.0.0-dev", path = "../../api" } +sp-application-crypto = { version = "2.0.0-dev", path = "../" } diff --git a/primitives/arithmetic/Cargo.toml b/primitives/arithmetic/Cargo.toml index 208525f6c193bdca2b9b5f9ba4a566999d523544..70efb4ac4a852df086d14a00f506c0ab826411d2 100644 --- a/primitives/arithmetic/Cargo.toml +++ b/primitives/arithmetic/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-arithmetic" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,19 +9,23 @@ repository = "https://github.com/paritytech/substrate/" description = "Minimal fixed point arithmetic primitives and types for runtime." documentation = "https://docs.rs/sp-arithmetic" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } integer-sqrt = "0.1.2" num-traits = { version = "0.2.8", default-features = false } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-debug-derive = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/debug-derive" } +sp-debug-derive = { version = "2.0.0-dev", default-features = false, path = "../../primitives/debug-derive" } +primitive-types = { version = "0.7.0", default-features = false } [dev-dependencies] -primitive-types = "0.7.0" rand = "0.7.2" criterion = "0.3" +serde_json = "1.0" [features] default = ["std"] @@ -31,11 +35,9 @@ std = [ "sp-std/std", "serde", "sp-debug-derive/std", + "primitive-types/std", ] [[bench]] name = "bench" harness = false - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/arithmetic/fuzzer/Cargo.toml b/primitives/arithmetic/fuzzer/Cargo.toml index 26b5f8c27b5085fd615dce5af70939e994c76767..f145c2cd9040c9594490a9674adca967d3dde447 100644 --- a/primitives/arithmetic/fuzzer/Cargo.toml +++ b/primitives/arithmetic/fuzzer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-arithmetic-fuzzer" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,9 +8,13 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Fuzzer for fixed point arithmetic primitives." documentation = "https://docs.rs/sp-arithmetic-fuzzer" +publish = false + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-arithmetic = { version = "2.0.0-alpha.5", path = ".." } +sp-arithmetic = { version = "2.0.0-dev", path = ".." } honggfuzz = "0.5" primitive-types = "0.7.0" num-bigint = "0.2" @@ -27,6 +31,3 @@ path = "src/per_thing_rational.rs" [[bin]] name = "rational128" path = "src/rational128.rs" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/arithmetic/src/fixed128.rs b/primitives/arithmetic/src/fixed128.rs new file mode 100644 index 0000000000000000000000000000000000000000..a0fafe5ee3ece14ebb02921c0223912d44dff835 --- /dev/null +++ b/primitives/arithmetic/src/fixed128.rs @@ -0,0 +1,731 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use codec::{Decode, Encode}; +use primitive_types::U256; +use crate::{ + traits::{Bounded, Saturating, UniqueSaturatedInto, SaturatedConversion}, + PerThing, Perquintill, +}; +use sp_std::{ + convert::{Into, TryFrom, TryInto}, + fmt, ops, + num::NonZeroI128, +}; + +#[cfg(feature = "std")] +use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; + +/// A signed fixed-point number. +/// Can hold any value in the range [-170_141_183_460_469_231_731, 170_141_183_460_469_231_731] +/// with fixed-point accuracy of 10 ** 18. +#[derive(Encode, Decode, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct Fixed128(i128); + +const DIV: i128 = 1_000_000_000_000_000_000; + +impl Fixed128 { + /// Create self from a natural number. + /// + /// Note that this might be lossy. + pub fn from_natural(int: i128) -> Self { + Self(int.saturating_mul(DIV)) + } + + /// Accuracy of `Fixed128`. + pub const fn accuracy() -> i128 { + DIV + } + + /// Raw constructor. Equal to `parts / DIV`. + pub const fn from_parts(parts: i128) -> Self { + Self(parts) + } + + /// Creates self from a rational number. Equal to `n/d`. + /// + /// Note that this might be lossy. Only use this if you are sure that `n * DIV` can fit into an + /// i128. + pub fn from_rational>(n: N, d: NonZeroI128) -> Self { + let n = n.unique_saturated_into(); + Self(n.saturating_mul(DIV.into()) / d.get()) + } + + /// Consume self and return the inner raw `i128` value. + /// + /// Note this is a low level function, as the returned value is represented with accuracy. + pub fn deconstruct(self) -> i128 { + self.0 + } + + /// Takes the reciprocal(inverse) of Fixed128, 1/x + pub fn recip(&self) -> Option { + Self::from_natural(1i128).checked_div(self) + } + + /// Checked add. Same semantic to `num_traits::CheckedAdd`. + pub fn checked_add(&self, rhs: &Self) -> Option { + self.0.checked_add(rhs.0).map(Self) + } + + /// Checked sub. Same semantic to `num_traits::CheckedSub`. + pub fn checked_sub(&self, rhs: &Self) -> Option { + self.0.checked_sub(rhs.0).map(Self) + } + + /// Checked mul. Same semantic to `num_traits::CheckedMul`. + pub fn checked_mul(&self, rhs: &Self) -> Option { + let signum = self.0.signum() * rhs.0.signum(); + let mut lhs = self.0; + if lhs.is_negative() { + lhs = lhs.saturating_mul(-1); + } + let mut rhs: i128 = rhs.0.saturated_into(); + if rhs.is_negative() { + rhs = rhs.saturating_mul(-1); + } + + U256::from(lhs) + .checked_mul(U256::from(rhs)) + .and_then(|n| n.checked_div(U256::from(DIV))) + .and_then(|n| TryInto::::try_into(n).ok()) + .map(|n| Self(n * signum)) + } + + /// Checked div. Same semantic to `num_traits::CheckedDiv`. + pub fn checked_div(&self, rhs: &Self) -> Option { + if rhs.0.signum() == 0 { + return None; + } + if self.0 == 0 { + return Some(*self); + } + + let signum = self.0.signum() / rhs.0.signum(); + let mut lhs: i128 = self.0; + if lhs.is_negative() { + lhs = lhs.saturating_mul(-1); + } + let mut rhs: i128 = rhs.0.saturated_into(); + if rhs.is_negative() { + rhs = rhs.saturating_mul(-1); + } + + U256::from(lhs) + .checked_mul(U256::from(DIV)) + .and_then(|n| n.checked_div(U256::from(rhs))) + .and_then(|n| TryInto::::try_into(n).ok()) + .map(|n| Self(n * signum)) + } + + /// Checked mul for int type `N`. + pub fn checked_mul_int(&self, other: &N) -> Option + where + N: Copy + TryFrom + TryInto, + { + N::try_into(*other).ok().and_then(|rhs| { + let mut lhs = self.0; + if lhs.is_negative() { + lhs = lhs.saturating_mul(-1); + } + let mut rhs: i128 = rhs.saturated_into(); + let signum = self.0.signum() * rhs.signum(); + if rhs.is_negative() { + rhs = rhs.saturating_mul(-1); + } + + U256::from(lhs) + .checked_mul(U256::from(rhs)) + .and_then(|n| n.checked_div(U256::from(DIV))) + .and_then(|n| TryInto::::try_into(n).ok()) + .and_then(|n| TryInto::::try_into(n * signum).ok()) + }) + } + + /// Checked mul for int type `N`. + pub fn saturating_mul_int(&self, other: &N) -> N + where + N: Copy + TryFrom + TryInto + Bounded, + { + self.checked_mul_int(other).unwrap_or_else(|| { + N::try_into(*other) + .map(|n| n.signum()) + .map(|n| n * self.0.signum()) + .map(|signum| { + if signum.is_negative() { + Bounded::min_value() + } else { + Bounded::max_value() + } + }) + .unwrap_or(Bounded::max_value()) + }) + } + + /// Checked div for int type `N`. + pub fn checked_div_int(&self, other: &N) -> Option + where + N: Copy + TryFrom + TryInto, + { + N::try_into(*other) + .ok() + .and_then(|n| self.0.checked_div(n)) + .and_then(|n| n.checked_div(DIV)) + .and_then(|n| TryInto::::try_into(n).ok()) + } + + pub fn zero() -> Self { + Self(0) + } + + pub fn is_zero(&self) -> bool { + self.0 == 0 + } + + /// Saturating absolute value. Returning MAX if `parts` == i128::MIN instead of overflowing. + pub fn saturating_abs(&self) -> Self { + if self.0 == i128::min_value() { + return Fixed128::max_value(); + } + + if self.0.is_negative() { + Fixed128::from_parts(self.0 * -1) + } else { + *self + } + } + + pub fn is_positive(&self) -> bool { + self.0.is_positive() + } + + pub fn is_negative(&self) -> bool { + self.0.is_negative() + } + + /// Performs a saturated multiply and accumulate by unsigned number. + /// + /// Returns a saturated `int + (self * int)`. + pub fn saturated_multiply_accumulate(self, int: N) -> N + where + N: TryFrom + From + UniqueSaturatedInto + Bounded + Clone + Saturating + + ops::Rem + ops::Div + ops::Mul + + ops::Add, + { + let div = DIV as u128; + let positive = self.0 > 0; + // safe to convert as absolute value. + let parts = self.0.checked_abs().map(|v| v as u128).unwrap_or(i128::max_value() as u128 + 1); + + + // will always fit. + let natural_parts = parts / div; + // might saturate. + let natural_parts: N = natural_parts.saturated_into(); + // fractional parts can always fit into u64. + let perquintill_parts = (parts % div) as u64; + + let n = int.clone().saturating_mul(natural_parts); + let p = Perquintill::from_parts(perquintill_parts) * int.clone(); + + // everything that needs to be either added or subtracted from the original weight. + let excess = n.saturating_add(p); + + if positive { + int.saturating_add(excess) + } else { + int.saturating_sub(excess) + } + } +} + +/// Note that this is a standard, _potentially-panicking_, implementation. Use `Saturating` trait +/// for safe addition. +impl ops::Add for Fixed128 { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + +/// Note that this is a standard, _potentially-panicking_, implementation. Use `Saturating` trait +/// for safe subtraction. +impl ops::Sub for Fixed128 { + type Output = Self; + + fn sub(self, rhs: Self) -> Self::Output { + Self(self.0 - rhs.0) + } +} + +impl Saturating for Fixed128 { + fn saturating_add(self, rhs: Self) -> Self { + Self(self.0.saturating_add(rhs.0)) + } + + fn saturating_sub(self, rhs: Self) -> Self { + Self(self.0.saturating_sub(rhs.0)) + } + + fn saturating_mul(self, rhs: Self) -> Self { + self.checked_mul(&rhs).unwrap_or_else(|| { + if (self.0.signum() * rhs.0.signum()).is_negative() { + Bounded::min_value() + } else { + Bounded::max_value() + } + }) + } + + fn saturating_pow(self, exp: usize) -> Self { + if exp == 0 { + return Self::from_natural(1); + } + + let exp = exp as u64; + let msb_pos = 64 - exp.leading_zeros(); + + let mut result = Self::from_natural(1); + let mut pow_val = self; + for i in 0..msb_pos { + if ((1 << i) & exp) > 0 { + result = result.saturating_mul(pow_val); + } + pow_val = pow_val.saturating_mul(pow_val); + } + result + } +} + +impl Bounded for Fixed128 { + fn min_value() -> Self { + Self(Bounded::min_value()) + } + + fn max_value() -> Self { + Self(Bounded::max_value()) + } +} + +impl fmt::Debug for Fixed128 { + #[cfg(feature = "std")] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let integral = { + let int = self.0 / DIV; + let signum_for_zero = if int == 0 && self.is_negative() { "-" } else { "" }; + format!("{}{}", signum_for_zero, int) + }; + let fractional = format!("{:0>18}", (self.0 % DIV).abs()); + write!(f, "Fixed128({}.{})", integral, fractional) + } + + #[cfg(not(feature = "std"))] + fn fmt(&self, _: &mut fmt::Formatter) -> fmt::Result { + Ok(()) + } +} + +impl From

for Fixed128 { + fn from(val: P) -> Self { + let accuracy = P::ACCURACY.saturated_into().max(1) as i128; + let value = val.deconstruct().saturated_into() as i128; + Fixed128::from_rational(value, NonZeroI128::new(accuracy).unwrap()) + } +} + +#[cfg(feature = "std")] +impl Fixed128 { + fn i128_str(&self) -> String { + format!("{}", &self.0) + } + + fn try_from_i128_str(s: &str) -> Result { + let parts: i128 = s.parse().map_err(|_| "invalid string input")?; + Ok(Self::from_parts(parts)) + } +} + +// Manual impl `Serialize` as serde_json does not support i128. +// TODO: remove impl if issue https://github.com/serde-rs/json/issues/548 fixed. +#[cfg(feature = "std")] +impl Serialize for Fixed128 { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&self.i128_str()) + } +} + +// Manual impl `Serialize` as serde_json does not support i128. +// TODO: remove impl if issue https://github.com/serde-rs/json/issues/548 fixed. +#[cfg(feature = "std")] +impl<'de> Deserialize<'de> for Fixed128 { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + Fixed128::try_from_i128_str(&s).map_err(|err_str| de::Error::custom(err_str)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Perbill, Percent, Permill, Perquintill}; + + fn max() -> Fixed128 { + Fixed128::max_value() + } + + fn min() -> Fixed128 { + Fixed128::min_value() + } + + #[test] + fn fixed128_semantics() { + let a = Fixed128::from_rational(5, NonZeroI128::new(2).unwrap()); + let b = Fixed128::from_rational(10, NonZeroI128::new(4).unwrap()); + assert_eq!(a.0, 5 * DIV / 2); + assert_eq!(a, b); + + let a = Fixed128::from_rational(-5, NonZeroI128::new(1).unwrap()); + assert_eq!(a, Fixed128::from_natural(-5)); + + let a = Fixed128::from_rational(5, NonZeroI128::new(-1).unwrap()); + assert_eq!(a, Fixed128::from_natural(-5)); + + // biggest value that can be created. + assert_ne!(max(), Fixed128::from_natural(170_141_183_460_469_231_731)); + assert_eq!(max(), Fixed128::from_natural(170_141_183_460_469_231_732)); + + // the smallest value that can be created. + assert_ne!(min(), Fixed128::from_natural(-170_141_183_460_469_231_731)); + assert_eq!(min(), Fixed128::from_natural(-170_141_183_460_469_231_732)); + } + + #[test] + fn fixed128_operation() { + let a = Fixed128::from_natural(2); + let b = Fixed128::from_natural(1); + assert_eq!(a.checked_add(&b), Some(Fixed128::from_natural(1 + 2))); + assert_eq!(a.checked_sub(&b), Some(Fixed128::from_natural(2 - 1))); + assert_eq!(a.checked_mul(&b), Some(Fixed128::from_natural(1 * 2))); + assert_eq!( + a.checked_div(&b), + Some(Fixed128::from_rational(2, NonZeroI128::new(1).unwrap())) + ); + + let a = Fixed128::from_rational(5, NonZeroI128::new(2).unwrap()); + let b = Fixed128::from_rational(3, NonZeroI128::new(2).unwrap()); + assert_eq!( + a.checked_add(&b), + Some(Fixed128::from_rational(8, NonZeroI128::new(2).unwrap())) + ); + assert_eq!( + a.checked_sub(&b), + Some(Fixed128::from_rational(2, NonZeroI128::new(2).unwrap())) + ); + assert_eq!( + a.checked_mul(&b), + Some(Fixed128::from_rational(15, NonZeroI128::new(4).unwrap())) + ); + assert_eq!( + a.checked_div(&b), + Some(Fixed128::from_rational(10, NonZeroI128::new(6).unwrap())) + ); + + let a = Fixed128::from_natural(120); + assert_eq!(a.checked_div_int(&2i32), Some(60)); + + let a = Fixed128::from_rational(20, NonZeroI128::new(1).unwrap()); + assert_eq!(a.checked_div_int(&2i32), Some(10)); + + let a = Fixed128::from_natural(120); + assert_eq!(a.checked_mul_int(&2i32), Some(240)); + + let a = Fixed128::from_rational(1, NonZeroI128::new(2).unwrap()); + assert_eq!(a.checked_mul_int(&20i32), Some(10)); + + let a = Fixed128::from_rational(-1, NonZeroI128::new(2).unwrap()); + assert_eq!(a.checked_mul_int(&20i32), Some(-10)); + } + + #[test] + fn saturating_mul_should_work() { + let a = Fixed128::from_natural(-1); + assert_eq!(min().saturating_mul(a), max()); + + assert_eq!(Fixed128::from_natural(125).saturating_mul(a).deconstruct(), -125 * DIV); + + let a = Fixed128::from_rational(1, NonZeroI128::new(5).unwrap()); + assert_eq!(Fixed128::from_natural(125).saturating_mul(a).deconstruct(), 25 * DIV); + } + + #[test] + fn saturating_mul_int_works() { + let a = Fixed128::from_rational(10, NonZeroI128::new(1).unwrap()); + assert_eq!(a.saturating_mul_int(&i32::max_value()), i32::max_value()); + + let a = Fixed128::from_rational(-10, NonZeroI128::new(1).unwrap()); + assert_eq!(a.saturating_mul_int(&i32::max_value()), i32::min_value()); + + let a = Fixed128::from_rational(3, NonZeroI128::new(1).unwrap()); + assert_eq!(a.saturating_mul_int(&100i8), i8::max_value()); + + let a = Fixed128::from_rational(10, NonZeroI128::new(1).unwrap()); + assert_eq!(a.saturating_mul_int(&123i128), 1230); + + let a = Fixed128::from_rational(-10, NonZeroI128::new(1).unwrap()); + assert_eq!(a.saturating_mul_int(&123i128), -1230); + + assert_eq!(max().saturating_mul_int(&2i128), 340_282_366_920_938_463_463); + + assert_eq!(max().saturating_mul_int(&i128::min_value()), i128::min_value()); + + assert_eq!(min().saturating_mul_int(&i128::max_value()), i128::min_value()); + + assert_eq!(min().saturating_mul_int(&i128::min_value()), i128::max_value()); + } + + #[test] + fn zero_works() { + assert_eq!(Fixed128::zero(), Fixed128::from_natural(0)); + } + + #[test] + fn is_zero_works() { + assert!(Fixed128::zero().is_zero()); + assert!(!Fixed128::from_natural(1).is_zero()); + } + + #[test] + fn checked_div_with_zero_should_be_none() { + let a = Fixed128::from_natural(1); + let b = Fixed128::from_natural(0); + assert_eq!(a.checked_div(&b), None); + assert_eq!(b.checked_div(&a), Some(b)); + } + + #[test] + fn checked_div_int_with_zero_should_be_none() { + let a = Fixed128::from_natural(1); + assert_eq!(a.checked_div_int(&0i32), None); + let a = Fixed128::from_natural(0); + assert_eq!(a.checked_div_int(&1i32), Some(0)); + } + + #[test] + fn checked_div_with_zero_dividend_should_be_zero() { + let a = Fixed128::zero(); + let b = Fixed128::from_parts(1); + + assert_eq!(a.checked_div(&b), Some(Fixed128::zero())); + } + + #[test] + fn under_flow_should_be_none() { + let b = Fixed128::from_natural(1); + assert_eq!(min().checked_sub(&b), None); + } + + #[test] + fn over_flow_should_be_none() { + let a = Fixed128::from_parts(i128::max_value() - 1); + let b = Fixed128::from_parts(2); + assert_eq!(a.checked_add(&b), None); + + let a = Fixed128::max_value(); + let b = Fixed128::from_rational(2, NonZeroI128::new(1).unwrap()); + assert_eq!(a.checked_mul(&b), None); + + let a = Fixed128::from_natural(255); + let b = 2u8; + assert_eq!(a.checked_mul_int(&b), None); + + let a = Fixed128::from_natural(256); + let b = 1u8; + assert_eq!(a.checked_div_int(&b), None); + + let a = Fixed128::from_natural(256); + let b = -1i8; + assert_eq!(a.checked_div_int(&b), None); + } + + #[test] + fn checked_div_int_should_work() { + // 256 / 10 = 25 (25.6 as int = 25) + let a = Fixed128::from_natural(256); + let result = a.checked_div_int(&10i128).unwrap(); + assert_eq!(result, 25); + + // 256 / 100 = 2 (2.56 as int = 2) + let a = Fixed128::from_natural(256); + let result = a.checked_div_int(&100i128).unwrap(); + assert_eq!(result, 2); + + // 256 / 1000 = 0 (0.256 as int = 0) + let a = Fixed128::from_natural(256); + let result = a.checked_div_int(&1000i128).unwrap(); + assert_eq!(result, 0); + + // 256 / -1 = -256 + let a = Fixed128::from_natural(256); + let result = a.checked_div_int(&-1i128).unwrap(); + assert_eq!(result, -256); + + // -256 / -1 = 256 + let a = Fixed128::from_natural(-256); + let result = a.checked_div_int(&-1i128).unwrap(); + assert_eq!(result, 256); + + // 10 / -5 = -2 + let a = Fixed128::from_rational(20, NonZeroI128::new(2).unwrap()); + let result = a.checked_div_int(&-5i128).unwrap(); + assert_eq!(result, -2); + + // -170_141_183_460_469_231_731 / -2 = 85_070_591_730_234_615_865 + let result = min().checked_div_int(&-2i128).unwrap(); + assert_eq!(result, 85_070_591_730_234_615_865); + + // 85_070_591_730_234_615_865 * -2 = -170_141_183_460_469_231_730 + let result = Fixed128::from_natural(result).checked_mul_int(&-2i128).unwrap(); + assert_eq!(result, -170_141_183_460_469_231_730); + } + + #[test] + fn perthing_into_fixed_i128() { + let ten_percent_percent: Fixed128 = Percent::from_percent(10).into(); + assert_eq!(ten_percent_percent.deconstruct(), DIV / 10); + + let ten_percent_permill: Fixed128 = Permill::from_percent(10).into(); + assert_eq!(ten_percent_permill.deconstruct(), DIV / 10); + + let ten_percent_perbill: Fixed128 = Perbill::from_percent(10).into(); + assert_eq!(ten_percent_perbill.deconstruct(), DIV / 10); + + let ten_percent_perquintill: Fixed128 = Perquintill::from_percent(10).into(); + assert_eq!(ten_percent_perquintill.deconstruct(), DIV / 10); + } + + #[test] + fn recip_should_work() { + let a = Fixed128::from_natural(2); + assert_eq!( + a.recip(), + Some(Fixed128::from_rational(1, NonZeroI128::new(2).unwrap())) + ); + + let a = Fixed128::from_natural(2); + assert_eq!(a.recip().unwrap().checked_mul_int(&4i32), Some(2i32)); + + let a = Fixed128::from_rational(100, NonZeroI128::new(121).unwrap()); + assert_eq!( + a.recip(), + Some(Fixed128::from_rational(121, NonZeroI128::new(100).unwrap())) + ); + + let a = Fixed128::from_rational(1, NonZeroI128::new(2).unwrap()); + assert_eq!(a.recip().unwrap().checked_mul(&a), Some(Fixed128::from_natural(1))); + + let a = Fixed128::from_natural(0); + assert_eq!(a.recip(), None); + + let a = Fixed128::from_rational(-1, NonZeroI128::new(2).unwrap()); + assert_eq!(a.recip(), Some(Fixed128::from_natural(-2))); + } + + #[test] + fn serialize_deserialize_should_work() { + let two_point_five = Fixed128::from_rational(5, NonZeroI128::new(2).unwrap()); + let serialized = serde_json::to_string(&two_point_five).unwrap(); + assert_eq!(serialized, "\"2500000000000000000\""); + let deserialized: Fixed128 = serde_json::from_str(&serialized).unwrap(); + assert_eq!(deserialized, two_point_five); + + let minus_two_point_five = Fixed128::from_rational(-5, NonZeroI128::new(2).unwrap()); + let serialized = serde_json::to_string(&minus_two_point_five).unwrap(); + assert_eq!(serialized, "\"-2500000000000000000\""); + let deserialized: Fixed128 = serde_json::from_str(&serialized).unwrap(); + assert_eq!(deserialized, minus_two_point_five); + } + + #[test] + fn saturating_abs_should_work() { + // normal + assert_eq!(Fixed128::from_parts(1).saturating_abs(), Fixed128::from_parts(1)); + assert_eq!(Fixed128::from_parts(-1).saturating_abs(), Fixed128::from_parts(1)); + + // saturating + assert_eq!(Fixed128::min_value().saturating_abs(), Fixed128::max_value()); + } + + #[test] + fn is_positive_negative_should_work() { + let positive = Fixed128::from_parts(1); + assert!(positive.is_positive()); + assert!(!positive.is_negative()); + + let negative = Fixed128::from_parts(-1); + assert!(!negative.is_positive()); + assert!(negative.is_negative()); + + let zero = Fixed128::zero(); + assert!(!zero.is_positive()); + assert!(!zero.is_negative()); + } + + #[test] + fn fmt_should_work() { + let positive = Fixed128::from_parts(1000000000000000001); + assert_eq!(format!("{:?}", positive), "Fixed128(1.000000000000000001)"); + let negative = Fixed128::from_parts(-1000000000000000001); + assert_eq!(format!("{:?}", negative), "Fixed128(-1.000000000000000001)"); + + let positive_fractional = Fixed128::from_parts(1); + assert_eq!(format!("{:?}", positive_fractional), "Fixed128(0.000000000000000001)"); + let negative_fractional = Fixed128::from_parts(-1); + assert_eq!(format!("{:?}", negative_fractional), "Fixed128(-0.000000000000000001)"); + + let zero = Fixed128::zero(); + assert_eq!(format!("{:?}", zero), "Fixed128(0.000000000000000000)"); + } + + #[test] + fn saturating_pow_should_work() { + assert_eq!(Fixed128::from_natural(2).saturating_pow(0), Fixed128::from_natural(1)); + assert_eq!(Fixed128::from_natural(2).saturating_pow(1), Fixed128::from_natural(2)); + assert_eq!(Fixed128::from_natural(2).saturating_pow(2), Fixed128::from_natural(4)); + assert_eq!(Fixed128::from_natural(2).saturating_pow(3), Fixed128::from_natural(8)); + assert_eq!(Fixed128::from_natural(2).saturating_pow(50), Fixed128::from_natural(1125899906842624)); + + assert_eq!(Fixed128::from_natural(1).saturating_pow(1000), Fixed128::from_natural(1)); + assert_eq!(Fixed128::from_natural(-1).saturating_pow(1000), Fixed128::from_natural(1)); + assert_eq!(Fixed128::from_natural(-1).saturating_pow(1001), Fixed128::from_natural(-1)); + assert_eq!(Fixed128::from_natural(1).saturating_pow(usize::max_value()), Fixed128::from_natural(1)); + assert_eq!(Fixed128::from_natural(-1).saturating_pow(usize::max_value()), Fixed128::from_natural(-1)); + assert_eq!(Fixed128::from_natural(-1).saturating_pow(usize::max_value() - 1), Fixed128::from_natural(1)); + + assert_eq!(Fixed128::from_natural(114209).saturating_pow(4), Fixed128::from_natural(170137997018538053761)); + assert_eq!(Fixed128::from_natural(114209).saturating_pow(5), Fixed128::max_value()); + + assert_eq!(Fixed128::from_natural(1).saturating_pow(usize::max_value()), Fixed128::from_natural(1)); + assert_eq!(Fixed128::from_natural(0).saturating_pow(usize::max_value()), Fixed128::from_natural(0)); + assert_eq!(Fixed128::from_natural(2).saturating_pow(usize::max_value()), Fixed128::max_value()); + } +} diff --git a/primitives/arithmetic/src/fixed64.rs b/primitives/arithmetic/src/fixed64.rs index 6b399b6aa5106d3889c1152c8cb1e7fc38a14585..63a69e66e7c9d6363e1c9eeba7b6c9e4c31b3164 100644 --- a/primitives/arithmetic/src/fixed64.rs +++ b/primitives/arithmetic/src/fixed64.rs @@ -49,9 +49,6 @@ impl Fixed64 { } /// Consume self and return the inner value. - /// - /// This should only be used for testing. - #[cfg(any(feature = "std", test))] pub fn into_inner(self) -> i64 { self.0 } /// Raw constructor. Equal to `parts / 1_000_000_000`. @@ -104,6 +101,10 @@ impl Fixed64 { int.saturating_sub(excess) } } + + pub fn is_negative(&self) -> bool { + self.0.is_negative() + } } impl Saturating for Fixed64 { @@ -112,7 +113,10 @@ impl Saturating for Fixed64 { } fn saturating_mul(self, rhs: Self) -> Self { - Self(self.0.saturating_mul(rhs.0) / DIV) + let a = self.0 as i128; + let b = rhs.0 as i128; + let res = a * b / DIV as i128; + Self(res.saturated_into()) } fn saturating_sub(self, rhs: Self) -> Self { @@ -120,12 +124,26 @@ impl Saturating for Fixed64 { } fn saturating_pow(self, exp: usize) -> Self { - Self(self.0.saturating_pow(exp as u32)) + if exp == 0 { + return Self::from_natural(1); + } + + let exp = exp as u64; + let msb_pos = 64 - exp.leading_zeros(); + + let mut result = Self::from_natural(1); + let mut pow_val = self; + for i in 0..msb_pos { + if ((1 << i) & exp) > 0 { + result = result.saturating_mul(pow_val); + } + pow_val = pow_val.saturating_mul(pow_val); + } + result } } -/// Note that this is a standard, _potentially-panicking_, implementation. Use `Saturating` trait -/// for safe addition. +/// Use `Saturating` trait for safe addition. impl ops::Add for Fixed64 { type Output = Self; @@ -134,8 +152,7 @@ impl ops::Add for Fixed64 { } } -/// Note that this is a standard, _potentially-panicking_, implementation. Use `Saturating` trait -/// for safe subtraction. +/// Use `Saturating` trait for safe subtraction. impl ops::Sub for Fixed64 { type Output = Self; @@ -144,8 +161,7 @@ impl ops::Sub for Fixed64 { } } -/// Note that this is a standard, _potentially-panicking_, implementation. Use `CheckedDiv` trait -/// for safe division. +/// Use `CheckedDiv` trait for safe division. impl ops::Div for Fixed64 { type Output = Self; @@ -188,7 +204,13 @@ impl CheckedDiv for Fixed64 { impl sp_std::fmt::Debug for Fixed64 { #[cfg(feature = "std")] fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { - write!(f, "Fixed64({},{})", self.0 / DIV, (self.0 % DIV) / 1000) + let integral = { + let int = self.0 / DIV; + let signum_for_zero = if int == 0 && self.is_negative() { "-" } else { "" }; + format!("{}{}", signum_for_zero, int) + }; + let fractional = format!("{:0>9}", (self.0 % DIV).abs()); + write!(f, "Fixed64({}.{})", integral, fractional) } #[cfg(not(feature = "std"))] @@ -329,4 +351,32 @@ mod tests { let b = Fixed64::from_rational(1, 100); assert_eq!(a.checked_div(&b), Some(Fixed64::from_rational(120, 1))); } + + #[test] + fn saturating_mul_should_work() { + assert_eq!(Fixed64::from_natural(100).saturating_mul(Fixed64::from_natural(100)), Fixed64::from_natural(10000)); + } + + #[test] + fn saturating_pow_should_work() { + assert_eq!(Fixed64::from_natural(2).saturating_pow(0), Fixed64::from_natural(1)); + assert_eq!(Fixed64::from_natural(2).saturating_pow(1), Fixed64::from_natural(2)); + assert_eq!(Fixed64::from_natural(2).saturating_pow(2), Fixed64::from_natural(4)); + assert_eq!(Fixed64::from_natural(2).saturating_pow(3), Fixed64::from_natural(8)); + assert_eq!(Fixed64::from_natural(2).saturating_pow(20), Fixed64::from_natural(1048576)); + + assert_eq!(Fixed64::from_natural(1).saturating_pow(1000), Fixed64::from_natural(1)); + assert_eq!(Fixed64::from_natural(-1).saturating_pow(1000), Fixed64::from_natural(1)); + assert_eq!(Fixed64::from_natural(-1).saturating_pow(1001), Fixed64::from_natural(-1)); + assert_eq!(Fixed64::from_natural(1).saturating_pow(usize::max_value()), Fixed64::from_natural(1)); + assert_eq!(Fixed64::from_natural(-1).saturating_pow(usize::max_value()), Fixed64::from_natural(-1)); + assert_eq!(Fixed64::from_natural(-1).saturating_pow(usize::max_value() - 1), Fixed64::from_natural(1)); + + assert_eq!(Fixed64::from_natural(309).saturating_pow(4), Fixed64::from_natural(9_116_621_361)); + assert_eq!(Fixed64::from_natural(309).saturating_pow(5), Fixed64::from_parts(i64::max_value())); + + assert_eq!(Fixed64::from_natural(1).saturating_pow(usize::max_value()), Fixed64::from_natural(1)); + assert_eq!(Fixed64::from_natural(0).saturating_pow(usize::max_value()), Fixed64::from_natural(0)); + assert_eq!(Fixed64::from_natural(2).saturating_pow(usize::max_value()), Fixed64::from_parts(i64::max_value())); + } } diff --git a/primitives/arithmetic/src/lib.rs b/primitives/arithmetic/src/lib.rs index f6d8b53e3499b340b27285964e8c366be39ca158..fb70b13a15312b4cee434c6dc9a58ef234bad444 100644 --- a/primitives/arithmetic/src/lib.rs +++ b/primitives/arithmetic/src/lib.rs @@ -37,9 +37,11 @@ pub mod helpers_128bit; pub mod traits; mod per_things; mod fixed64; +mod fixed128; mod rational128; pub use fixed64::Fixed64; +pub use fixed128::Fixed128; pub use per_things::{PerThing, Percent, PerU16, Permill, Perbill, Perquintill}; pub use rational128::Rational128; diff --git a/primitives/arithmetic/src/per_things.rs b/primitives/arithmetic/src/per_things.rs index ad529fbf32e249ef1fb0e5838c30fce52aab4b25..56fc562cd1a926cd3d33b8ccee48dabe6c13bb24 100644 --- a/primitives/arithmetic/src/per_things.rs +++ b/primitives/arithmetic/src/per_things.rs @@ -1096,7 +1096,7 @@ macro_rules! implement_per_thing { <$type>::max_value(), super::Rounding::Nearest, ), - (<$type>::max_value() - 1).into(), + <$upper_type>::from((<$type>::max_value() - 1)), ); // (max % 2) * max / 2 == max / 2 assert_eq!( diff --git a/primitives/arithmetic/src/traits.rs b/primitives/arithmetic/src/traits.rs index 23f8f23f0bd77b783dfc1908b0efc6354f9fc4a8..6b5e32446462323cdae371e323efebc3a167abc8 100644 --- a/primitives/arithmetic/src/traits.rs +++ b/primitives/arithmetic/src/traits.rs @@ -117,7 +117,7 @@ pub trait Saturating { /// Saturating multiply. Compute `self * rhs`, saturating at the numeric bounds instead of /// overflowing. fn saturating_mul(self, rhs: Self) -> Self; - + /// Saturating exponentiation. Compute `self.pow(exp)`, saturating at the numeric bounds /// instead of overflowing. fn saturating_pow(self, exp: usize) -> Self; diff --git a/primitives/authority-discovery/Cargo.toml b/primitives/authority-discovery/Cargo.toml index f37b67fab1ce8d476fef66656b3c5013701edbf9..32f67d5e6e0261a2310b70c0fea8d4f9d21a1dec 100644 --- a/primitives/authority-discovery/Cargo.toml +++ b/primitives/authority-discovery/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-authority-discovery" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] description = "Authority discovery primitives" edition = "2018" @@ -8,12 +8,15 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../application-crypto" } +sp-application-crypto = { version = "2.0.0-dev", default-features = false, path = "../application-crypto" } codec = { package = "parity-scale-codec", default-features = false, version = "1.3.0" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../api" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../api" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../runtime" } [features] default = ["std"] @@ -24,6 +27,3 @@ std = [ "sp-api/std", "sp-runtime/std" ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/authorship/Cargo.toml b/primitives/authorship/Cargo.toml index a4d5aa03c2382c3c93f278bca825c7ca08688951..7bc01953ef8dd1c59d5ed9c0e8222d97259b177e 100644 --- a/primitives/authorship/Cargo.toml +++ b/primitives/authorship/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-authorship" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] description = "Authorship primitives" edition = "2018" @@ -8,10 +8,13 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../inherents" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../inherents" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../runtime" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } [features] @@ -22,6 +25,3 @@ std = [ "sp-inherents/std", "sp-runtime/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/block-builder/Cargo.toml b/primitives/block-builder/Cargo.toml index df33b2c955ff096b1bd861769d1dd5c56497a867..70bb5e12d37a202330d6d98af82f092f4e578671 100644 --- a/primitives/block-builder/Cargo.toml +++ b/primitives/block-builder/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-block-builder" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,12 +8,15 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "The block builder runtime api." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../api" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../runtime" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../api" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../inherents" } +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../inherents" } [features] default = [ "std" ] @@ -24,6 +27,3 @@ std = [ "sp-api/std", "sp-std/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/blockchain/Cargo.toml b/primitives/blockchain/Cargo.toml index 49e6baead1fb82a919b3d7ca40fe0a6045ee0524..d5cf80b775196a0892af2a33325fd1653d799388 100644 --- a/primitives/blockchain/Cargo.toml +++ b/primitives/blockchain/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-blockchain" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,6 +9,9 @@ repository = "https://github.com/paritytech/substrate/" description = "Substrate blockchain traits and primitives." documentation = "https://docs.rs/sp-blockchain" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] log = "0.4.8" @@ -16,10 +19,7 @@ lru = "0.4.0" parking_lot = "0.10.0" derive_more = "0.99.2" codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-consensus = { version = "0.8.0-alpha.5", path = "../consensus/common" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../runtime" } -sp-block-builder = { version = "2.0.0-alpha.5", path = "../block-builder" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../state-machine" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sp-consensus = { version = "0.8.0-dev", path = "../consensus/common" } +sp-runtime = { version = "2.0.0-dev", path = "../runtime" } +sp-block-builder = { version = "2.0.0-dev", path = "../block-builder" } +sp-state-machine = { version = "0.8.0-dev", path = "../state-machine" } diff --git a/primitives/blockchain/src/backend.rs b/primitives/blockchain/src/backend.rs index e92dfd8c98e89397ee2e3d10e6f85e71b0c5ac00..45d627a1c27224a6c0349b9b5f2c11353b5d3a0b 100644 --- a/primitives/blockchain/src/backend.rs +++ b/primitives/blockchain/src/backend.rs @@ -242,7 +242,7 @@ pub trait Cache: Send + Sync { } /// Blockchain info -#[derive(Debug)] +#[derive(Debug, Eq, PartialEq)] pub struct Info { /// Best block hash. pub best_hash: Block::Hash, diff --git a/primitives/chain-spec/Cargo.toml b/primitives/chain-spec/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..585decc68d02b86cfe9a7631b796bc82a16cf9f2 --- /dev/null +++ b/primitives/chain-spec/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "sp-chain-spec" +version = "2.0.0-dev" +authors = ["Parity Technologies "] +edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Substrate chain configurations types." + +[dependencies] +serde = { version = "1.0.101", features = ["derive"] } +serde_json = "1.0.41" diff --git a/primitives/chain-spec/src/lib.rs b/primitives/chain-spec/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..13ebc09b6c0f0646b92d21056bbc1ef217f0ca3d --- /dev/null +++ b/primitives/chain-spec/src/lib.rs @@ -0,0 +1,42 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Types and traits related to chain specifications. + +/// The type of a chain. +/// +/// This can be used by tools to determine the type of a chain for displaying +/// additional information or enabling additional features. +#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq, Clone)] +pub enum ChainType { + /// A development chain that runs mainly on one node. + Development, + /// A local chain that runs locally on multiple nodes for testing purposes. + Local, + /// A live chain. + Live, + /// Some custom chain type. + Custom(String), +} + +impl Default for ChainType { + fn default() -> Self { + Self::Live + } +} + +/// Arbitrary properties defined in chain spec as a JSON object +pub type Properties = serde_json::map::Map; diff --git a/primitives/consensus/aura/Cargo.toml b/primitives/consensus/aura/Cargo.toml index 99ce51a229c72e045e97b92f9e8c8f8d3aff2513..574b80bd3d3902b959c55429d44e2fc2f21d96f3 100644 --- a/primitives/consensus/aura/Cargo.toml +++ b/primitives/consensus/aura/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-consensus-aura" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "Primitives for Aura consensus" edition = "2018" @@ -8,14 +8,17 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../../application-crypto" } +sp-application-crypto = { version = "2.0.0-dev", default-features = false, path = "../../application-crypto" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../std" } -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../api" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../runtime" } -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../inherents" } -sp-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../../timestamp" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../std" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../../api" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../runtime" } +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../../inherents" } +sp-timestamp = { version = "2.0.0-dev", default-features = false, path = "../../timestamp" } [features] default = ["std"] @@ -28,6 +31,3 @@ std = [ "sp-inherents/std", "sp-timestamp/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/consensus/babe/Cargo.toml b/primitives/consensus/babe/Cargo.toml index 195a54a5910c419957f053a6f928aa369b467110..ba7e7fffb623be2e76f2a49d65a387f7a8bc7406 100644 --- a/primitives/consensus/babe/Cargo.toml +++ b/primitives/consensus/babe/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-consensus-babe" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "Primitives for BABE consensus" edition = "2018" @@ -8,16 +8,19 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../../application-crypto" } +sp-application-crypto = { version = "2.0.0-dev", default-features = false, path = "../../application-crypto" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../std" } -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../api" } -sp-consensus = { version = "0.8.0-alpha.5", optional = true, path = "../common" } -sp-consensus-vrf = { version = "0.8.0-alpha.5", path = "../vrf", default-features = false } -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../inherents" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../runtime" } -sp-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../../timestamp" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../std" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../../api" } +sp-consensus = { version = "0.8.0-dev", optional = true, path = "../common" } +sp-consensus-vrf = { version = "0.8.0-dev", path = "../vrf", default-features = false } +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../../inherents" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../runtime" } +sp-timestamp = { version = "2.0.0-dev", default-features = false, path = "../../timestamp" } [features] default = ["std"] @@ -32,6 +35,3 @@ std = [ "sp-runtime/std", "sp-timestamp/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/consensus/babe/src/digests.rs b/primitives/consensus/babe/src/digests.rs index 6079aa88c87492538fbf3caefdd3775d4cdd9292..141d6cf4bd72739913fcf814b4fe6b4e55013ae9 100644 --- a/primitives/consensus/babe/src/digests.rs +++ b/primitives/consensus/babe/src/digests.rs @@ -18,7 +18,7 @@ #[cfg(feature = "std")] use super::{BABE_ENGINE_ID, AuthoritySignature}; -use super::{AuthorityId, AuthorityIndex, SlotNumber, BabeAuthorityWeight}; +use super::{AuthorityId, AuthorityIndex, SlotNumber, BabeAuthorityWeight, BabeEpochConfiguration, AllowedSlots}; #[cfg(feature = "std")] use sp_runtime::{DigestItem, generic::OpaqueDigestItemId}; #[cfg(feature = "std")] @@ -65,7 +65,7 @@ impl TryFrom for PrimaryPreDigest { /// BABE secondary slot assignment pre-digest. #[derive(Clone, RuntimeDebug, Encode, Decode)] -pub struct SecondaryPreDigest { +pub struct SecondaryPlainPreDigest { /// Authority index /// /// This is not strictly-speaking necessary, since the secondary slots @@ -77,6 +77,37 @@ pub struct SecondaryPreDigest { pub slot_number: SlotNumber, } +/// BABE secondary deterministic slot assignment with VRF outputs. +#[derive(Clone, RuntimeDebug, Encode, Decode)] +pub struct RawSecondaryVRFPreDigest { + /// Authority index + pub authority_index: super::AuthorityIndex, + /// Slot number + pub slot_number: SlotNumber, + /// VRF output + pub vrf_output: VRFOutput, + /// VRF proof + pub vrf_proof: VRFProof, +} + +#[cfg(feature = "std")] +/// BABE secondary slot assignment with VRF outputs pre-digest, for std environment. +pub type SecondaryVRFPreDigest = RawSecondaryVRFPreDigest; + +#[cfg(feature = "std")] +impl TryFrom for SecondaryVRFPreDigest { + type Error = SignatureError; + + fn try_from(raw: RawSecondaryVRFPreDigest) -> Result { + Ok(SecondaryVRFPreDigest { + authority_index: raw.authority_index, + slot_number: raw.slot_number, + vrf_output: raw.vrf_output.try_into()?, + vrf_proof: raw.vrf_proof.try_into()?, + }) + } +} + /// A BABE pre-runtime digest. This contains all data required to validate a /// block and for the BABE runtime module. Slots can be assigned to a primary /// (VRF based) and to a secondary (slot number based). @@ -87,7 +118,10 @@ pub enum RawPreDigest), /// A secondary deterministic slot assignment. #[codec(index = "2")] - Secondary(SecondaryPreDigest), + SecondaryPlain(SecondaryPlainPreDigest), + /// A secondary deterministic slot assignment with VRF outputs. + #[codec(index = "3")] + SecondaryVRF(RawSecondaryVRFPreDigest), } #[cfg(feature = "std")] @@ -99,7 +133,8 @@ impl RawPreDigest { pub fn authority_index(&self) -> AuthorityIndex { match self { RawPreDigest::Primary(primary) => primary.authority_index, - RawPreDigest::Secondary(secondary) => secondary.authority_index, + RawPreDigest::SecondaryPlain(secondary) => secondary.authority_index, + RawPreDigest::SecondaryVRF(secondary) => secondary.authority_index, } } @@ -107,7 +142,8 @@ impl RawPreDigest { pub fn slot_number(&self) -> SlotNumber { match self { RawPreDigest::Primary(primary) => primary.slot_number, - RawPreDigest::Secondary(secondary) => secondary.slot_number, + RawPreDigest::SecondaryPlain(secondary) => secondary.slot_number, + RawPreDigest::SecondaryVRF(secondary) => secondary.slot_number, } } @@ -116,7 +152,7 @@ impl RawPreDigest { pub fn added_weight(&self) -> crate::BabeBlockWeight { match self { RawPreDigest::Primary(_) => 1, - RawPreDigest::Secondary(_) => 0, + RawPreDigest::SecondaryPlain(_) | RawPreDigest::SecondaryVRF(_) => 0, } } } @@ -128,14 +164,15 @@ impl TryFrom for PreDigest { fn try_from(raw: RawPreDigest) -> Result { Ok(match raw { RawPreDigest::Primary(primary) => PreDigest::Primary(primary.try_into()?), - RawPreDigest::Secondary(secondary) => PreDigest::Secondary(secondary), + RawPreDigest::SecondaryPlain(secondary) => PreDigest::SecondaryPlain(secondary), + RawPreDigest::SecondaryVRF(secondary) => PreDigest::SecondaryVRF(secondary.try_into()?), }) } } /// Information about the next epoch. This is broadcast in the first block /// of the epoch. -#[derive(Decode, Encode, Default, PartialEq, Eq, Clone, RuntimeDebug)] +#[derive(Decode, Encode, PartialEq, Eq, Clone, RuntimeDebug)] pub struct NextEpochDescriptor { /// The authorities. pub authorities: Vec<(AuthorityId, BabeAuthorityWeight)>, @@ -144,6 +181,29 @@ pub struct NextEpochDescriptor { pub randomness: Randomness, } +/// Information about the next epoch config, if changed. This is broadcast in the first +/// block of the epoch, and applies using the same rules as `NextEpochDescriptor`. +#[derive(Decode, Encode, PartialEq, Eq, Clone, RuntimeDebug)] +pub enum NextConfigDescriptor { + /// Version 1. + #[codec(index = "1")] + V1 { + /// Value of `c` in `BabeEpochConfiguration`. + c: (u64, u64), + /// Value of `allowed_slots` in `BabeEpochConfiguration`. + allowed_slots: AllowedSlots, + } +} + +impl From for BabeEpochConfiguration { + fn from(desc: NextConfigDescriptor) -> Self { + match desc { + NextConfigDescriptor::V1 { c, allowed_slots } => + Self { c, allowed_slots }, + } + } +} + /// A digest item which is usable with BABE consensus. #[cfg(feature = "std")] pub trait CompatibleDigestItem: Sized { @@ -159,8 +219,11 @@ pub trait CompatibleDigestItem: Sized { /// If this item is a BABE signature, return the signature. fn as_babe_seal(&self) -> Option; - /// If this item is a BABE epoch, return it. + /// If this item is a BABE epoch descriptor, return it. fn as_next_epoch_descriptor(&self) -> Option; + + /// If this item is a BABE config descriptor, return it. + fn as_next_config_descriptor(&self) -> Option; } #[cfg(feature = "std")] @@ -190,4 +253,12 @@ impl CompatibleDigestItem for DigestItem where _ => None, }) } + + fn as_next_config_descriptor(&self) -> Option { + self.try_to(OpaqueDigestItemId::Consensus(&BABE_ENGINE_ID)) + .and_then(|x: super::ConsensusLog| match x { + super::ConsensusLog::NextConfigData(n) => Some(n), + _ => None, + }) + } } diff --git a/primitives/consensus/babe/src/lib.rs b/primitives/consensus/babe/src/lib.rs index 33701860d1f10234a823c4d625acfc02094f89f5..7cf9483e6f6b77179e3e99c1b157dd607cb169ee 100644 --- a/primitives/consensus/babe/src/lib.rs +++ b/primitives/consensus/babe/src/lib.rs @@ -29,7 +29,7 @@ pub use sp_consensus_vrf::schnorrkel::{ use codec::{Encode, Decode}; use sp_std::vec::Vec; use sp_runtime::{ConsensusEngineId, RuntimeDebug}; -use crate::digests::NextEpochDescriptor; +use crate::digests::{NextEpochDescriptor, NextConfigDescriptor}; mod app { use sp_application_crypto::{app_crypto, key_types::BABE, sr25519}; @@ -87,11 +87,15 @@ pub enum ConsensusLog { /// Disable the authority with given index. #[codec(index = "2")] OnDisabled(AuthorityIndex), + /// The epoch has changed, and the epoch after the current one will + /// enact different epoch configurations. + #[codec(index = "3")] + NextConfigData(NextConfigDescriptor), } /// Configuration data used by the BABE consensus engine. #[derive(Clone, PartialEq, Eq, Encode, Decode, RuntimeDebug)] -pub struct BabeConfiguration { +pub struct BabeGenesisConfigurationV1 { /// The slot duration in milliseconds for BABE. Currently, only /// the value provided by this type at genesis will be used. /// @@ -120,8 +124,78 @@ pub struct BabeConfiguration { pub secondary_slots: bool, } +impl From for BabeGenesisConfiguration { + fn from(v1: BabeGenesisConfigurationV1) -> Self { + Self { + slot_duration: v1.slot_duration, + epoch_length: v1.epoch_length, + c: v1.c, + genesis_authorities: v1.genesis_authorities, + randomness: v1.randomness, + allowed_slots: if v1.secondary_slots { + AllowedSlots::PrimaryAndSecondaryPlainSlots + } else { + AllowedSlots::PrimarySlots + }, + } + } +} + +/// Configuration data used by the BABE consensus engine. +#[derive(Clone, PartialEq, Eq, Encode, Decode, RuntimeDebug)] +pub struct BabeGenesisConfiguration { + /// The slot duration in milliseconds for BABE. Currently, only + /// the value provided by this type at genesis will be used. + /// + /// Dynamic slot duration may be supported in the future. + pub slot_duration: u64, + + /// The duration of epochs in slots. + pub epoch_length: SlotNumber, + + /// A constant value that is used in the threshold calculation formula. + /// Expressed as a rational where the first member of the tuple is the + /// numerator and the second is the denominator. The rational should + /// represent a value between 0 and 1. + /// In the threshold formula calculation, `1 - c` represents the probability + /// of a slot being empty. + pub c: (u64, u64), + + /// The authorities for the genesis epoch. + pub genesis_authorities: Vec<(AuthorityId, BabeAuthorityWeight)>, + + /// The randomness for the genesis epoch. + pub randomness: Randomness, + + /// Type of allowed slots. + pub allowed_slots: AllowedSlots, +} + +/// Types of allowed slots. +#[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, RuntimeDebug)] +pub enum AllowedSlots { + /// Only allow primary slots. + PrimarySlots, + /// Allow primary and secondary plain slots. + PrimaryAndSecondaryPlainSlots, + /// Allow primary and secondary VRF slots. + PrimaryAndSecondaryVRFSlots, +} + +impl AllowedSlots { + /// Whether plain secondary slots are allowed. + pub fn is_secondary_plain_slots_allowed(&self) -> bool { + *self == Self::PrimaryAndSecondaryPlainSlots + } + + /// Whether VRF secondary slots are allowed. + pub fn is_secondary_vrf_slots_allowed(&self) -> bool { + *self == Self::PrimaryAndSecondaryVRFSlots + } +} + #[cfg(feature = "std")] -impl sp_consensus::SlotData for BabeConfiguration { +impl sp_consensus::SlotData for BabeGenesisConfiguration { fn slot_duration(&self) -> u64 { self.slot_duration } @@ -129,14 +203,32 @@ impl sp_consensus::SlotData for BabeConfiguration { const SLOT_KEY: &'static [u8] = b"babe_configuration"; } +/// Configuration data used by the BABE consensus engine. +#[derive(Clone, PartialEq, Eq, Encode, Decode, RuntimeDebug)] +pub struct BabeEpochConfiguration { + /// A constant value that is used in the threshold calculation formula. + /// Expressed as a rational where the first member of the tuple is the + /// numerator and the second is the denominator. The rational should + /// represent a value between 0 and 1. + /// In the threshold formula calculation, `1 - c` represents the probability + /// of a slot being empty. + pub c: (u64, u64), + + /// Whether this chain should run with secondary slots, which are assigned + /// in round-robin manner. + pub allowed_slots: AllowedSlots, +} + sp_api::decl_runtime_apis! { /// API necessary for block authorship with BABE. + #[api_version(2)] pub trait BabeApi { - /// Return the configuration for BABE. Currently, - /// only the value provided by this type at genesis will be used. - /// - /// Dynamic configuration may be supported in the future. - fn configuration() -> BabeConfiguration; + /// Return the genesis configuration for BABE. The configuration is only read on genesis. + fn configuration() -> BabeGenesisConfiguration; + + /// Return the configuration for BABE. Version 1. + #[changed_in(2)] + fn configuration() -> BabeGenesisConfigurationV1; /// Returns the slot number that started the current epoch. fn current_epoch_start() -> SlotNumber; diff --git a/primitives/consensus/common/Cargo.toml b/primitives/consensus/common/Cargo.toml index 112b9499a7485a289893965c644918b1451165a1..d34333c8831494191590e5b7e1da6ba1ba968b99 100644 --- a/primitives/consensus/common/Cargo.toml +++ b/primitives/consensus/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-consensus" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,21 +9,24 @@ repository = "https://github.com/paritytech/substrate/" description = "Common utilities for building and using consensus engines in substrate." documentation = "https://docs.rs/sp-consensus/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] derive_more = "0.99.2" -libp2p = { version = "0.16.2", default-features = false } +libp2p = { version = "0.18.1", default-features = false } log = "0.4.8" -sp-core = { path= "../../core" , version = "2.0.0-alpha.5"} -sp-inherents = { version = "2.0.0-alpha.5", path = "../../inherents" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../../primitives/state-machine" } +sp-core = { path= "../../core", version = "2.0.0-dev"} +sp-inherents = { version = "2.0.0-dev", path = "../../inherents" } +sp-state-machine = { version = "0.8.0-dev", path = "../../../primitives/state-machine" } futures = { version = "0.3.1", features = ["thread-pool"] } futures-timer = "3.0.1" futures-diagnose = "1.0" -sp-std = { version = "2.0.0-alpha.5", path = "../../std" } -sp-version = { version = "2.0.0-alpha.5", path = "../../version" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../runtime" } -sp-utils = { version = "2.0.0-alpha.5", path = "../../utils" } +sp-std = { version = "2.0.0-dev", path = "../../std" } +sp-version = { version = "2.0.0-dev", path = "../../version" } +sp-runtime = { version = "2.0.0-dev", path = "../../runtime" } +sp-utils = { version = "2.0.0-dev", path = "../../utils" } codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } parking_lot = "0.10.0" serde = { version = "1.0", features = ["derive"] } @@ -33,6 +36,3 @@ sp-test-primitives = { version = "2.0.0-dev", path = "../../test-primitives" } [features] default = [] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/consensus/pow/Cargo.toml b/primitives/consensus/pow/Cargo.toml index 5ca60bb2155590fa34e1368ee1758ce7639871aa..a7bcb6a000fed5d30b74e027cc32ef56a3f82d2b 100644 --- a/primitives/consensus/pow/Cargo.toml +++ b/primitives/consensus/pow/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-consensus-pow" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "Primitives for Aura consensus" edition = "2018" @@ -8,11 +8,14 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../api" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../runtime" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../core" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../../api" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../runtime" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../core" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } [features] @@ -24,6 +27,3 @@ std = [ "sp-core/std", "codec/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/consensus/vrf/Cargo.toml b/primitives/consensus/vrf/Cargo.toml index cf194ec38b50ac75b638044936ceca73172309c8..71b647df593d3eb79e490472c67f5803b5a8faf9 100644 --- a/primitives/consensus/vrf/Cargo.toml +++ b/primitives/consensus/vrf/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-consensus-vrf" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "Primitives for VRF based consensus" edition = "2018" @@ -8,12 +8,15 @@ license = "GPL-3.0" repository = "https://github.com/paritytech/substrate/" homepage = "https://substrate.dev" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { version = "1.0.0", package = "parity-scale-codec", default-features = false } schnorrkel = { version = "0.9.1", features = ["preaudit_deprecated"], optional = true } -sp-std = { version = "2.0.0-alpha.5", path = "../../std", default-features = false } -sp-core = { version = "2.0.0-alpha.5", path = "../../core", default-features = false } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../runtime" } +sp-std = { version = "2.0.0-dev", path = "../../std", default-features = false } +sp-core = { version = "2.0.0-dev", path = "../../core", default-features = false } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../runtime" } [features] default = ["std"] @@ -24,6 +27,3 @@ std = [ "sp-core/std", "sp-runtime/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/core/Cargo.toml b/primitives/core/Cargo.toml index 7708188ea4bf230e40a06814e5ac0773773da173..2ab23d23c4a0e691770941cb74bef3ac3d6a405b 100644 --- a/primitives/core/Cargo.toml +++ b/primitives/core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-core" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,8 +9,11 @@ repository = "https://github.com/paritytech/substrate/" description = "Shareable Substrate types." documentation = "https://docs.rs/sp-core" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } log = { version = "0.4.8", default-features = false } serde = { version = "1.0.101", optional = true, features = ["derive"] } @@ -21,7 +24,7 @@ wasmi = { version = "0.6.2", optional = true } hash-db = { version = "0.15.2", default-features = false } hash256-std-hasher = { version = "0.15.2", default-features = false } base58 = { version = "0.1.0", optional = true } -rand = { version = "0.7.2", optional = true } +rand = { version = "0.7.3", optional = true, features = ["small_rng"] } substrate-bip39 = { version = "0.4.1", optional = true } tiny-bip39 = { version = "0.7", optional = true } regex = { version = "1.3.1", optional = true } @@ -29,10 +32,10 @@ num-traits = { version = "0.2.8", default-features = false } zeroize = { version = "1.0.0", default-features = false } lazy_static = { version = "1.4.0", default-features = false, optional = true } parking_lot = { version = "0.10.0", optional = true } -sp-debug-derive = { version = "2.0.0-alpha.5", path = "../debug-derive" } -sp-externalities = { version = "0.8.0-alpha.5", optional = true, path = "../externalities" } -sp-storage = { version = "2.0.0-alpha.5", default-features = false, path = "../storage" } -parity-util-mem = { version = "0.6.0", default-features = false, features = ["primitive-types"] } +sp-debug-derive = { version = "2.0.0-dev", path = "../debug-derive" } +sp-externalities = { version = "0.8.0-dev", optional = true, path = "../externalities" } +sp-storage = { version = "2.0.0-dev", default-features = false, path = "../storage" } +parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } futures = { version = "0.3.1", optional = true } # full crypto @@ -44,11 +47,12 @@ sha2 = { version = "0.8.0", default-features = false, optional = true } hex = { version = "0.4", default-features = false, optional = true } twox-hash = { version = "1.5.0", default-features = false, optional = true } libsecp256k1 = { version = "0.3.2", default-features = false, features = ["hmac"], optional = true } +merlin = { version = "2.0", default-features = false, optional = true } -sp-runtime-interface = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime-interface" } +sp-runtime-interface = { version = "2.0.0-dev", default-features = false, path = "../runtime-interface" } [dev-dependencies] -sp-serializer = { version = "2.0.0-alpha.5", path = "../serializer" } +sp-serializer = { version = "2.0.0-dev", path = "../serializer" } pretty_assertions = "0.6.1" hex-literal = "0.2.1" rand = "0.7.2" @@ -94,7 +98,6 @@ std = [ "schnorrkel/std", "regex", "num-traits/std", - "libsecp256k1/std", "tiny-keccak", "sp-debug-derive/std", "sp-externalities", @@ -103,6 +106,7 @@ std = [ "zeroize/alloc", "futures", "futures/thread-pool", + "libsecp256k1/std", ] # This feature enables all crypto primitives for `no_std` builds like microcontrollers @@ -118,7 +122,5 @@ full_crypto = [ "twox-hash", "libsecp256k1", "sp-runtime-interface/disable_target_static_assertions", + "merlin", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/core/src/crypto.rs b/primitives/core/src/crypto.rs index 79a36b2ad2353887aa8c581de552e34f2fb34d3e..a76110c7147c378fcc3fff5f5ebd23780d1f6894 100644 --- a/primitives/core/src/crypto.rs +++ b/primitives/core/src/crypto.rs @@ -455,6 +455,8 @@ ss58_address_format!( (10, "acala", "Acala mainnet, standard account (*25519).") LaminarAccount => (11, "laminar", "Laminar mainnet, standard account (*25519).") + PolymathAccount => + (12, "polymath", "Polymath network, standard account (*25519).") KulupuAccount => (16, "kulupu", "Kulupu mainnet, standard account (*25519).") DarwiniaAccount => @@ -990,19 +992,19 @@ impl sp_std::fmt::Display for CryptoTypePublicPair { pub mod key_types { use super::KeyTypeId; - /// Key type for Babe module, build-in. + /// Key type for Babe module, built-in. Identified as `babe`. pub const BABE: KeyTypeId = KeyTypeId(*b"babe"); - /// Key type for Grandpa module, build-in. + /// Key type for Grandpa module, built-in. Identified as `gran`. pub const GRANDPA: KeyTypeId = KeyTypeId(*b"gran"); - /// Key type for controlling an account in a Substrate runtime, built-in. + /// Key type for controlling an account in a Substrate runtime, built-in. Identified as `acco`. pub const ACCOUNT: KeyTypeId = KeyTypeId(*b"acco"); - /// Key type for Aura module, built-in. + /// Key type for Aura module, built-in. Identified as `aura`. pub const AURA: KeyTypeId = KeyTypeId(*b"aura"); - /// Key type for ImOnline module, built-in. + /// Key type for ImOnline module, built-in. Identified as `imon`. pub const IM_ONLINE: KeyTypeId = KeyTypeId(*b"imon"); - /// Key type for AuthorityDiscovery module, built-in. + /// Key type for AuthorityDiscovery module, built-in. Identified as `audi`. pub const AUTHORITY_DISCOVERY: KeyTypeId = KeyTypeId(*b"audi"); - /// Key type for staking, built-in. + /// Key type for staking, built-in. Identified as `stak`. pub const STAKING: KeyTypeId = KeyTypeId(*b"stak"); /// A key type ID useful for tests. pub const DUMMY: KeyTypeId = KeyTypeId(*b"dumy"); diff --git a/primitives/core/src/hashing.rs b/primitives/core/src/hashing.rs index 87f6469b573e1a54f4eaa6f700183fa3457ea245..d958da6c321380301d087cd40951bc3c50c12b2f 100644 --- a/primitives/core/src/hashing.rs +++ b/primitives/core/src/hashing.rs @@ -57,6 +57,18 @@ pub fn blake2_128(data: &[u8]) -> [u8; 16] { r } +/// Do a Blake2 64-bit hash and place result in `dest`. +pub fn blake2_64_into(data: &[u8], dest: &mut [u8; 8]) { + dest.copy_from_slice(blake2_rfc::blake2b::blake2b(8, &[], data).as_bytes()); +} + +/// Do a Blake2 64-bit hash and return result. +pub fn blake2_64(data: &[u8]) -> [u8; 8] { + let mut r = [0; 8]; + blake2_64_into(data, &mut r); + r +} + /// Do a XX 64-bit hash and place result in `dest`. pub fn twox_64_into(data: &[u8], dest: &mut [u8; 8]) { use ::core::hash::Hasher; diff --git a/primitives/core/src/offchain/mod.rs b/primitives/core/src/offchain/mod.rs index c393b0f9f88821e8c9f58c9e93b3d6544ba4c7e3..e792d71afca3b41bf8f312d34dba2859ae324986 100644 --- a/primitives/core/src/offchain/mod.rs +++ b/primitives/core/src/offchain/mod.rs @@ -28,6 +28,9 @@ pub mod storage; #[cfg(feature = "std")] pub mod testing; +/// Local storage prefix used by the Offchain Worker API to +pub const STORAGE_PREFIX : &'static [u8] = b"storage"; + /// Offchain workers local storage. pub trait OffchainStorage: Clone + Send + Sync { /// Persist a value in storage under given key and prefix. @@ -482,8 +485,8 @@ pub trait Externalities: Send { buffer: &mut [u8], deadline: Option ) -> Result; - } + impl Externalities for Box { fn is_validator(&self) -> bool { (& **self).is_validator() @@ -557,6 +560,7 @@ impl Externalities for Box { (&mut **self).http_response_read_body(request_id, buffer, deadline) } } + /// An `OffchainExternalities` implementation with limited capabilities. pub struct LimitedExternalities { capabilities: Capabilities, diff --git a/primitives/core/src/offchain/storage.rs b/primitives/core/src/offchain/storage.rs index 31b6423e5d81ce5d7257b3e1505bcb113e87117f..830c25392b7426e1f48257487fbbe09fdf8fdd99 100644 --- a/primitives/core/src/offchain/storage.rs +++ b/primitives/core/src/offchain/storage.rs @@ -18,6 +18,7 @@ use std::collections::hash_map::{HashMap, Entry}; use crate::offchain::OffchainStorage; +use std::iter::Iterator; /// In-memory storage for offchain workers. #[derive(Debug, Clone, Default)] @@ -25,6 +26,24 @@ pub struct InMemOffchainStorage { storage: HashMap, Vec>, } +impl InMemOffchainStorage { + /// Consume the offchain storage and iterate over all key value pairs. + pub fn into_iter(self) -> impl Iterator,Vec)> { + self.storage.into_iter() + } + + /// Iterate over all key value pairs by reference. + pub fn iter<'a>(&'a self) -> impl Iterator,&'a Vec)> { + self.storage.iter() + } + + /// Remove a key and its associated value from the offchain database. + pub fn remove(&mut self, prefix: &[u8], key: &[u8]) { + let key: Vec = prefix.iter().chain(key).cloned().collect(); + let _ = self.storage.remove(&key); + } +} + impl OffchainStorage for InMemOffchainStorage { fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { let key = prefix.iter().chain(key).cloned().collect(); @@ -58,3 +77,212 @@ impl OffchainStorage for InMemOffchainStorage { } } } + + + + +/// Change to be applied to the offchain worker db in regards to a key. +#[derive(Debug,Clone,Hash,Eq,PartialEq)] +pub enum OffchainOverlayedChange { + /// Remove the data associated with the key + Remove, + /// Overwrite the value of an associated key + SetValue(Vec), +} + +/// In-memory storage for offchain workers recoding changes for the actual offchain storage implementation. +#[derive(Debug, Clone)] +pub enum OffchainOverlayedChanges { + /// Writing overlay changes to the offchain worker database is disabled by configuration. + Disabled, + /// Overlay changes can be recorded using the inner collection of this variant. + Enabled(HashMap, OffchainOverlayedChange>), +} + +impl Default for OffchainOverlayedChanges { + fn default() -> Self { + Self::Disabled + } +} + +impl OffchainOverlayedChanges { + /// Create the disabled variant. + pub fn disabled() -> Self { + Self::Disabled + } + + /// Create the enabled variant. + pub fn enabled() -> Self { + Self::Enabled(HashMap::new()) + } + + /// Consume the offchain storage and iterate over all key value pairs. + pub fn into_iter(self) -> OffchainOverlayedChangesIntoIter { + OffchainOverlayedChangesIntoIter::new(self) + } + + /// Iterate over all key value pairs by reference. + pub fn iter<'a>(&'a self) -> OffchainOverlayedChangesIter { + OffchainOverlayedChangesIter::new(&self) + } + + /// Drain all elements of changeset. + pub fn drain<'a, 'd>(&'a mut self) -> OffchainOverlayedChangesDrain<'d> where 'a: 'd { + OffchainOverlayedChangesDrain::new(self) + } + + /// Remove a key and its associated value from the offchain database. + pub fn remove(&mut self, prefix: &[u8], key: &[u8]) { + if let Self::Enabled(ref mut storage) = self { + let key: Vec = prefix.iter().chain(key).cloned().collect(); + let _ = storage.insert(key, OffchainOverlayedChange::Remove); + } + } + + /// Set the value associated with a key under a prefix to the value provided. + pub fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { + if let Self::Enabled(ref mut storage) = self { + let key = prefix.iter().chain(key).cloned().collect(); + let _ = storage.insert(key, OffchainOverlayedChange::SetValue(value.to_vec())); + } + } + + /// Obtain a associated value to the given key in storage with prefix. + pub fn get(&self, prefix: &[u8], key: &[u8]) -> Option { + if let Self::Enabled(ref storage) = self { + let key: Vec = prefix.iter().chain(key).cloned().collect(); + storage.get(&key).cloned() + } else { + None + } + } +} + +use std::collections::hash_map; + +/// Iterate by reference over the prepared offchain worker storage changes. +pub struct OffchainOverlayedChangesIter<'i> { + inner: Option, OffchainOverlayedChange>>, +} + +impl<'i> Iterator for OffchainOverlayedChangesIter<'i> { + type Item = (&'i Vec, &'i OffchainOverlayedChange); + fn next(&mut self) -> Option { + if let Some(ref mut iter) = self.inner { + iter.next() + } else { + None + } + } +} + +impl<'i> OffchainOverlayedChangesIter<'i> { + /// Create a new iterator based on a refernce to the parent container. + pub fn new(container: &'i OffchainOverlayedChanges) -> Self { + match container { + OffchainOverlayedChanges::Enabled(inner) => Self { + inner: Some(inner.iter()) + }, + OffchainOverlayedChanges::Disabled => Self { inner: None, }, + } + } +} + + +/// Iterate by value over the prepared offchain worker storage changes. +pub struct OffchainOverlayedChangesIntoIter { + inner: Option,OffchainOverlayedChange>>, +} + +impl Iterator for OffchainOverlayedChangesIntoIter { + type Item = (Vec, OffchainOverlayedChange); + fn next(&mut self) -> Option { + if let Some(ref mut iter) = self.inner { + iter.next() + } else { + None + } + } +} + +impl OffchainOverlayedChangesIntoIter { + /// Create a new iterator by consuming the collection. + pub fn new(container: OffchainOverlayedChanges) -> Self { + match container { + OffchainOverlayedChanges::Enabled(inner) => Self { + inner: Some(inner.into_iter()) + }, + OffchainOverlayedChanges::Disabled => Self { inner: None, }, + } + } +} + +/// Iterate over all items while draining them from the collection. +pub struct OffchainOverlayedChangesDrain<'d> { + inner: Option,OffchainOverlayedChange>>, +} + +impl<'d> Iterator for OffchainOverlayedChangesDrain<'d> { + type Item = (Vec, OffchainOverlayedChange); + fn next(&mut self) -> Option { + if let Some(ref mut iter) = self.inner { + iter.next() + } else { + None + } + } +} + +impl<'d> OffchainOverlayedChangesDrain<'d> { + /// Create a new iterator by taking a mut reference to the collection, + /// for the lifetime of the created drain iterator. + pub fn new(container: &'d mut OffchainOverlayedChanges) -> Self { + match container { + OffchainOverlayedChanges::Enabled(ref mut inner) => Self { + inner: Some(inner.drain()) + }, + OffchainOverlayedChanges::Disabled => Self { inner: None, }, + } + } +} + + +#[cfg(test)] +mod test { + use super::*; + use super::super::STORAGE_PREFIX; + + #[test] + fn test_drain() { + let mut ooc = OffchainOverlayedChanges::enabled(); + ooc.set(STORAGE_PREFIX,b"kkk", b"vvv"); + let drained = ooc.drain().count(); + assert_eq!(drained, 1); + let leftover = ooc.iter().count(); + assert_eq!(leftover, 0); + + ooc.set(STORAGE_PREFIX, b"a", b"v"); + ooc.set(STORAGE_PREFIX, b"b", b"v"); + ooc.set(STORAGE_PREFIX, b"c", b"v"); + ooc.set(STORAGE_PREFIX, b"d", b"v"); + ooc.set(STORAGE_PREFIX, b"e", b"v"); + assert_eq!(ooc.iter().count(), 5); + } + + #[test] + fn test_accumulated_set_remove_set() { + let mut ooc = OffchainOverlayedChanges::enabled(); + ooc.set(STORAGE_PREFIX, b"ppp", b"qqq"); + ooc.remove(STORAGE_PREFIX, b"ppp"); + // keys are equiv, so it will overwrite the value and the overlay will contain + // one item + assert_eq!(ooc.iter().count(), 1); + + ooc.set(STORAGE_PREFIX, b"ppp", b"rrr"); + let mut iter = ooc.into_iter(); + let mut k = STORAGE_PREFIX.to_vec(); + k.extend_from_slice(&b"ppp"[..]); + assert_eq!(iter.next(), Some((k, OffchainOverlayedChange::SetValue(b"rrr".to_vec())))); + assert_eq!(iter.next(), None); + } +} diff --git a/primitives/core/src/offchain/testing.rs b/primitives/core/src/offchain/testing.rs index f4faee6b026eb70f069149993b9b197246230da5..d93528093fc0768aa188995e0fb5e42c0df9509a 100644 --- a/primitives/core/src/offchain/testing.rs +++ b/primitives/core/src/offchain/testing.rs @@ -74,6 +74,8 @@ pub struct OffchainState { pub local_storage: InMemOffchainStorage, /// Current timestamp (unix millis) pub timestamp: u64, + /// A supposedly random seed. + pub seed: [u8; 32], } impl OffchainState { @@ -165,7 +167,7 @@ impl offchain::Externalities for TestOffchainExt { } fn random_seed(&mut self) -> [u8; 32] { - unimplemented!("not needed in tests so far") + self.0.read().seed } fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { diff --git a/primitives/core/src/sr25519.rs b/primitives/core/src/sr25519.rs index 717952eb01c71b7cb398c5c57efc040704e07d24..cadfb25776b83552ee2ef8794660dd3a4ef34f73 100644 --- a/primitives/core/src/sr25519.rs +++ b/primitives/core/src/sr25519.rs @@ -611,6 +611,45 @@ impl CryptoType for Pair { type Pair = Pair; } +/// Batch verification. +/// +/// `messages`, `signatures` and `pub_keys` should all have equal length. +/// +/// Returns `true` if all signatures are correct, `false` otherwise. +#[cfg(feature = "std")] +pub fn verify_batch( + messages: Vec<&[u8]>, + signatures: Vec<&Signature>, + pub_keys: Vec<&Public>, +) -> bool { + let mut sr_pub_keys = Vec::with_capacity(pub_keys.len()); + for pub_key in pub_keys { + match schnorrkel::PublicKey::from_bytes(pub_key.as_ref()) { + Ok(pk) => sr_pub_keys.push(pk), + Err(_) => return false, + }; + } + + let mut sr_signatures = Vec::with_capacity(signatures.len()); + for signature in signatures { + match schnorrkel::Signature::from_bytes(signature.as_ref()) { + Ok(s) => sr_signatures.push(s), + Err(_) => return false + }; + } + + let mut messages: Vec = messages.into_iter().map( + |msg| signing_context(SIGNING_CTX).bytes(msg) + ).collect(); + + schnorrkel::verify_batch( + &mut messages, + &sr_signatures, + &sr_pub_keys, + true, + ).is_ok() +} + #[cfg(test)] mod compatibility_test { use super::*; diff --git a/primitives/core/src/testing.rs b/primitives/core/src/testing.rs index b5e6f4c7aff37427573de40587b9bfe6bf311281..9e83090bcff26ca3adf2f4ee0d9b4a0a84a23f7f 100644 --- a/primitives/core/src/testing.rs +++ b/primitives/core/src/testing.rs @@ -16,10 +16,10 @@ //! Types that should only be used for testing! -use crate::crypto::{KeyTypeId, CryptoTypePublicPair}; +use crate::crypto::KeyTypeId; #[cfg(feature = "std")] use crate::{ - crypto::{Pair, Public}, + crypto::{Pair, Public, CryptoTypePublicPair}, ed25519, sr25519, traits::BareCryptoStoreError }; diff --git a/primitives/core/src/traits.rs b/primitives/core/src/traits.rs index 14839fb58562a3ec1bcff7219e6dbfb7e1b47b4d..1724f5e9877ee8bae51d8dcec1433963be248be9 100644 --- a/primitives/core/src/traits.rs +++ b/primitives/core/src/traits.rs @@ -262,6 +262,23 @@ impl std::fmt::Display for CodeNotFound { } } +/// `Allow` or `Disallow` missing host functions when instantiating a WASM blob. +#[derive(Clone, Copy, Debug)] +pub enum MissingHostFunctions { + /// Any missing host function will be replaced by a stub that returns an error when + /// being called. + Allow, + /// Any missing host function will result in an error while instantiating the WASM blob, + Disallow, +} + +impl MissingHostFunctions { + /// Are missing host functions allowed? + pub fn allowed(self) -> bool { + matches!(self, Self::Allow) + } +} + /// Something that can call a method in a WASM blob. pub trait CallInWasm: Send + Sync { /// Call the given `method` in the given `wasm_blob` using `call_data` (SCALE encoded arguments) @@ -280,6 +297,7 @@ pub trait CallInWasm: Send + Sync { method: &str, call_data: &[u8], ext: &mut dyn Externalities, + missing_host_functions: MissingHostFunctions, ) -> Result, String>; } diff --git a/primitives/database/Cargo.toml b/primitives/database/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..2ab86de61d65f12c597ce9db46e23f06b0b71ed2 --- /dev/null +++ b/primitives/database/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "sp-database" +version = "2.0.0-dev" +authors = ["Parity Technologies "] +edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Substrate database trait." +documentation = "https://docs.rs/sp-database" + +[dependencies] +parking_lot = "0.10.0" +kvdb = "0.5.0" diff --git a/primitives/database/src/kvdb.rs b/primitives/database/src/kvdb.rs new file mode 100644 index 0000000000000000000000000000000000000000..85a324b5c105f757e9c264b7b9a696ac612faefb --- /dev/null +++ b/primitives/database/src/kvdb.rs @@ -0,0 +1,59 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +/// A wrapper around `kvdb::Database` that implements `sp_database::Database` trait + +use ::kvdb::{DBTransaction, KeyValueDB}; + +use crate::{Database, Change, Transaction, ColumnId}; + +struct DbAdapter(D); + +fn handle_err(result: std::io::Result) -> T { + match result { + Ok(r) => r, + Err(e) => { + panic!("Critical database eror: {:?}", e); + } + } +} + +/// Wrap RocksDb database into a trait object that implements `sp_database::Database` +pub fn as_database(db: D) -> std::sync::Arc> { + std::sync::Arc::new(DbAdapter(db)) +} + +impl Database for DbAdapter { + fn commit(&self, transaction: Transaction) { + let mut tx = DBTransaction::new(); + for change in transaction.0.into_iter() { + match change { + Change::Set(col, key, value) => tx.put_vec(col, &key, value), + Change::Remove(col, key) => tx.delete(col, &key), + _ => unimplemented!(), + } + } + handle_err(self.0.write(tx)); + } + + fn get(&self, col: ColumnId, key: &[u8]) -> Option> { + handle_err(self.0.get(col, key)) + } + + fn lookup(&self, _hash: &H) -> Option> { + unimplemented!(); + } +} diff --git a/primitives/database/src/lib.rs b/primitives/database/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..bd9bd2eb54c28c9189891e06689723f86284062e --- /dev/null +++ b/primitives/database/src/lib.rs @@ -0,0 +1,187 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! The main database trait, allowing Substrate to store data persistently. + +mod mem; +mod kvdb; + +pub use mem::MemDb; +pub use crate::kvdb::as_database; + +/// An identifier for a column. +pub type ColumnId = u32; + +/// An alteration to the database. +#[derive(Clone)] +pub enum Change { + Set(ColumnId, Vec, Vec), + Remove(ColumnId, Vec), + Store(H, Vec), + Release(H), +} + +/// An alteration to the database that references the data. +pub enum ChangeRef<'a, H> { + Set(ColumnId, &'a [u8], &'a [u8]), + Remove(ColumnId, &'a [u8]), + Store(H, &'a [u8]), + Release(H), +} + +/// A series of changes to the database that can be committed atomically. They do not take effect +/// until passed into `Database::commit`. +#[derive(Default, Clone)] +pub struct Transaction(pub Vec>); + +impl Transaction { + /// Create a new transaction to be prepared and committed atomically. + pub fn new() -> Self { + Transaction(Vec::new()) + } + /// Set the value of `key` in `col` to `value`, replacing anything that is there currently. + pub fn set(&mut self, col: ColumnId, key: &[u8], value: &[u8]) { + self.0.push(Change::Set(col, key.to_vec(), value.to_vec())) + } + /// Set the value of `key` in `col` to `value`, replacing anything that is there currently. + pub fn set_from_vec(&mut self, col: ColumnId, key: &[u8], value: Vec) { + self.0.push(Change::Set(col, key.to_vec(), value)) + } + /// Remove the value of `key` in `col`. + pub fn remove(&mut self, col: ColumnId, key: &[u8]) { + self.0.push(Change::Remove(col, key.to_vec())) + } + /// Store the `preimage` of `hash` into the database, so that it may be looked up later with + /// `Database::lookup`. This may be called multiple times, but `Database::lookup` but subsequent + /// calls will ignore `preimage` and simply increase the number of references on `hash`. + pub fn store(&mut self, hash: H, preimage: &[u8]) { + self.0.push(Change::Store(hash, preimage.to_vec())) + } + /// Release the preimage of `hash` from the database. An equal number of these to the number of + /// corresponding `store`s must have been given before it is legal for `Database::lookup` to + /// be unable to provide the preimage. + pub fn release(&mut self, hash: H) { + self.0.push(Change::Release(hash)) + } +} + +pub trait Database: Send + Sync { + /// Commit the `transaction` to the database atomically. Any further calls to `get` or `lookup` + /// will reflect the new state. + fn commit(&self, transaction: Transaction) { + for change in transaction.0.into_iter() { + match change { + Change::Set(col, key, value) => self.set(col, &key, &value), + Change::Remove(col, key) => self.remove(col, &key), + Change::Store(hash, preimage) => self.store(&hash, &preimage), + Change::Release(hash) => self.release(&hash), + } + } + } + + /// Commit the `transaction` to the database atomically. Any further calls to `get` or `lookup` + /// will reflect the new state. + fn commit_ref<'a>(&self, transaction: &mut dyn Iterator>) { + let mut tx = Transaction::new(); + for change in transaction { + match change { + ChangeRef::Set(col, key, value) => tx.set(col, key, value), + ChangeRef::Remove(col, key) => tx.remove(col, key), + ChangeRef::Store(hash, preimage) => tx.store(hash, preimage), + ChangeRef::Release(hash) => tx.release(hash), + } + } + self.commit(tx); + } + + /// Retrieve the value previously stored against `key` or `None` if + /// `key` is not currently in the database. + fn get(&self, col: ColumnId, key: &[u8]) -> Option>; + + /// Call `f` with the value previously stored against `key`. + /// + /// This may be faster than `get` since it doesn't allocate. + /// Use `with_get` helper function if you need `f` to return a value from `f` + fn with_get(&self, col: ColumnId, key: &[u8], f: &mut dyn FnMut(&[u8])) { + self.get(col, key).map(|v| f(&v)); + } + + /// Set the value of `key` in `col` to `value`, replacing anything that is there currently. + fn set(&self, col: ColumnId, key: &[u8], value: &[u8]) { + let mut t = Transaction::new(); + t.set(col, key, value); + self.commit(t); + } + /// Remove the value of `key` in `col`. + fn remove(&self, col: ColumnId, key: &[u8]) { + let mut t = Transaction::new(); + t.remove(col, key); + self.commit(t); + } + + /// Retrieve the first preimage previously `store`d for `hash` or `None` if no preimage is + /// currently stored. + fn lookup(&self, hash: &H) -> Option>; + + /// Call `f` with the preimage stored for `hash` and return the result, or `None` if no preimage + /// is currently stored. + /// + /// This may be faster than `lookup` since it doesn't allocate. + /// Use `with_lookup` helper function if you need `f` to return a value from `f` + fn with_lookup(&self, hash: &H, f: &mut dyn FnMut(&[u8])) { + self.lookup(hash).map(|v| f(&v)); + } + + /// Store the `preimage` of `hash` into the database, so that it may be looked up later with + /// `Database::lookup`. This may be called multiple times, but `Database::lookup` but subsequent + /// calls will ignore `preimage` and simply increase the number of references on `hash`. + fn store(&self, hash: &H, preimage: &[u8]) { + let mut t = Transaction::new(); + t.store(hash.clone(), preimage); + self.commit(t); + } + + /// Release the preimage of `hash` from the database. An equal number of these to the number of + /// corresponding `store`s must have been given before it is legal for `Database::lookup` to + /// be unable to provide the preimage. + fn release(&self, hash: &H) { + let mut t = Transaction::new(); + t.release(hash.clone()); + self.commit(t); + } +} + +/// Call `f` with the value previously stored against `key` and return the result, or `None` if +/// `key` is not currently in the database. +/// +/// This may be faster than `get` since it doesn't allocate. +pub fn with_get(db: &dyn Database, col: ColumnId, key: &[u8], mut f: impl FnMut(&[u8]) -> R) -> Option { + let mut result: Option = None; + let mut adapter = |k: &_| { result = Some(f(k)); }; + db.with_get(col, key, &mut adapter); + result +} + +/// Call `f` with the preimage stored for `hash` and return the result, or `None` if no preimage +/// is currently stored. +/// +/// This may be faster than `lookup` since it doesn't allocate. +pub fn with_lookup(db: &dyn Database, hash: &H, mut f: impl FnMut(&[u8]) -> R) -> Option { + let mut result: Option = None; + let mut adapter = |k: &_| { result = Some(f(k)); }; + db.with_lookup(hash, &mut adapter); + result +} diff --git a/primitives/database/src/mem.rs b/primitives/database/src/mem.rs new file mode 100644 index 0000000000000000000000000000000000000000..09d6149bed174b9225d31ca53fc24a59893c8171 --- /dev/null +++ b/primitives/database/src/mem.rs @@ -0,0 +1,68 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! In-memory implementation of `Database` + +use std::collections::HashMap; +use crate::{Database, Transaction, ColumnId, Change}; +use parking_lot::RwLock; + +#[derive(Default)] +/// This implements `Database` as an in-memory hash map. `commit` is not atomic. +pub struct MemDb + (RwLock<(HashMap, Vec>>, HashMap>)>); + +impl Database for MemDb + where H: Clone + Send + Sync + Eq + PartialEq + Default + std::hash::Hash +{ + fn commit(&self, transaction: Transaction) { + let mut s = self.0.write(); + for change in transaction.0.into_iter() { + match change { + Change::Set(col, key, value) => { s.0.entry(col).or_default().insert(key, value); }, + Change::Remove(col, key) => { s.0.entry(col).or_default().remove(&key); }, + Change::Store(hash, preimage) => { s.1.insert(hash, preimage); }, + Change::Release(hash) => { s.1.remove(&hash); }, + } + } + } + + fn get(&self, col: ColumnId, key: &[u8]) -> Option> { + let s = self.0.read(); + s.0.get(&col).and_then(|c| c.get(key).cloned()) + } + + fn lookup(&self, hash: &H) -> Option> { + let s = self.0.read(); + s.1.get(hash).cloned() + } +} + +impl MemDb + where H: Clone + Send + Sync + Eq + PartialEq + Default + std::hash::Hash +{ + /// Create a new instance + pub fn new() -> Self { + MemDb::default() + } + + /// Count number of values in a column + pub fn count(&self, col: ColumnId) -> usize { + let s = self.0.read(); + s.0.get(&col).map(|c| c.len()).unwrap_or(0) + } +} + diff --git a/primitives/debug-derive/Cargo.toml b/primitives/debug-derive/Cargo.toml index 0079b6219f5b618c709c322cf5639b9f6eec85f9..a3e9d91fd2825dbe70da48ee2e4b78e6a549e267 100644 --- a/primitives/debug-derive/Cargo.toml +++ b/primitives/debug-derive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-debug-derive" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,6 +9,10 @@ repository = "https://github.com/paritytech/substrate/" description = "Macros to derive runtime debug implementation." documentation = "https://docs.rs/sp-debug-derive" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [lib] proc-macro = true @@ -21,7 +25,3 @@ proc-macro2 = "1.0" std = [] [dev-dependencies] - - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/externalities/Cargo.toml b/primitives/externalities/Cargo.toml index af6e516fbfcd32eec31dd9d446785a5028a70ddd..9d7cd1df6b7fe5f652feec480fe25fd3346b416f 100644 --- a/primitives/externalities/Cargo.toml +++ b/primitives/externalities/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-externalities" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" @@ -9,10 +9,11 @@ repository = "https://github.com/paritytech/substrate/" description = "Substrate externalities abstraction" documentation = "https://docs.rs/sp-externalities" -[dependencies] -sp-storage = { version = "2.0.0-alpha.5", path = "../storage" } -sp-std = { version = "2.0.0-alpha.5", path = "../std" } -environmental = { version = "1.1.1" } - [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +sp-storage = { version = "2.0.0-dev", path = "../storage" } +sp-std = { version = "2.0.0-dev", path = "../std" } +environmental = { version = "1.1.1" } +codec = { package = "parity-scale-codec", version = "1.3.0" } diff --git a/primitives/externalities/src/extensions.rs b/primitives/externalities/src/extensions.rs index a61c03534fb0dbaeaad4e91bfcf2affa9555c10d..f38f256bb9e68456d14da43c80ed24eefeeabb0f 100644 --- a/primitives/externalities/src/extensions.rs +++ b/primitives/externalities/src/extensions.rs @@ -21,7 +21,8 @@ //! //! It is required that each extension implements the [`Extension`] trait. -use std::{collections::HashMap, any::{Any, TypeId}, ops::DerefMut}; +use std::{collections::HashMap, collections::hash_map::Entry, any::{Any, TypeId}, ops::DerefMut}; +use crate::Error; /// Marker trait for types that should be registered as [`Externalities`](crate::Externalities) extension. /// @@ -87,6 +88,16 @@ pub trait ExtensionStore { /// It is advised to use [`ExternalitiesExt::extension`](crate::ExternalitiesExt::extension) /// instead of this function to get type system support and automatic type downcasting. fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any>; + + /// Register extension `extension` with speciifed `type_id`. + /// + /// It should return error if extension is already registered. + fn register_extension_with_type_id(&mut self, type_id: TypeId, extension: Box) -> Result<(), Error>; + + /// Deregister extension with speicifed 'type_id' and drop it. + /// + /// It should return error if extension is not registered. + fn deregister_extension_by_type_id(&mut self, type_id: TypeId) -> Result<(), Error>; } /// Stores extensions that should be made available through the externalities. @@ -95,6 +106,12 @@ pub struct Extensions { extensions: HashMap>, } +impl std::fmt::Debug for Extensions { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Extensions: ({})", self.extensions.len()) + } +} + impl Extensions { /// Create new instance of `Self`. pub fn new() -> Self { @@ -106,10 +123,23 @@ impl Extensions { self.extensions.insert(ext.type_id(), Box::new(ext)); } + /// Register extension `ext`. + pub fn register_with_type_id(&mut self, type_id: TypeId, extension: Box) -> Result<(), Error> { + match self.extensions.entry(type_id) { + Entry::Vacant(vacant) => { vacant.insert(extension); Ok(()) }, + Entry::Occupied(_) => Err(Error::ExtensionAlreadyRegistered), + } + } + /// Return a mutable reference to the requested extension. pub fn get_mut(&mut self, ext_type_id: TypeId) -> Option<&mut dyn Any> { self.extensions.get_mut(&ext_type_id).map(DerefMut::deref_mut).map(Extension::as_mut_any) } + + /// Deregister extension of type `E`. + pub fn deregister(&mut self, type_id: TypeId) -> Option> { + self.extensions.remove(&type_id) + } } #[cfg(test)] diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 6fbd239b89cef8995d2ab05d7192e710fcc87c34..2b584b512e4993b363127124f2866cfcb7b1565d 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -24,7 +24,7 @@ use std::any::{Any, TypeId}; -use sp_storage::{ChildStorageKey, ChildInfo}; +use sp_storage::ChildInfo; pub use scope_limited::{set_and_run_with_externalities, with_externalities}; pub use extensions::{Extension, Extensions, ExtensionStore}; @@ -32,23 +32,42 @@ pub use extensions::{Extension, Extensions, ExtensionStore}; mod extensions; mod scope_limited; +/// Externalities error. +#[derive(Debug)] +pub enum Error { + /// Same extension cannot be registered twice. + ExtensionAlreadyRegistered, + /// Extensions are not supported. + ExtensionsAreNotSupported, + /// Extension `TypeId` is not registered. + ExtensionIsNotRegistered(TypeId), + /// Failed to update storage, + StorageUpdateFailed(&'static str), +} + /// The Substrate externalities. /// /// Provides access to the storage and to other registered extensions. pub trait Externalities: ExtensionStore { + /// Write a key value pair to the offchain storage database. + fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>); + /// Read runtime storage. fn storage(&self, key: &[u8]) -> Option>; - /// Get storage value hash. This may be optimized for large values. + /// Get storage value hash. + /// + /// This may be optimized for large values. fn storage_hash(&self, key: &[u8]) -> Option>; - /// Get child storage value hash. This may be optimized for large values. + /// Get child storage value hash. + /// + /// This may be optimized for large values. /// /// Returns an `Option` that holds the SCALE encoded hash. fn child_storage_hash( &self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Option>; @@ -57,8 +76,7 @@ pub trait Externalities: ExtensionStore { /// Returns an `Option` that holds the SCALE encoded hash. fn child_storage( &self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Option>; @@ -70,12 +88,11 @@ pub trait Externalities: ExtensionStore { /// Set child storage entry `key` of current contract being called (effective immediately). fn set_child_storage( &mut self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: Vec, value: Vec, ) { - self.place_child_storage(storage_key, child_info, key, Some(value)) + self.place_child_storage(child_info, key, Some(value)) } /// Clear a storage entry (`key`) of current contract being called (effective immediately). @@ -86,11 +103,10 @@ pub trait Externalities: ExtensionStore { /// Clear a child storage entry (`key`) of current contract being called (effective immediately). fn clear_child_storage( &mut self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) { - self.place_child_storage(storage_key, child_info, key.to_vec(), None) + self.place_child_storage(child_info, key.to_vec(), None) } /// Whether a storage entry exists. @@ -101,11 +117,10 @@ pub trait Externalities: ExtensionStore { /// Whether a child storage entry exists. fn exists_child_storage( &self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> bool { - self.child_storage(storage_key, child_info, key).is_some() + self.child_storage(child_info, key).is_some() } /// Returns the key immediately following the given key, if it exists. @@ -114,13 +129,12 @@ pub trait Externalities: ExtensionStore { /// Returns the key immediately following the given key, if it exists, in child storage. fn next_child_storage_key( &self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Option>; /// Clear an entire child storage. - fn kill_child_storage(&mut self, storage_key: ChildStorageKey, child_info: ChildInfo); + fn kill_child_storage(&mut self, child_info: &ChildInfo); /// Clear storage entries which keys are start with the given prefix. fn clear_prefix(&mut self, prefix: &[u8]); @@ -128,19 +142,17 @@ pub trait Externalities: ExtensionStore { /// Clear child storage entries which keys are start with the given prefix. fn clear_child_prefix( &mut self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], ); /// Set or clear a storage entry (`key`) of current contract being called (effective immediately). fn place_storage(&mut self, key: Vec, value: Option>); - /// Set or clear a child storage entry. Return whether the operation succeeds. + /// Set or clear a child storage entry. fn place_child_storage( &mut self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: Vec, value: Option>, ); @@ -148,29 +160,38 @@ pub trait Externalities: ExtensionStore { /// Get the identity of the chain. fn chain_id(&self) -> u64; - /// Get the trie root of the current storage map. This will also update all child storage keys - /// in the top-level storage map. + /// Get the trie root of the current storage map. /// - /// The hash is defined by the `Block`. + /// This will also update all child storage keys in the top-level storage map. /// - /// Returns the SCALE encoded hash. + /// The returned hash is defined by the `Block` and is SCALE encoded. fn storage_root(&mut self) -> Vec; - /// Get the trie root of a child storage map. This will also update the value of the child - /// storage keys in the top-level storage map. + /// Get the trie root of a child storage map. + /// + /// This will also update the value of the child storage keys in the top-level storage map. + /// /// If the storage root equals the default hash as defined by the trie, the key in the top-level /// storage map will be removed. fn child_storage_root( &mut self, - storage_key: ChildStorageKey, + child_info: &ChildInfo, ) -> Vec; - /// Get the change trie root of the current storage overlay at a block with given parent. - /// `parent` is expects a SCALE encoded hash. + /// Append storage item. /// - /// The hash is defined by the `Block`. + /// This assumes specific format of the storage item. Also there is no way to undo this operation. + fn storage_append( + &mut self, + key: Vec, + value: Vec, + ); + + /// Get the changes trie root of the current storage overlay at a block with given `parent`. + /// + /// `parent` expects a SCALE encoded hash. /// - /// Returns the SCALE encoded hash. + /// The returned hash is defined by the `Block` and is SCALE encoded. fn storage_changes_root(&mut self, parent: &[u8]) -> Result>, ()>; /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -194,10 +215,29 @@ pub trait Externalities: ExtensionStore { pub trait ExternalitiesExt { /// Tries to find a registered extension and returns a mutable reference. fn extension(&mut self) -> Option<&mut T>; + + /// Register extension `ext`. + /// + /// Should return error if extension is already registered or extensions are not supported. + fn register_extension(&mut self, ext: T) -> Result<(), Error>; + + /// Deregister and drop extension of `T` type. + /// + /// Should return error if extension of type `T` is not registered or + /// extensions are not supported. + fn deregister_extension(&mut self) -> Result<(), Error>; } impl ExternalitiesExt for &mut dyn Externalities { fn extension(&mut self) -> Option<&mut T> { self.extension_by_type_id(TypeId::of::()).and_then(Any::downcast_mut) } + + fn register_extension(&mut self, ext: T) -> Result<(), Error> { + self.register_extension_with_type_id(TypeId::of::(), Box::new(ext)) + } + + fn deregister_extension(&mut self) -> Result<(), Error> { + self.deregister_extension_by_type_id(TypeId::of::()) + } } diff --git a/primitives/finality-grandpa/Cargo.toml b/primitives/finality-grandpa/Cargo.toml index 0595fa7ba700649fa2c4df5ae1ad3178648fdc1f..b5ce970c0d82c6ef0043a229c2e4eb7bf7bd71a0 100644 --- a/primitives/finality-grandpa/Cargo.toml +++ b/primitives/finality-grandpa/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-finality-grandpa" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,14 +9,17 @@ repository = "https://github.com/paritytech/substrate/" description = "Primitives for GRANDPA integration, suitable for WASM compilation." documentation = "https://docs.rs/sp-finality-grandpa" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../application-crypto" } +sp-application-crypto = { version = "2.0.0-dev", default-features = false, path = "../application-crypto" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../api" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../api" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../runtime" } [features] default = ["std"] @@ -28,6 +31,3 @@ std = [ "sp-api/std", "sp-runtime/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/finality-tracker/Cargo.toml b/primitives/finality-tracker/Cargo.toml index 4e6cf6c92d324c36278c18ef0b5b5703b21b56e3..a111c6267526aeaabc29a019364a793356d65376 100644 --- a/primitives/finality-tracker/Cargo.toml +++ b/primitives/finality-tracker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-finality-tracker" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,10 +8,13 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME module that tracks the last finalized block, as perceived by block authors." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../../primitives/inherents" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } [features] default = ["std"] @@ -20,6 +23,3 @@ std = [ "sp-std/std", "sp-inherents/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/inherents/Cargo.toml b/primitives/inherents/Cargo.toml index dd640f00ec1a272d87511bd2f22f88916abeebbb..084d275882a197afb829eb05cd02f956860f7390 100644 --- a/primitives/inherents/Cargo.toml +++ b/primitives/inherents/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-inherents" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,11 +9,14 @@ repository = "https://github.com/paritytech/substrate/" description = "Provides types and traits for creating and checking inherents." documentation = "https://docs.rs/sp-inherents" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] parking_lot = { version = "0.10.0", optional = true } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../core" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } derive_more = { version = "0.99.2", optional = true } @@ -26,6 +29,3 @@ std = [ "sp-core/std", "derive_more", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/inherents/src/lib.rs b/primitives/inherents/src/lib.rs index e8df2c49e51931659859b16912664ab3fcd4c208..e63382cdf2e4622b76b08c88dcae0b87c867ed97 100644 --- a/primitives/inherents/src/lib.rs +++ b/primitives/inherents/src/lib.rs @@ -408,6 +408,11 @@ pub trait ProvideInherent { /// Create an inherent out of the given `InherentData`. fn create_inherent(data: &InherentData) -> Option; + /// If `Some`, indicates that an inherent is required. Check will return the inner error if no + /// inherent is found. If `Err`, indicates that the check failed and further operations should + /// be aborted. + fn is_inherent_required(_: &InherentData) -> Result, Self::Error> { Ok(None) } + /// Check the given inherent if it is valid. /// Checking the inherent is optional and can be omitted. fn check_inherent(_: &Self::Call, _: &InherentData) -> Result<(), Self::Error> { diff --git a/primitives/io/Cargo.toml b/primitives/io/Cargo.toml index 9cda2120cc2b45290da425f14f758ba483445b43..93c8cc9d38b40cc4d1404f97874a9642daca628d 100644 --- a/primitives/io/Cargo.toml +++ b/primitives/io/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-io" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,19 +9,24 @@ repository = "https://github.com/paritytech/substrate/" description = "I/O for Substrate runtimes" documentation = "https://docs.rs/sp-io" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } hash-db = { version = "0.15.2", default-features = false } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../core" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } libsecp256k1 = { version = "0.3.4", optional = true } -sp-state-machine = { version = "0.8.0-alpha.5", optional = true, path = "../../primitives/state-machine" } -sp-wasm-interface = { version = "2.0.0-alpha.5", path = "../../primitives/wasm-interface", default-features = false } -sp-runtime-interface = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime-interface" } -sp-trie = { version = "2.0.0-alpha.5", optional = true, path = "../../primitives/trie" } -sp-externalities = { version = "0.8.0-alpha.5", optional = true, path = "../externalities" } +sp-state-machine = { version = "0.8.0-dev", optional = true, path = "../../primitives/state-machine" } +sp-wasm-interface = { version = "2.0.0-dev", path = "../../primitives/wasm-interface", default-features = false } +sp-runtime-interface = { version = "2.0.0-dev", default-features = false, path = "../runtime-interface" } +sp-trie = { version = "2.0.0-dev", optional = true, path = "../../primitives/trie" } +sp-externalities = { version = "0.8.0-dev", optional = true, path = "../externalities" } log = { version = "0.4.8", optional = true } +futures = { version = "0.3.1", features = ["thread-pool"], optional = true } +parking_lot = { version = "0.10.0", optional = true } [features] default = ["std"] @@ -37,6 +42,8 @@ std = [ "sp-externalities", "sp-wasm-interface/std", "log", + "futures", + "parking_lot", ] # These two features are used for `no_std` builds for the environments which already provides @@ -46,6 +53,3 @@ std = [ disable_panic_handler = [] disable_oom = [] disable_allocator = [] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/io/src/batch_verifier.rs b/primitives/io/src/batch_verifier.rs new file mode 100644 index 0000000000000000000000000000000000000000..2a2ad4dd5c53f0e0abe5138713a00c6c21371c53 --- /dev/null +++ b/primitives/io/src/batch_verifier.rs @@ -0,0 +1,168 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Batch/parallel verification. + +use sp_core::{ed25519, sr25519, crypto::Pair, traits::CloneableSpawn}; +use std::sync::{Arc, atomic::{AtomicBool, Ordering as AtomicOrdering}}; +use futures::{future::FutureExt, task::FutureObj, channel::oneshot}; + +#[derive(Debug, Clone)] +struct Sr25519BatchItem { + signature: sr25519::Signature, + pub_key: sr25519::Public, + message: Vec, +} + +/// Batch verifier. +/// +/// Used to parallel-verify signatures for runtime host. Provide task executor and +/// just push (`push_ed25519`, `push_sr25519`) as many signature as you need. At the end, +/// call `verify_and_clear to get a result. After that, batch verifier is ready for the +/// next batching job. +pub struct BatchVerifier { + scheduler: Box, + sr25519_items: Vec, + invalid: Arc, + pending_tasks: Vec>, +} + +impl BatchVerifier { + pub fn new(scheduler: Box) -> Self { + BatchVerifier { + scheduler, + sr25519_items: Default::default(), + invalid: Arc::new(false.into()), + pending_tasks: vec![], + } + } + + fn spawn_verification_task( + &mut self, f: impl FnOnce() -> bool + Send + 'static, + ) -> Result<(), ()> { + // there is already invalid transaction encountered + if self.invalid.load(AtomicOrdering::Relaxed) { return Err(()); } + + let invalid_clone = self.invalid.clone(); + let (sender, receiver) = oneshot::channel(); + self.pending_tasks.push(receiver); + + self.scheduler.spawn_obj(FutureObj::new(async move { + if !f() { + invalid_clone.store(true, AtomicOrdering::Relaxed); + } + if sender.send(()).is_err() { + // sanity + log::warn!("Verification halted while result was pending"); + invalid_clone.store(true, AtomicOrdering::Relaxed); + } + }.boxed())).map_err(drop) + } + + /// Push ed25519 signature to verify. + /// + /// Returns false if some of the pushed signatures before already failed the check + /// (in this case it won't verify anything else) + pub fn push_ed25519( + &mut self, + signature: ed25519::Signature, + pub_key: ed25519::Public, + message: Vec, + ) -> bool { + if self.invalid.load(AtomicOrdering::Relaxed) { return false; } + + if self.spawn_verification_task(move || ed25519::Pair::verify(&signature, &message, &pub_key)).is_err() { + log::debug!( + target: "runtime", + "Batch-verification returns false because failed to spawn background task.", + ); + + return false; + } + true + } + + /// Push sr25519 signature to verify. + /// + /// Returns false if some of the pushed signatures before already failed the check. + /// (in this case it won't verify anything else) + pub fn push_sr25519( + &mut self, + signature: sr25519::Signature, + pub_key: sr25519::Public, + message: Vec, + ) -> bool { + if self.invalid.load(AtomicOrdering::Relaxed) { return false; } + self.sr25519_items.push(Sr25519BatchItem { signature, pub_key, message }); + true + } + + /// Verify all previously pushed signatures since last call and return + /// aggregated result. + #[must_use] + pub fn verify_and_clear(&mut self) -> bool { + let pending = std::mem::replace(&mut self.pending_tasks, vec![]); + let started = std::time::Instant::now(); + + log::trace!( + target: "runtime", + "Batch-verification: {} pending tasks, {} sr25519 signatures", + pending.len(), + self.sr25519_items.len(), + ); + + let messages = self.sr25519_items.iter().map(|item| &item.message[..]).collect(); + let signatures = self.sr25519_items.iter().map(|item| &item.signature).collect(); + let pub_keys = self.sr25519_items.iter().map(|item| &item.pub_key).collect(); + + if !sr25519::verify_batch(messages, signatures, pub_keys) { + self.sr25519_items.clear(); + + return false; + } + + self.sr25519_items.clear(); + + if pending.len() > 0 { + let (sender, receiver) = std::sync::mpsc::channel(); + if self.scheduler.spawn_obj(FutureObj::new(async move { + futures::future::join_all(pending).await; + sender.send(()) + .expect("Channel never panics if receiver is live. \ + Receiver is always live until received this data; qed. "); + }.boxed())).is_err() { + log::debug!( + target: "runtime", + "Batch-verification returns false because failed to spawn background task.", + ); + + return false; + } + if receiver.recv().is_err() { + log::warn!(target: "runtime", "Haven't received async result from verification task. Returning false."); + return false; + } + } + + log::trace!( + target: "runtime", + "Finalization of batch verification took {} ms", + started.elapsed().as_millis(), + ); + + !self.invalid.swap(false, AtomicOrdering::Relaxed) + } +} diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index bc49df159eb6f873d5a9ac7e011a529cc2422cc0..e4787dade0e7ad31803de2100db30b6bdb8100a9 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! This is part of the Substrate runtime. +//! I/O host interface for substrate runtime. #![warn(missing_docs)] @@ -34,10 +34,10 @@ use sp_std::ops::Deref; #[cfg(feature = "std")] use sp_core::{ crypto::Pair, - traits::{KeystoreExt, CallInWasmExt}, + traits::{KeystoreExt, CallInWasmExt, TaskExecutorExt}, offchain::{OffchainExt, TransactionPoolExt}, hexdisplay::HexDisplay, - storage::{ChildStorageKey, ChildInfo}, + storage::ChildInfo, }; use sp_core::{ @@ -57,6 +57,12 @@ use codec::{Encode, Decode}; #[cfg(feature = "std")] use sp_externalities::{ExternalitiesExt, Externalities}; +#[cfg(feature = "std")] +mod batch_verifier; + +#[cfg(feature = "std")] +use batch_verifier::BatchVerifier; + /// Error verifying ECDSA signature #[derive(Encode, Decode)] pub enum EcdsaVerifyError { @@ -68,19 +74,6 @@ pub enum EcdsaVerifyError { BadSignature, } -/// Returns a `ChildStorageKey` if the given `storage_key` slice is a valid storage -/// key or panics otherwise. -/// -/// Panicking here is aligned with what the `without_std` environment would do -/// in the case of an invalid child storage key. -#[cfg(feature = "std")] -fn child_storage_key_or_panic(storage_key: &[u8]) -> ChildStorageKey { - match ChildStorageKey::from_slice(storage_key) { - Some(storage_key) => storage_key, - None => panic!("child storage key is invalid"), - } -} - /// Interface for accessing the storage from within the runtime. #[runtime_interface] pub trait Storage { @@ -89,30 +82,6 @@ pub trait Storage { self.storage(key).map(|s| s.to_vec()) } - /// All Child api uses : - /// - A `child_storage_key` to define the anchor point for the child proof - /// (commonly the location where the child root is stored in its parent trie). - /// - A `child_storage_types` to identify the kind of the child type and how its - /// `child definition` parameter is encoded. - /// - A `child_definition_parameter` which is the additional information required - /// to use the child trie. For instance defaults child tries requires this to - /// contain a collision free unique id. - /// - /// This function specifically returns the data for `key` in the child storage or `None` - /// if the key can not be found. - fn child_get( - &self, - child_storage_key: &[u8], - child_definition: &[u8], - child_type: u32, - key: &[u8], - ) -> Option> { - let storage_key = child_storage_key_or_panic(child_storage_key); - let child_info = ChildInfo::resolve_child_info(child_type, child_definition) - .expect("Invalid child definition"); - self.child_storage(storage_key, child_info, key).map(|s| s.to_vec()) - } - /// Get `key` from storage, placing the value into `value_out` and return the number of /// bytes that the entry in storage has beyond the offset or `None` if the storage entry /// doesn't exist at all. @@ -128,26 +97,92 @@ pub trait Storage { }) } + /// Set `key` to `value` in the storage. + fn set(&mut self, key: &[u8], value: &[u8]) { + self.set_storage(key.to_vec(), value.to_vec()); + } + + /// Clear the storage of the given `key` and its value. + fn clear(&mut self, key: &[u8]) { + self.clear_storage(key) + } + + /// Check whether the given `key` exists in storage. + fn exists(&self, key: &[u8]) -> bool { + self.exists_storage(key) + } + + /// Clear the storage of each key-value pair where the key starts with the given `prefix`. + fn clear_prefix(&mut self, prefix: &[u8]) { + Externalities::clear_prefix(*self, prefix) + } + + /// Append to storage item (assumes it is in "Vec" format). + fn append(&mut self, key: &[u8], value: Vec) { + self.storage_append(key.to_vec(), value); + } + + /// "Commit" all existing operations and compute the resulting storage root. + /// + /// The hashing algorithm is defined by the `Block`. + /// + /// Returns the SCALE encoded hash. + fn root(&mut self) -> Vec { + self.storage_root() + } + + /// "Commit" all existing operations and get the resulting storage change root. + /// `parent_hash` is a SCALE encoded hash. + /// + /// The hashing algorithm is defined by the `Block`. + /// + /// Returns an `Some(_)` which holds the SCALE encoded hash or `None` when + /// changes trie is disabled. + fn changes_root(&mut self, parent_hash: &[u8]) -> Option> { + self.storage_changes_root(parent_hash) + .expect("Invalid `parent_hash` given to `changes_root`.") + } + + /// Get the next key in storage after the given one in lexicographic order. + fn next_key(&mut self, key: &[u8]) -> Option> { + self.next_storage_key(&key) + } +} + +/// Interface for accessing the child storage for default child trie, +/// from within the runtime. +#[runtime_interface] +pub trait DefaultChildStorage { + + /// Get a default child storage value for a given key. + /// + /// Parameter `storage_key` is the unprefixed location of the root of the child trie in the parent trie. + /// Result is `None` if the value for `key` in the child storage can not be found. + fn get( + &self, + storage_key: &[u8], + key: &[u8], + ) -> Option> { + let child_info = ChildInfo::new_default(storage_key); + self.child_storage(&child_info, key).map(|s| s.to_vec()) + } + + /// Allocation efficient variant of `get`. + /// /// Get `key` from child storage, placing the value into `value_out` and return the number /// of bytes that the entry in storage has beyond the offset or `None` if the storage entry /// doesn't exist at all. /// If `value_out` length is smaller than the returned length, only `value_out` length bytes /// are copied into `value_out`. - /// - /// See `child_get` for common child api parameters. - fn child_read( + fn read( &self, - child_storage_key: &[u8], - child_definition: &[u8], - child_type: u32, + storage_key: &[u8], key: &[u8], value_out: &mut [u8], value_offset: u32, ) -> Option { - let storage_key = child_storage_key_or_panic(child_storage_key); - let child_info = ChildInfo::resolve_child_info(child_type, child_definition) - .expect("Invalid child definition"); - self.child_storage(storage_key, child_info, key) + let child_info = ChildInfo::new_default(storage_key); + self.child_storage(&child_info, key) .map(|value| { let value_offset = value_offset as usize; let data = &value[value_offset.min(value.len())..]; @@ -157,159 +192,91 @@ pub trait Storage { }) } - /// Set `key` to `value` in the storage. - fn set(&mut self, key: &[u8], value: &[u8]) { - self.set_storage(key.to_vec(), value.to_vec()); - } - - /// Set `key` to `value` in the child storage denoted by `child_storage_key`. + /// Set a child storage value. /// - /// See `child_get` for common child api parameters. - fn child_set( + /// Set `key` to `value` in the child storage denoted by `storage_key`. + fn set( &mut self, - child_storage_key: &[u8], - child_definition: &[u8], - child_type: u32, + storage_key: &[u8], key: &[u8], value: &[u8], ) { - let storage_key = child_storage_key_or_panic(child_storage_key); - let child_info = ChildInfo::resolve_child_info(child_type, child_definition) - .expect("Invalid child definition"); - self.set_child_storage(storage_key, child_info, key.to_vec(), value.to_vec()); - } - - /// Clear the storage of the given `key` and its value. - fn clear(&mut self, key: &[u8]) { - self.clear_storage(key) + let child_info = ChildInfo::new_default(storage_key); + self.set_child_storage(&child_info, key.to_vec(), value.to_vec()); } - /// Clear the given child storage of the given `key` and its value. + /// Clear a child storage key. /// - /// See `child_get` for common child api parameters. - fn child_clear( + /// For the default child storage at `storage_key`, clear value at `key`. + fn clear ( &mut self, - child_storage_key: &[u8], - child_definition: &[u8], - child_type: u32, + storage_key: &[u8], key: &[u8], ) { - let storage_key = child_storage_key_or_panic(child_storage_key); - let child_info = ChildInfo::resolve_child_info(child_type, child_definition) - .expect("Invalid child definition"); - self.clear_child_storage(storage_key, child_info, key); + let child_info = ChildInfo::new_default(storage_key); + self.clear_child_storage(&child_info, key); } /// Clear an entire child storage. /// - /// See `child_get` for common child api parameters. - fn child_storage_kill( + /// If it exists, the child storage for `storage_key` + /// is removed. + fn storage_kill( &mut self, - child_storage_key: &[u8], - child_definition: &[u8], - child_type: u32, + storage_key: &[u8], ) { - let storage_key = child_storage_key_or_panic(child_storage_key); - let child_info = ChildInfo::resolve_child_info(child_type, child_definition) - .expect("Invalid child definition"); - self.kill_child_storage(storage_key, child_info); + let child_info = ChildInfo::new_default(storage_key); + self.kill_child_storage(&child_info); } - /// Check whether the given `key` exists in storage. - fn exists(&self, key: &[u8]) -> bool { - self.exists_storage(key) - } - - /// Check whether the given `key` exists in storage. + /// Check a child storage key. /// - /// See `child_get` for common child api parameters. - fn child_exists( + /// Check whether the given `key` exists in default child defined at `storage_key`. + fn exists( &self, - child_storage_key: &[u8], - child_definition: &[u8], - child_type: u32, + storage_key: &[u8], key: &[u8], ) -> bool { - let storage_key = child_storage_key_or_panic(child_storage_key); - let child_info = ChildInfo::resolve_child_info(child_type, child_definition) - .expect("Invalid child definition"); - self.exists_child_storage(storage_key, child_info, key) + let child_info = ChildInfo::new_default(storage_key); + self.exists_child_storage(&child_info, key) } - /// Clear the storage of each key-value pair where the key starts with the given `prefix`. - fn clear_prefix(&mut self, prefix: &[u8]) { - Externalities::clear_prefix(*self, prefix) - } - - /// Clear the child storage of each key-value pair where the key starts with the given `prefix`. + /// Clear child default key by prefix. /// - /// See `child_get` for common child api parameters. - fn child_clear_prefix( + /// Clear the child storage of each key-value pair where the key starts with the given `prefix`. + fn clear_prefix( &mut self, - child_storage_key: &[u8], - child_definition: &[u8], - child_type: u32, + storage_key: &[u8], prefix: &[u8], ) { - let storage_key = child_storage_key_or_panic(child_storage_key); - let child_info = ChildInfo::resolve_child_info(child_type, child_definition) - .expect("Invalid child definition"); - self.clear_child_prefix(storage_key, child_info, prefix); + let child_info = ChildInfo::new_default(storage_key); + self.clear_child_prefix(&child_info, prefix); } - /// "Commit" all existing operations and compute the resulting storage root. - /// - /// The hashing algorithm is defined by the `Block`. + /// Default child root calculation. /// - /// Returns the SCALE encoded hash. - fn root(&mut self) -> Vec { - self.storage_root() - } - /// "Commit" all existing operations and compute the resulting child storage root. - /// /// The hashing algorithm is defined by the `Block`. /// /// Returns the SCALE encoded hash. - /// - /// See `child_get` for common child api parameters. - fn child_root( + fn root( &mut self, - child_storage_key: &[u8], + storage_key: &[u8], ) -> Vec { - let storage_key = child_storage_key_or_panic(child_storage_key); - self.child_storage_root(storage_key) + let child_info = ChildInfo::new_default(storage_key); + self.child_storage_root(&child_info) } - /// "Commit" all existing operations and get the resulting storage change root. - /// `parent_hash` is a SCALE encoded hash. + /// Child storage key iteration. /// - /// The hashing algorithm is defined by the `Block`. - /// - /// Returns an `Some(_)` which holds the SCALE encoded hash or `None` when - /// changes trie is disabled. - fn changes_root(&mut self, parent_hash: &[u8]) -> Option> { - self.storage_changes_root(parent_hash) - .expect("Invalid `parent_hash` given to `changes_root`.") - } - - /// Get the next key in storage after the given one in lexicographic order. - fn next_key(&mut self, key: &[u8]) -> Option> { - self.next_storage_key(&key) - } - /// Get the next key in storage after the given one in lexicographic order in child storage. - fn child_next_key( + fn next_key( &mut self, - child_storage_key: &[u8], - child_definition: &[u8], - child_type: u32, + storage_key: &[u8], key: &[u8], ) -> Option> { - let storage_key = child_storage_key_or_panic(child_storage_key); - let child_info = ChildInfo::resolve_child_info(child_type, child_definition) - .expect("Invalid child definition"); - self.next_child_storage_key(storage_key, child_info, key) + let child_info = ChildInfo::new_default(storage_key); + self.next_child_storage_key(&child_info, key) } } @@ -367,7 +334,18 @@ pub trait Misc { self.extension::() .expect("No `CallInWasmExt` associated for the current context!") - .call_in_wasm(wasm, None, "Core_version", &[], &mut ext) + .call_in_wasm( + wasm, + None, + "Core_version", + &[], + &mut ext, + // If a runtime upgrade introduces new host functions that are not provided by + // the node, we should not fail at instantiation. Otherwise nodes that are + // updated could run this successfully and it could lead to a storage root + // mismatch when importing this block. + sp_core::traits::MissingHostFunctions::Allow, + ) .ok() } } @@ -416,16 +394,98 @@ pub trait Crypto { .ok() } - /// Verify an `ed25519` signature. + /// Verify `ed25519` signature. /// - /// Returns `true` when the verification in successful. + /// Returns `true` when the verification is either successful or batched. + /// If no batching verification extension registered, this will return the result + /// of verification immediately. If batching verification extension is registered + /// caller should call `crypto::finish_batch_verify` to actualy check all submitted + /// signatures. fn ed25519_verify( - &self, sig: &ed25519::Signature, msg: &[u8], pub_key: &ed25519::Public, ) -> bool { - ed25519::Pair::verify(sig, msg, pub_key) + // TODO: see #5554, this is used outside of externalities context/runtime, thus this manual + // `with_externalities`. + // + // This `with_externalities(..)` block returns Some(Some(result)) if signature verification was successfully + // batched, everything else (Some(None)/None) means it was not batched and needs to be verified. + let evaluated = sp_externalities::with_externalities(|mut instance| + instance.extension::().map( + |extension| extension.push_ed25519( + sig.clone(), + pub_key.clone(), + msg.to_vec(), + ) + ) + ); + + match evaluated { + Some(Some(val)) => val, + _ => ed25519::Pair::verify(sig, msg, pub_key), + } + } + + /// Verify `sr25519` signature. + /// + /// Returns `true` when the verification is either successful or batched. + /// If no batching verification extension registered, this will return the result + /// of verification immediately. If batching verification extension is registered, + /// caller should call `crypto::finish_batch_verify` to actualy check all submitted + #[version(2)] + fn sr25519_verify( + sig: &sr25519::Signature, + msg: &[u8], + pub_key: &sr25519::Public, + ) -> bool { + // TODO: see #5554, this is used outside of externalities context/runtime, thus this manual + // `with_externalities`. + // + // This `with_externalities(..)` block returns Some(Some(result)) if signature verification was successfully + // batched, everything else (Some(None)/None) means it was not batched and needs to be verified. + let evaluated = sp_externalities::with_externalities(|mut instance| + instance.extension::().map( + |extension| extension.push_sr25519( + sig.clone(), + pub_key.clone(), + msg.to_vec(), + ) + ) + ); + + match evaluated { + Some(Some(val)) => val, + _ => sr25519::Pair::verify(sig, msg, pub_key), + } + } + + /// Start verification extension. + fn start_batch_verify(&mut self) { + let scheduler = self.extension::() + .expect("No task executor associated with the current context!") + .0 + .clone(); + + self.register_extension(VerificationExt(BatchVerifier::new(scheduler))) + .expect("Failed to register required extension: `VerificationExt`"); + } + + /// Finish batch-verification of signatures. + /// + /// Verify or wait for verification to finish for all signatures which were previously + /// deferred by `sr25519_verify`/`ed25519_verify`. + /// + /// Will panic if no `VerificationExt` is registered (`start_batch_verify` was not called). + fn finish_batch_verify(&mut self) -> bool { + let result = self.extension::() + .expect("`finish_batch_verify` should only be called after `start_batch_verify`") + .verify_and_clear(); + + self.deregister_extension::() + .expect("No verification extension in current context!"); + + result } /// Returns all `sr25519` public keys for the given key id from the keystore. @@ -477,14 +537,6 @@ pub trait Crypto { sr25519::Pair::verify_deprecated(sig, msg, pubkey) } - /// Verify an `sr25519` signature. - /// - /// Returns `true` when the verification in successful. - #[version(2)] - fn sr25519_verify(sig: &sr25519::Signature, msg: &[u8], pubkey: &sr25519::Public) -> bool { - sr25519::Pair::verify(sig, msg, pubkey) - } - /// Verify and recover a SECP256k1 ECDSA signature. /// /// - `sig` is passed in RSV format. V should be either `0/1` or `27/28`. @@ -566,7 +618,29 @@ pub trait Hashing { } } +/// Interface that provides functions to access the Offchain DB. +#[runtime_interface] +pub trait OffchainIndex { + /// Write a key value pair to the Offchain DB database in a buffered fashion. + fn set(&mut self, key: &[u8], value: &[u8]) { + self.set_offchain_storage(key, Some(value)); + } + + /// Remove a key and its associated value from the Offchain DB. + fn clear(&mut self, key: &[u8]) { + self.set_offchain_storage(key, None); + } +} + +#[cfg(feature = "std")] +sp_externalities::decl_extension! { + /// The keystore extension to register/retrieve from the externalities. + pub struct VerificationExt(BatchVerifier); +} + /// Interface that provides functions to access the offchain functionality. +/// +/// These functions are being made available to the runtime and are called by the runtime. #[runtime_interface] pub trait Offchain { /// Returns if the local node is a potential validator. @@ -933,6 +1007,7 @@ pub type TestExternalities = sp_state_machine::TestExternalities b"bar".to_vec()], - children: map![], + children_default: map![], }); t.execute_with(|| { @@ -976,7 +1053,7 @@ mod tests { fn read_storage_works() { let mut t = BasicExternalities::new(Storage { top: map![b":test".to_vec() => b"\x0b\0\0\0Hello world".to_vec()], - children: map![], + children_default: map![], }); t.execute_with(|| { @@ -998,7 +1075,7 @@ mod tests { b":abc".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), b":abdd".to_vec() => b"\x0b\0\0\0Hello world".to_vec() ], - children: map![], + children_default: map![], }); t.execute_with(|| { @@ -1010,4 +1087,132 @@ mod tests { assert!(storage::get(b":abc").is_none()); }); } + + #[test] + fn dynamic_extensions_work() { + let mut ext = BasicExternalities::with_tasks_executor(); + ext.execute_with(|| { + crypto::start_batch_verify(); + }); + + assert!(ext.extensions().get_mut(TypeId::of::()).is_some()); + + ext.execute_with(|| { + crypto::finish_batch_verify(); + }); + + assert!(ext.extensions().get_mut(TypeId::of::()).is_none()); + } + + #[test] + fn long_sr25519_batching() { + let mut ext = BasicExternalities::with_tasks_executor(); + ext.execute_with(|| { + let pair = sr25519::Pair::generate_with_phrase(None).0; + crypto::start_batch_verify(); + for it in 0..70 { + let msg = format!("Schnorrkel {}!", it); + let signature = pair.sign(msg.as_bytes()); + crypto::sr25519_verify(&signature, msg.as_bytes(), &pair.public()); + } + + // push invlaid + crypto::sr25519_verify( + &Default::default(), + &Vec::new(), + &Default::default(), + ); + assert!(!crypto::finish_batch_verify()); + + crypto::start_batch_verify(); + for it in 0..70 { + let msg = format!("Schnorrkel {}!", it); + let signature = pair.sign(msg.as_bytes()); + crypto::sr25519_verify(&signature, msg.as_bytes(), &pair.public()); + } + assert!(crypto::finish_batch_verify()); + }); + } + + #[test] + fn batching_works() { + let mut ext = BasicExternalities::with_tasks_executor(); + ext.execute_with(|| { + // invalid ed25519 signature + crypto::start_batch_verify(); + crypto::ed25519_verify( + &Default::default(), + &Vec::new(), + &Default::default(), + ); + assert!(!crypto::finish_batch_verify()); + + // 2 valid ed25519 signatures + crypto::start_batch_verify(); + + let pair = ed25519::Pair::generate_with_phrase(None).0; + let msg = b"Important message"; + let signature = pair.sign(msg); + crypto::ed25519_verify(&signature, msg, &pair.public()); + + let pair = ed25519::Pair::generate_with_phrase(None).0; + let msg = b"Even more important message"; + let signature = pair.sign(msg); + crypto::ed25519_verify(&signature, msg, &pair.public()); + + assert!(crypto::finish_batch_verify()); + + // 1 valid, 1 invalid ed25519 signature + crypto::start_batch_verify(); + + let pair = ed25519::Pair::generate_with_phrase(None).0; + let msg = b"Important message"; + let signature = pair.sign(msg); + crypto::ed25519_verify(&signature, msg, &pair.public()); + + crypto::ed25519_verify( + &Default::default(), + &Vec::new(), + &Default::default(), + ); + + assert!(!crypto::finish_batch_verify()); + + // 1 valid ed25519, 2 valid sr25519 + crypto::start_batch_verify(); + + let pair = ed25519::Pair::generate_with_phrase(None).0; + let msg = b"Ed25519 batching"; + let signature = pair.sign(msg); + crypto::ed25519_verify(&signature, msg, &pair.public()); + + let pair = sr25519::Pair::generate_with_phrase(None).0; + let msg = b"Schnorrkel rules"; + let signature = pair.sign(msg); + crypto::sr25519_verify(&signature, msg, &pair.public()); + + let pair = sr25519::Pair::generate_with_phrase(None).0; + let msg = b"Schnorrkel batches!"; + let signature = pair.sign(msg); + crypto::sr25519_verify(&signature, msg, &pair.public()); + + assert!(crypto::finish_batch_verify()); + + // 1 valid sr25519, 1 invalid sr25519 + crypto::start_batch_verify(); + + let pair = sr25519::Pair::generate_with_phrase(None).0; + let msg = b"Schnorrkcel!"; + let signature = pair.sign(msg); + crypto::sr25519_verify(&signature, msg, &pair.public()); + + crypto::sr25519_verify( + &Default::default(), + &Vec::new(), + &Default::default(), + ); + + assert!(!crypto::finish_batch_verify()); + }); + } } diff --git a/primitives/keyring/Cargo.toml b/primitives/keyring/Cargo.toml index 0764146250446c367c0c0bfbd53147e0a9dd4134..23e243239a553322b75878422f3c8e22d8e9ce29 100644 --- a/primitives/keyring/Cargo.toml +++ b/primitives/keyring/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-keyring" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,12 +9,12 @@ repository = "https://github.com/paritytech/substrate/" description = "Keyring support code for the runtime. A set of test accounts." documentation = "https://docs.rs/sp-keyring" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-core = { version = "2.0.0-alpha.5", path = "../core" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../runtime" } +sp-core = { version = "2.0.0-dev", path = "../core" } +sp-runtime = { version = "2.0.0-dev", path = "../runtime" } lazy_static = "1.4.0" strum = { version = "0.16.0", features = ["derive"] } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/offchain/Cargo.toml b/primitives/offchain/Cargo.toml index 66febccd59b9b6ac5a9111cfbd4fcb8027e8113a..ddac54372ea98412e33bbee10f6d41ebaea3d391 100644 --- a/primitives/offchain/Cargo.toml +++ b/primitives/offchain/Cargo.toml @@ -1,23 +1,28 @@ [package] description = "Substrate offchain workers primitives" name = "sp-offchain" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../api" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../core" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../api" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../runtime" } + +[dev-dependencies] +sp-state-machine = { version = "0.8.0-alpha.5", default-features = false, path = "../state-machine" } [features] default = ["std"] std = [ + "sp-core/std", "sp-api/std", "sp-runtime/std" ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/offchain/src/lib.rs b/primitives/offchain/src/lib.rs index ae02fed4967685e7cb419d49064a63634497a749..8f043d712a8404ebecfec138210fd72490303fea 100644 --- a/primitives/offchain/src/lib.rs +++ b/primitives/offchain/src/lib.rs @@ -19,8 +19,8 @@ #![cfg_attr(not(feature = "std"), no_std)] #![warn(missing_docs)] -/// Local Storage Prefix used by the Offchain Worker API to -pub const STORAGE_PREFIX: &[u8] = b"storage"; +/// Re-export of parent module scope storage prefix. +pub use sp_core::offchain::STORAGE_PREFIX as STORAGE_PREFIX; sp_api::decl_runtime_apis! { /// The offchain worker api. diff --git a/primitives/panic-handler/Cargo.toml b/primitives/panic-handler/Cargo.toml index 169443f6c42cee9f40de81baeb04fe5d49a8ba76..b5adb9cb5483fd32b13840a23377aa52a1ad7bce 100644 --- a/primitives/panic-handler/Cargo.toml +++ b/primitives/panic-handler/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-panic-handler" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,9 +9,9 @@ repository = "https://github.com/paritytech/substrate/" description = "Custom panic hook with bug report link" documentation = "https://docs.rs/sp-panic-handler" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] backtrace = "0.3.38" log = "0.4.8" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/phragmen/Cargo.toml b/primitives/phragmen/Cargo.toml index f5d26e8a40b80427360bfd56915a540d84d584bc..29fc0ed41e40878bfd6981efb0ae551d7a28ab05 100644 --- a/primitives/phragmen/Cargo.toml +++ b/primitives/phragmen/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-phragmen" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,17 +8,21 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Phragmen primitives" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -sp-phragmen-compact = { version = "2.0.0-alpha.4", path = "./compact" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } +sp-phragmen-compact = { version = "2.0.0-dev", path = "./compact" } +sp-arithmetic = { version = "2.0.0-dev", default-features = false, path = "../arithmetic" } [dev-dependencies] -substrate-test-utils = { version = "2.0.0-alpha.5", path = "../../test-utils" } +substrate-test-utils = { version = "2.0.0-dev", path = "../../test-utils" } rand = "0.7.3" -sp-phragmen = { path = "." } +sp-phragmen = { version = "2.0.0-dev", path = "." } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } [features] default = ["std"] @@ -27,8 +31,5 @@ std = [ "codec/std", "serde", "sp-std/std", - "sp-runtime/std", + "sp-arithmetic/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/phragmen/benches/phragmen.rs b/primitives/phragmen/benches/phragmen.rs index 33da0b19563a8d3f7ef226077862558b5e0fd285..e274586f601d5588ddeeedf2e88c8149c473bd7c 100644 --- a/primitives/phragmen/benches/phragmen.rs +++ b/primitives/phragmen/benches/phragmen.rs @@ -24,133 +24,139 @@ extern crate test; use test::Bencher; use rand::{self, Rng}; -extern crate sp_phragmen as phragmen; -use phragmen::{Support, SupportMap, PhragmenStakedAssignment}; +use sp_phragmen::{PhragmenResult, VoteWeight}; use std::collections::BTreeMap; -use sp_runtime::traits::{Convert, SaturatedConversion}; +use sp_runtime::{Perbill, traits::Zero}; -const VALIDATORS: u64 = 1000; -const NOMINATORS: u64 = 10_000; +// default params. Each will be scaled by the benchmarks individually. +const VALIDATORS: u64 = 100; +const NOMINATORS: u64 = 1_000; const EDGES: u64 = 2; -const TO_ELECT: usize = 100; -const STAKE: Balance = 1000; +const TO_ELECT: usize = 10; +const STAKE: VoteWeight = 1000; + +const PREFIX: AccountId = 1000_000; -type Balance = u128; type AccountId = u64; -pub struct TestCurrencyToVote; -impl Convert for TestCurrencyToVote { - fn convert(x: Balance) -> u64 { x.saturated_into() } -} -impl Convert for TestCurrencyToVote { - fn convert(x: u128) -> Balance { x.saturated_into() } -} +mod bench_closure_and_slice { + use sp_phragmen::{ + VoteWeight, ExtendedBalance, Assignment, StakedAssignment, IdentifierT, + assignment_ratio_to_staked, + }; + use sp_runtime::{Perbill, PerThing}; + use rand::{self, Rng, RngCore}; + use test::Bencher; + + fn random_assignment() -> Assignment { + let mut rng = rand::thread_rng(); + let who = rng.next_u32(); + let distribution = (0..5) + .map(|x| (x + rng.next_u32(), Perbill::from_percent(rng.next_u32() % 100))) + .collect::>(); + Assignment { who, distribution } + } -fn do_phragmen( - b: &mut Bencher, - num_vals: u64, - num_noms: u64, - count: usize, - votes_per: u64, - eq_iters: usize, - _eq_tolerance: u128, -) { - assert!(num_vals > votes_per); - let rr = |a, b| rand::thread_rng().gen_range(a as usize, b as usize) as Balance; + /// Converts a vector of ratio assignments into ones with absolute budget value. + pub fn assignment_ratio_to_staked_slice( + ratio: Vec>, + stakes: &[VoteWeight], + ) -> Vec> + where + T: sp_std::ops::Mul, + ExtendedBalance: From<::Inner>, + { + ratio + .into_iter() + .zip(stakes.into_iter().map(|x| *x as ExtendedBalance)) + .map(|(a, stake)| { + a.into_staked(stake.into(), true) + }) + .collect() + } - // prefix to distinguish the validator and nominator account ranges. - let np = 10_000; + #[bench] + fn closure(b: &mut Bencher) { + let assignments = (0..1000).map(|_| random_assignment()).collect::>>(); + let stake_of = |x: &u32| -> VoteWeight { (x * 2 + 100).into() }; - let mut candidates = Vec::with_capacity(num_vals as usize); - let mut slashable_balance_of: BTreeMap = BTreeMap::new(); + // each have one clone of assignments + b.iter(|| assignment_ratio_to_staked(assignments.clone(), stake_of)); + } - (1 ..= num_vals) - .for_each(|acc| { - candidates.push(acc); - slashable_balance_of.insert(acc, STAKE + rr(10, 50)); - }); + #[bench] + fn slice(b: &mut Bencher) { + let assignments = (0..1000).map(|_| random_assignment()).collect::>>(); + let stake_of = |x: &u32| -> VoteWeight { (x * 2 + 100).into() }; - let mut voters = Vec::with_capacity(num_noms as usize); - (np ..= (np + num_noms)) - .for_each(|acc| { - let mut stashes_to_vote = candidates.clone(); - let votes = (0 .. votes_per) - .map(|_| { - stashes_to_vote.remove(rr(0, stashes_to_vote.len()) as usize) - }) - .collect::>(); - voters.push((acc, votes)); - slashable_balance_of.insert(acc, STAKE + rr(10, 50)); + b.iter(|| { + let local = assignments.clone(); + let stakes = local.iter().map(|x| stake_of(&x.who)).collect::>(); + assignment_ratio_to_staked_slice(local, stakes.as_ref()); }); + } +} - let slashable_balance = |who: &AccountId| -> Balance { - *slashable_balance_of.get(who).unwrap() - }; +fn do_phragmen( + b: &mut Bencher, + num_validators: u64, + num_nominators: u64, + to_elect: usize, + edge_per_voter: u64, + eq_iters: usize, + eq_tolerance: u128, +) { + assert!(num_validators > edge_per_voter); + let rr = |a, b| rand::thread_rng().gen_range(a as usize, b as usize) as VoteWeight; + + let mut candidates = Vec::with_capacity(num_validators as usize); + let mut stake_of_tree: BTreeMap = BTreeMap::new(); + + (1 ..= num_validators).for_each(|acc| { + candidates.push(acc); + stake_of_tree.insert(acc, STAKE + rr(10, 1000)); + }); + + let mut voters = Vec::with_capacity(num_nominators as usize); + (PREFIX ..= (PREFIX + num_nominators)).for_each(|acc| { + // all possible targets + let mut all_targets = candidates.clone(); + // we remove and pop into `targets` `edge_per_voter` times. + let targets = (0 .. edge_per_voter).map(|_| { + all_targets.remove(rr(0, all_targets.len()) as usize) + }) + .collect::>(); + + let stake = STAKE + rr(10, 1000); + stake_of_tree.insert(acc, stake); + voters.push((acc, stake, targets)); + }); b.iter(|| { - let r = phragmen::elect::( - count, - 1_usize, + let PhragmenResult { winners, assignments } = sp_phragmen::elect::( + to_elect, + Zero::zero(), candidates.clone(), voters.clone(), - slashable_balance, - true, ).unwrap(); + let stake_of = |who: &AccountId| -> VoteWeight { + *stake_of_tree.get(who).unwrap() + }; + // Do the benchmarking with equalize. if eq_iters > 0 { - let elected_stashes = r.winners; - let assignments = r.assignments; - - let to_votes = |b: Balance| - >::convert(b) as u128; - - // Initialize the support of each candidate. - let mut supports = >::new(); - elected_stashes - .iter() - .map(|(e, _)| (e, to_votes(slashable_balance(e)))) - .for_each(|(e, s)| { - let item = Support { own: s, total: s, ..Default::default() }; - supports.insert(e.clone(), item); - }); - - // build support struct. - for (n, assignment) in assignments.iter() { - for (c, per_thing) in assignment.iter() { - let nominator_stake = to_votes(slashable_balance(n)); - let other_stake = *per_thing * nominator_stake; - if let Some(support) = supports.get_mut(c) { - support.total = support.total.saturating_add(other_stake); - support.others.push((n.clone(), other_stake)); - } - } - } - - let mut staked_assignments - : Vec<(AccountId, Vec>)> - = Vec::with_capacity(assignments.len()); - for (n, assignment) in assignments.iter() { - let mut staked_assignment - : Vec> - = Vec::with_capacity(assignment.len()); - for (c, per_thing) in assignment.iter() { - let nominator_stake = to_votes(slashable_balance(n)); - let other_stake = *per_thing * nominator_stake; - staked_assignment.push((c.clone(), other_stake)); - } - staked_assignments.push((n.clone(), staked_assignment)); - } - - let tolerance = 0_u128; - let iterations = 2_usize; - phragmen::equalize::<_, _, TestCurrencyToVote, _>( - staked_assignments, - &mut supports, - tolerance, - iterations, - slashable_balance, + use sp_phragmen::{equalize, assignment_ratio_to_staked, build_support_map, to_without_backing}; + let staked = assignment_ratio_to_staked(assignments, &stake_of); + let winners = to_without_backing(winners); + let mut support = build_support_map(winners.as_ref(), staked.as_ref()).0; + + equalize( + staked.into_iter().map(|a| (a.clone(), stake_of(&a.who))).collect(), + &mut support, + eq_tolerance, + eq_iters, ); } }) @@ -176,13 +182,13 @@ macro_rules! phragmen_benches { phragmen_benches! { bench_1_1: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES, 0, 0), - bench_1_2: (VALIDATORS*2, NOMINATORS, TO_ELECT, EDGES, 0, 0), - bench_1_3: (VALIDATORS*4, NOMINATORS, TO_ELECT, EDGES, 0, 0), - bench_1_4: (VALIDATORS*8, NOMINATORS, TO_ELECT, EDGES, 0, 0), + bench_1_2: (VALIDATORS * 2, NOMINATORS, TO_ELECT, EDGES, 0, 0), + bench_1_3: (VALIDATORS * 4, NOMINATORS, TO_ELECT, EDGES, 0, 0), + bench_1_4: (VALIDATORS * 8, NOMINATORS, TO_ELECT, EDGES, 0, 0), bench_1_1_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES, 2, 0), - bench_1_2_eq: (VALIDATORS*2, NOMINATORS, TO_ELECT, EDGES, 2, 0), - bench_1_3_eq: (VALIDATORS*4, NOMINATORS, TO_ELECT, EDGES, 2, 0), - bench_1_4_eq: (VALIDATORS*8, NOMINATORS, TO_ELECT, EDGES, 2, 0), + bench_1_2_eq: (VALIDATORS * 2, NOMINATORS, TO_ELECT, EDGES, 2, 0), + bench_1_3_eq: (VALIDATORS * 4, NOMINATORS, TO_ELECT, EDGES, 2, 0), + bench_1_4_eq: (VALIDATORS * 8, NOMINATORS, TO_ELECT, EDGES, 2, 0), bench_0_1: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES, 0, 0), bench_0_2: (VALIDATORS, NOMINATORS, TO_ELECT * 4, EDGES, 0, 0), @@ -194,20 +200,20 @@ phragmen_benches! { bench_0_4_eq: (VALIDATORS, NOMINATORS, TO_ELECT * 16, EDGES , 2, 0), bench_2_1: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES, 0, 0), - bench_2_2: (VALIDATORS, NOMINATORS*2, TO_ELECT, EDGES, 0, 0), - bench_2_3: (VALIDATORS, NOMINATORS*4, TO_ELECT, EDGES, 0, 0), - bench_2_4: (VALIDATORS, NOMINATORS*8, TO_ELECT, EDGES, 0, 0), + bench_2_2: (VALIDATORS, NOMINATORS * 2, TO_ELECT, EDGES, 0, 0), + bench_2_3: (VALIDATORS, NOMINATORS * 4, TO_ELECT, EDGES, 0, 0), + bench_2_4: (VALIDATORS, NOMINATORS * 8, TO_ELECT, EDGES, 0, 0), bench_2_1_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES, 2, 0), - bench_2_2_eq: (VALIDATORS, NOMINATORS*2, TO_ELECT, EDGES, 2, 0), - bench_2_3_eq: (VALIDATORS, NOMINATORS*4, TO_ELECT, EDGES, 2, 0), - bench_2_4_eq: (VALIDATORS, NOMINATORS*8, TO_ELECT, EDGES, 2, 0), + bench_2_2_eq: (VALIDATORS, NOMINATORS * 2, TO_ELECT, EDGES, 2, 0), + bench_2_3_eq: (VALIDATORS, NOMINATORS * 4, TO_ELECT, EDGES, 2, 0), + bench_2_4_eq: (VALIDATORS, NOMINATORS * 8, TO_ELECT, EDGES, 2, 0), bench_3_1: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES, 0, 0 ), - bench_3_2: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES*2, 0, 0), - bench_3_3: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES*4, 0, 0), - bench_3_4: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES*8, 0, 0), + bench_3_2: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES * 2, 0, 0), + bench_3_3: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES * 4, 0, 0), + bench_3_4: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES * 8, 0, 0), bench_3_1_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES, 2, 0), - bench_3_2_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES*2, 2, 0), - bench_3_3_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES*4, 2, 0), - bench_3_4_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES*8, 2, 0), + bench_3_2_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES * 2, 2, 0), + bench_3_3_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES * 4, 2, 0), + bench_3_4_eq: (VALIDATORS, NOMINATORS, TO_ELECT, EDGES * 8, 2, 0), } diff --git a/primitives/phragmen/compact/Cargo.toml b/primitives/phragmen/compact/Cargo.toml index 56b4520c54277825b7dc340b8fef7a9f132ca43a..386116f2683ae9998e5b02e70477065f903cfd59 100644 --- a/primitives/phragmen/compact/Cargo.toml +++ b/primitives/phragmen/compact/Cargo.toml @@ -8,6 +8,9 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Phragmen Compact Solution" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [lib] proc-macro = true @@ -16,6 +19,3 @@ syn = { version = "1.0.7", features = ["full", "visit"] } quote = "1.0" proc-macro2 = "1.0.6" proc-macro-crate = "0.1.4" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/phragmen/compact/src/assignment.rs b/primitives/phragmen/compact/src/assignment.rs index 587e482ccb22096182404dd04823c05749d4575b..4630a494fc77562dfd4aab6bbc8bf83ca75bd961 100644 --- a/primitives/phragmen/compact/src/assignment.rs +++ b/primitives/phragmen/compact/src/assignment.rs @@ -91,7 +91,7 @@ fn into_impl(count: usize) -> TokenStream2 { } // defensive only. Since Percent doesn't have `Sub`. - let p2 = _phragmen::sp_runtime::traits::Saturating::saturating_sub( + let p2 = _phragmen::sp_arithmetic::traits::Saturating::saturating_sub( Accuracy::one(), p1, ); @@ -115,7 +115,7 @@ fn into_impl(count: usize) -> TokenStream2 { let mut inners_parsed = inners .iter() .map(|(ref t_idx, p)| { - sum = _phragmen::sp_runtime::traits::Saturating::saturating_add(sum, *p); + sum = _phragmen::sp_arithmetic::traits::Saturating::saturating_add(sum, *p); let target = target_at(*t_idx).ok_or(_phragmen::Error::CompactInvalidIndex)?; Ok((target, *p)) }) @@ -126,7 +126,7 @@ fn into_impl(count: usize) -> TokenStream2 { } // defensive only. Since Percent doesn't have `Sub`. - let p_last = _phragmen::sp_runtime::traits::Saturating::saturating_sub( + let p_last = _phragmen::sp_arithmetic::traits::Saturating::saturating_sub( Accuracy::one(), sum, ); @@ -163,7 +163,7 @@ pub(crate) fn assignment( #voter_type: _phragmen::codec::Codec + Default + Copy, #target_type: _phragmen::codec::Codec + Default + Copy, Accuracy: - _phragmen::codec::Codec + Default + Clone + _phragmen::sp_runtime::PerThing + + _phragmen::codec::Codec + Default + Clone + _phragmen::sp_arithmetic::PerThing + PartialOrd, > #ident<#voter_type, #target_type, Accuracy> diff --git a/primitives/phragmen/compact/src/lib.rs b/primitives/phragmen/compact/src/lib.rs index 114aeaeb32ed92ae4badbb59d38dc77cbcf309e7..9406f944c396ebd170f9e393d7383de878ac69aa 100644 --- a/primitives/phragmen/compact/src/lib.rs +++ b/primitives/phragmen/compact/src/lib.rs @@ -164,7 +164,7 @@ fn struct_def( PartialEq, Eq, Clone, - _phragmen::sp_runtime::RuntimeDebug, + Debug, _phragmen::codec::Encode, _phragmen::codec::Decode, )] diff --git a/primitives/phragmen/compact/src/staked.rs b/primitives/phragmen/compact/src/staked.rs index a7cf853f17086651ef4be37dcf2538cdce936dc6..81ccb5c55927c5633a07feccad842c4c3119a4e6 100644 --- a/primitives/phragmen/compact/src/staked.rs +++ b/primitives/phragmen/compact/src/staked.rs @@ -73,7 +73,7 @@ fn into_impl(count: usize) -> TokenStream2 { quote!( for (voter_index, target_index) in self.#name { let who = voter_at(voter_index).ok_or(_phragmen::Error::CompactInvalidIndex)?; - let all_stake = max_of(&who); + let all_stake: u128 = max_of(&who).into(); assignments.push(_phragmen::StakedAssignment { who, distribution: vec![(target_at(target_index).ok_or(_phragmen::Error::CompactInvalidIndex)?, all_stake)], @@ -87,7 +87,7 @@ fn into_impl(count: usize) -> TokenStream2 { quote!( for (voter_index, (t1_idx, w1), t2_idx) in self.#name { let who = voter_at(voter_index).ok_or(_phragmen::Error::CompactInvalidIndex)?; - let all_stake = max_of(&who); + let all_stake: u128 = max_of(&who).into(); if w1 >= all_stake { return Err(_phragmen::Error::CompactStakeOverflow); @@ -112,7 +112,7 @@ fn into_impl(count: usize) -> TokenStream2 { for (voter_index, inners, t_last_idx) in self.#name { let who = voter_at(voter_index).ok_or(_phragmen::Error::CompactInvalidIndex)?; let mut sum = u128::min_value(); - let all_stake = max_of(&who); + let all_stake: u128 = max_of(&who).into(); let mut inners_parsed = inners .iter() @@ -154,6 +154,7 @@ pub(crate) fn staked( let from_impl = from_impl(count); let into_impl = into_impl(count); + quote!( impl< #voter_type: _phragmen::codec::Codec + Default + Copy, @@ -196,7 +197,7 @@ pub(crate) fn staked( ) -> Result>, _phragmen::Error> where - for<'r> FM: Fn(&'r A) -> u128, + for<'r> FM: Fn(&'r A) -> u64, A: _phragmen::IdentifierT, { let mut assignments: Vec<_phragmen::StakedAssignment> = Default::default(); diff --git a/primitives/phragmen/fuzzer/Cargo.toml b/primitives/phragmen/fuzzer/Cargo.toml index 90af69a707f75a84b417e2729abf6a245061bb41..a26e6e9f8975b37af1fc14bb105e1befcc1091d4 100644 --- a/primitives/phragmen/fuzzer/Cargo.toml +++ b/primitives/phragmen/fuzzer/Cargo.toml @@ -1,19 +1,29 @@ [package] name = "sp-phragmen-fuzzer" -version = "2.0.0" +version = "2.0.0-alpha.5" authors = ["Parity Technologies "] edition = "2018" +license = "GPL-3.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Fuzzer for phragmén implementation." +documentation = "https://docs.rs/sp-phragmen-fuzzer" +publish = false [dependencies] -sp-phragmen = { version = "2.0.0-alpha.3", path = ".." } +sp-phragmen = { version = "2.0.0-alpha.5", path = ".." } +sp-std = { version = "2.0.0-alpha.5", path = "../../std" } +sp-runtime = { version = "2.0.0-alpha.5", path = "../../runtime" } honggfuzz = "0.5" -rand = "0.7.3" - -[workspace] +rand = { version = "0.7.3", features = ["std", "small_rng"] } [[bin]] name = "reduce" path = "src/reduce.rs" +[[bin]] +name = "equalize" +path = "src/equalize.rs" + [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/phragmen/fuzzer/src/common.rs b/primitives/phragmen/fuzzer/src/common.rs new file mode 100644 index 0000000000000000000000000000000000000000..3429dcb20adde2e260d66cf9be76df9f0420ec39 --- /dev/null +++ b/primitives/phragmen/fuzzer/src/common.rs @@ -0,0 +1,29 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Common fuzzing utils. + +/// converts x into the range [a, b] in a pseudo-fair way. +pub fn to_range(x: usize, a: usize, b: usize) -> usize { + // does not work correctly if b < 2*a + assert!(b > 2 * a); + let collapsed = x % b; + if collapsed >= a { + collapsed + } else { + collapsed + a + } +} diff --git a/primitives/phragmen/fuzzer/src/equalize.rs b/primitives/phragmen/fuzzer/src/equalize.rs new file mode 100644 index 0000000000000000000000000000000000000000..cb4f98c4eb15284be7de03bbbefa90b5d027e82c --- /dev/null +++ b/primitives/phragmen/fuzzer/src/equalize.rs @@ -0,0 +1,146 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Fuzzing fro the equalize algorithm +//! +//! It ensures that any solution which gets equalized will lead into a better or equally scored +//! one. + +mod common; +use common::to_range; +use honggfuzz::fuzz; +use sp_phragmen::{ + equalize, assignment_ratio_to_staked, build_support_map, to_without_backing, elect, + PhragmenResult, VoteWeight, evaluate_support, is_score_better, +}; +use sp_std::collections::btree_map::BTreeMap; +use sp_runtime::Perbill; +use rand::{self, Rng, SeedableRng, RngCore}; + +type AccountId = u64; + +fn generate_random_phragmen_result( + voter_count: u64, + target_count: u64, + to_elect: usize, + edge_per_voter: u64, + mut rng: impl RngCore, +) -> (PhragmenResult, BTreeMap) { + let prefix = 100_000; + // Note, it is important that stakes are always bigger than ed and + let base_stake: u64 = 1_000_000_000; + let ed: u64 = base_stake; + + let mut candidates = Vec::with_capacity(target_count as usize); + let mut stake_of_tree: BTreeMap = BTreeMap::new(); + + (1..=target_count).for_each(|acc| { + candidates.push(acc); + let stake_var = rng.gen_range(ed, 100 * ed); + stake_of_tree.insert(acc, base_stake + stake_var); + }); + + let mut voters = Vec::with_capacity(voter_count as usize); + (prefix ..= (prefix + voter_count)).for_each(|acc| { + // all possible targets + let mut all_targets = candidates.clone(); + // we remove and pop into `targets` `edge_per_voter` times. + let targets = (0..edge_per_voter).map(|_| { + let upper = all_targets.len() - 1; + let idx = rng.gen_range(0, upper); + all_targets.remove(idx) + }) + .collect::>(); + + let stake_var = rng.gen_range(ed, 100 * ed) ; + let stake = base_stake + stake_var; + stake_of_tree.insert(acc, stake); + voters.push((acc, stake, targets)); + }); + + ( + elect::( + to_elect, + 0, + candidates, + voters, + ).unwrap(), + stake_of_tree, + ) +} + +fn main() { + loop { + fuzz!(|data: (usize, usize, usize, usize, usize, u64)| { + let (mut target_count, mut voter_count, mut iterations, mut edge_per_voter, mut to_elect, seed) = data; + let rng = rand::rngs::SmallRng::seed_from_u64(seed); + target_count = to_range(target_count, 50, 2000); + voter_count = to_range(voter_count, 50, 1000); + iterations = to_range(iterations, 1, 20); + to_elect = to_range(to_elect, 25, target_count); + edge_per_voter = to_range(edge_per_voter, 1, target_count); + + println!("++ [{} / {} / {} / {}]", voter_count, target_count, to_elect, iterations); + let (PhragmenResult { winners, assignments }, stake_of_tree) = generate_random_phragmen_result( + voter_count as u64, + target_count as u64, + to_elect, + edge_per_voter as u64, + rng, + ); + + let stake_of = |who: &AccountId| -> VoteWeight { + *stake_of_tree.get(who).unwrap() + }; + + let mut staked = assignment_ratio_to_staked(assignments.clone(), &stake_of); + let winners = to_without_backing(winners); + let mut support = build_support_map(winners.as_ref(), staked.as_ref()).0; + + let initial_score = evaluate_support(&support); + if initial_score[0] == 0 { + // such cases cannot be improved by reduce. + return; + } + + let i = equalize( + &mut staked, + &mut support, + 10, + iterations, + ); + + let final_score = evaluate_support(&support); + if final_score[0] == initial_score[0] { + // such solutions can only be improved by such a tiny fiction that it is most often + // wrong due to rounding errors. + return; + } + + let enhance = is_score_better(initial_score, final_score); + + println!( + "iter = {} // {:?} -> {:?} [{}]", + i, + initial_score, + final_score, + enhance, + ); + // if more than one iteration has been done, or they must be equal. + assert!(enhance || initial_score == final_score || i == 0) + }); + } +} diff --git a/primitives/phragmen/fuzzer/src/reduce.rs b/primitives/phragmen/fuzzer/src/reduce.rs index 4bf08590a149feafae1fc9ab33372c24440377de..f0a16466636e3fe6e271ea039113c59d46345b4d 100644 --- a/primitives/phragmen/fuzzer/src/reduce.rs +++ b/primitives/phragmen/fuzzer/src/reduce.rs @@ -14,6 +14,12 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +//! Fuzzing for the reduce algorithm. +//! +//! It that reduce always return a new set og edges in which the bound is kept (`edges_after <= m + +//! n,`) and the result must effectively be the same, meaning that the same support map should be +//! computable from both. +//! //! # Running //! //! Run with `cargo hfuzz run reduce`. `honggfuzz`. @@ -24,8 +30,11 @@ //! `cargo hfuzz run-debug reduce hfuzz_workspace/reduce/*.fuzz`. use honggfuzz::fuzz; + +mod common; +use common::to_range; use sp_phragmen::{StakedAssignment, ExtendedBalance, build_support_map, reduce}; -use rand::{self, Rng}; +use rand::{self, Rng, SeedableRng, RngCore}; type Balance = u128; type AccountId = u64; @@ -35,15 +44,20 @@ const KSM: Balance = 1_000_000_000_000; fn main() { loop { - fuzz!(|_data: _| { - let (assignments, winners) = generate_random_phragmen_assignment( - rr(100, 1000), - rr(100, 2000), - 8, - 8, - ); + fuzz!(|data: (usize, usize, u64)| { + let (mut voter_count, mut target_count, seed) = data; + let rng = rand::rngs::SmallRng::seed_from_u64(seed); + target_count = to_range(target_count, 100, 1000); + voter_count = to_range(voter_count, 100, 2000); + let (assignments, winners) = generate_random_phragmen_assignment( + voter_count, + target_count, + 8, + 8, + rng + ); reduce_and_compare(&assignments, &winners); - }); + }); } } @@ -52,13 +66,10 @@ fn generate_random_phragmen_assignment( target_count: usize, avg_edge_per_voter: usize, edge_per_voter_var: usize, + mut rng: impl RngCore, ) -> (Vec>, Vec) { - // random in range of (a, b) - let rr_128 = |a: u128, b: u128| -> u128 { rand::thread_rng().gen_range(a, b) }; - // prefix to distinguish the voter and target account ranges. let target_prefix = 1_000_000; - // let target_prefix = 1000; assert!(voter_count < target_prefix); let mut assignments = Vec::with_capacity(voter_count as usize); @@ -70,17 +81,17 @@ fn generate_random_phragmen_assignment( (1..=voter_count).for_each(|acc| { let mut targets_to_chose_from = all_targets.clone(); - let targets_to_chose = if edge_per_voter_var > 0 { rr( + let targets_to_chose = if edge_per_voter_var > 0 { rng.gen_range( avg_edge_per_voter - edge_per_voter_var, avg_edge_per_voter + edge_per_voter_var, ) } else { avg_edge_per_voter }; let distribution = (0..targets_to_chose).map(|_| { - let target = targets_to_chose_from.remove(rr(0, targets_to_chose_from.len())); + let target = targets_to_chose_from.remove(rng.gen_range(0, targets_to_chose_from.len())); if winners.iter().find(|w| **w == target).is_none() { winners.push(target.clone()); } - (target, rr_128(1 * KSM, 100 * KSM)) + (target, rng.gen_range(1 * KSM, 100 * KSM)) }).collect::>(); assignments.push(StakedAssignment { @@ -139,7 +150,3 @@ fn assignment_len(assignments: &[StakedAssignment]) -> u32 { assignments.iter().for_each(|x| x.distribution.iter().for_each(|_| counter += 1)); counter } - -fn rr(a: usize, b: usize) -> usize { - rand::thread_rng().gen_range(a, b) -} diff --git a/primitives/phragmen/src/helpers.rs b/primitives/phragmen/src/helpers.rs index 27f51b4a05fe2c2c22a02f8442efdd9d9f86124d..6b1497e3ad752e5c428ad3b0544913af95cd4d53 100644 --- a/primitives/phragmen/src/helpers.rs +++ b/primitives/phragmen/src/helpers.rs @@ -16,8 +16,8 @@ //! Helper methods for phragmen. -use crate::{Assignment, ExtendedBalance, IdentifierT, StakedAssignment}; -use sp_runtime::PerThing; +use crate::{Assignment, ExtendedBalance, VoteWeight, IdentifierT, StakedAssignment, WithApprovalOf}; +use sp_arithmetic::PerThing; use sp_std::prelude::*; /// Converts a vector of ratio assignments into ones with absolute budget value. @@ -26,7 +26,7 @@ pub fn assignment_ratio_to_staked( stake_of: FS, ) -> Vec> where - for<'r> FS: Fn(&'r A) -> ExtendedBalance, + for<'r> FS: Fn(&'r A) -> VoteWeight, T: sp_std::ops::Mul, ExtendedBalance: From<::Inner>, { @@ -34,30 +34,34 @@ where .into_iter() .map(|a| { let stake = stake_of(&a.who); - a.into_staked(stake, true) + a.into_staked(stake.into(), true) }) .collect() } /// Converts a vector of staked assignments into ones with ratio values. pub fn assignment_staked_to_ratio( - ratio: Vec>, + staked: Vec>, ) -> Vec> where ExtendedBalance: From<::Inner>, { - ratio.into_iter().map(|a| a.into_assignment(true)).collect() + staked.into_iter().map(|a| a.into_assignment(true)).collect() +} + +/// consumes a vector of winners with backing stake to just winners. +pub fn to_without_backing(winners: Vec>) -> Vec { + winners.into_iter().map(|(who, _)| who).collect::>() } #[cfg(test)] mod tests { use super::*; - use crate::ExtendedBalance; - use sp_runtime::Perbill; + use sp_arithmetic::Perbill; #[test] fn into_staked_works() { - let ratio = vec![ + let assignments = vec![ Assignment { who: 1u32, distribution: vec![ @@ -74,8 +78,8 @@ mod tests { }, ]; - let stake_of = |_: &u32| -> ExtendedBalance { 100u128 }; - let staked = assignment_ratio_to_staked(ratio, stake_of); + let stake_of = |_: &u32| -> VoteWeight { 100 }; + let staked = assignment_ratio_to_staked(assignments, stake_of); assert_eq!( staked, diff --git a/primitives/phragmen/src/lib.rs b/primitives/phragmen/src/lib.rs index c0d94a71e1fc209f4db7085bd8fcba0262d66ecd..cf972eb1ed7babd138693f6ccd42fc2c61c24ab1 100644 --- a/primitives/phragmen/src/lib.rs +++ b/primitives/phragmen/src/lib.rs @@ -34,8 +34,11 @@ #![cfg_attr(not(feature = "std"), no_std)] use sp_std::{prelude::*, collections::btree_map::BTreeMap, fmt::Debug, cmp::Ordering, convert::TryFrom}; -use sp_runtime::{helpers_128bit::multiply_by_rational, PerThing, Rational128, RuntimeDebug, SaturatedConversion}; -use sp_runtime::traits::{Zero, Convert, Member, AtLeast32Bit, Saturating, Bounded}; +use sp_arithmetic::{ + PerThing, Rational128, + helpers_128bit::multiply_by_rational, + traits::{Zero, Saturating, Bounded, SaturatedConversion}, +}; #[cfg(test)] mod mock; @@ -60,7 +63,7 @@ pub use helpers::*; #[doc(hidden)] pub use codec; #[doc(hidden)] -pub use sp_runtime; +pub use sp_arithmetic; // re-export the compact solution type. pub use sp_phragmen_compact::generate_compact_solution_type; @@ -88,24 +91,26 @@ pub enum Error { CompactInvalidIndex, } -/// A type in which performing operations on balances and stakes of candidates and voters are safe. -/// -/// This module's functions expect a `Convert` type to convert all balances to u64. Hence, u128 is -/// a safe type for arithmetic operations over them. -/// -/// Balance types converted to `ExtendedBalance` are referred to as `Votes`. +/// A type which is used in the API of this crate as a numeric weight of a vote, most often the +/// stake of the voter. It is always converted to [`ExtendedBalance`] for computation. +pub type VoteWeight = u64; + +/// A type in which performing operations on vote weights are safe. pub type ExtendedBalance = u128; /// The score of an assignment. This can be computed from the support map via [`evaluate_support`]. pub type PhragmenScore = [ExtendedBalance; 3]; +/// A winner, with their respective approval stake. +pub type WithApprovalOf = (A, ExtendedBalance); + /// The denominator used for loads. Since votes are collected as u64, the smallest ratio that we /// might collect is `1/approval_stake` where approval stake is the sum of votes. Hence, some number /// bigger than u64::max_value() is needed. For maximum accuracy we simply use u128; const DEN: u128 = u128::max_value(); /// A candidate entity for phragmen election. -#[derive(Clone, Default, RuntimeDebug)] +#[derive(Clone, Default, Debug)] struct Candidate { /// Identifier. who: AccountId, @@ -118,7 +123,7 @@ struct Candidate { } /// A voter entity. -#[derive(Clone, Default, RuntimeDebug)] +#[derive(Clone, Default, Debug)] struct Voter { /// Identifier. who: AccountId, @@ -131,7 +136,7 @@ struct Voter { } /// A candidate being backed by a voter. -#[derive(Clone, Default, RuntimeDebug)] +#[derive(Clone, Default, Debug)] struct Edge { /// Identifier. who: AccountId, @@ -142,21 +147,21 @@ struct Edge { } /// Final result of the phragmen election. -#[derive(RuntimeDebug)] +#[derive(Debug)] pub struct PhragmenResult { /// Just winners zipped with their approval stake. Note that the approval stake is merely the /// sub of their received stake and could be used for very basic sorting and approval voting. - pub winners: Vec<(AccountId, ExtendedBalance)>, + pub winners: Vec>, /// Individual assignments. for each tuple, the first elements is a voter and the second /// is the list of candidates that it supports. pub assignments: Vec>, } /// A voter's stake assignment among a set of targets, represented as ratios. -#[derive(RuntimeDebug, Clone, Default)] +#[derive(Debug, Clone, Default)] #[cfg_attr(feature = "std", derive(PartialEq, Eq, Encode, Decode))] pub struct Assignment { - /// Voter's identifier + /// Voter's identifier. pub who: AccountId, /// The distribution of the voter's stake. pub distribution: Vec<(AccountId, T)>, @@ -221,7 +226,7 @@ where /// A voter's stake assignment among a set of targets, represented as absolute values in the scale /// of [`ExtendedBalance`]. -#[derive(RuntimeDebug, Clone, Default)] +#[derive(Debug, Clone, Default)] #[cfg_attr(feature = "std", derive(PartialEq, Eq, Encode, Decode))] pub struct StakedAssignment { /// Voter's identifier @@ -285,6 +290,11 @@ impl StakedAssignment { distribution, } } + + /// Get the total stake of this assignment (aka voter budget). + pub fn total(&self) -> ExtendedBalance { + self.distribution.iter().fold(Zero::zero(), |a, b| a.saturating_add(b.1)) + } } /// A structure to demonstrate the phragmen result from the perspective of the candidate, i.e. how @@ -294,7 +304,7 @@ impl StakedAssignment { /// /// This, at the current version, resembles the `Exposure` defined in the Staking pallet, yet /// they do not necessarily have to be the same. -#[derive(Default, RuntimeDebug)] +#[derive(Default, Debug)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Eq, PartialEq))] pub struct Support { /// Total support. @@ -316,25 +326,20 @@ pub type SupportMap = BTreeMap>; /// `None` is returned. /// * `initial_candidates`: candidates list to be elected from. /// * `initial_voters`: voters list. -/// * `stake_of`: something that can return the stake stake of a particular candidate or voter. /// /// This function does not strip out candidates who do not have any backing stake. It is the /// responsibility of the caller to make sure only those candidates who have a sensible economic /// value are passed in. From the perspective of this function, a candidate can easily be among the /// winner with no backing stake. -pub fn elect( +pub fn elect( candidate_count: usize, minimum_candidate_count: usize, initial_candidates: Vec, - initial_voters: Vec<(AccountId, Balance, Vec)>, + initial_voters: Vec<(AccountId, VoteWeight, Vec)>, ) -> Option> where - AccountId: Default + Ord + Member, - Balance: Default + Copy + AtLeast32Bit, - C: Convert + Convert, + AccountId: Default + Ord + Clone, R: PerThing, { - let to_votes = |b: Balance| >::convert(b) as ExtendedBalance; - // return structures let mut elected_candidates: Vec<(AccountId, ExtendedBalance)>; let mut assigned: Vec>; @@ -368,14 +373,14 @@ pub fn elect( if let Some(idx) = c_idx_cache.get(&v) { // This candidate is valid + already cached. candidates[*idx].approval_stake = candidates[*idx].approval_stake - .saturating_add(to_votes(voter_stake)); + .saturating_add(voter_stake.into()); edges.push(Edge { who: v.clone(), candidate_index: *idx, ..Default::default() }); } // else {} would be wrong votes. We don't really care about it. } Voter { who, edges: edges, - budget: to_votes(voter_stake), + budget: voter_stake.into(), load: Rational128::zero(), } })); @@ -559,7 +564,7 @@ pub fn build_support_map( winners: &[AccountId], assignments: &[StakedAssignment], ) -> (SupportMap, u32) where - AccountId: Default + Ord + Member, + AccountId: Default + Ord + Clone, { let mut errors = 0; // Initialize the support of each candidate. @@ -633,32 +638,27 @@ pub fn is_score_better(this: PhragmenScore, that: PhragmenScore) -> bool { /// rounds. The number of rounds and the maximum diff-per-round tolerance can be tuned through input /// parameters. /// -/// No value is returned from the function and the `supports` parameter is updated. +/// Returns the number of iterations that were preformed. /// /// - `assignments`: exactly the same is the output of phragmen. /// - `supports`: mutable reference to s `SupportMap`. This parameter is updated. /// - `tolerance`: maximum difference that can occur before an early quite happens. /// - `iterations`: maximum number of iterations that will be processed. -/// - `stake_of`: something that can return the stake stake of a particular candidate or voter. -pub fn equalize( - mut assignments: Vec>, +pub fn equalize( + assignments: &mut Vec>, supports: &mut SupportMap, tolerance: ExtendedBalance, iterations: usize, - stake_of: FS, -) where - C: Convert + Convert, - for<'r> FS: Fn(&'r AccountId) -> Balance, - AccountId: Ord + Clone, -{ - // prepare the data for equalise - for _i in 0..iterations { - let mut max_diff = 0; +) -> usize where AccountId: Ord + Clone { + if iterations == 0 { return 0; } - for StakedAssignment { who, distribution } in assignments.iter_mut() { - let voter_budget = stake_of(&who); - - let diff = do_equalize::<_, _, C>( + let mut i = 0 ; + loop { + let mut max_diff = 0; + for assignment in assignments.iter_mut() { + let voter_budget = assignment.total(); + let StakedAssignment { who, distribution } = assignment; + let diff = do_equalize( who, voter_budget, distribution, @@ -668,28 +668,22 @@ pub fn equalize( if diff > max_diff { max_diff = diff; } } - if max_diff < tolerance { - break; + i += 1; + if max_diff <= tolerance || i >= iterations { + break i; } } } /// actually perform equalize. same interface is `equalize`. Just called in loops with a check for /// maximum difference. -fn do_equalize( +fn do_equalize( voter: &AccountId, - budget_balance: Balance, + budget: ExtendedBalance, elected_edges: &mut Vec<(AccountId, ExtendedBalance)>, support_map: &mut SupportMap, tolerance: ExtendedBalance -) -> ExtendedBalance where - C: Convert + Convert, - AccountId: Ord + Clone, -{ - let to_votes = |b: Balance| - >::convert(b) as ExtendedBalance; - let budget = to_votes(budget_balance); - +) -> ExtendedBalance where AccountId: Ord + Clone { // Nothing to do. This voter had nothing useful. // Defensive only. Assignment list should always be populated. 1 might happen for self vote. if elected_edges.is_empty() || elected_edges.len() == 1 { return 0; } diff --git a/primitives/phragmen/src/mock.rs b/primitives/phragmen/src/mock.rs index 31ce3d38c3500ef930d3676a141beabb8694b275..cf9c90334bc588bb21855e1ed346ec99a3d1fe95 100644 --- a/primitives/phragmen/src/mock.rs +++ b/primitives/phragmen/src/mock.rs @@ -18,20 +18,10 @@ #![cfg(test)] -use crate::{elect, PhragmenResult, Assignment}; -use sp_runtime::{ - assert_eq_error_rate, PerThing, - traits::{Convert, Member, SaturatedConversion, Zero, One} -}; +use crate::{elect, PhragmenResult, Assignment, VoteWeight, ExtendedBalance}; +use sp_arithmetic::{PerThing, traits::{SaturatedConversion, Zero, One}}; use sp_std::collections::btree_map::BTreeMap; - -pub(crate) struct TestCurrencyToVote; -impl Convert for TestCurrencyToVote { - fn convert(x: Balance) -> u64 { x.saturated_into() } -} -impl Convert for TestCurrencyToVote { - fn convert(x: u128) -> Balance { x } -} +use sp_runtime::assert_eq_error_rate; #[derive(Default, Debug)] pub(crate) struct _Candidate { @@ -66,12 +56,11 @@ pub(crate) struct _Support { pub(crate) type _PhragmenAssignment = (A, f64); pub(crate) type _SupportMap = BTreeMap>; -pub(crate) type Balance = u128; pub(crate) type AccountId = u64; #[derive(Debug, Clone)] pub(crate) struct _PhragmenResult { - pub winners: Vec<(A, Balance)>, + pub winners: Vec<(A, ExtendedBalance)>, pub assignments: Vec<(A, Vec<_PhragmenAssignment>)> } @@ -86,10 +75,10 @@ pub(crate) fn elect_float( initial_voters: Vec<(A, Vec)>, stake_of: FS, ) -> Option<_PhragmenResult> where - A: Default + Ord + Member + Copy, - for<'r> FS: Fn(&'r A) -> Balance, + A: Default + Ord + Copy, + for<'r> FS: Fn(&'r A) -> VoteWeight, { - let mut elected_candidates: Vec<(A, Balance)>; + let mut elected_candidates: Vec<(A, ExtendedBalance)>; let mut assigned: Vec<(A, Vec<_PhragmenAssignment>)>; let mut c_idx_cache = BTreeMap::::new(); let num_voters = initial_candidates.len() + initial_voters.len(); @@ -161,7 +150,7 @@ pub(crate) fn elect_float( } } - elected_candidates.push((winner.who.clone(), winner.approval_stake as Balance)); + elected_candidates.push((winner.who.clone(), winner.approval_stake as ExtendedBalance)); } else { break } @@ -195,7 +184,7 @@ pub(crate) fn equalize_float( iterations: usize, stake_of: FS, ) where - for<'r> FS: Fn(&'r A) -> Balance, + for<'r> FS: Fn(&'r A) -> VoteWeight, A: Ord + Clone + std::fmt::Debug, { for _i in 0..iterations { @@ -220,7 +209,7 @@ pub(crate) fn equalize_float( pub(crate) fn do_equalize_float( voter: &A, - budget_balance: Balance, + budget_balance: VoteWeight, elected_edges: &mut Vec<_PhragmenAssignment>, support_map: &mut _SupportMap, tolerance: f64 @@ -310,12 +299,12 @@ pub(crate) fn do_equalize_float( } -pub(crate) fn create_stake_of(stakes: &[(AccountId, Balance)]) - -> Box Balance> +pub(crate) fn create_stake_of(stakes: &[(AccountId, VoteWeight)]) + -> Box VoteWeight> { - let mut storage = BTreeMap::::new(); + let mut storage = BTreeMap::::new(); stakes.iter().for_each(|s| { storage.insert(s.0, s.1); }); - let stake_of = move |who: &AccountId| -> Balance { storage.get(who).unwrap().to_owned() }; + let stake_of = move |who: &AccountId| -> VoteWeight { storage.get(who).unwrap().to_owned() }; Box::new(stake_of) } @@ -331,12 +320,12 @@ pub fn check_assignments_sum(assignments: Vec( candidates: Vec, voters: Vec<(AccountId, Vec)>, - stake_of: &Box Balance>, + stake_of: &Box VoteWeight>, to_elect: usize, min_to_elect: usize, ) { // run fixed point code. - let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Output>( + let PhragmenResult { winners, assignments } = elect::<_, Output>( to_elect, min_to_elect, candidates.clone(), @@ -352,7 +341,7 @@ pub(crate) fn run_and_compare( &stake_of, ).unwrap(); - assert_eq!(winners, truth_value.winners); + assert_eq!(winners.iter().map(|(x, _)| x).collect::>(), truth_value.winners.iter().map(|(x, _)| x).collect::>()); for Assignment { who, distribution } in assignments.clone() { if let Some(float_assignments) = truth_value.assignments.iter().find(|x| x.0 == who) { @@ -379,7 +368,7 @@ pub(crate) fn build_support_map_float( result: &mut _PhragmenResult, stake_of: FS, ) -> _SupportMap - where for<'r> FS: Fn(&'r AccountId) -> Balance + where for<'r> FS: Fn(&'r AccountId) -> VoteWeight { let mut supports = <_SupportMap>::new(); result.winners diff --git a/primitives/phragmen/src/node.rs b/primitives/phragmen/src/node.rs index 92ef325a348724762f3c92f29849963c1cfe4edb..432c537052ebb5c3e9c5921233554f545cc50ffa 100644 --- a/primitives/phragmen/src/node.rs +++ b/primitives/phragmen/src/node.rs @@ -16,11 +16,10 @@ //! (very) Basic implementation of a graph node used in the reduce algorithm. -use sp_runtime::RuntimeDebug; use sp_std::{cell::RefCell, fmt, prelude::*, rc::Rc}; /// The role that a node can accept. -#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, RuntimeDebug)] +#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Debug)] pub(crate) enum NodeRole { /// A voter. This is synonym to a nominator in a staking context. Voter, diff --git a/primitives/phragmen/src/reduce.rs b/primitives/phragmen/src/reduce.rs index 54a71a7ff29aabdf80b35565f4f09c34fec5b808..ac4441ddc0eac9d786394aa361797d9e69e482c0 100644 --- a/primitives/phragmen/src/reduce.rs +++ b/primitives/phragmen/src/reduce.rs @@ -48,7 +48,7 @@ use crate::node::{Node, NodeId, NodeRef, NodeRole}; use crate::{ExtendedBalance, IdentifierT, StakedAssignment}; -use sp_runtime::traits::{Bounded, Zero}; +use sp_arithmetic::traits::{Bounded, Zero}; use sp_std::{ collections::btree_map::{BTreeMap, Entry::*}, prelude::*, @@ -305,7 +305,7 @@ fn reduce_4(assignments: &mut Vec>) -> u32 { } (false, false) => { // Neither of the edges was removed? impossible. - debug_assert!(false, "Duplicate voter (or other corrupt input)."); + panic!("Duplicate voter (or other corrupt input)."); } } } diff --git a/primitives/phragmen/src/tests.rs b/primitives/phragmen/src/tests.rs index e9861ede723cf9f31baf5f6c5d91b79e361b75d9..720a0a3f7524a754d4bd7f4112ff3978b4d436ae 100644 --- a/primitives/phragmen/src/tests.rs +++ b/primitives/phragmen/src/tests.rs @@ -20,11 +20,11 @@ use crate::mock::*; use crate::{ - elect, equalize, build_support_map, is_score_better, + elect, equalize, build_support_map, is_score_better, helpers::*, Support, StakedAssignment, Assignment, PhragmenResult, ExtendedBalance, }; use substrate_test_utils::assert_eq_uvec; -use sp_runtime::{Perbill, Permill, Percent, PerU16, traits::Convert}; +use sp_arithmetic::{Perbill, Permill, Percent, PerU16}; #[test] fn float_phragmen_poc_works() { @@ -82,7 +82,7 @@ fn phragmen_poc_works() { ]; let stake_of = create_stake_of(&[(10, 10), (20, 20), (30, 30)]); - let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Perbill>( + let PhragmenResult { winners, assignments } = elect::<_, Perbill>( 2, 2, candidates, @@ -110,6 +110,77 @@ fn phragmen_poc_works() { }, ] ); + + let mut staked = assignment_ratio_to_staked(assignments, &stake_of); + let winners = to_without_backing(winners); + let mut support_map = build_support_map::(&winners, &staked).0; + + assert_eq_uvec!( + staked, + vec![ + StakedAssignment { + who: 10u64, + distribution: vec![(2, 10)], + }, + StakedAssignment { + who: 20, + distribution: vec![(3, 20)], + }, + StakedAssignment { + who: 30, + distribution: vec![ + (2, 15), + (3, 15), + ], + }, + ] + ); + + assert_eq!( + *support_map.get(&2).unwrap(), + Support:: { total: 25, voters: vec![(10, 10), (30, 15)] }, + ); + assert_eq!( + *support_map.get(&3).unwrap(), + Support:: { total: 35, voters: vec![(20, 20), (30, 15)] }, + ); + + equalize( + &mut staked, + &mut support_map, + 0, + 2, + ); + + assert_eq_uvec!( + staked, + vec![ + StakedAssignment { + who: 10u64, + distribution: vec![(2, 10)], + }, + StakedAssignment { + who: 20, + distribution: vec![(3, 20)], + }, + StakedAssignment { + who: 30, + distribution: vec![ + (2, 20), + (3, 10), + ], + }, + ] + ); + + assert_eq!( + *support_map.get(&2).unwrap(), + Support:: { total: 30, voters: vec![(10, 10), (30, 20)] }, + ); + assert_eq!( + *support_map.get(&3).unwrap(), + Support:: { total: 30, voters: vec![(20, 20), (30, 10)] }, + ); } #[test] @@ -168,7 +239,7 @@ fn phragmen_accuracy_on_large_scale_only_validators() { (5, (u64::max_value() - 2).into()), ]); - let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Perbill>( + let PhragmenResult { winners, assignments } = elect::<_, Perbill>( 2, 2, candidates.clone(), @@ -198,7 +269,7 @@ fn phragmen_accuracy_on_large_scale_validators_and_nominators() { (14, u64::max_value().into()), ]); - let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Perbill>( + let PhragmenResult { winners, assignments } = elect::<_, Perbill>( 2, 2, candidates, @@ -241,7 +312,7 @@ fn phragmen_accuracy_on_small_scale_self_vote() { (30, 1), ]); - let PhragmenResult { winners, assignments: _ } = elect::<_, _, TestCurrencyToVote, Perbill>( + let PhragmenResult { winners, assignments: _ } = elect::<_, Perbill>( 3, 3, candidates, @@ -271,7 +342,7 @@ fn phragmen_accuracy_on_small_scale_no_self_vote() { (3, 1), ]); - let PhragmenResult { winners, assignments: _ } = elect::<_, _, TestCurrencyToVote, Perbill>( + let PhragmenResult { winners, assignments: _ } = elect::<_, Perbill>( 3, 3, candidates, @@ -304,7 +375,7 @@ fn phragmen_large_scale_test() { (50, 990000000000000000), ]); - let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Perbill>( + let PhragmenResult { winners, assignments } = elect::<_, Perbill>( 2, 2, candidates, @@ -330,7 +401,7 @@ fn phragmen_large_scale_test_2() { (50, nom_budget.into()), ]); - let PhragmenResult { winners, assignments } = elect::<_, _, TestCurrencyToVote, Perbill>( + let PhragmenResult { winners, assignments } = elect::<_, Perbill>( 2, 2, candidates, @@ -406,7 +477,7 @@ fn elect_has_no_entry_barrier() { (2, 10), ]); - let PhragmenResult { winners, assignments: _ } = elect::<_, _, TestCurrencyToVote, Perbill>( + let PhragmenResult { winners, assignments: _ } = elect::<_, Perbill>( 3, 3, candidates, @@ -433,7 +504,7 @@ fn minimum_to_elect_is_respected() { (2, 10), ]); - let maybe_result = elect::<_, _, TestCurrencyToVote, Perbill>( + let maybe_result = elect::<_, Perbill>( 10, 10, candidates, @@ -459,7 +530,7 @@ fn self_votes_should_be_kept() { (1, 8), ]); - let result = elect::<_, _, TestCurrencyToVote, Perbill>( + let result = elect::<_, Perbill>( 2, 2, candidates, @@ -480,16 +551,11 @@ fn self_votes_should_be_kept() { ], ); - let staked_assignments: Vec> = result.assignments - .into_iter() - .map(|a| { - let stake = >::convert(stake_of(&a.who)) as ExtendedBalance; - a.into_staked(stake, true) - }).collect(); + let mut staked_assignments = assignment_ratio_to_staked(result.assignments, &stake_of); + let winners = to_without_backing(result.winners); - let winners = result.winners.into_iter().map(|(who, _)| who).collect::>(); let (mut supports, _) = build_support_map::( - winners.as_slice(), + &winners, &staked_assignments, ); @@ -503,12 +569,11 @@ fn self_votes_should_be_kept() { &Support { total: 24u128, voters: vec![(20u64, 20u128), (1u64, 4u128)] }, ); - equalize::( - staked_assignments, + equalize( + &mut staked_assignments, &mut supports, 0, 2usize, - &stake_of, ); assert_eq!( @@ -526,7 +591,7 @@ fn assignment_convert_works() { let staked = StakedAssignment { who: 1 as AccountId, distribution: vec![ - (20, 100 as Balance), + (20, 100 as ExtendedBalance), (30, 25), ], }; @@ -578,12 +643,12 @@ fn score_comparison_is_lexicographical() { mod compact { use codec::{Decode, Encode}; - use crate::generate_compact_solution_type; - use super::{AccountId, Balance}; + use crate::{generate_compact_solution_type, VoteWeight}; + use super::{AccountId}; // these need to come from the same dev-dependency `sp-phragmen`, not from the crate. - use sp_phragmen::{Assignment, StakedAssignment, Error as PhragmenError}; + use sp_phragmen::{Assignment, StakedAssignment, Error as PhragmenError, ExtendedBalance}; use sp_std::{convert::{TryInto, TryFrom}, fmt::Debug}; - use sp_runtime::Percent; + use sp_arithmetic::Percent; type Accuracy = Percent; @@ -736,7 +801,7 @@ mod compact { let assignments = vec![ StakedAssignment { who: 2 as AccountId, - distribution: vec![(20, 100 as Balance)] + distribution: vec![(20, 100 as ExtendedBalance)] }, StakedAssignment { who: 4, @@ -773,7 +838,7 @@ mod compact { targets.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() }; - let compacted = >::from_staked( + let compacted = >::from_staked( assignments.clone(), voter_index, target_index, @@ -794,7 +859,7 @@ mod compact { } ); - let max_of_fn = |_: &AccountId| -> Balance { 100u128 }; + let max_of_fn = |_: &AccountId| -> VoteWeight { 100 }; let voter_at = |a: u16| -> Option { voters.get(a as usize).cloned() }; let target_at = |a: u16| -> Option { targets.get(a as usize).cloned() }; @@ -812,14 +877,14 @@ mod compact { fn compact_into_stake_must_report_overflow() { // The last edge which is computed from the rest should ALWAYS be positive. // in votes2 - let compact = TestCompact:: { + let compact = TestCompact:: { votes1: Default::default(), votes2: vec![(0, (1, 10), 2)], ..Default::default() }; let entity_at = |a: u16| -> Option { Some(a as AccountId) }; - let max_of = |_: &AccountId| -> Balance { 5 }; + let max_of = |_: &AccountId| -> VoteWeight { 5 }; assert_eq!( compact.into_staked(&max_of, &entity_at, &entity_at).unwrap_err(), @@ -827,7 +892,7 @@ mod compact { ); // in votes3 onwards - let compact = TestCompact:: { + let compact = TestCompact:: { votes1: Default::default(), votes2: Default::default(), votes3: vec![(0, [(1, 7), (2, 8)], 3)], @@ -840,7 +905,7 @@ mod compact { ); // Also if equal - let compact = TestCompact:: { + let compact = TestCompact:: { votes1: Default::default(), votes2: Default::default(), // 5 is total, we cannot leave none for 30 here. @@ -889,13 +954,13 @@ mod compact { let assignments = vec![ StakedAssignment { who: 1 as AccountId, - distribution: (10..26).map(|i| (i as AccountId, i as Balance)).collect::>(), + distribution: (10..26).map(|i| (i as AccountId, i as ExtendedBalance)).collect::>(), }, ]; let entity_index = |a: &AccountId| -> Option { Some(*a as u16) }; - let compacted = >::from_staked( + let compacted = >::from_staked( assignments.clone(), entity_index, entity_index, @@ -906,11 +971,11 @@ mod compact { let assignments = vec![ StakedAssignment { who: 1 as AccountId, - distribution: (10..27).map(|i| (i as AccountId, i as Balance)).collect::>(), + distribution: (10..27).map(|i| (i as AccountId, i as ExtendedBalance)).collect::>(), }, ]; - let compacted = >::from_staked( + let compacted = >::from_staked( assignments.clone(), entity_index, entity_index, @@ -948,7 +1013,7 @@ mod compact { let assignments = vec![ StakedAssignment { who: 1 as AccountId, - distribution: vec![(10, 100 as Balance), (11, 100)] + distribution: vec![(10, 100 as ExtendedBalance), (11, 100)] }, StakedAssignment { who: 2, @@ -963,7 +1028,7 @@ mod compact { targets.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() }; - let compacted = >::from_staked( + let compacted = >::from_staked( assignments.clone(), voter_index, target_index, diff --git a/primitives/rpc/Cargo.toml b/primitives/rpc/Cargo.toml index a7975fb4754b9ee049b7f76f45ca0876a4b91006..740b429c0c8fb6a30b845da7ae4a7321af8dfe89 100644 --- a/primitives/rpc/Cargo.toml +++ b/primitives/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-rpc" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,12 +8,12 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Substrate RPC primitives and utilities." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", features = ["derive"] } -sp-core = { version = "2.0.0-alpha.5", path = "../core" } +sp-core = { version = "2.0.0-dev", path = "../core" } [dev-dependencies] serde_json = "1.0.41" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/runtime-interface/Cargo.toml b/primitives/runtime-interface/Cargo.toml index 01f7f2f19c98169dbebd86e841fd37473ea6d011..1d0ae8f95120d1d6157ebf1c3a553a1eefe75b60 100644 --- a/primitives/runtime-interface/Cargo.toml +++ b/primitives/runtime-interface/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-runtime-interface" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,20 +9,24 @@ repository = "https://github.com/paritytech/substrate/" description = "Substrate runtime interface" documentation = "https://docs.rs/sp-runtime-interface/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-wasm-interface = { version = "2.0.0-alpha.5", path = "../wasm-interface", default-features = false } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } -sp-runtime-interface-proc-macro = { version = "2.0.0-alpha.5", path = "proc-macro" } -sp-externalities = { version = "0.8.0-alpha.5", optional = true, path = "../externalities" } +sp-wasm-interface = { version = "2.0.0-dev", path = "../wasm-interface", default-features = false } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } +sp-tracing = { version = "2.0.0-dev", default-features = false, path = "../tracing" } +sp-runtime-interface-proc-macro = { version = "2.0.0-dev", path = "proc-macro" } +sp-externalities = { version = "0.8.0-dev", optional = true, path = "../externalities" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } static_assertions = "1.0.0" primitive-types = { version = "0.7.0", default-features = false } [dev-dependencies] sp-runtime-interface-test-wasm = { version = "2.0.0-dev", path = "test-wasm" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../primitives/state-machine" } -sp-core = { version = "2.0.0-alpha.5", path = "../core" } -sp-io = { version = "2.0.0-alpha.5", path = "../io" } +sp-state-machine = { version = "0.8.0-dev", path = "../../primitives/state-machine" } +sp-core = { version = "2.0.0-dev", path = "../core" } +sp-io = { version = "2.0.0-dev", path = "../io" } rustversion = "1.0.0" trybuild = "1.0.23" @@ -31,6 +35,7 @@ default = [ "std" ] std = [ "sp-wasm-interface/std", "sp-std/std", + "sp-tracing/std", "codec/std", "sp-externalities", "primitive-types/std", @@ -43,6 +48,3 @@ std = [ # Disables static assertions in `impls.rs` that checks the word size. To prevent any footgun, the # check is changed into a runtime check. disable_target_static_assertions = [] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/runtime-interface/proc-macro/Cargo.toml b/primitives/runtime-interface/proc-macro/Cargo.toml index 6d0b7ee5fb7262c4a55922c0b1cef6549424c935..28fe00cc390dde62b1cef847e9019ec6917a4479 100644 --- a/primitives/runtime-interface/proc-macro/Cargo.toml +++ b/primitives/runtime-interface/proc-macro/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-runtime-interface-proc-macro" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,6 +9,9 @@ repository = "https://github.com/paritytech/substrate/" description = "This crate provides procedural macros for usage within the context of the Substrate runtime interface." documentation = "https://docs.rs/sp-runtime-interface-proc-macro" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [lib] proc-macro = true @@ -18,6 +21,3 @@ quote = "1.0.3" proc-macro2 = "1.0.3" Inflector = "0.11.4" proc-macro-crate = "0.1.4" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs b/primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs index e7c34fbf9934211df7ea874e08d2d4382b1c7fe2..8e83556f045085a887cbc4a68838a8988f23045f 100644 --- a/primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs +++ b/primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs @@ -146,6 +146,7 @@ fn function_std_impl( is_wasm_only: bool, ) -> Result { let function_name = create_function_ident_with_version(&method.sig.ident, version); + let function_name_str = function_name.to_string(); let crate_ = generate_crate_access(); let args = get_function_arguments(&method.sig).map(FnArg::Typed).chain( @@ -172,6 +173,7 @@ fn function_std_impl( #[cfg(feature = "std")] #( #attrs )* fn #function_name( #( #args, )* ) #return_value { + #crate_::sp_tracing::enter_span!(#function_name_str); #call_to_trait } } diff --git a/primitives/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs b/primitives/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs index 205ee87105c41c9a1d5166f2a6b182ae28dd308b..46de98c3c3f706ea30f3ca4f95ed5f7ab53d9f32 100644 --- a/primitives/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs +++ b/primitives/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs @@ -226,6 +226,7 @@ fn generate_host_function_implementation( __function_context__: &mut dyn #crate_::sp_wasm_interface::FunctionContext, args: &mut dyn Iterator, ) -> std::result::Result, String> { + #crate_::sp_tracing::enter_span!(#name); #( #wasm_to_ffi_values )* #( #ffi_to_host_values )* #host_function_call diff --git a/primitives/runtime-interface/src/lib.rs b/primitives/runtime-interface/src/lib.rs index fd158d4b8aa9609df1206b2ce5a640e95b5e3092..4f748825e5bc1d889bf2eaa024516d0ce722a632 100644 --- a/primitives/runtime-interface/src/lib.rs +++ b/primitives/runtime-interface/src/lib.rs @@ -109,6 +109,9 @@ extern crate self as sp_runtime_interface; #[cfg(feature = "std")] pub use sp_wasm_interface; +#[doc(hidden)] +pub use sp_tracing; + #[doc(hidden)] pub use sp_std; diff --git a/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml b/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml index 6f2d66bd77c4b2d672c54d3dc6f760830ccc570b..f992cad69b0f8df53a9cace9db3be21044637907 100644 --- a/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml +++ b/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml @@ -9,11 +9,14 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" publish = false +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-runtime-interface = { version = "2.0.0-alpha.5", default-features = false, path = "../" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../io" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../core" } +sp-runtime-interface = { version = "2.0.0-dev", default-features = false, path = "../" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../io" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../core" } [build-dependencies] wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } @@ -21,6 +24,3 @@ wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-run [features] default = [ "std" ] std = [ "sp-runtime-interface/std", "sp-std/std", "sp-core/std", "sp-io/std" ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/runtime-interface/test-wasm/Cargo.toml b/primitives/runtime-interface/test-wasm/Cargo.toml index 4eb4f01c9f0c810ab42f9016e1f7db66147e1f65..f9e64a5027a68a7befd47b4cd58523411187266f 100644 --- a/primitives/runtime-interface/test-wasm/Cargo.toml +++ b/primitives/runtime-interface/test-wasm/Cargo.toml @@ -9,11 +9,14 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" publish = false +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-runtime-interface = { version = "2.0.0-alpha.5", default-features = false, path = "../" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../io" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../core" } +sp-runtime-interface = { version = "2.0.0-dev", default-features = false, path = "../" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../io" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../core" } [build-dependencies] wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" } @@ -21,6 +24,3 @@ wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-run [features] default = [ "std" ] std = [ "sp-runtime-interface/std", "sp-std/std", "sp-core/std", "sp-io/std" ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/runtime-interface/test/Cargo.toml b/primitives/runtime-interface/test/Cargo.toml index f3bee038c8204c612444c8c3d6e34a4e7f3e2126..03f0122b2200b2e8684d06599b60a1a9a599320f 100644 --- a/primitives/runtime-interface/test/Cargo.toml +++ b/primitives/runtime-interface/test/Cargo.toml @@ -8,14 +8,16 @@ publish = false homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-runtime-interface = { version = "2.0.0-alpha.5", path = "../" } -sc-executor = { version = "0.8.0-alpha.5", path = "../../../client/executor" } +sp-runtime-interface = { version = "2.0.0-dev", path = "../" } +sc-executor = { version = "0.8.0-dev", path = "../../../client/executor" } sp-runtime-interface-test-wasm = { version = "2.0.0-dev", path = "../test-wasm" } sp-runtime-interface-test-wasm-deprecated = { version = "2.0.0-dev", path = "../test-wasm-deprecated" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../../primitives/state-machine" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../runtime" } -sp-io = { version = "2.0.0-alpha.5", path = "../../io" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sp-state-machine = { version = "0.8.0-dev", path = "../../../primitives/state-machine" } +sp-runtime = { version = "2.0.0-dev", path = "../../runtime" } +sp-core = { version = "2.0.0-dev", path = "../../core" } +sp-io = { version = "2.0.0-dev", path = "../../io" } +tracing = "0.1.13" diff --git a/primitives/runtime-interface/test/src/lib.rs b/primitives/runtime-interface/test/src/lib.rs index 110eda980fd6947166768150e55c464c31c9ac80..e7ef1934e25d8b644fe201b9ead6b4a6fb13918c 100644 --- a/primitives/runtime-interface/test/src/lib.rs +++ b/primitives/runtime-interface/test/src/lib.rs @@ -27,6 +27,8 @@ use sp_runtime_interface_test_wasm_deprecated::WASM_BINARY as WASM_BINARY_DEPREC use sp_wasm_interface::HostFunctions as HostFunctionsT; use sc_executor::CallInWasm; +use std::{collections::HashSet, sync::{Arc, Mutex}}; + type TestExternalities = sp_state_machine::TestExternalities; fn call_wasm_method(binary: &[u8], method: &str) -> TestExternalities { @@ -39,7 +41,6 @@ fn call_wasm_method(binary: &[u8], method: &str) -> TestExte sc_executor::WasmExecutionMethod::Interpreted, Some(8), host_functions, - false, 8, ); executor.call_in_wasm( @@ -48,6 +49,7 @@ fn call_wasm_method(binary: &[u8], method: &str) -> TestExte method, &[], &mut ext_ext, + sp_core::traits::MissingHostFunctions::Disallow, ).expect(&format!("Executes `{}`", method)); ext @@ -150,3 +152,47 @@ fn test_versionining_with_new_host_works() { "test_versionning_works", ); } + +#[test] +fn test_tracing() { + use tracing::span::Id as SpanId; + + #[derive(Clone)] + struct TracingSubscriber(Arc>); + + #[derive(Default)] + struct Inner { + spans: HashSet<&'static str>, + } + + impl tracing::subscriber::Subscriber for TracingSubscriber { + fn enabled(&self, _: &tracing::Metadata) -> bool { true } + + fn new_span(&self, span: &tracing::span::Attributes) -> tracing::Id { + let mut inner = self.0.lock().unwrap(); + let id = SpanId::from_u64((inner.spans.len() + 1) as _); + inner.spans.insert(span.metadata().name()); + id + } + + fn record(&self, _: &SpanId, _: &tracing::span::Record) {} + + fn record_follows_from(&self, _: &SpanId, _: &SpanId) {} + + fn event(&self, _: &tracing::Event) {} + + fn enter(&self, _: &SpanId) {} + + fn exit(&self, _: &SpanId) {} + } + + let subscriber = TracingSubscriber(Default::default()); + let _guard = tracing::subscriber::set_default(subscriber.clone()); + + // Call some method to generate a trace + call_wasm_method::(&WASM_BINARY[..], "test_return_data"); + + let inner = subscriber.0.lock().unwrap(); + assert!(inner.spans.contains("return_input_version_1")); + assert!(inner.spans.contains("ext_test_api_return_input_version_1")); +} diff --git a/primitives/runtime/Cargo.toml b/primitives/runtime/Cargo.toml index 0e71e8becd61e4ea18a77e9ae4b5afcc031d731d..030f0a61cbf74a0ccf1dfa61905d1738d36ebc80 100644 --- a/primitives/runtime/Cargo.toml +++ b/primitives/runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-runtime" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,26 +9,30 @@ repository = "https://github.com/paritytech/substrate/" description = "Runtime Modules shared primitive types." documentation = "https://docs.rs/sp-runtime" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = { version = "1.0.101", optional = true, features = ["derive"] } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core" } -sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../application-crypto" } -sp-arithmetic = { version = "2.0.0-alpha.5", default-features = false, path = "../arithmetic" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../io" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../core" } +sp-application-crypto = { version = "2.0.0-dev", default-features = false, path = "../application-crypto" } +sp-arithmetic = { version = "2.0.0-dev", default-features = false, path = "../arithmetic" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../io" } log = { version = "0.4.8", optional = true } paste = "0.1.6" rand = { version = "0.7.2", optional = true } impl-trait-for-tuples = "0.1.3" -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../inherents" } -parity-util-mem = { version = "0.6.0", default-features = false, features = ["primitive-types"] } +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../inherents" } +parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } hash256-std-hasher = { version = "0.15.2", default-features = false } [dev-dependencies] serde_json = "1.0.41" rand = "0.7.2" +sp-state-machine = { version = "0.8.0-alpha.5", path = "../../primitives/state-machine" } [features] bench = [] @@ -48,6 +52,3 @@ std = [ "parity-util-mem/std", "hash256-std-hasher/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/runtime/src/generic/checked_extrinsic.rs b/primitives/runtime/src/generic/checked_extrinsic.rs index 673501bb913c0c7d1acac5b1c74108b401375328..a329f334c0d7707edc8a4e07ada8eac443accce2 100644 --- a/primitives/runtime/src/generic/checked_extrinsic.rs +++ b/primitives/runtime/src/generic/checked_extrinsic.rs @@ -18,9 +18,8 @@ //! stage. use crate::traits::{ - self, Member, MaybeDisplay, SignedExtension, Dispatchable, + self, Member, MaybeDisplay, SignedExtension, Dispatchable, DispatchInfoOf, ValidateUnsigned, }; -use crate::traits::ValidateUnsigned; use crate::transaction_validity::{TransactionValidity, TransactionSource}; /// Definition of something that the external world might want to say; its @@ -36,28 +35,26 @@ pub struct CheckedExtrinsic { pub function: Call, } -impl traits::Applyable for +impl traits::Applyable for CheckedExtrinsic where AccountId: Member + MaybeDisplay, Call: Member + Dispatchable, - Extra: SignedExtension, + Extra: SignedExtension, Origin: From>, - Info: Clone, { type Call = Call; - type DispatchInfo = Info; fn validate>( &self, // TODO [#5006;ToDr] should source be passed to `SignedExtension`s? // Perhaps a change for 2.0 to avoid breaking too much APIs? source: TransactionSource, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { if let Some((ref id, ref extra)) = self.signed { - Extra::validate(extra, id, &self.function, info.clone(), len) + Extra::validate(extra, id, &self.function, info, len) } else { let valid = Extra::validate_unsigned(&self.function, info, len)?; let unsigned_validation = U::validate_unsigned(source, &self.function)?; @@ -67,21 +64,24 @@ where fn apply>( self, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> crate::ApplyExtrinsicResult { let (maybe_who, pre) = if let Some((id, extra)) = self.signed { - let pre = Extra::pre_dispatch(extra, &id, &self.function, info.clone(), len)?; + let pre = Extra::pre_dispatch(extra, &id, &self.function, info, len)?; (Some(id), pre) } else { - let pre = Extra::pre_dispatch_unsigned(&self.function, info.clone(), len)?; + let pre = Extra::pre_dispatch_unsigned(&self.function, info, len)?; U::pre_dispatch(&self.function)?; (None, pre) }; - let res = self.function.dispatch(Origin::from(maybe_who)) - .map(|_| ()) - .map_err(|e| e.error); - Extra::post_dispatch(pre, info.clone(), len, &res)?; + let res = self.function.dispatch(Origin::from(maybe_who)); + let post_info = match res { + Ok(info) => info, + Err(err) => err.post_info, + }; + let res = res.map(|_| ()).map_err(|e| e.error); + Extra::post_dispatch(pre, info, &post_info, len, &res)?; Ok(res) } } diff --git a/primitives/runtime/src/generic/unchecked_extrinsic.rs b/primitives/runtime/src/generic/unchecked_extrinsic.rs index a516bc1f7fa999f1087718f93f072ff8eff45099..4eb96ff960bcc72d65fd12d0567bd7206945d131 100644 --- a/primitives/runtime/src/generic/unchecked_extrinsic.rs +++ b/primitives/runtime/src/generic/unchecked_extrinsic.rs @@ -24,7 +24,8 @@ use crate::{ self, Member, MaybeDisplay, SignedExtension, Checkable, Extrinsic, ExtrinsicMetadata, IdentifyAccount, }, - generic::CheckedExtrinsic, transaction_validity::{TransactionValidityError, InvalidTransaction}, + generic::CheckedExtrinsic, + transaction_validity::{TransactionValidityError, InvalidTransaction}, }; const TRANSACTION_VERSION: u8 = 4; @@ -125,9 +126,7 @@ where Some((signed, signature, extra)) => { let signed = lookup.lookup(signed)?; let raw_payload = SignedPayload::new(self.function, extra)?; - if !raw_payload.using_encoded(|payload| { - signature.verify(payload, &signed) - }) { + if !raw_payload.using_encoded(|payload| signature.verify(payload, &signed)) { return Err(InvalidTransaction::BadProof.into()) } @@ -321,29 +320,10 @@ mod tests { use super::*; use sp_io::hashing::blake2_256; use crate::codec::{Encode, Decode}; - use crate::traits::{SignedExtension, IdentifyAccount, IdentityLookup}; - use serde::{Serialize, Deserialize}; + use crate::traits::{SignedExtension, IdentityLookup}; + use crate::testing::TestSignature as TestSig; type TestContext = IdentityLookup; - - #[derive(Eq, PartialEq, Clone, Copy, Debug, Serialize, Deserialize, Encode, Decode)] - pub struct TestSigner(pub u64); - impl From for TestSigner { fn from(x: u64) -> Self { Self(x) } } - impl From for u64 { fn from(x: TestSigner) -> Self { x.0 } } - impl IdentifyAccount for TestSigner { - type AccountId = u64; - fn into_account(self) -> u64 { self.into() } - } - - #[derive(Eq, PartialEq, Clone, Debug, Serialize, Deserialize, Encode, Decode)] - struct TestSig(u64, Vec); - impl traits::Verify for TestSig { - type Signer = TestSigner; - fn verify>(&self, mut msg: L, signer: &u64) -> bool { - signer == &self.0 && msg.get() == &self.1[..] - } - } - type TestAccountId = u64; type TestCall = Vec; @@ -357,7 +337,6 @@ mod tests { type AccountId = u64; type Call = (); type AdditionalSigned = (); - type DispatchInfo = (); type Pre = (); fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index c80971b576df2d9c6312bb3df0966ebc942cc680..a5b3e71edcde3d9f1d258730f58ba3749daf2f46 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -42,7 +42,8 @@ pub use sp_core::storage::{Storage, StorageChild}; use sp_std::prelude::*; use sp_std::convert::TryFrom; -use sp_core::{crypto, ed25519, sr25519, ecdsa, hash::{H256, H512}}; +use sp_core::{crypto::{self, Public}, ed25519, sr25519, ecdsa, hash::{H256, H512}}; + use codec::{Encode, Decode}; pub mod curve; @@ -69,8 +70,8 @@ pub use sp_core::RuntimeDebug; /// Re-export top-level arithmetic stuff. pub use sp_arithmetic::{ - Perquintill, Perbill, Permill, Percent, PerU16, Rational128, Fixed64, PerThing, - traits::SaturatedConversion, + Perquintill, Perbill, Permill, Percent, PerU16, Rational128, Fixed64, Fixed128, + PerThing, traits::SaturatedConversion, }; /// Re-export 128 bit helpers. pub use sp_arithmetic::helpers_128bit; @@ -135,15 +136,15 @@ impl BuildStorage for sp_core::storage::Storage { storage: &mut sp_core::storage::Storage, )-> Result<(), String> { storage.top.extend(self.top.iter().map(|(k, v)| (k.clone(), v.clone()))); - for (k, other_map) in self.children.iter() { + for (k, other_map) in self.children_default.iter() { let k = k.clone(); - if let Some(map) = storage.children.get_mut(&k) { + if let Some(map) = storage.children_default.get_mut(&k) { map.data.extend(other_map.data.iter().map(|(k, v)| (k.clone(), v.clone()))); - if !map.child_info.try_update(other_map.child_info.as_ref()) { + if !map.child_info.try_update(&other_map.child_info) { return Err("Incompatible child info update".to_string()); } } else { - storage.children.insert(k, other_map.clone()); + storage.children_default.insert(k, other_map.clone()); } } Ok(()) @@ -181,18 +182,39 @@ impl From for MultiSignature { } } +impl TryFrom for ed25519::Signature { + type Error = (); + fn try_from(m: MultiSignature) -> Result { + if let MultiSignature::Ed25519(x) = m { Ok(x) } else { Err(()) } + } +} + impl From for MultiSignature { fn from(x: sr25519::Signature) -> Self { MultiSignature::Sr25519(x) } } +impl TryFrom for sr25519::Signature { + type Error = (); + fn try_from(m: MultiSignature) -> Result { + if let MultiSignature::Sr25519(x) = m { Ok(x) } else { Err(()) } + } +} + impl From for MultiSignature { fn from(x: ecdsa::Signature) -> Self { MultiSignature::Ecdsa(x) } } +impl TryFrom for ecdsa::Signature { + type Error = (); + fn try_from(m: MultiSignature) -> Result { + if let MultiSignature::Ecdsa(x) = m { Ok(x) } else { Err(()) } + } +} + impl Default for MultiSignature { fn default() -> Self { MultiSignature::Ed25519(Default::default()) @@ -299,7 +321,6 @@ impl std::fmt::Display for MultiSigner { impl Verify for MultiSignature { type Signer = MultiSigner; fn verify>(&self, mut msg: L, signer: &AccountId32) -> bool { - use sp_core::crypto::Public; match (self, signer) { (MultiSignature::Ed25519(ref sig), who) => sig.verify(msg, &ed25519::Public::from_slice(who.as_ref())), (MultiSignature::Sr25519(ref sig), who) => sig.verify(msg, &sr25519::Public::from_slice(who.as_ref())), @@ -324,7 +345,6 @@ pub struct AnySignature(H512); impl Verify for AnySignature { type Signer = sr25519::Public; fn verify>(&self, mut msg: L, signer: &sr25519::Public) -> bool { - use sp_core::crypto::Public; let msg = msg.get(); sr25519::Signature::try_from(self.0.as_fixed_bytes().as_ref()) .map(|s| s.verify(msg, signer)) @@ -735,6 +755,39 @@ pub fn print(print: impl traits::Printable) { print.print(); } + +/// Batching session. +/// +/// To be used in runtime only. Outside of runtime, just construct +/// `BatchVerifier` directly. +#[must_use = "`verify()` needs to be called to finish batch signature verification!"] +pub struct SignatureBatching(bool); + +impl SignatureBatching { + /// Start new batching session. + pub fn start() -> Self { + sp_io::crypto::start_batch_verify(); + SignatureBatching(false) + } + + /// Verify all signatures submitted during the batching session. + #[must_use] + pub fn verify(mut self) -> bool { + self.0 = true; + sp_io::crypto::finish_batch_verify() + } +} + +impl Drop for SignatureBatching { + fn drop(&mut self) { + // Sanity check. If user forgets to actually call `verify()`. + if !self.0 { + panic!("Signature verification has not been called before `SignatureBatching::drop`") + } + } +} + + #[cfg(test)] mod tests { use super::*; @@ -782,4 +835,19 @@ mod tests { let multi_signer = MultiSigner::from(pair.public()); assert!(multi_sig.verify(msg, &multi_signer.into_account())); } + + + #[test] + #[should_panic(expected = "Signature verification has not been called")] + fn batching_still_finishes_when_not_called_directly() { + let mut ext = sp_state_machine::BasicExternalities::with_tasks_executor(); + ext.execute_with(|| { + let _batching = SignatureBatching::start(); + sp_io::crypto::sr25519_verify( + &Default::default(), + &Vec::new(), + &Default::default(), + ); + }); + } } diff --git a/primitives/runtime/src/testing.rs b/primitives/runtime/src/testing.rs index f5fc35fe8ecef1203d3ed865a374b221b0cde42f..40b4e23e3fa0a6b94bb340a1d640f6461c543ca5 100644 --- a/primitives/runtime/src/testing.rs +++ b/primitives/runtime/src/testing.rs @@ -17,11 +17,11 @@ //! Testing utilities. use serde::{Serialize, Serializer, Deserialize, de::Error as DeError, Deserializer}; -use std::{fmt::Debug, ops::Deref, fmt, cell::RefCell}; +use std::{fmt::{self, Debug}, ops::Deref, cell::RefCell}; use crate::codec::{Codec, Encode, Decode}; use crate::traits::{ self, Checkable, Applyable, BlakeTwo256, OpaqueKeys, - SignedExtension, Dispatchable, + SignedExtension, Dispatchable, DispatchInfoOf, }; use crate::traits::ValidateUnsigned; use crate::{generic, KeyTypeId, ApplyExtrinsicResult}; @@ -29,7 +29,12 @@ pub use sp_core::{H256, sr25519}; use sp_core::{crypto::{CryptoType, Dummy, key_types, Public}, U256}; use crate::transaction_validity::{TransactionValidity, TransactionValidityError, TransactionSource}; -/// Authority Id +/// A dummy type which can be used instead of regular cryptographic primitives. +/// +/// 1. Wraps a `u64` `AccountId` and is able to `IdentifyAccount`. +/// 2. Can be converted to any `Public` key. +/// 3. Implements `RuntimeAppPublic` so it can be used instead of regular application-specific +/// crypto. #[derive(Default, PartialEq, Eq, Clone, Encode, Decode, Debug, Hash, Serialize, Deserialize, PartialOrd, Ord)] pub struct UintAuthorityId(pub u64); @@ -82,7 +87,7 @@ impl UintAuthorityId { impl sp_application_crypto::RuntimeAppPublic for UintAuthorityId { const ID: KeyTypeId = key_types::DUMMY; - type Signature = u64; + type Signature = TestSignature; fn all() -> Vec { ALL_KEYS.with(|l| l.borrow().clone()) @@ -94,25 +99,11 @@ impl sp_application_crypto::RuntimeAppPublic for UintAuthorityId { } fn sign>(&self, msg: &M) -> Option { - let mut signature = [0u8; 8]; - msg.as_ref().iter() - .chain(std::iter::repeat(&42u8)) - .take(8) - .enumerate() - .for_each(|(i, v)| { signature[i] = *v; }); - - Some(u64::from_le_bytes(signature)) + Some(TestSignature(self.0, msg.as_ref().to_vec())) } fn verify>(&self, msg: &M, signature: &Self::Signature) -> bool { - let mut msg_signature = [0u8; 8]; - msg.as_ref().iter() - .chain(std::iter::repeat(&42)) - .take(8) - .enumerate() - .for_each(|(i, v)| { msg_signature[i] = *v; }); - - u64::from_le_bytes(msg_signature) == *signature + traits::Verify::verify(signature, msg.as_ref(), &self.0) } fn to_raw_vec(&self) -> Vec { @@ -140,6 +131,26 @@ impl crate::BoundToRuntimeAppPublic for UintAuthorityId { type Public = Self; } +impl traits::IdentifyAccount for UintAuthorityId { + type AccountId = u64; + + fn into_account(self) -> Self::AccountId { + self.0 + } +} + +/// A dummy signature type, to match `UintAuthorityId`. +#[derive(Eq, PartialEq, Clone, Debug, Hash, Serialize, Deserialize, Encode, Decode)] +pub struct TestSignature(pub u64, pub Vec); + +impl traits::Verify for TestSignature { + type Signer = UintAuthorityId; + + fn verify>(&self, mut msg: L, signer: &u64) -> bool { + signer == &self.0 && msg.get() == &self.1[..] + } +} + /// Digest item pub type DigestItem = generic::DigestItem; @@ -332,6 +343,7 @@ impl Checkable for TestXt Result { Ok(self) } } + impl traits::Extrinsic for TestXt { type Call = Call; type SignaturePayload = (u64, Extra); @@ -345,20 +357,18 @@ impl traits::Extrinsic for TestXt } } -impl Applyable for TestXt where +impl Applyable for TestXt where Call: 'static + Sized + Send + Sync + Clone + Eq + Codec + Debug + Dispatchable, - Extra: SignedExtension, + Extra: SignedExtension, Origin: From>, - Info: Clone, { type Call = Call; - type DispatchInfo = Info; /// Checks to see if this is a valid *transaction*. It returns information on it if so. fn validate>( &self, _source: TransactionSource, - _info: Self::DispatchInfo, + _info: &DispatchInfoOf, _len: usize, ) -> TransactionValidity { Ok(Default::default()) @@ -368,7 +378,7 @@ impl Applyable for TestXt where /// index and sender. fn apply>( self, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> ApplyExtrinsicResult { let maybe_who = if let Some((who, extra)) = self.signature { diff --git a/primitives/runtime/src/traits.rs b/primitives/runtime/src/traits.rs index da5d5c4a81c0db367624fd020c55f86783ff8fcb..d843bdc478c49f6cf5713386be278110549f222e 100644 --- a/primitives/runtime/src/traits.rs +++ b/primitives/runtime/src/traits.rs @@ -81,12 +81,15 @@ impl IdentifyAccount for sp_core::ecdsa::Public { pub trait Verify { /// Type of the signer. type Signer: IdentifyAccount; - /// Verify a signature. Return `true` if signature is valid for the value. + /// Verify a signature. + /// + /// Return `true` if signature is valid for the value. fn verify>(&self, msg: L, signer: &::AccountId) -> bool; } impl Verify for sp_core::ed25519::Signature { type Signer = sp_core::ed25519::Public; + fn verify>(&self, mut msg: L, signer: &sp_core::ed25519::Public) -> bool { sp_io::crypto::ed25519_verify(self, msg.get(), signer) } @@ -94,6 +97,7 @@ impl Verify for sp_core::ed25519::Signature { impl Verify for sp_core::sr25519::Signature { type Signer = sp_core::sr25519::Public; + fn verify>(&self, mut msg: L, signer: &sp_core::sr25519::Public) -> bool { sp_io::crypto::sr25519_verify(self, msg.get(), signer) } @@ -625,6 +629,10 @@ pub trait Dispatchable { type Origin; /// ... type Trait; + /// An opaque set of information attached to the transaction. This could be constructed anywhere + /// down the line in a runtime. The current Substrate runtime uses a struct with the same name + /// to represent the dispatch class and weight. + type Info; /// Additional information that is returned by `dispatch`. Can be used to supply the caller /// with information about a `Dispatchable` that is ownly known post dispatch. type PostInfo: Eq + PartialEq + Clone + Copy + Encode + Decode + Printable; @@ -632,6 +640,21 @@ pub trait Dispatchable { fn dispatch(self, origin: Self::Origin) -> crate::DispatchResultWithInfo; } +/// Shortcut to reference the `Info` type of a `Dispatchable`. +pub type DispatchInfoOf = ::Info; +/// Shortcut to reference the `PostInfo` type of a `Dispatchable`. +pub type PostDispatchInfoOf = ::PostInfo; + +impl Dispatchable for () { + type Origin = (); + type Trait = (); + type Info = (); + type PostInfo = (); + fn dispatch(self, _origin: Self::Origin) -> crate::DispatchResultWithInfo { + panic!("This implemention should not be used for actual dispatch."); + } +} + /// Means by which a transaction may be extended. This type embodies both the data and the logic /// that should be additionally associated with the transaction. It should be plain old data. pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq { @@ -645,7 +668,7 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq type AccountId; /// The type which encodes the call to be dispatched. - type Call; + type Call: Dispatchable; /// Any additional data that will go into the signed payload. This may be created dynamically /// from the transaction using the `additional_signed` function. @@ -654,11 +677,6 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq /// The type that encodes information that can be passed from pre_dispatch to post-dispatch. type Pre: Default; - /// An opaque set of information attached to the transaction. This could be constructed anywhere - /// down the line in a runtime. The current Substrate runtime uses a struct with the same name - /// to represent the dispatch class and weight. - type DispatchInfo: Clone; - /// Construct any additional data that should be in the signed payload of the transaction. Can /// also perform any pre-signature-verification checks and return an error if needed. fn additional_signed(&self) -> Result; @@ -676,7 +694,7 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq &self, _who: &Self::AccountId, _call: &Self::Call, - _info: Self::DispatchInfo, + _info: &DispatchInfoOf, _len: usize, ) -> TransactionValidity { Ok(ValidTransaction::default()) @@ -694,7 +712,7 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq self, who: &Self::AccountId, call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> Result { self.validate(who, call, info.clone(), len) @@ -712,7 +730,7 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq /// Make sure to perform the same checks in `pre_dispatch_unsigned` function. fn validate_unsigned( _call: &Self::Call, - _info: Self::DispatchInfo, + _info: &DispatchInfoOf, _len: usize, ) -> TransactionValidity { Ok(ValidTransaction::default()) @@ -728,7 +746,7 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq /// perform the same validation as in `validate_unsigned`. fn pre_dispatch_unsigned( call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> Result { Self::validate_unsigned(call, info.clone(), len) @@ -751,7 +769,8 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq /// will come from either an offchain-worker or via `InherentData`. fn post_dispatch( _pre: Self::Pre, - _info: Self::DispatchInfo, + _info: &DispatchInfoOf, + _post_info: &PostDispatchInfoOf, _len: usize, _result: &DispatchResult, ) -> Result<(), TransactionValidityError> { @@ -771,11 +790,10 @@ pub trait SignedExtension: Codec + Debug + Sync + Send + Clone + Eq + PartialEq } #[impl_for_tuples(1, 12)] -impl SignedExtension for Tuple { - for_tuples!( where #( Tuple: SignedExtension )* ); +impl SignedExtension for Tuple { + for_tuples!( where #( Tuple: SignedExtension )* ); type AccountId = AccountId; type Call = Call; - type DispatchInfo = Info; const IDENTIFIER: &'static str = "You should call `identifier()`!"; for_tuples!( type AdditionalSigned = ( #( Tuple::AdditionalSigned ),* ); ); for_tuples!( type Pre = ( #( Tuple::Pre ),* ); ); @@ -788,45 +806,46 @@ impl SignedExtension for Tuple { &self, who: &Self::AccountId, call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { let valid = ValidTransaction::default(); - for_tuples!( #( let valid = valid.combine_with(Tuple.validate(who, call, info.clone(), len)?); )* ); + for_tuples!( #( let valid = valid.combine_with(Tuple.validate(who, call, info, len)?); )* ); Ok(valid) } - fn pre_dispatch(self, who: &Self::AccountId, call: &Self::Call, info: Self::DispatchInfo, len: usize) + fn pre_dispatch(self, who: &Self::AccountId, call: &Self::Call, info: &DispatchInfoOf, len: usize) -> Result { - Ok(for_tuples!( ( #( Tuple.pre_dispatch(who, call, info.clone(), len)? ),* ) )) + Ok(for_tuples!( ( #( Tuple.pre_dispatch(who, call, info, len)? ),* ) )) } fn validate_unsigned( call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { let valid = ValidTransaction::default(); - for_tuples!( #( let valid = valid.combine_with(Tuple::validate_unsigned(call, info.clone(), len)?); )* ); + for_tuples!( #( let valid = valid.combine_with(Tuple::validate_unsigned(call, info, len)?); )* ); Ok(valid) } fn pre_dispatch_unsigned( call: &Self::Call, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> Result { - Ok(for_tuples!( ( #( Tuple::pre_dispatch_unsigned(call, info.clone(), len)? ),* ) )) + Ok(for_tuples!( ( #( Tuple::pre_dispatch_unsigned(call, info, len)? ),* ) )) } fn post_dispatch( pre: Self::Pre, - info: Self::DispatchInfo, + info: &DispatchInfoOf, + post_info: &PostDispatchInfoOf, len: usize, result: &DispatchResult, ) -> Result<(), TransactionValidityError> { - for_tuples!( #( Tuple::post_dispatch(pre.Tuple, info.clone(), len, result)?; )* ); + for_tuples!( #( Tuple::post_dispatch(pre.Tuple, info, post_info, len, result)?; )* ); Ok(()) } @@ -844,7 +863,6 @@ impl SignedExtension for () { type AdditionalSigned = (); type Call = (); type Pre = (); - type DispatchInfo = (); const IDENTIFIER: &'static str = "UnitSignedExtension"; fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } } @@ -857,16 +875,13 @@ impl SignedExtension for () { /// each piece of attributable information to be disambiguated. pub trait Applyable: Sized + Send + Sync { /// Type by which we can dispatch. Restricts the `UnsignedValidator` type. - type Call; - - /// An opaque set of information attached to the transaction. - type DispatchInfo: Clone; + type Call: Dispatchable; /// Checks to see if this is a valid *transaction*. It returns information on it if so. fn validate>( &self, source: TransactionSource, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> TransactionValidity; @@ -874,7 +889,7 @@ pub trait Applyable: Sized + Send + Sync { /// index and sender. fn apply>( self, - info: Self::DispatchInfo, + info: &DispatchInfoOf, len: usize, ) -> crate::ApplyExtrinsicResult; } @@ -1273,6 +1288,12 @@ impl Printable for bool { } } +impl Printable for () { + fn print(&self) { + "()".print() + } +} + #[impl_for_tuples(1, 12)] impl Printable for Tuple { fn print(&self) { diff --git a/primitives/runtime/src/transaction_validity.rs b/primitives/runtime/src/transaction_validity.rs index 94cf44384def1078693919071e75d3f32a482d1d..95903b4876262d29b97d7362782eed07290aa4cf 100644 --- a/primitives/runtime/src/transaction_validity.rs +++ b/primitives/runtime/src/transaction_validity.rs @@ -264,6 +264,17 @@ impl Default for ValidTransaction { } impl ValidTransaction { + /// Initiate `ValidTransaction` builder object with a particular prefix for tags. + /// + /// To avoid conflicts between different parts in runtime it's recommended to build `requires` + /// and `provides` tags with a unique prefix. + pub fn with_tag_prefix(prefix: &'static str) -> ValidTransactionBuilder { + ValidTransactionBuilder { + prefix: Some(prefix), + validity: Default::default(), + } + } + /// Combine two instances into one, as a best effort. This will take the superset of each of the /// `provides` and `requires` tags, it will sum the priorities, take the minimum longevity and /// the logic *And* of the propagate flags. @@ -278,6 +289,104 @@ impl ValidTransaction { } } +/// `ValidTransaction` builder. +/// +/// +/// Allows to easily construct `ValidTransaction` and most importantly takes care of +/// prefixing `requires` and `provides` tags to avoid conflicts. +#[derive(Default, Clone, RuntimeDebug)] +pub struct ValidTransactionBuilder { + prefix: Option<&'static str>, + validity: ValidTransaction, +} + +impl ValidTransactionBuilder { + /// Set the priority of a transaction. + /// + /// Note that the final priority for `FRAME` is combined from all `SignedExtension`s. + /// Most likely for unsigned transactions you want the priority to be higher + /// than for regular transactions. We recommend exposing a base priority for unsigned + /// transactions as a runtime module parameter, so that the runtime can tune inter-module + /// priorities. + pub fn priority(mut self, priority: TransactionPriority) -> Self { + self.validity.priority = priority; + self + } + + /// Set the longevity of a transaction. + /// + /// By default the transaction will be considered valid forever and will not be revalidated + /// by the transaction pool. It's recommended though to set the longevity to a finite value + /// though. If unsure, it's also reasonable to expose this parameter via module configuration + /// and let the runtime decide. + pub fn longevity(mut self, longevity: TransactionLongevity) -> Self { + self.validity.longevity = longevity; + self + } + + /// Set the propagate flag. + /// + /// Set to `false` if the transaction is not meant to be gossiped to peers. Combined with + /// `TransactionSource::Local` validation it can be used to have special kind of + /// transactions that are only produced and included by the validator nodes. + pub fn propagate(mut self, propagate: bool) -> Self { + self.validity.propagate = propagate; + self + } + + /// Add a `TransactionTag` to the set of required tags. + /// + /// The tag will be encoded and prefixed with module prefix (if any). + /// If you'd rather add a raw `require` tag, consider using `#combine_with` method. + pub fn and_requires(mut self, tag: impl Encode) -> Self { + self.validity.requires.push(match self.prefix.as_ref() { + Some(prefix) => (prefix, tag).encode(), + None => tag.encode(), + }); + self + } + + /// Add a `TransactionTag` to the set of provided tags. + /// + /// The tag will be encoded and prefixed with module prefix (if any). + /// If you'd rather add a raw `require` tag, consider using `#combine_with` method. + pub fn and_provides(mut self, tag: impl Encode) -> Self { + self.validity.provides.push(match self.prefix.as_ref() { + Some(prefix) => (prefix, tag).encode(), + None => tag.encode(), + }); + self + } + + /// Augment the builder with existing `ValidTransaction`. + /// + /// This method does add the prefix to `require` or `provides` tags. + pub fn combine_with(mut self, validity: ValidTransaction) -> Self { + self.validity = core::mem::take(&mut self.validity).combine_with(validity); + self + } + + /// Finalize the builder and produce `TransactionValidity`. + /// + /// Note the result will always be `Ok`. Use `Into` to produce `ValidTransaction`. + pub fn build(self) -> TransactionValidity { + self.into() + } +} + +impl From for TransactionValidity { + fn from(builder: ValidTransactionBuilder) -> Self { + Ok(builder.into()) + } +} + +impl From for ValidTransaction { + fn from(builder: ValidTransactionBuilder) -> Self { + builder.validity + } +} + + #[cfg(test)] mod tests { use super::*; @@ -301,4 +410,26 @@ mod tests { // decode back assert_eq!(TransactionValidity::decode(&mut &*encoded), Ok(v)); } + + #[test] + fn builder_should_prefix_the_tags() { + const PREFIX: &str = "test"; + let a: ValidTransaction = ValidTransaction::with_tag_prefix(PREFIX) + .and_requires(1) + .and_requires(2) + .and_provides(3) + .and_provides(4) + .propagate(false) + .longevity(5) + .priority(3) + .priority(6) + .into(); + assert_eq!(a, ValidTransaction { + propagate: false, + longevity: 5, + priority: 6, + requires: vec![(PREFIX, 1).encode(), (PREFIX, 2).encode()], + provides: vec![(PREFIX, 3).encode(), (PREFIX, 4).encode()], + }); + } } diff --git a/primitives/sandbox/Cargo.toml b/primitives/sandbox/Cargo.toml index 5c1595027f86fd129ebec97ffebe63b2b1263ca7..a64854424ecc2a80c6654111d91e2210b067eea8 100755 --- a/primitives/sandbox/Cargo.toml +++ b/primitives/sandbox/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-sandbox" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,12 +8,15 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "This crate provides means to instantiate and execute wasm modules." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] wasmi = { version = "0.6.2", optional = true } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../io" } -sp-wasm-interface = { version = "2.0.0-alpha.5", default-features = false, path = "../wasm-interface" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../core" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } +sp-io = { version = "2.0.0-dev", default-features = false, path = "../io" } +sp-wasm-interface = { version = "2.0.0-dev", default-features = false, path = "../wasm-interface" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } [dev-dependencies] @@ -31,6 +34,3 @@ std = [ "sp-wasm-interface/std", ] strict = [] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/serializer/Cargo.toml b/primitives/serializer/Cargo.toml index 75263321b805c85793e09f90ec0616ab1a6c4203..187c0e532fedf4b1183a9483d1c7516769b307d3 100644 --- a/primitives/serializer/Cargo.toml +++ b/primitives/serializer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-serializer" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,9 +9,9 @@ repository = "https://github.com/paritytech/substrate/" description = "Substrate customizable serde serializer." documentation = "https://docs.rs/sp-serializer" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] serde = "1.0.101" serde_json = "1.0.41" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/session/Cargo.toml b/primitives/session/Cargo.toml index ffe1bc327f7ffd9edb5f3c003abeb0f4d66077f3..6d210b341f43b31e49cd887d288079ff31573d5c 100644 --- a/primitives/session/Cargo.toml +++ b/primitives/session/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-session" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,15 +8,15 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Primitives for sessions" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../api" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core" } -sp-runtime = { version = "2.0.0-alpha.5", optional = true, path = "../runtime" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../api" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../core" } +sp-runtime = { version = "2.0.0-dev", optional = true, path = "../runtime" } [features] default = [ "std" ] std = [ "sp-api/std", "sp-std/std", "sp-runtime", "sp-core/std" ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/sr-api/proc-macro/src/lib.rs b/primitives/sr-api/proc-macro/src/lib.rs index adb3b9636d08673370d427aa2bb8efa92f6fe32b..0c506a1455dbe243b97b59d969252230c6a369b1 100644 --- a/primitives/sr-api/proc-macro/src/lib.rs +++ b/primitives/sr-api/proc-macro/src/lib.rs @@ -103,6 +103,7 @@ mod utils; /// impl_version: 0, /// // Here we are exposing the runtime api versions. /// apis: RUNTIME_API_VERSIONS, +/// transaction_version: 1, /// }; /// /// # fn main() {} diff --git a/primitives/staking/Cargo.toml b/primitives/staking/Cargo.toml index 60bf3f759e8dfcf0f3d2efe6bec7a3d231d291b8..28907a75d39f140a1fd4cf4e2accbe907075a3a5 100644 --- a/primitives/staking/Cargo.toml +++ b/primitives/staking/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-staking" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,10 +8,13 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "A crate which contains primitives that are useful for implementation that uses staking approaches in general. Definitions related to sessions, slashing, etc go here." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../runtime" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } [features] default = ["std"] @@ -20,6 +23,3 @@ std = [ "sp-runtime/std", "sp-std/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/state-machine/Cargo.toml b/primitives/state-machine/Cargo.toml index 548f3f43086e928986dd5c546f105e86e2ebf0c1..1d608ea6fcb7a78745e76bbd00a60bd5efa4cbe5 100644 --- a/primitives/state-machine/Cargo.toml +++ b/primitives/state-machine/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-state-machine" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "Substrate State Machine" edition = "2018" @@ -9,26 +9,26 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sp-state-machine" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] log = "0.4.8" parking_lot = "0.10.0" hash-db = "0.15.2" -trie-db = "0.20.0" +trie-db = "0.20.1" trie-root = "0.16.0" -sp-trie = { version = "2.0.0-alpha.5", path = "../trie" } -sp-core = { version = "2.0.0-alpha.5", path = "../core" } -sp-panic-handler = { version = "2.0.0-alpha.5", path = "../panic-handler" } +sp-trie = { version = "2.0.0-dev", path = "../trie" } +sp-core = { version = "2.0.0-dev", path = "../core" } +sp-panic-handler = { version = "2.0.0-dev", path = "../panic-handler" } codec = { package = "parity-scale-codec", version = "1.3.0" } num-traits = "0.2.8" rand = "0.7.2" -sp-externalities = { version = "0.8.0-alpha.5", path = "../externalities" } +sp-externalities = { version = "0.8.0-dev", path = "../externalities" } [dev-dependencies] hex-literal = "0.2.1" -sp-runtime = { version = "2.0.0-alpha.5", path = "../runtime" } +sp-runtime = { version = "2.0.0-dev", path = "../runtime" } [features] default = [] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 94144fdb90fda5e04746e584993a678698b8e5f6..d45e822be83a9631f59ab2a84c832f8bbcd2fc6a 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -16,12 +16,9 @@ //! State machine backends. These manage the code and storage of contracts. -use log::warn; use hash_db::Hasher; use codec::{Decode, Encode}; - -use sp_core::{traits::RuntimeCode, storage::{ChildInfo, OwnedChildInfo, well_known_keys}}; -use sp_trie::{TrieMut, MemoryDB, trie_types::TrieDBMut}; +use sp_core::{traits::RuntimeCode, storage::{ChildInfo, well_known_keys}}; use crate::{ trie_backend::TrieBackend, @@ -54,19 +51,17 @@ pub trait Backend: std::fmt::Debug { /// Get keyed child storage or None if there is nothing associated. fn child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result, Self::Error>; /// Get child keyed storage value hash or None if there is nothing associated. fn child_storage_hash( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result, Self::Error> { - self.child_storage(storage_key, child_info, key).map(|v| v.map(|v| H::hash(&v))) + self.child_storage(child_info, key).map(|v| v.map(|v| H::hash(&v))) } /// true if a key exists in storage. @@ -77,11 +72,10 @@ pub trait Backend: std::fmt::Debug { /// true if a key exists in child storage. fn exists_child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result { - Ok(self.child_storage(storage_key, child_info, key)?.is_some()) + Ok(self.child_storage(child_info, key)?.is_some()) } /// Return the next key in storage in lexicographic order or `None` if there is no value. @@ -90,16 +84,14 @@ pub trait Backend: std::fmt::Debug { /// Return the next key in child storage in lexicographic order or `None` if there is no value. fn next_child_storage_key( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8] ) -> Result, Self::Error>; /// Retrieve all entries keys of child storage and call `f` for each of those keys. fn for_keys_in_child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, f: F, ); @@ -118,8 +110,7 @@ pub trait Backend: std::fmt::Debug { /// call `f` for each of those keys. fn for_child_keys_with_prefix( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], f: F, ); @@ -137,8 +128,7 @@ pub trait Backend: std::fmt::Debug { /// is true if child storage root equals default storage root. fn child_storage_root( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, delta: I, ) -> (H::Out, bool, Self::Transaction) where @@ -158,12 +148,11 @@ pub trait Backend: std::fmt::Debug { /// Get all keys of child storage with given prefix fn child_keys( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], ) -> Vec { let mut all = Vec::new(); - self.for_child_keys_with_prefix(storage_key, child_info, prefix, |k| all.push(k.to_vec())); + self.for_child_keys_with_prefix(child_info, prefix, |k| all.push(k.to_vec())); all } @@ -183,20 +172,21 @@ pub trait Backend: std::fmt::Debug { where I1: IntoIterator)>, I2i: IntoIterator)>, - I2: IntoIterator, + I2: IntoIterator, H::Out: Ord + Encode, { let mut txs: Self::Transaction = Default::default(); let mut child_roots: Vec<_> = Default::default(); // child first - for (storage_key, child_delta, child_info) in child_deltas { + for (child_info, child_delta) in child_deltas { let (child_root, empty, child_txs) = - self.child_storage_root(&storage_key[..], child_info.as_ref(), child_delta); + self.child_storage_root(&child_info, child_delta); + let prefixed_storage_key = child_info.prefixed_storage_key(); txs.consolidate(child_txs); if empty { - child_roots.push((storage_key, None)); + child_roots.push((prefixed_storage_key.into_inner(), None)); } else { - child_roots.push((storage_key, Some(child_root.encode()))); + child_roots.push((prefixed_storage_key.into_inner(), Some(child_root.encode()))); } } let (root, parent_txs) = self.storage_root( @@ -239,20 +229,18 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { fn child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result, Self::Error> { - (*self).child_storage(storage_key, child_info, key) + (*self).child_storage(child_info, key) } fn for_keys_in_child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, f: F, ) { - (*self).for_keys_in_child_storage(storage_key, child_info, f) + (*self).for_keys_in_child_storage(child_info, f) } fn next_storage_key(&self, key: &[u8]) -> Result, Self::Error> { @@ -261,11 +249,10 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { fn next_child_storage_key( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result, Self::Error> { - (*self).next_child_storage_key(storage_key, child_info, key) + (*self).next_child_storage_key(child_info, key) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { @@ -274,12 +261,11 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { fn for_child_keys_with_prefix( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], f: F, ) { - (*self).for_child_keys_with_prefix(storage_key, child_info, prefix, f) + (*self).for_child_keys_with_prefix(child_info, prefix, f) } fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) @@ -292,15 +278,14 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { fn child_storage_root( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, delta: I, ) -> (H::Out, bool, Self::Transaction) where I: IntoIterator)>, H::Out: Ord, { - (*self).child_storage_root(storage_key, child_info, delta) + (*self).child_storage_root(child_info, delta) } fn pairs(&self) -> Vec<(StorageKey, StorageValue)> { @@ -331,7 +316,7 @@ impl Consolidate for () { } impl Consolidate for Vec<( - Option<(StorageKey, OwnedChildInfo)>, + Option, StorageCollection, )> { fn consolidate(&mut self, mut other: Self) { @@ -346,17 +331,20 @@ impl> Consolidate for sp_trie::GenericMem } /// Insert input pairs into memory db. -pub(crate) fn insert_into_memory_db(mdb: &mut MemoryDB, input: I) -> Option +#[cfg(test)] +pub(crate) fn insert_into_memory_db(mdb: &mut sp_trie::MemoryDB, input: I) -> Option where H: Hasher, I: IntoIterator, { + use sp_trie::{TrieMut, trie_types::TrieDBMut}; + let mut root = ::Out::default(); { let mut trie = TrieDBMut::::new(mdb, &mut root); for (key, value) in input { if let Err(e) = trie.insert(&key, &value) { - warn!(target: "trie", "Failed to write to trie: {}", e); + log::warn!(target: "trie", "Failed to write to trie: {}", e); return None; } } diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 819244050ba4c32370ee9b5398481f02db7e2372..a5d573f5145bd6d8f6495eec42b1701b1173927f 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -19,30 +19,48 @@ use std::{ collections::BTreeMap, any::{TypeId, Any}, iter::FromIterator, ops::Bound }; -use crate::{Backend, InMemoryBackend, StorageKey, StorageValue}; +use crate::{Backend, StorageKey, StorageValue}; use hash_db::Hasher; -use sp_trie::{TrieConfiguration, default_child_trie_root}; +use sp_trie::{TrieConfiguration, empty_child_trie_root}; use sp_trie::trie_types::Layout; use sp_core::{ storage::{ - well_known_keys::is_child_storage_key, ChildStorageKey, Storage, + well_known_keys::is_child_storage_key, Storage, ChildInfo, StorageChild, }, traits::Externalities, Blake2Hasher, }; use log::warn; use codec::Encode; +use sp_externalities::Extensions; /// Simple Map-based Externalities impl. #[derive(Debug)] pub struct BasicExternalities { inner: Storage, + extensions: Extensions, } impl BasicExternalities { /// Create a new instance of `BasicExternalities` pub fn new(inner: Storage) -> Self { - BasicExternalities { inner } + BasicExternalities { inner, extensions: Default::default() } + } + + /// New basic externalities with empty storage. + pub fn new_empty() -> Self { + Self::new(Storage::default()) + } + + /// New basic extternalities with tasks executor. + pub fn with_tasks_executor() -> Self { + let mut extensions = Extensions::default(); + extensions.register(sp_core::traits::TaskExecutorExt(sp_core::tasks::executor())); + + Self { + inner: Storage::default(), + extensions, + } } /// Insert key/value @@ -62,10 +80,13 @@ impl BasicExternalities { storage: &mut sp_core::storage::Storage, f: impl FnOnce() -> R, ) -> R { - let mut ext = Self { inner: Storage { - top: std::mem::replace(&mut storage.top, Default::default()), - children: std::mem::replace(&mut storage.children, Default::default()), - }}; + let mut ext = Self { + inner: Storage { + top: std::mem::replace(&mut storage.top, Default::default()), + children_default: std::mem::replace(&mut storage.children_default, Default::default()), + }, + extensions: Default::default(), + }; let r = ext.execute_with(f); @@ -80,12 +101,17 @@ impl BasicExternalities { pub fn execute_with(&mut self, f: impl FnOnce() -> R) -> R { sp_externalities::set_and_run_with_externalities(self, f) } + + /// List of active extensions. + pub fn extensions(&mut self) -> &mut Extensions { + &mut self.extensions + } } impl PartialEq for BasicExternalities { fn eq(&self, other: &BasicExternalities) -> bool { self.inner.top.eq(&other.inner.top) - && self.inner.children.eq(&other.inner.children) + && self.inner.children_default.eq(&other.inner.children_default) } } @@ -103,14 +129,19 @@ impl Default for BasicExternalities { impl From> for BasicExternalities { fn from(hashmap: BTreeMap) -> Self { - BasicExternalities { inner: Storage { - top: hashmap, - children: Default::default(), - }} + BasicExternalities { + inner: Storage { + top: hashmap, + children_default: Default::default(), + }, + extensions: Default::default(), + } } } impl Externalities for BasicExternalities { + fn set_offchain_storage(&mut self, _key: &[u8], _value: Option<&[u8]>) {} + fn storage(&self, key: &[u8]) -> Option { self.inner.top.get(key).cloned() } @@ -121,20 +152,19 @@ impl Externalities for BasicExternalities { fn child_storage( &self, - storage_key: ChildStorageKey, - _child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Option { - self.inner.children.get(storage_key.as_ref()).and_then(|child| child.data.get(key)).cloned() + self.inner.children_default.get(child_info.storage_key()) + .and_then(|child| child.data.get(key)).cloned() } fn child_storage_hash( &self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Option> { - self.child_storage(storage_key, child_info, key).map(|v| Blake2Hasher::hash(&v).encode()) + self.child_storage(child_info, key).map(|v| Blake2Hasher::hash(&v).encode()) } fn next_storage_key(&self, key: &[u8]) -> Option { @@ -144,12 +174,11 @@ impl Externalities for BasicExternalities { fn next_child_storage_key( &self, - storage_key: ChildStorageKey, - _child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Option { let range = (Bound::Excluded(key), Bound::Unbounded); - self.inner.children.get(storage_key.as_ref()) + self.inner.children_default.get(child_info.storage_key()) .and_then(|child| child.data.range::<[u8], _>(range).next().map(|(k, _)| k).cloned()) } @@ -167,12 +196,11 @@ impl Externalities for BasicExternalities { fn place_child_storage( &mut self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: StorageKey, value: Option, ) { - let child_map = self.inner.children.entry(storage_key.into_owned()) + let child_map = self.inner.children_default.entry(child_info.storage_key().to_vec()) .or_insert_with(|| StorageChild { data: Default::default(), child_info: child_info.to_owned(), @@ -186,10 +214,9 @@ impl Externalities for BasicExternalities { fn kill_child_storage( &mut self, - storage_key: ChildStorageKey, - _child_info: ChildInfo, + child_info: &ChildInfo, ) { - self.inner.children.remove(storage_key.as_ref()); + self.inner.children_default.remove(child_info.storage_key()); } fn clear_prefix(&mut self, prefix: &[u8]) { @@ -214,11 +241,10 @@ impl Externalities for BasicExternalities { fn clear_child_prefix( &mut self, - storage_key: ChildStorageKey, - _child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], ) { - if let Some(child) = self.inner.children.get_mut(storage_key.as_ref()) { + if let Some(child) = self.inner.children_default.get_mut(child_info.storage_key()) { let to_remove = child.data.range::<[u8], _>((Bound::Included(prefix), Bound::Unbounded)) .map(|(k, _)| k) .take_while(|k| k.starts_with(prefix)) @@ -231,24 +257,33 @@ impl Externalities for BasicExternalities { } } + fn storage_append( + &mut self, + key: Vec, + value: Vec, + ) { + let previous = self.storage(&key).unwrap_or_default(); + let new = crate::ext::append_to_storage(previous, value).expect("Failed to append to storage"); + self.place_storage(key.clone(), Some(new)); + } + fn chain_id(&self) -> u64 { 42 } fn storage_root(&mut self) -> Vec { let mut top = self.inner.top.clone(); - let keys: Vec<_> = self.inner.children.keys().map(|k| k.to_vec()).collect(); + let prefixed_keys: Vec<_> = self.inner.children_default.iter().map(|(_k, v)| { + (v.child_info.prefixed_storage_key(), v.child_info.clone()) + }).collect(); // Single child trie implementation currently allows using the same child // empty root for all child trie. Using null storage key until multiple // type of child trie support. - let empty_hash = default_child_trie_root::>(&[]); - for storage_key in keys { - let child_root = self.child_storage_root( - ChildStorageKey::from_slice(storage_key.as_slice()) - .expect("Map only feed by valid keys; qed"), - ); + let empty_hash = empty_child_trie_root::>(); + for (prefixed_storage_key, child_info) in prefixed_keys { + let child_root = self.child_storage_root(&child_info); if &empty_hash[..] == &child_root[..] { - top.remove(storage_key.as_slice()); + top.remove(prefixed_storage_key.as_slice()); } else { - top.insert(storage_key, child_root); + top.insert(prefixed_storage_key.into_inner(), child_root); } } @@ -257,15 +292,14 @@ impl Externalities for BasicExternalities { fn child_storage_root( &mut self, - storage_key: ChildStorageKey, + child_info: &ChildInfo, ) -> Vec { - if let Some(child) = self.inner.children.get(storage_key.as_ref()) { + if let Some(child) = self.inner.children_default.get(child_info.storage_key()) { let delta = child.data.clone().into_iter().map(|(k, v)| (k, Some(v))); - - InMemoryBackend::::default() - .child_storage_root(storage_key.as_ref(), child.child_info.as_ref(), delta).0 + crate::in_memory_backend::new_in_mem::() + .child_storage_root(&child.child_info, delta).0 } else { - default_child_trie_root::>(storage_key.as_ref()) + empty_child_trie_root::>() }.encode() } @@ -279,9 +313,23 @@ impl Externalities for BasicExternalities { } impl sp_externalities::ExtensionStore for BasicExternalities { - fn extension_by_type_id(&mut self, _: TypeId) -> Option<&mut dyn Any> { - warn!("Extensions are not supported by `BasicExternalities`."); - None + fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> { + self.extensions.get_mut(type_id) + } + + fn register_extension_with_type_id( + &mut self, + type_id: TypeId, + extension: Box, + ) -> Result<(), sp_externalities::Error> { + self.extensions.register_with_type_id(type_id, extension) + } + + fn deregister_extension_by_type_id(&mut self, type_id: TypeId) -> Result<(), sp_externalities::Error> { + self.extensions + .deregister(type_id) + .ok_or(sp_externalities::Error::ExtensionIsNotRegistered(type_id)) + .map(drop) } } @@ -293,8 +341,6 @@ mod tests { use sp_core::storage::well_known_keys::CODE; use hex_literal::hex; - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); - #[test] fn commit_should_work() { let mut ext = BasicExternalities::default(); @@ -318,37 +364,35 @@ mod tests { #[test] fn children_works() { - let child_storage = b":child_storage:default:test".to_vec(); - + let child_info = ChildInfo::new_default(b"storage_key"); + let child_info = &child_info; let mut ext = BasicExternalities::new(Storage { top: Default::default(), - children: map![ - child_storage.clone() => StorageChild { - data: map![ b"doe".to_vec() => b"reindeer".to_vec() ], - child_info: CHILD_INFO_1.to_owned(), + children_default: map![ + child_info.storage_key().to_vec() => StorageChild { + data: map![ b"doe".to_vec() => b"reindeer".to_vec() ], + child_info: child_info.to_owned(), } ] }); - let child = || ChildStorageKey::from_vec(child_storage.clone()).unwrap(); - - assert_eq!(ext.child_storage(child(), CHILD_INFO_1, b"doe"), Some(b"reindeer".to_vec())); + assert_eq!(ext.child_storage(child_info, b"doe"), Some(b"reindeer".to_vec())); - ext.set_child_storage(child(), CHILD_INFO_1, b"dog".to_vec(), b"puppy".to_vec()); - assert_eq!(ext.child_storage(child(), CHILD_INFO_1, b"dog"), Some(b"puppy".to_vec())); + ext.set_child_storage(child_info, b"dog".to_vec(), b"puppy".to_vec()); + assert_eq!(ext.child_storage(child_info, b"dog"), Some(b"puppy".to_vec())); - ext.clear_child_storage(child(), CHILD_INFO_1, b"dog"); - assert_eq!(ext.child_storage(child(), CHILD_INFO_1, b"dog"), None); + ext.clear_child_storage(child_info, b"dog"); + assert_eq!(ext.child_storage(child_info, b"dog"), None); - ext.kill_child_storage(child(), CHILD_INFO_1); - assert_eq!(ext.child_storage(child(), CHILD_INFO_1, b"doe"), None); + ext.kill_child_storage(child_info); + assert_eq!(ext.child_storage(child_info, b"doe"), None); } #[test] fn basic_externalities_is_empty() { // Make sure no values are set by default in `BasicExternalities`. - let storage = BasicExternalities::new(Default::default()).into_storages(); + let storage = BasicExternalities::new_empty().into_storages(); assert!(storage.top.is_empty()); - assert!(storage.children.is_empty()); + assert!(storage.children_default.is_empty()); } } diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index 39ad81ed59a0eb3dc6746f922be36f334299801a..45535204e088434d486e8e3fd6da2037f2926457 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -32,6 +32,7 @@ use crate::{ input::{InputKey, InputPair, DigestIndex, ExtrinsicIndex, ChildIndex}, }, }; +use sp_core::storage::{ChildInfo, ChildType, PrefixedStorageKey}; /// Prepare input pairs for building a changes trie of given block. /// @@ -105,19 +106,19 @@ fn prepare_extrinsics_input<'a, B, H, Number>( Number: BlockNumber, { - let mut children_keys = BTreeSet::::new(); + let mut children_info = BTreeSet::::new(); let mut children_result = BTreeMap::new(); - for (storage_key, _) in changes.prospective.children.iter() - .chain(changes.committed.children.iter()) { - children_keys.insert(storage_key.clone()); + for (_storage_key, (_map, child_info)) in changes.prospective.children_default.iter() + .chain(changes.committed.children_default.iter()) { + children_info.insert(child_info.clone()); } - for storage_key in children_keys { + for child_info in children_info { let child_index = ChildIndex:: { block: block.clone(), - storage_key: storage_key.clone(), + storage_key: child_info.prefixed_storage_key(), }; - let iter = prepare_extrinsics_input_inner(backend, block, changes, Some(storage_key))?; + let iter = prepare_extrinsics_input_inner(backend, block, changes, Some(child_info))?; children_result.insert(child_index, iter); } @@ -130,22 +131,22 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>( backend: &'a B, block: &Number, changes: &'a OverlayedChanges, - storage_key: Option, + child_info: Option, ) -> Result> + 'a, String> where B: Backend, H: Hasher, Number: BlockNumber, { - let (committed, prospective, child_info) = if let Some(sk) = storage_key.as_ref() { - let child_info = changes.child_info(sk).cloned(); - ( - changes.committed.children.get(sk).map(|c| &c.0), - changes.prospective.children.get(sk).map(|c| &c.0), - child_info, - ) + let (committed, prospective) = if let Some(child_info) = child_info.as_ref() { + match child_info.child_type() { + ChildType::ParentKeyId => ( + changes.committed.children_default.get(child_info.storage_key()).map(|c| &c.0), + changes.prospective.children_default.get(child_info.storage_key()).map(|c| &c.0), + ), + } } else { - (Some(&changes.committed.top), Some(&changes.prospective.top), None) + (Some(&changes.committed.top), Some(&changes.prospective.top)) }; committed.iter().flat_map(|c| c.iter()) .chain(prospective.iter().flat_map(|c| c.iter())) @@ -155,13 +156,11 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>( Entry::Vacant(entry) => { // ignore temporary values (values that have null value at the end of operation // AND are not in storage at the beginning of operation - if let Some(sk) = storage_key.as_ref() { - if !changes.child_storage(sk, k).map(|v| v.is_some()).unwrap_or_default() { - if let Some(child_info) = child_info.as_ref() { - if !backend.exists_child_storage(sk, child_info.as_ref(), k) - .map_err(|e| format!("{}", e))? { - return Ok(map); - } + if let Some(child_info) = child_info.as_ref() { + if !changes.child_storage(child_info, k).map(|v| v.is_some()).unwrap_or_default() { + if !backend.exists_child_storage(&child_info, k) + .map_err(|e| format!("{}", e))? { + return Ok(map); } } } else { @@ -281,7 +280,7 @@ fn prepare_digest_input<'a, H, Number>( return Ok((map, child_map)); } - let mut children_roots = BTreeMap::::new(); + let mut children_roots = BTreeMap::::new(); { let trie_storage = TrieBackendEssence::<_, H>::new( crate::changes_trie::TrieBackendStorageAdapter(storage), @@ -344,22 +343,20 @@ mod test { use codec::Encode; use sp_core::Blake2Hasher; use sp_core::storage::well_known_keys::EXTRINSIC_INDEX; - use sp_core::storage::ChildInfo; use crate::InMemoryBackend; use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage}; use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; use super::*; - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); - const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2"); - fn prepare_for_build(zero: u64) -> ( InMemoryBackend, InMemoryStorage, OverlayedChanges, Configuration, ) { + let child_info_1 = ChildInfo::new_default(b"storage_key1"); + let child_info_2 = ChildInfo::new_default(b"storage_key2"); let backend: InMemoryBackend<_> = vec![ (vec![100], vec![255]), (vec![101], vec![255]), @@ -368,8 +365,9 @@ mod test { (vec![104], vec![255]), (vec![105], vec![255]), ].into_iter().collect::>().into(); - let child_trie_key1 = b"1".to_vec(); - let child_trie_key2 = b"2".to_vec(); + let prefixed_child_trie_key1 = child_info_1.prefixed_storage_key(); + let child_trie_key1 = child_info_1.storage_key().to_vec(); + let child_trie_key2 = child_info_2.storage_key().to_vec(); let storage = InMemoryStorage::with_inputs(vec![ (zero + 1, vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![100] }, vec![1, 3]), @@ -403,7 +401,7 @@ mod test { ]), (zero + 9, Vec::new()), (zero + 10, Vec::new()), (zero + 11, Vec::new()), (zero + 12, Vec::new()), (zero + 13, Vec::new()), (zero + 14, Vec::new()), (zero + 15, Vec::new()), - ], vec![(child_trie_key1.clone(), vec![ + ], vec![(prefixed_child_trie_key1.clone(), vec![ (zero + 1, vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![100] }, vec![1, 3]), InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![101] }, vec![0, 2]), @@ -430,19 +428,19 @@ mod test { extrinsics: Some(vec![0, 1].into_iter().collect()) }), ].into_iter().collect(), - children: vec![ + children_default: vec![ (child_trie_key1.clone(), (vec![ (vec![100], OverlayedValue { value: Some(vec![200]), extrinsics: Some(vec![0, 2].into_iter().collect()) }) - ].into_iter().collect(), CHILD_INFO_1.to_owned())), + ].into_iter().collect(), child_info_1.to_owned())), (child_trie_key2, (vec![ (vec![100], OverlayedValue { value: Some(vec![200]), extrinsics: Some(vec![0, 2].into_iter().collect()) }) - ].into_iter().collect(), CHILD_INFO_2.to_owned())), + ].into_iter().collect(), child_info_2.to_owned())), ].into_iter().collect() }, committed: OverlayedChangeSet { top: vec![ @@ -459,13 +457,13 @@ mod test { extrinsics: Some(vec![1].into_iter().collect()) }), ].into_iter().collect(), - children: vec![ + children_default: vec![ (child_trie_key1, (vec![ (vec![100], OverlayedValue { value: Some(vec![202]), extrinsics: Some(vec![3].into_iter().collect()) }) - ].into_iter().collect(), CHILD_INFO_1.to_owned())), + ].into_iter().collect(), child_info_1.to_owned())), ].into_iter().collect(), }, collect_extrinsics: true, @@ -487,6 +485,8 @@ mod test { #[test] fn build_changes_trie_nodes_on_non_digest_block() { fn test_with_zero(zero: u64) { + let child_trie_key1 = ChildInfo::new_default(b"storage_key1").prefixed_storage_key(); + let child_trie_key2 = ChildInfo::new_default(b"storage_key2").prefixed_storage_key(); let (backend, storage, changes, config) = prepare_for_build(zero); let parent = AnchorBlockId { hash: Default::default(), number: zero + 4 }; let changes_trie_nodes = prepare_input( @@ -503,11 +503,11 @@ mod test { ]); assert_eq!(changes_trie_nodes.1.into_iter() .map(|(k,v)| (k, v.collect::>())).collect::>(), vec![ - (ChildIndex { block: zero + 5u64, storage_key: b"1".to_vec() }, + (ChildIndex { block: zero + 5u64, storage_key: child_trie_key1 }, vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 5u64, key: vec![100] }, vec![0, 2, 3]), ]), - (ChildIndex { block: zero + 5, storage_key: b"2".to_vec() }, + (ChildIndex { block: zero + 5, storage_key: child_trie_key2 }, vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 5, key: vec![100] }, vec![0, 2]), ]), @@ -523,6 +523,8 @@ mod test { #[test] fn build_changes_trie_nodes_on_digest_block_l1() { fn test_with_zero(zero: u64) { + let child_trie_key1 = ChildInfo::new_default(b"storage_key1").prefixed_storage_key(); + let child_trie_key2 = ChildInfo::new_default(b"storage_key2").prefixed_storage_key(); let (backend, storage, changes, config) = prepare_for_build(zero); let parent = AnchorBlockId { hash: Default::default(), number: zero + 3 }; let changes_trie_nodes = prepare_input( @@ -544,7 +546,7 @@ mod test { ]); assert_eq!(changes_trie_nodes.1.into_iter() .map(|(k,v)| (k, v.collect::>())).collect::>(), vec![ - (ChildIndex { block: zero + 4u64, storage_key: b"1".to_vec() }, + (ChildIndex { block: zero + 4u64, storage_key: child_trie_key1.clone() }, vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4u64, key: vec![100] }, vec![0, 2, 3]), @@ -553,7 +555,7 @@ mod test { InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]), InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![105] }, vec![zero + 1]), ]), - (ChildIndex { block: zero + 4, storage_key: b"2".to_vec() }, + (ChildIndex { block: zero + 4, storage_key: child_trie_key2.clone() }, vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2]), ]), @@ -568,6 +570,8 @@ mod test { #[test] fn build_changes_trie_nodes_on_digest_block_l2() { fn test_with_zero(zero: u64) { + let child_trie_key1 = ChildInfo::new_default(b"storage_key1").prefixed_storage_key(); + let child_trie_key2 = ChildInfo::new_default(b"storage_key2").prefixed_storage_key(); let (backend, storage, changes, config) = prepare_for_build(zero); let parent = AnchorBlockId { hash: Default::default(), number: zero + 15 }; let changes_trie_nodes = prepare_input( @@ -590,13 +594,13 @@ mod test { ]); assert_eq!(changes_trie_nodes.1.into_iter() .map(|(k,v)| (k, v.collect::>())).collect::>(), vec![ - (ChildIndex { block: zero + 16u64, storage_key: b"1".to_vec() }, + (ChildIndex { block: zero + 16u64, storage_key: child_trie_key1.clone() }, vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16u64, key: vec![100] }, vec![0, 2, 3]), InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![102] }, vec![zero + 4]), ]), - (ChildIndex { block: zero + 16, storage_key: b"2".to_vec() }, + (ChildIndex { block: zero + 16, storage_key: child_trie_key2.clone() }, vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16, key: vec![100] }, vec![0, 2]), ]), @@ -657,6 +661,8 @@ mod test { #[test] fn build_changes_trie_nodes_ignores_temporary_storage_values() { fn test_with_zero(zero: u64) { + let child_trie_key1 = ChildInfo::new_default(b"storage_key1").prefixed_storage_key(); + let child_trie_key2 = ChildInfo::new_default(b"storage_key2").prefixed_storage_key(); let (backend, storage, mut changes, config) = prepare_for_build(zero); // 110: missing from backend, set to None in overlay @@ -685,7 +691,7 @@ mod test { ]); assert_eq!(changes_trie_nodes.1.into_iter() .map(|(k,v)| (k, v.collect::>())).collect::>(), vec![ - (ChildIndex { block: zero + 4u64, storage_key: b"1".to_vec() }, + (ChildIndex { block: zero + 4u64, storage_key: child_trie_key1.clone() }, vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4u64, key: vec![100] }, vec![0, 2, 3]), @@ -694,7 +700,7 @@ mod test { InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]), InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![105] }, vec![zero + 1]), ]), - (ChildIndex { block: zero + 4, storage_key: b"2".to_vec() }, + (ChildIndex { block: zero + 4, storage_key: child_trie_key2.clone() }, vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2]), ]), @@ -709,6 +715,8 @@ mod test { #[test] fn cache_is_used_when_changes_trie_is_built() { + let child_trie_key1 = ChildInfo::new_default(b"storage_key1").prefixed_storage_key(); + let child_trie_key2 = ChildInfo::new_default(b"storage_key2").prefixed_storage_key(); let (backend, mut storage, changes, config) = prepare_for_build(0); let parent = AnchorBlockId { hash: Default::default(), number: 15 }; @@ -728,8 +736,8 @@ mod test { let cached_data4 = IncompleteCacheAction::CacheBuildData(IncompleteCachedBuildData::new()) .set_digest_input_blocks(vec![1, 2, 3]) .insert(None, vec![vec![100], vec![102]].into_iter().collect()) - .insert(Some(b"1".to_vec()), vec![vec![103], vec![104]].into_iter().collect()) - .insert(Some(b"2".to_vec()), vec![vec![105], vec![106]].into_iter().collect()) + .insert(Some(child_trie_key1.clone()), vec![vec![103], vec![104]].into_iter().collect()) + .insert(Some(child_trie_key2.clone()), vec![vec![105], vec![106]].into_iter().collect()) .complete(4, &trie_root4); storage.cache_mut().perform(cached_data4); @@ -755,7 +763,10 @@ mod test { .map(|(k, i)| (k, i.collect::>())) .collect::>(); assert_eq!( - child_changes_tries_nodes.get(&ChildIndex { block: 16u64, storage_key: b"1".to_vec() }).unwrap(), + child_changes_tries_nodes.get(&ChildIndex { + block: 16u64, + storage_key: child_trie_key1.clone(), + }).unwrap(), &vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16u64, key: vec![100] }, vec![0, 2, 3]), @@ -764,7 +775,7 @@ mod test { ], ); assert_eq!( - child_changes_tries_nodes.get(&ChildIndex { block: 16u64, storage_key: b"2".to_vec() }).unwrap(), + child_changes_tries_nodes.get(&ChildIndex { block: 16u64, storage_key: child_trie_key2.clone() }).unwrap(), &vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16u64, key: vec![100] }, vec![0, 2]), diff --git a/primitives/state-machine/src/changes_trie/build_cache.rs b/primitives/state-machine/src/changes_trie/build_cache.rs index 9d0dbb4c1f3108a4b0c9ff15c19c1991852edd17..aebebf3a17f591708fd06eec0fad2ebacd91fa73 100644 --- a/primitives/state-machine/src/changes_trie/build_cache.rs +++ b/primitives/state-machine/src/changes_trie/build_cache.rs @@ -19,6 +19,7 @@ use std::collections::{HashMap, HashSet}; use crate::StorageKey; +use sp_core::storage::PrefixedStorageKey; /// Changes trie build cache. /// @@ -38,7 +39,7 @@ pub struct BuildCache { /// The `Option>` in inner `HashMap` stands for the child storage key. /// If it is `None`, then the `HashSet` contains keys changed in top-level storage. /// If it is `Some`, then the `HashSet` contains keys changed in child storage, identified by the key. - changed_keys: HashMap, HashSet>>, + changed_keys: HashMap, HashSet>>, } /// The action to perform when block-with-changes-trie is imported. @@ -56,7 +57,7 @@ pub struct CachedBuildData { block: N, trie_root: H, digest_input_blocks: Vec, - changed_keys: HashMap, HashSet>, + changed_keys: HashMap, HashSet>, } /// The action to perform when block-with-changes-trie is imported. @@ -72,7 +73,7 @@ pub(crate) enum IncompleteCacheAction { #[derive(Debug, PartialEq)] pub(crate) struct IncompleteCachedBuildData { digest_input_blocks: Vec, - changed_keys: HashMap, HashSet>, + changed_keys: HashMap, HashSet>, } impl BuildCache @@ -89,7 +90,7 @@ impl BuildCache } /// Get cached changed keys for changes trie with given root. - pub fn get(&self, root: &H) -> Option<&HashMap, HashSet>> { + pub fn get(&self, root: &H) -> Option<&HashMap, HashSet>> { self.changed_keys.get(&root) } @@ -98,7 +99,7 @@ impl BuildCache pub fn with_changed_keys( &self, root: &H, - functor: &mut dyn FnMut(&HashMap, HashSet>), + functor: &mut dyn FnMut(&HashMap, HashSet>), ) -> bool { match self.changed_keys.get(&root) { Some(changed_keys) => { @@ -164,7 +165,7 @@ impl IncompleteCacheAction { /// Insert changed keys of given storage into cached data. pub(crate) fn insert( self, - storage_key: Option, + storage_key: Option, changed_keys: HashSet, ) -> Self { match self { @@ -200,7 +201,7 @@ impl IncompleteCachedBuildData { fn insert( mut self, - storage_key: Option, + storage_key: Option, changed_keys: HashSet, ) -> Self { self.changed_keys.insert(storage_key, changed_keys); diff --git a/primitives/state-machine/src/changes_trie/changes_iterator.rs b/primitives/state-machine/src/changes_trie/changes_iterator.rs index 685786218c75f42100742726e22ada3d651792be..f5a936069ba40d24b29a02f441871a1410e796f4 100644 --- a/primitives/state-machine/src/changes_trie/changes_iterator.rs +++ b/primitives/state-machine/src/changes_trie/changes_iterator.rs @@ -22,6 +22,7 @@ use std::collections::VecDeque; use codec::{Decode, Encode, Codec}; use hash_db::Hasher; use num_traits::Zero; +use sp_core::storage::PrefixedStorageKey; use sp_trie::Recorder; use crate::changes_trie::{AnchorBlockId, ConfigurationRange, RootsStorage, Storage, BlockNumber}; use crate::changes_trie::input::{DigestIndex, ExtrinsicIndex, DigestIndexValue, ExtrinsicIndexValue}; @@ -40,7 +41,7 @@ pub fn key_changes<'a, H: Hasher, Number: BlockNumber>( begin: Number, end: &'a AnchorBlockId, max: Number, - storage_key: Option<&'a [u8]>, + storage_key: Option<&'a PrefixedStorageKey>, key: &'a [u8], ) -> Result, String> { // we can't query any roots before root @@ -79,7 +80,7 @@ pub fn key_changes_proof<'a, H: Hasher, Number: BlockNumber>( begin: Number, end: &AnchorBlockId, max: Number, - storage_key: Option<&[u8]>, + storage_key: Option<&PrefixedStorageKey>, key: &[u8], ) -> Result>, String> where H::Out: Codec { // we can't query any roots before root @@ -127,7 +128,7 @@ pub fn key_changes_proof_check<'a, H: Hasher, Number: BlockNumber>( begin: Number, end: &AnchorBlockId, max: Number, - storage_key: Option<&[u8]>, + storage_key: Option<&PrefixedStorageKey>, key: &[u8] ) -> Result, String> where H::Out: Encode { key_changes_proof_check_with_db( @@ -150,7 +151,7 @@ pub fn key_changes_proof_check_with_db<'a, H: Hasher, Number: BlockNumber>( begin: Number, end: &AnchorBlockId, max: Number, - storage_key: Option<&[u8]>, + storage_key: Option<&PrefixedStorageKey>, key: &[u8] ) -> Result, String> where H::Out: Encode { // we can't query any roots before root @@ -188,7 +189,7 @@ pub struct DrilldownIteratorEssence<'a, H, Number> Number: BlockNumber, H::Out: 'a, { - storage_key: Option<&'a [u8]>, + storage_key: Option<&'a PrefixedStorageKey>, key: &'a [u8], roots_storage: &'a dyn RootsStorage, storage: &'a dyn Storage, @@ -238,7 +239,7 @@ impl<'a, H, Number> DrilldownIteratorEssence<'a, H, Number> let trie_root = if let Some(storage_key) = self.storage_key { let child_key = ChildIndex { block: block.clone(), - storage_key: storage_key.to_vec(), + storage_key: storage_key.clone(), }.encode(); if let Some(trie_root) = trie_reader(self.storage, trie_root, &child_key)? .and_then(|v| >::decode(&mut &v[..]).ok()) @@ -382,6 +383,11 @@ mod tests { use sp_runtime::traits::BlakeTwo256; use super::*; + fn child_key() -> PrefixedStorageKey { + let child_info = sp_core::storage::ChildInfo::new_default(&b"1"[..]); + child_info.prefixed_storage_key() + } + fn prepare_for_drilldown() -> (Configuration, InMemoryStorage) { let config = Configuration { digest_interval: 4, digest_levels: 2 }; let backend = InMemoryStorage::with_inputs(vec![ @@ -418,7 +424,7 @@ mod tests { (16, vec![ InputPair::DigestIndex(DigestIndex { block: 16, key: vec![42] }, vec![4, 8]), ]), - ], vec![(b"1".to_vec(), vec![ + ], vec![(child_key(), vec![ (1, vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 1, key: vec![42] }, vec![0]), ]), @@ -535,7 +541,7 @@ mod tests { 1, &AnchorBlockId { hash: Default::default(), number: 100 }, 1000, - Some(&b"1"[..]), + Some(&child_key()), &[42], ).and_then(|i| i.collect::, _>>()).is_err()); } @@ -577,7 +583,7 @@ mod tests { let (remote_config, remote_storage) = prepare_for_drilldown(); let remote_proof_child = key_changes_proof::( configuration_range(&remote_config, 0), &remote_storage, 1, - &AnchorBlockId { hash: Default::default(), number: 16 }, 16, Some(&b"1"[..]), &[42]).unwrap(); + &AnchorBlockId { hash: Default::default(), number: 16 }, 16, Some(&child_key()), &[42]).unwrap(); // happens on local light node: @@ -592,7 +598,7 @@ mod tests { local_storage.clear_storage(); let local_result_child = key_changes_proof_check::( configuration_range(&local_config, 0), &local_storage, remote_proof_child, 1, - &AnchorBlockId { hash: Default::default(), number: 16 }, 16, Some(&b"1"[..]), &[42]); + &AnchorBlockId { hash: Default::default(), number: 16 }, 16, Some(&child_key()), &[42]); // check that drilldown result is the same as if it was happening at the full node assert_eq!(local_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)])); diff --git a/primitives/state-machine/src/changes_trie/input.rs b/primitives/state-machine/src/changes_trie/input.rs index 4a1420f8486f99b189e952a31d1ce9aa333579ab..4f0f3da40c52bff30be127c29aec25320980a3db 100644 --- a/primitives/state-machine/src/changes_trie/input.rs +++ b/primitives/state-machine/src/changes_trie/input.rs @@ -21,6 +21,7 @@ use crate::{ StorageKey, StorageValue, changes_trie::BlockNumber }; +use sp_core::storage::PrefixedStorageKey; /// Key of { changed key => set of extrinsic indices } mapping. #[derive(Clone, Debug, PartialEq, Eq)] @@ -49,7 +50,7 @@ pub struct ChildIndex { /// Block at which this key has been inserted in the trie. pub block: Number, /// Storage key this node is responsible for. - pub storage_key: StorageKey, + pub storage_key: PrefixedStorageKey, } /// Value of { changed key => block/digest block numbers } mapping. @@ -178,7 +179,7 @@ impl Decode for InputKey { })), 3 => Ok(InputKey::ChildIndex(ChildIndex { block: Decode::decode(input)?, - storage_key: Decode::decode(input)?, + storage_key: PrefixedStorageKey::new(Decode::decode(input)?), })), _ => Err("Invalid input key variant".into()), } diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs index d614992df303320cefabadf80585ae47f2123206..ee6c6778e0aad10c269f1e3c0eb89f852e7e176d 100644 --- a/primitives/state-machine/src/changes_trie/mod.rs +++ b/primitives/state-machine/src/changes_trie/mod.rs @@ -71,6 +71,7 @@ use hash_db::{Hasher, Prefix}; use num_traits::{One, Zero}; use codec::{Decode, Encode}; use sp_core; +use sp_core::storage::PrefixedStorageKey; use sp_trie::{MemoryDB, DBValue, TrieMut}; use sp_trie::trie_types::TrieDBMut; use crate::{ @@ -156,7 +157,7 @@ pub trait Storage: RootsStorage { fn with_cached_changed_keys( &self, root: &H::Out, - functor: &mut dyn FnMut(&HashMap, HashSet>), + functor: &mut dyn FnMut(&HashMap, HashSet>), ) -> bool; /// Get a trie node. fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String>; diff --git a/primitives/state-machine/src/changes_trie/prune.rs b/primitives/state-machine/src/changes_trie/prune.rs index 87923dc2f593c183ac4fd791e3cbe742be51e0c9..05555df305b7c458bc96d602b02a9c6818c617a1 100644 --- a/primitives/state-machine/src/changes_trie/prune.rs +++ b/primitives/state-machine/src/changes_trie/prune.rs @@ -137,7 +137,8 @@ mod tests { #[test] fn prune_works() { fn prepare_storage() -> InMemoryStorage { - let child_key = ChildIndex { block: 67u64, storage_key: b"1".to_vec() }.encode(); + let child_info = sp_core::storage::ChildInfo::new_default(&b"1"[..]); + let child_key = ChildIndex { block: 67u64, storage_key: child_info.prefixed_storage_key() }.encode(); let mut mdb1 = MemoryDB::::default(); let root1 = insert_into_memory_db::( &mut mdb1, vec![(vec![10], vec![20])]).unwrap(); diff --git a/primitives/state-machine/src/changes_trie/storage.rs b/primitives/state-machine/src/changes_trie/storage.rs index 7fb418672872bafc40f8994d09a1d0bfb66b8b13..81651dd2e719bec94b9c1d56bc02604c9b916fe8 100644 --- a/primitives/state-machine/src/changes_trie/storage.rs +++ b/primitives/state-machine/src/changes_trie/storage.rs @@ -18,6 +18,7 @@ use std::collections::{BTreeMap, HashSet, HashMap}; use hash_db::{Hasher, Prefix, EMPTY_PREFIX}; +use sp_core::storage::PrefixedStorageKey; use sp_trie::DBValue; use sp_trie::MemoryDB; use parking_lot::RwLock; @@ -96,7 +97,7 @@ impl InMemoryStorage { #[cfg(test)] pub fn with_inputs( mut top_inputs: Vec<(Number, Vec>)>, - children_inputs: Vec<(StorageKey, Vec<(Number, Vec>)>)>, + children_inputs: Vec<(PrefixedStorageKey, Vec<(Number, Vec>)>)>, ) -> Self { let mut mdb = MemoryDB::default(); let mut roots = BTreeMap::new(); @@ -182,7 +183,7 @@ impl Storage for InMemoryStorage, HashSet>), + functor: &mut dyn FnMut(&HashMap, HashSet>), ) -> bool { self.cache.with_changed_keys(root, functor) } diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 133af7ccd9b5f1234b4b9e8a823ea56e43953d8a..a92a10bf0e99b17bb4069169fa82beb97b4bb280 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -24,12 +24,13 @@ use crate::{ use hash_db::Hasher; use sp_core::{ - storage::{ChildStorageKey, well_known_keys::is_child_storage_key, ChildInfo}, + offchain::storage::OffchainOverlayedChanges, + storage::{well_known_keys::is_child_storage_key, ChildInfo}, traits::Externalities, hexdisplay::HexDisplay, }; -use sp_trie::{trie_types::Layout, default_child_trie_root}; -use sp_externalities::Extensions; -use codec::{Decode, Encode}; +use sp_trie::{trie_types::Layout, empty_child_trie_root}; +use sp_externalities::{Extensions, Extension}; +use codec::{Compact, Decode, Encode}; use std::{error, fmt, any::{Any, TypeId}}; use log::{warn, trace}; @@ -74,6 +75,8 @@ pub struct Ext<'a, H, N, B> { /// The overlayed changes to write to. overlay: &'a mut OverlayedChanges, + /// The overlayed changes destined for the Offchain DB. + offchain_overlay: &'a mut OffchainOverlayedChanges, /// The storage backend to read from. backend: &'a B, /// The cache for the storage transactions. @@ -99,13 +102,15 @@ where /// Create a new `Ext` from overlayed changes and read-only backend pub fn new( overlay: &'a mut OverlayedChanges, + offchain_overlay: &'a mut OffchainOverlayedChanges, storage_transaction_cache: &'a mut StorageTransactionCache, backend: &'a B, changes_trie_state: Option>, extensions: Option<&'a mut Extensions>, ) -> Self { - Ext { + Self { overlay, + offchain_overlay, backend, changes_trie_state, storage_transaction_cache, @@ -121,6 +126,11 @@ where fn mark_dirty(&mut self) { self.storage_transaction_cache.reset(); } + + /// Read only accessor for the scheduled overlay changes. + pub fn get_offchain_storage_changes(&self) -> &OffchainOverlayedChanges { + &*self.offchain_overlay + } } #[cfg(test)] @@ -152,11 +162,20 @@ where B: 'a + Backend, N: crate::changes_trie::BlockNumber, { + + fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) { + use ::sp_core::offchain::STORAGE_PREFIX; + match value { + Some(value) => self.offchain_overlay.set(STORAGE_PREFIX, key, value), + None => self.offchain_overlay.remove(STORAGE_PREFIX, key), + } + } + fn storage(&self, key: &[u8]) -> Option { let _guard = sp_panic_handler::AbortGuard::force_abort(); let result = self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(|| self.backend.storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL)); - trace!(target: "state-trace", "{:04x}: Get {}={:?}", + trace!(target: "state", "{:04x}: Get {}={:?}", self.id, HexDisplay::from(&key), result.as_ref().map(HexDisplay::from) @@ -171,7 +190,7 @@ where .map(|x| x.map(|x| H::hash(x))) .unwrap_or_else(|| self.backend.storage_hash(key).expect(EXT_NOT_ALLOWED_TO_FAIL)); - trace!(target: "state-trace", "{:04x}: Hash {}={:?}", + trace!(target: "state", "{:04x}: Hash {}={:?}", self.id, HexDisplay::from(&key), result, @@ -181,22 +200,21 @@ where fn child_storage( &self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Option { let _guard = sp_panic_handler::AbortGuard::force_abort(); let result = self.overlay - .child_storage(storage_key.as_ref(), key) + .child_storage(child_info, key) .map(|x| x.map(|x| x.to_vec())) .unwrap_or_else(|| - self.backend.child_storage(storage_key.as_ref(), child_info, key) + self.backend.child_storage(child_info, key) .expect(EXT_NOT_ALLOWED_TO_FAIL) ); - trace!(target: "state-trace", "{:04x}: GetChild({}) {}={:?}", + trace!(target: "state", "{:04x}: GetChild({}) {}={:?}", self.id, - HexDisplay::from(&storage_key.as_ref()), + HexDisplay::from(&child_info.storage_key()), HexDisplay::from(&key), result.as_ref().map(HexDisplay::from) ); @@ -206,22 +224,21 @@ where fn child_storage_hash( &self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Option> { let _guard = sp_panic_handler::AbortGuard::force_abort(); let result = self.overlay - .child_storage(storage_key.as_ref(), key) + .child_storage(child_info, key) .map(|x| x.map(|x| H::hash(x))) .unwrap_or_else(|| - self.backend.child_storage_hash(storage_key.as_ref(), child_info, key) + self.backend.child_storage_hash(child_info, key) .expect(EXT_NOT_ALLOWED_TO_FAIL) ); - trace!(target: "state-trace", "{:04x}: ChildHash({}) {}={:?}", + trace!(target: "state", "{:04x}: ChildHash({}) {}={:?}", self.id, - HexDisplay::from(&storage_key.as_ref()), + HexDisplay::from(&child_info.storage_key()), HexDisplay::from(&key), result, ); @@ -236,7 +253,7 @@ where _ => self.backend.exists_storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL), }; - trace!(target: "state-trace", "{:04x}: Exists {}={:?}", + trace!(target: "state", "{:04x}: Exists {}={:?}", self.id, HexDisplay::from(&key), result, @@ -247,22 +264,21 @@ where fn exists_child_storage( &self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> bool { let _guard = sp_panic_handler::AbortGuard::force_abort(); - let result = match self.overlay.child_storage(storage_key.as_ref(), key) { + let result = match self.overlay.child_storage(child_info, key) { Some(x) => x.is_some(), _ => self.backend - .exists_child_storage(storage_key.as_ref(), child_info, key) + .exists_child_storage(child_info, key) .expect(EXT_NOT_ALLOWED_TO_FAIL), }; - trace!(target: "state-trace", "{:04x}: ChildExists({}) {}={:?}", + trace!(target: "state", "{:04x}: ChildExists({}) {}={:?}", self.id, - HexDisplay::from(&storage_key.as_ref()), + HexDisplay::from(&child_info.storage_key()), HexDisplay::from(&key), result, ); @@ -286,15 +302,14 @@ where fn next_child_storage_key( &self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Option { let next_backend_key = self.backend - .next_child_storage_key(storage_key.as_ref(), child_info, key) + .next_child_storage_key(child_info, key) .expect(EXT_NOT_ALLOWED_TO_FAIL); let next_overlay_key_change = self.overlay.next_child_storage_key_change( - storage_key.as_ref(), + child_info.storage_key(), key ); @@ -305,7 +320,6 @@ where Some(overlay_key.0.to_vec()) } else { self.next_child_storage_key( - storage_key, child_info, &overlay_key.0[..], ) @@ -314,7 +328,7 @@ where } fn place_storage(&mut self, key: StorageKey, value: Option) { - trace!(target: "state-trace", "{:04x}: Put {}={:?}", + trace!(target: "state", "{:04x}: Put {}={:?}", self.id, HexDisplay::from(&key), value.as_ref().map(HexDisplay::from) @@ -331,43 +345,41 @@ where fn place_child_storage( &mut self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: StorageKey, value: Option, ) { - trace!(target: "state-trace", "{:04x}: PutChild({}) {}={:?}", + trace!(target: "state", "{:04x}: PutChild({}) {}={:?}", self.id, - HexDisplay::from(&storage_key.as_ref()), + HexDisplay::from(&child_info.storage_key()), HexDisplay::from(&key), value.as_ref().map(HexDisplay::from) ); let _guard = sp_panic_handler::AbortGuard::force_abort(); self.mark_dirty(); - self.overlay.set_child_storage(storage_key.into_owned(), child_info, key, value); + self.overlay.set_child_storage(child_info, key, value); } fn kill_child_storage( &mut self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, ) { - trace!(target: "state-trace", "{:04x}: KillChild({})", + trace!(target: "state", "{:04x}: KillChild({})", self.id, - HexDisplay::from(&storage_key.as_ref()), + HexDisplay::from(&child_info.storage_key()), ); let _guard = sp_panic_handler::AbortGuard::force_abort(); self.mark_dirty(); - self.overlay.clear_child_storage(storage_key.as_ref(), child_info); - self.backend.for_keys_in_child_storage(storage_key.as_ref(), child_info, |key| { - self.overlay.set_child_storage(storage_key.as_ref().to_vec(), child_info, key.to_vec(), None); + self.overlay.clear_child_storage(child_info); + self.backend.for_keys_in_child_storage(child_info, |key| { + self.overlay.set_child_storage(child_info, key.to_vec(), None); }); } fn clear_prefix(&mut self, prefix: &[u8]) { - trace!(target: "state-trace", "{:04x}: ClearPrefix {}", + trace!(target: "state", "{:04x}: ClearPrefix {}", self.id, HexDisplay::from(&prefix), ); @@ -386,24 +398,43 @@ where fn clear_child_prefix( &mut self, - storage_key: ChildStorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], ) { - trace!(target: "state-trace", "{:04x}: ClearChildPrefix({}) {}", + trace!(target: "state", "{:04x}: ClearChildPrefix({}) {}", self.id, - HexDisplay::from(&storage_key.as_ref()), + HexDisplay::from(&child_info.storage_key()), HexDisplay::from(&prefix), ); let _guard = sp_panic_handler::AbortGuard::force_abort(); self.mark_dirty(); - self.overlay.clear_child_prefix(storage_key.as_ref(), child_info, prefix); - self.backend.for_child_keys_with_prefix(storage_key.as_ref(), child_info, prefix, |key| { - self.overlay.set_child_storage(storage_key.as_ref().to_vec(), child_info, key.to_vec(), None); + self.overlay.clear_child_prefix(child_info, prefix); + self.backend.for_child_keys_with_prefix(child_info, prefix, |key| { + self.overlay.set_child_storage(child_info, key.to_vec(), None); }); } + fn storage_append( + &mut self, + key: Vec, + value: Vec, + ) { + let _guard = sp_panic_handler::AbortGuard::force_abort(); + self.mark_dirty(); + + let current_value = self.overlay + .value_mut(&key) + .map(|v| v.take()) + .unwrap_or_else(|| self.backend.storage(&key).expect(EXT_NOT_ALLOWED_TO_FAIL)) + .unwrap_or_default(); + + self.overlay.set_storage( + key, + Some(append_to_storage(current_value, value).expect(EXT_NOT_ALLOWED_TO_FAIL)), + ); + } + fn chain_id(&self) -> u64 { 42 } @@ -411,7 +442,7 @@ where fn storage_root(&mut self) -> Vec { let _guard = sp_panic_handler::AbortGuard::force_abort(); if let Some(ref root) = self.storage_transaction_cache.transaction_storage_root { - trace!(target: "state-trace", "{:04x}: Root (cached) {}", + trace!(target: "state", "{:04x}: Root (cached) {}", self.id, HexDisplay::from(&root.as_ref()), ); @@ -419,43 +450,44 @@ where } let root = self.overlay.storage_root(self.backend, self.storage_transaction_cache); - trace!(target: "state-trace", "{:04x}: Root {}", self.id, HexDisplay::from(&root.as_ref())); + trace!(target: "state", "{:04x}: Root {}", self.id, HexDisplay::from(&root.as_ref())); root.encode() } fn child_storage_root( &mut self, - storage_key: ChildStorageKey, + child_info: &ChildInfo, ) -> Vec { let _guard = sp_panic_handler::AbortGuard::force_abort(); + let storage_key = child_info.storage_key(); + let prefixed_storage_key = child_info.prefixed_storage_key(); if self.storage_transaction_cache.transaction_storage_root.is_some() { let root = self - .storage(storage_key.as_ref()) + .storage(prefixed_storage_key.as_slice()) .and_then(|k| Decode::decode(&mut &k[..]).ok()) .unwrap_or( - default_child_trie_root::>(storage_key.as_ref()) + empty_child_trie_root::>() ); - trace!(target: "state-trace", "{:04x}: ChildRoot({}) (cached) {}", + trace!(target: "state", "{:04x}: ChildRoot({}) (cached) {}", self.id, - HexDisplay::from(&storage_key.as_ref()), + HexDisplay::from(&storage_key), HexDisplay::from(&root.as_ref()), ); root.encode() } else { - let storage_key = storage_key.as_ref(); - if let Some(child_info) = self.overlay.child_info(storage_key).cloned() { + if let Some(child_info) = self.overlay.default_child_info(storage_key).cloned() { let (root, is_empty, _) = { - let delta = self.overlay.committed.children.get(storage_key) + let delta = self.overlay.committed.children_default.get(storage_key) .into_iter() .flat_map(|(map, _)| map.clone().into_iter().map(|(k, v)| (k, v.value))) .chain( - self.overlay.prospective.children.get(storage_key) + self.overlay.prospective.children_default.get(storage_key) .into_iter() .flat_map(|(map, _)| map.clone().into_iter().map(|(k, v)| (k, v.value))) ); - self.backend.child_storage_root(storage_key, child_info.as_ref(), delta) + self.backend.child_storage_root(&child_info, delta) }; let root = root.encode(); @@ -465,12 +497,12 @@ where // A better design would be to manage 'child_storage_transaction' in a // similar way as 'storage_transaction' but for each child trie. if is_empty { - self.overlay.set_storage(storage_key.into(), None); + self.overlay.set_storage(prefixed_storage_key.into_inner(), None); } else { - self.overlay.set_storage(storage_key.into(), Some(root.clone())); + self.overlay.set_storage(prefixed_storage_key.into_inner(), Some(root.clone())); } - trace!(target: "state-trace", "{:04x}: ChildRoot({}) {}", + trace!(target: "state", "{:04x}: ChildRoot({}) {}", self.id, HexDisplay::from(&storage_key.as_ref()), HexDisplay::from(&root.as_ref()), @@ -479,12 +511,12 @@ where } else { // empty overlay let root = self - .storage(storage_key.as_ref()) + .storage(prefixed_storage_key.as_slice()) .and_then(|k| Decode::decode(&mut &k[..]).ok()) .unwrap_or( - default_child_trie_root::>(storage_key.as_ref()) + empty_child_trie_root::>() ); - trace!(target: "state-trace", "{:04x}: ChildRoot({}) (no change) {}", + trace!(target: "state", "{:04x}: ChildRoot({}) (no change) {}", self.id, HexDisplay::from(&storage_key.as_ref()), HexDisplay::from(&root.as_ref()), @@ -501,7 +533,7 @@ where self.changes_trie_state.as_ref(), Decode::decode(&mut &parent_hash[..]).map_err(|e| trace!( - target: "state-trace", + target: "state", "Failed to decode changes root parent hash: {}", e, ) @@ -510,7 +542,7 @@ where self.storage_transaction_cache, ); - trace!(target: "state-trace", "{:04x}: ChangesRoot({}) {:?}", + trace!(target: "state", "{:04x}: ChangesRoot({}) {:?}", self.id, HexDisplay::from(&parent_hash), root, @@ -539,6 +571,67 @@ where } } +fn extract_length_data(data: &[u8]) -> Result<(u32, usize, usize), &'static str> { + use codec::CompactLen; + + let len = u32::from( + Compact::::decode(&mut &data[..]) + .map_err(|_| "Incorrect updated item encoding")? + ); + let new_len = len + .checked_add(1) + .ok_or_else(|| "New vec length greater than `u32::max_value()`.")?; + + let encoded_len = Compact::::compact_len(&len); + let encoded_new_len = Compact::::compact_len(&new_len); + + Ok((new_len, encoded_len, encoded_new_len)) +} + +pub fn append_to_storage( + mut self_encoded: Vec, + value: Vec, +) -> Result, &'static str> { + // No data present, just encode the given input data. + if self_encoded.is_empty() { + Compact::from(1u32).encode_to(&mut self_encoded); + self_encoded.extend(value); + return Ok(self_encoded); + } + + let (new_len, encoded_len, encoded_new_len) = extract_length_data(&self_encoded)?; + + let replace_len = |dest: &mut Vec| { + Compact(new_len).using_encoded(|e| { + dest[..encoded_new_len].copy_from_slice(e); + }) + }; + + let append_new_elems = |dest: &mut Vec| dest.extend(&value[..]); + + // If old and new encoded len is equal, we don't need to copy the + // already encoded data. + if encoded_len == encoded_new_len { + replace_len(&mut self_encoded); + append_new_elems(&mut self_encoded); + + Ok(self_encoded) + } else { + let size = encoded_new_len + self_encoded.len() - encoded_len; + + let mut res = Vec::with_capacity(size + value.len()); + unsafe { res.set_len(size); } + + // Insert the new encoded len, copy the already encoded data and + // add the new element. + replace_len(&mut res); + res[encoded_new_len..size].copy_from_slice(&self_encoded[encoded_len..]); + append_new_elems(&mut res); + + Ok(res) + } +} + impl<'a, H, B, N> sp_externalities::ExtensionStore for Ext<'a, H, N, B> where H: Hasher, @@ -548,6 +641,29 @@ where fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> { self.extensions.as_mut().and_then(|exts| exts.get_mut(type_id)) } + + fn register_extension_with_type_id( + &mut self, + type_id: TypeId, + extension: Box, + ) -> Result<(), sp_externalities::Error> { + if let Some(ref mut extensions) = self.extensions { + extensions.register_with_type_id(type_id, extension) + } else { + Err(sp_externalities::Error::ExtensionsAreNotSupported) + } + } + + fn deregister_extension_by_type_id(&mut self, type_id: TypeId) -> Result<(), sp_externalities::Error> { + if let Some(ref mut extensions) = self.extensions { + match extensions.deregister(type_id) { + Some(_) => Ok(()), + None => Err(sp_externalities::Error::ExtensionIsNotRegistered(type_id)) + } + } else { + Err(sp_externalities::Error::ExtensionsAreNotSupported) + } + } } #[cfg(test)] @@ -556,23 +672,27 @@ mod tests { use hex_literal::hex; use num_traits::Zero; use codec::Encode; - use sp_core::{H256, Blake2Hasher, storage::well_known_keys::EXTRINSIC_INDEX, map}; + use sp_core::{ + H256, + Blake2Hasher, + map, + offchain, + storage::{ + Storage, + StorageChild, + well_known_keys::EXTRINSIC_INDEX, + }, + }; use crate::{ changes_trie::{ Configuration as ChangesTrieConfiguration, InMemoryStorage as TestChangesTrieStorage, }, InMemoryBackend, overlayed_changes::OverlayedValue, }; - use sp_core::storage::{Storage, StorageChild}; type TestBackend = InMemoryBackend; type TestExt<'a> = Ext<'a, Blake2Hasher, u64, TestBackend>; - const CHILD_KEY_1: &[u8] = b":child_storage:default:Child1"; - - const CHILD_UUID_1: &[u8] = b"unique_id_1"; - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(CHILD_UUID_1); - fn prepare_overlay_with_changes() -> OverlayedChanges { OverlayedChanges { prospective: vec![ @@ -591,6 +711,13 @@ mod tests { } } + fn prepare_offchain_overlay_with_changes() -> OffchainOverlayedChanges { + let mut ooc = OffchainOverlayedChanges::enabled(); + ooc.set(offchain::STORAGE_PREFIX, b"k1", b"v1"); + ooc.set(offchain::STORAGE_PREFIX, b"k2", b"v2"); + ooc + } + fn changes_trie_config() -> ChangesTrieConfiguration { ChangesTrieConfiguration { digest_interval: 0, @@ -601,29 +728,32 @@ mod tests { #[test] fn storage_changes_root_is_none_when_storage_is_not_provided() { let mut overlay = prepare_overlay_with_changes(); + let mut offchain_overlay = prepare_offchain_overlay_with_changes(); let mut cache = StorageTransactionCache::default(); let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None); } #[test] fn storage_changes_root_is_none_when_state_is_not_provided() { let mut overlay = prepare_overlay_with_changes(); + let mut offchain_overlay = prepare_offchain_overlay_with_changes(); let mut cache = StorageTransactionCache::default(); let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None); } #[test] fn storage_changes_root_is_some_when_extrinsic_changes_are_non_empty() { let mut overlay = prepare_overlay_with_changes(); + let mut offchain_overlay = prepare_offchain_overlay_with_changes(); let mut cache = StorageTransactionCache::default(); let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage)); let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None); + let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, state, None); assert_eq!( ext.storage_changes_root(&H256::default().encode()).unwrap(), Some(hex!("bb0c2ef6e1d36d5490f9766cfcc7dfe2a6ca804504c3bb206053890d6dd02376").to_vec()), @@ -633,12 +763,13 @@ mod tests { #[test] fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() { let mut overlay = prepare_overlay_with_changes(); + let mut offchain_overlay = prepare_offchain_overlay_with_changes(); let mut cache = StorageTransactionCache::default(); overlay.prospective.top.get_mut(&vec![1]).unwrap().value = None; let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage)); let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None); + let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, state, None); assert_eq!( ext.storage_changes_root(&H256::default().encode()).unwrap(), Some(hex!("96f5aae4690e7302737b6f9b7f8567d5bbb9eac1c315f80101235a92d9ec27f4").to_vec()), @@ -651,16 +782,17 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_storage(vec![20], None); overlay.set_storage(vec![30], Some(vec![31])); + let mut offchain_overlay = prepare_offchain_overlay_with_changes(); let backend = Storage { top: map![ vec![10] => vec![10], vec![20] => vec![20], vec![40] => vec![40] ], - children: map![] + children_default: map![] }.into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); // next_backend < next_overlay assert_eq!(ext.next_storage_key(&[5]), Some(vec![10])); @@ -676,7 +808,7 @@ mod tests { drop(ext); overlay.set_storage(vec![50], Some(vec![50])); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); // next_overlay exist but next_backend doesn't exist assert_eq!(ext.next_storage_key(&[40]), Some(vec![50])); @@ -684,93 +816,93 @@ mod tests { #[test] fn next_child_storage_key_works() { - const CHILD_KEY_1: &[u8] = b":child_storage:default:Child1"; - - const CHILD_UUID_1: &[u8] = b"unique_id_1"; - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(CHILD_UUID_1); + let child_info = ChildInfo::new_default(b"Child1"); + let child_info = &child_info; let mut cache = StorageTransactionCache::default(); - let child = || ChildStorageKey::from_slice(CHILD_KEY_1).unwrap(); let mut overlay = OverlayedChanges::default(); - overlay.set_child_storage(child().as_ref().to_vec(), CHILD_INFO_1, vec![20], None); - overlay.set_child_storage(child().as_ref().to_vec(), CHILD_INFO_1, vec![30], Some(vec![31])); + overlay.set_child_storage(child_info, vec![20], None); + overlay.set_child_storage(child_info, vec![30], Some(vec![31])); let backend = Storage { top: map![], - children: map![ - child().as_ref().to_vec() => StorageChild { + children_default: map![ + child_info.storage_key().to_vec() => StorageChild { data: map![ vec![10] => vec![10], vec![20] => vec![20], vec![40] => vec![40] ], - child_info: CHILD_INFO_1.to_owned(), + child_info: child_info.to_owned(), } ], }.into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let mut offchain_overlay = prepare_offchain_overlay_with_changes(); + + let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); // next_backend < next_overlay - assert_eq!(ext.next_child_storage_key(child(), CHILD_INFO_1, &[5]), Some(vec![10])); + assert_eq!(ext.next_child_storage_key(child_info, &[5]), Some(vec![10])); // next_backend == next_overlay but next_overlay is a delete - assert_eq!(ext.next_child_storage_key(child(), CHILD_INFO_1, &[10]), Some(vec![30])); + assert_eq!(ext.next_child_storage_key(child_info, &[10]), Some(vec![30])); // next_overlay < next_backend - assert_eq!(ext.next_child_storage_key(child(), CHILD_INFO_1, &[20]), Some(vec![30])); + assert_eq!(ext.next_child_storage_key(child_info, &[20]), Some(vec![30])); // next_backend exist but next_overlay doesn't exist - assert_eq!(ext.next_child_storage_key(child(), CHILD_INFO_1, &[30]), Some(vec![40])); + assert_eq!(ext.next_child_storage_key(child_info, &[30]), Some(vec![40])); drop(ext); - overlay.set_child_storage(child().as_ref().to_vec(), CHILD_INFO_1, vec![50], Some(vec![50])); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + overlay.set_child_storage(child_info, vec![50], Some(vec![50])); + let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); // next_overlay exist but next_backend doesn't exist - assert_eq!(ext.next_child_storage_key(child(), CHILD_INFO_1, &[40]), Some(vec![50])); + assert_eq!(ext.next_child_storage_key(child_info, &[40]), Some(vec![50])); } #[test] fn child_storage_works() { + let child_info = ChildInfo::new_default(b"Child1"); + let child_info = &child_info; let mut cache = StorageTransactionCache::default(); - let child = || ChildStorageKey::from_slice(CHILD_KEY_1).unwrap(); let mut overlay = OverlayedChanges::default(); - overlay.set_child_storage(child().as_ref().to_vec(), CHILD_INFO_1, vec![20], None); - overlay.set_child_storage(child().as_ref().to_vec(), CHILD_INFO_1, vec![30], Some(vec![31])); + overlay.set_child_storage(child_info, vec![20], None); + overlay.set_child_storage(child_info, vec![30], Some(vec![31])); + let mut offchain_overlay = prepare_offchain_overlay_with_changes(); let backend = Storage { top: map![], - children: map![ - child().as_ref().to_vec() => StorageChild { + children_default: map![ + child_info.storage_key().to_vec() => StorageChild { data: map![ vec![10] => vec![10], vec![20] => vec![20], vec![30] => vec![40] ], - child_info: CHILD_INFO_1.to_owned(), + child_info: child_info.to_owned(), } ], }.into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); - assert_eq!(ext.child_storage(child(), CHILD_INFO_1, &[10]), Some(vec![10])); + assert_eq!(ext.child_storage(child_info, &[10]), Some(vec![10])); assert_eq!( - ext.child_storage_hash(child(), CHILD_INFO_1, &[10]), + ext.child_storage_hash(child_info, &[10]), Some(Blake2Hasher::hash(&[10]).as_ref().to_vec()), ); - assert_eq!(ext.child_storage(child(), CHILD_INFO_1, &[20]), None); + assert_eq!(ext.child_storage(child_info, &[20]), None); assert_eq!( - ext.child_storage_hash(child(), CHILD_INFO_1, &[20]), + ext.child_storage_hash(child_info, &[20]), None, ); - assert_eq!(ext.child_storage(child(), CHILD_INFO_1, &[30]), Some(vec![31])); + assert_eq!(ext.child_storage(child_info, &[30]), Some(vec![31])); assert_eq!( - ext.child_storage_hash(child(), CHILD_INFO_1, &[30]), + ext.child_storage_hash(child_info, &[30]), Some(Blake2Hasher::hash(&[31]).as_ref().to_vec()), ); - } } diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index 8cbed90e9afc2803a7bd11d78b761719cb8def57..8c0ae1ec8bf419f729e0ed53f912be8c7cc9926a 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -19,138 +19,182 @@ use crate::{ StorageKey, StorageValue, StorageCollection, trie_backend::TrieBackend, - backend::{Backend, insert_into_memory_db}, - stats::UsageInfo, }; -use std::{error, fmt, collections::{BTreeMap, HashMap}, marker::PhantomData, ops}; +use std::{collections::{BTreeMap, HashMap}}; use hash_db::Hasher; use sp_trie::{ - MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration, trie_types::Layout, + MemoryDB, TrieMut, + trie_types::TrieDBMut, }; use codec::Codec; -use sp_core::storage::{ChildInfo, OwnedChildInfo, Storage}; +use sp_core::storage::{ChildInfo, Storage}; -/// Error impossible. -// FIXME: use `!` type when stabilized. https://github.com/rust-lang/rust/issues/35121 -#[derive(Debug)] -pub enum Void {} - -impl fmt::Display for Void { - fn fmt(&self, _: &mut fmt::Formatter) -> fmt::Result { - match *self {} +/// Insert input pairs into memory db. +fn insert_into_memory_db(mut root: H::Out, mdb: &mut MemoryDB, input: I) -> H::Out +where + H: Hasher, + I: IntoIterator)>, +{ + { + let mut trie = if root == Default::default() { + TrieDBMut::::new(mdb, &mut root) + } else { + TrieDBMut::::from_existing(mdb, &mut root).unwrap() + }; + for (key, value) in input { + if let Err(e) = match value { + Some(value) => { + trie.insert(&key, &value) + }, + None => { + trie.remove(&key) + }, + } { + panic!("Failed to write to trie: {}", e); + } + } + trie.commit(); } + root } -impl error::Error for Void { - fn description(&self) -> &str { "unreachable error" } -} - -/// In-memory backend. Fully recomputes tries each time `as_trie_backend` is called but useful for -/// tests and proof checking. -pub struct InMemory { - inner: HashMap, BTreeMap>, - // This field is only needed for returning reference in `as_trie_backend`. - trie: Option, H>>, - _hasher: PhantomData, +/// Create a new empty instance of in-memory backend. +pub fn new_in_mem() -> TrieBackend, H> +where + H::Out: Codec + Ord, +{ + let db = MemoryDB::default(); + let mut backend = TrieBackend::new(db, Default::default()); + backend.insert(std::iter::empty()); + backend } -impl std::fmt::Debug for InMemory { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "InMemory ({} values)", self.inner.len()) +impl TrieBackend, H> +where + H::Out: Codec + Ord, +{ + /// Copy the state, with applied updates + pub fn update< + T: IntoIterator, StorageCollection)> + >( + &self, + changes: T, + ) -> Self { + let mut clone = self.clone(); + clone.insert(changes); + clone } -} -impl Default for InMemory { - fn default() -> Self { - InMemory { - inner: Default::default(), - trie: None, - _hasher: PhantomData, + /// Insert values into backend trie. + pub fn insert< + T: IntoIterator, StorageCollection)> + >( + &mut self, + changes: T, + ) { + let mut new_child_roots = Vec::new(); + let mut root_map = None; + let root = self.root().clone(); + for (child_info, map) in changes { + if let Some(child_info) = child_info.as_ref() { + let prefix_storage_key = child_info.prefixed_storage_key(); + let ch = insert_into_memory_db::(root, self.backend_storage_mut(), map.clone().into_iter()); + new_child_roots.push((prefix_storage_key.into_inner(), Some(ch.as_ref().into()))); + } else { + root_map = Some(map); + } } + + let root = match root_map { + Some(map) => insert_into_memory_db::( + root, + self.backend_storage_mut(), + map.clone().into_iter().chain(new_child_roots.into_iter()), + ), + None => insert_into_memory_db::( + root, + self.backend_storage_mut(), + new_child_roots.into_iter(), + ), + }; + self.essence.set_root(root); } -} -impl Clone for InMemory { - fn clone(&self) -> Self { - InMemory { - inner: self.inner.clone(), - trie: None, - _hasher: PhantomData, - } + /// Merge trie nodes into this backend. + pub fn update_backend(&self, root: H::Out, changes: MemoryDB) -> Self { + let mut clone = self.backend_storage().clone(); + clone.consolidate(changes); + Self::new(clone, root) + } + + /// Compare with another in-memory backend. + pub fn eq(&self, other: &Self) -> bool { + self.root() == other.root() } } -impl PartialEq for InMemory { - fn eq(&self, other: &Self) -> bool { - self.inner.eq(&other.inner) +impl Clone for TrieBackend, H> +where + H::Out: Codec + Ord, +{ + fn clone(&self) -> Self { + TrieBackend::new(self.backend_storage().clone(), self.root().clone()) } } -impl InMemory { - /// Copy the state, with applied updates - pub fn update< - T: IntoIterator, StorageCollection)> - >( - &self, - changes: T, - ) -> Self { - let mut inner = self.inner.clone(); - for (child_info, key_values) in changes.into_iter() { - let entry = inner.entry(child_info).or_default(); - for (key, val) in key_values { - match val { - Some(v) => { entry.insert(key, v); }, - None => { entry.remove(&key); }, - } - } - } - inner.into() +impl Default for TrieBackend, H> +where + H::Out: Codec + Ord, +{ + fn default() -> Self { + new_in_mem() } } -impl From, BTreeMap>> - for InMemory +impl From, BTreeMap>> + for TrieBackend, H> +where + H::Out: Codec + Ord, { - fn from(inner: HashMap, BTreeMap>) -> Self { - InMemory { - inner, - trie: None, - _hasher: PhantomData, - } + fn from(inner: HashMap, BTreeMap>) -> Self { + let mut backend = new_in_mem(); + backend.insert(inner.into_iter().map(|(k, m)| (k, m.into_iter().map(|(k, v)| (k, Some(v))).collect()))); + backend } } -impl From for InMemory { +impl From for TrieBackend, H> +where + H::Out: Codec + Ord, +{ fn from(inners: Storage) -> Self { - let mut inner: HashMap, BTreeMap> - = inners.children.into_iter().map(|(k, c)| (Some((k, c.child_info)), c.data)).collect(); + let mut inner: HashMap, BTreeMap> + = inners.children_default.into_iter().map(|(_k, c)| (Some(c.child_info), c.data)).collect(); inner.insert(None, inners.top); - InMemory { - inner, - trie: None, - _hasher: PhantomData, - } + inner.into() } } -impl From> for InMemory { +impl From> for TrieBackend, H> +where + H::Out: Codec + Ord, +{ fn from(inner: BTreeMap) -> Self { let mut expanded = HashMap::new(); expanded.insert(None, inner); - InMemory { - inner: expanded, - trie: None, - _hasher: PhantomData, - } + expanded.into() } } -impl From, StorageCollection)>> - for InMemory { +impl From, StorageCollection)>> + for TrieBackend, H> +where + H::Out: Codec + Ord, +{ fn from( - inner: Vec<(Option<(StorageKey, OwnedChildInfo)>, StorageCollection)>, + inner: Vec<(Option, StorageCollection)>, ) -> Self { - let mut expanded: HashMap, BTreeMap> + let mut expanded: HashMap, BTreeMap> = HashMap::new(); for (child_info, key_values) in inner { let entry = expanded.entry(child_info).or_default(); @@ -164,227 +208,28 @@ impl From, StorageCollectio } } -impl InMemory { - /// child storage key iterator - pub fn child_storage_keys(&self) -> impl Iterator { - self.inner.iter().filter_map(|item| - item.0.as_ref().map(|v|(&v.0[..], v.1.as_ref())) - ) - } -} - -impl Backend for InMemory where H::Out: Codec { - type Error = Void; - type Transaction = Vec<( - Option<(StorageKey, OwnedChildInfo)>, - StorageCollection, - )>; - type TrieBackendStorage = MemoryDB; - - fn storage(&self, key: &[u8]) -> Result, Self::Error> { - Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone))) - } - - fn child_storage( - &self, - storage_key: &[u8], - child_info: ChildInfo, - key: &[u8], - ) -> Result, Self::Error> { - Ok(self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned()))) - .and_then(|map| map.get(key).map(Clone::clone))) - } - - fn exists_storage(&self, key: &[u8]) -> Result { - Ok(self.inner.get(&None).map(|map| map.get(key).is_some()).unwrap_or(false)) - } - - fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { - self.inner.get(&None) - .map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f)); - } - - fn for_key_values_with_prefix(&self, prefix: &[u8], mut f: F) { - self.inner.get(&None).map(|map| map.iter().filter(|(key, _val)| key.starts_with(prefix)) - .for_each(|(k, v)| f(k, v))); - } - - fn for_keys_in_child_storage( - &self, - storage_key: &[u8], - child_info: ChildInfo, - mut f: F, - ) { - self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned()))) - .map(|map| map.keys().for_each(|k| f(&k))); - } - - fn for_child_keys_with_prefix( - &self, - storage_key: &[u8], - child_info: ChildInfo, - prefix: &[u8], - f: F, - ) { - self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned()))) - .map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f)); - } - - fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) - where - I: IntoIterator, Option>)>, - ::Out: Ord, - { - let existing_pairs = self.inner.get(&None) - .into_iter() - .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); - - let transaction: Vec<_> = delta.into_iter().collect(); - let root = Layout::::trie_root(existing_pairs.chain(transaction.iter().cloned()) - .collect::>() - .into_iter() - .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) - ); - - let full_transaction = transaction.into_iter().collect(); - - (root, vec![(None, full_transaction)]) - } - - fn child_storage_root( - &self, - storage_key: &[u8], - child_info: ChildInfo, - delta: I, - ) -> (H::Out, bool, Self::Transaction) - where - I: IntoIterator, Option>)>, - H::Out: Ord - { - let storage_key = storage_key.to_vec(); - let child_info = Some((storage_key.clone(), child_info.to_owned())); - - let existing_pairs = self.inner.get(&child_info) - .into_iter() - .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); - - let transaction: Vec<_> = delta.into_iter().collect(); - let root = child_trie_root::, _, _, _>( - &storage_key, - existing_pairs.chain(transaction.iter().cloned()) - .collect::>() - .into_iter() - .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) - ); - - let full_transaction = transaction.into_iter().collect(); - - let is_default = root == default_child_trie_root::>(&storage_key); - - (root, is_default, vec![(child_info, full_transaction)]) - } - - fn next_storage_key(&self, key: &[u8]) -> Result, Self::Error> { - let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded); - let next_key = self.inner.get(&None) - .and_then(|map| map.range::<[u8], _>(range).next().map(|(k, _)| k).cloned()); - - Ok(next_key) - } - - fn next_child_storage_key( - &self, - storage_key: &[u8], - child_info: ChildInfo, - key: &[u8], - ) -> Result, Self::Error> { - let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded); - let next_key = self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned()))) - .and_then(|map| map.range::<[u8], _>(range).next().map(|(k, _)| k).cloned()); - - Ok(next_key) - } - - fn pairs(&self) -> Vec<(StorageKey, StorageValue)> { - self.inner.get(&None) - .into_iter() - .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone()))) - .collect() - } - - fn keys(&self, prefix: &[u8]) -> Vec { - self.inner.get(&None) - .into_iter() - .flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned()) - .collect() - } - - fn child_keys( - &self, - storage_key: &[u8], - child_info: ChildInfo, - prefix: &[u8], - ) -> Vec { - self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned()))) - .into_iter() - .flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned()) - .collect() - } - - fn as_trie_backend(&mut self)-> Option<&TrieBackend> { - let mut mdb = MemoryDB::default(); - let mut new_child_roots = Vec::new(); - let mut root_map = None; - for (child_info, map) in &self.inner { - if let Some((storage_key, _child_info)) = child_info.as_ref() { - // no need to use child_info at this point because we use a MemoryDB for - // proof (with PrefixedMemoryDB it would be needed). - let ch = insert_into_memory_db::(&mut mdb, map.clone().into_iter())?; - new_child_roots.push((storage_key.clone(), ch.as_ref().into())); - } else { - root_map = Some(map); - } - } - let root = match root_map { - Some(map) => insert_into_memory_db::( - &mut mdb, - map.clone().into_iter().chain(new_child_roots.into_iter()), - )?, - None => insert_into_memory_db::( - &mut mdb, - new_child_roots.into_iter(), - )?, - }; - self.trie = Some(TrieBackend::new(mdb, root)); - self.trie.as_ref() - } - - fn register_overlay_stats(&mut self, _stats: &crate::stats::StateMachineStats) { } - - fn usage_info(&self) -> UsageInfo { - UsageInfo::empty() - } -} - #[cfg(test)] mod tests { use super::*; use sp_runtime::traits::BlakeTwo256; + use crate::backend::Backend; /// Assert in memory backend with only child trie keys works as trie backend. #[test] fn in_memory_with_child_trie_only() { - let storage = InMemory::::default(); - let child_info = OwnedChildInfo::new_default(b"unique_id_1".to_vec()); + let storage = new_in_mem::(); + let child_info = ChildInfo::new_default(b"1"); + let child_info = &child_info; let mut storage = storage.update( vec![( - Some((b"1".to_vec(), child_info.clone())), + Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))] )] ); let trie_backend = storage.as_trie_backend().unwrap(); - assert_eq!(trie_backend.child_storage(b"1", child_info.as_ref(), b"2").unwrap(), + assert_eq!(trie_backend.child_storage(child_info, b"2").unwrap(), Some(b"3".to_vec())); - assert!(trie_backend.storage(b"1").unwrap().is_some()); + let storage_key = child_info.prefixed_storage_key(); + assert!(trie_backend.storage(storage_key.as_slice()).unwrap().is_some()); } } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 9a2dc52cca20a91ef3b545f0a7eec3ad2220e6a0..aa3a3ad5739102ab0c846823e6638635b9cf52bc 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -23,6 +23,7 @@ use log::{warn, trace}; use hash_db::Hasher; use codec::{Decode, Encode, Codec}; use sp_core::{ + offchain::storage::OffchainOverlayedChanges, storage::ChildInfo, NativeOrEncoded, NeverNativeValue, hexdisplay::HexDisplay, traits::{CodeExecutor, CallInWasmExt, RuntimeCode}, }; @@ -72,7 +73,7 @@ pub use proving_backend::{ pub use trie_backend_essence::{TrieBackendStorage, Storage}; pub use trie_backend::TrieBackend; pub use error::{Error, ExecutionError}; -pub use in_memory_backend::InMemory as InMemoryBackend; +pub use in_memory_backend::new_in_mem; pub use stats::{UsageInfo, UsageUnit, StateMachineStats}; pub use sp_core::traits::CloneableSpawn; @@ -87,6 +88,9 @@ pub type ChangesTrieTransaction = ( ChangesTrieCacheAction<::Out, N>, ); +/// Trie backend with in-memory storage. +pub type InMemoryBackend = TrieBackend, H>; + /// Strategy for executing a call into the runtime. #[derive(Copy, Clone, Eq, PartialEq, Debug)] pub enum ExecutionStrategy { @@ -187,6 +191,7 @@ pub struct StateMachine<'a, B, H, N, Exec> method: &'a str, call_data: &'a [u8], overlay: &'a mut OverlayedChanges, + offchain_overlay: &'a mut OffchainOverlayedChanges, extensions: Extensions, changes_trie_state: Option>, storage_transaction_cache: Option<&'a mut StorageTransactionCache>, @@ -216,6 +221,7 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where backend: &'a B, changes_trie_state: Option>, overlay: &'a mut OverlayedChanges, + offchain_overlay: &'a mut OffchainOverlayedChanges, exec: &'a Exec, method: &'a str, call_data: &'a [u8], @@ -233,6 +239,7 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where call_data, extensions, overlay, + offchain_overlay, changes_trie_state, storage_transaction_cache: None, runtime_code, @@ -290,6 +297,7 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where let mut ext = Ext::new( self.overlay, + self.offchain_overlay, cache, self.backend, self.changes_trie_state.clone(), @@ -298,7 +306,7 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where let id = ext.id; trace!( - target: "state-trace", "{:04x}: Call {} at {:?}. Input={:?}", + target: "state", "{:04x}: Call {} at {:?}. Input={:?}", id, self.method, self.backend, @@ -315,7 +323,7 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where ); trace!( - target: "state-trace", "{:04x}: Return. Native={:?}, Result={:?}", + target: "state", "{:04x}: Return. Native={:?}, Result={:?}", id, was_native, result, @@ -500,11 +508,13 @@ where Exec: CodeExecutor + 'static + Clone, N: crate::changes_trie::BlockNumber, { + let mut offchain_overlay = OffchainOverlayedChanges::default(); let proving_backend = proving_backend::ProvingBackend::new(trie_backend); let mut sm = StateMachine::<_, H, N, Exec>::new( &proving_backend, None, overlay, + &mut offchain_overlay, exec, method, call_data, @@ -566,10 +576,12 @@ where Exec: CodeExecutor + Clone + 'static, N: crate::changes_trie::BlockNumber, { + let mut offchain_overlay = OffchainOverlayedChanges::default(); let mut sm = StateMachine::<_, H, N, Exec>::new( trie_backend, None, overlay, + &mut offchain_overlay, exec, method, call_data, @@ -606,8 +618,7 @@ where /// Generate child storage read proof. pub fn prove_child_read( mut backend: B, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, keys: I, ) -> Result> where @@ -619,7 +630,7 @@ where { let trie_backend = backend.as_trie_backend() .ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box)?; - prove_child_read_on_trie_backend(trie_backend, storage_key, child_info, keys) + prove_child_read_on_trie_backend(trie_backend, child_info, keys) } /// Generate storage read proof on pre-created trie backend. @@ -646,8 +657,7 @@ where /// Generate storage read proof on pre-created trie backend. pub fn prove_child_read_on_trie_backend( trie_backend: &TrieBackend, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, keys: I, ) -> Result> where @@ -660,7 +670,7 @@ where let proving_backend = proving_backend::ProvingBackend::<_, H>::new(trie_backend); for key in keys.into_iter() { proving_backend - .child_storage(storage_key, child_info.clone(), key.as_ref()) + .child_storage(child_info, key.as_ref()) .map_err(|e| Box::new(e) as Box)?; } Ok(proving_backend.extract_proof()) @@ -691,7 +701,7 @@ where pub fn read_child_proof_check( root: H::Out, proof: StorageProof, - storage_key: &[u8], + child_info: &ChildInfo, keys: I, ) -> Result, Option>>, Box> where @@ -705,7 +715,7 @@ where for key in keys.into_iter() { let value = read_child_proof_check_on_proving_backend( &proving_backend, - storage_key, + child_info, key.as_ref(), )?; result.insert(key.as_ref().to_vec(), value); @@ -728,15 +738,14 @@ where /// Check child storage read proof on pre-created proving backend. pub fn read_child_proof_check_on_proving_backend( proving_backend: &TrieBackend, H>, - storage_key: &[u8], + child_info: &ChildInfo, key: &[u8], ) -> Result>, Box> where H: Hasher, H::Out: Ord + Codec, { - // Not a prefixed memory db, using empty unique id and include root resolution. - proving_backend.child_storage(storage_key, ChildInfo::new_default(&[]), key) + proving_backend.child_storage(child_info, key) .map_err(|e| Box::new(e) as Box) } @@ -748,7 +757,7 @@ mod tests { use super::*; use super::ext::Ext; use super::changes_trie::Configuration as ChangesTrieConfig; - use sp_core::{map, traits::{Externalities, RuntimeCode}, storage::ChildStorageKey}; + use sp_core::{map, traits::{Externalities, RuntimeCode}}; use sp_runtime::traits::BlakeTwo256; #[derive(Clone)] @@ -759,8 +768,6 @@ mod tests { fallback_succeeds: bool, } - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); - impl CodeExecutor for DummyCodeExecutor { type Error = u8; @@ -816,6 +823,7 @@ mod tests { _: &str, _: &[u8], _: &mut dyn Externalities, + _: sp_core::traits::MissingHostFunctions, ) -> std::result::Result, String> { unimplemented!("Not required in tests.") } @@ -825,12 +833,14 @@ mod tests { fn execute_works() { let backend = trie_backend::tests::test_trie(); let mut overlayed_changes = Default::default(); + let mut offchain_overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, changes_trie::disabled_state::<_, u64>(), &mut overlayed_changes, + &mut offchain_overlayed_changes, &DummyCodeExecutor { change_changes_trie_config: false, native_available: true, @@ -855,12 +865,14 @@ mod tests { fn execute_works_with_native_else_wasm() { let backend = trie_backend::tests::test_trie(); let mut overlayed_changes = Default::default(); + let mut offchain_overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, changes_trie::disabled_state::<_, u64>(), &mut overlayed_changes, + &mut offchain_overlayed_changes, &DummyCodeExecutor { change_changes_trie_config: false, native_available: true, @@ -882,12 +894,14 @@ mod tests { let mut consensus_failed = false; let backend = trie_backend::tests::test_trie(); let mut overlayed_changes = Default::default(); + let mut offchain_overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, changes_trie::disabled_state::<_, u64>(), &mut overlayed_changes, + &mut offchain_overlayed_changes, &DummyCodeExecutor { change_changes_trie_config: false, native_available: true, @@ -973,11 +987,14 @@ mod tests { ], ..Default::default() }; + let mut offchain_overlay = Default::default(); + { let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, + &mut offchain_overlay, &mut cache, backend, changes_trie::disabled_state::<_, u64>(), @@ -1003,12 +1020,16 @@ mod tests { #[test] fn set_child_storage_works() { - let mut state = InMemoryBackend::::default(); + let child_info = ChildInfo::new_default(b"sub1"); + let child_info = &child_info; + let mut state = new_in_mem::(); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, + &mut offchain_overlay, &mut cache, backend, changes_trie::disabled_state::<_, u64>(), @@ -1016,27 +1037,23 @@ mod tests { ); ext.set_child_storage( - ChildStorageKey::from_slice(b":child_storage:default:testchild").unwrap(), - CHILD_INFO_1, + child_info, b"abc".to_vec(), b"def".to_vec() ); assert_eq!( ext.child_storage( - ChildStorageKey::from_slice(b":child_storage:default:testchild").unwrap(), - CHILD_INFO_1, + child_info, b"abc" ), Some(b"def".to_vec()) ); ext.kill_child_storage( - ChildStorageKey::from_slice(b":child_storage:default:testchild").unwrap(), - CHILD_INFO_1, + child_info, ); assert_eq!( ext.child_storage( - ChildStorageKey::from_slice(b":child_storage:default:testchild").unwrap(), - CHILD_INFO_1, + child_info, b"abc" ), None @@ -1045,6 +1062,8 @@ mod tests { #[test] fn prove_read_and_proof_check_works() { + let child_info = ChildInfo::new_default(b"sub1"); + let child_info = &child_info; // fetch read proof from 'remote' full node let remote_backend = trie_backend::tests::test_trie(); let remote_root = remote_backend.storage_root(::std::iter::empty()).0; @@ -1071,20 +1090,19 @@ mod tests { let remote_root = remote_backend.storage_root(::std::iter::empty()).0; let remote_proof = prove_child_read( remote_backend, - b":child_storage:default:sub1", - CHILD_INFO_1, + child_info, &[b"value3"], ).unwrap(); let local_result1 = read_child_proof_check::( remote_root, remote_proof.clone(), - b":child_storage:default:sub1", + child_info, &[b"value3"], ).unwrap(); let local_result2 = read_child_proof_check::( remote_root, remote_proof.clone(), - b":child_storage:default:sub1", + child_info, &[b"value2"], ).unwrap(); assert_eq!( @@ -1099,25 +1117,27 @@ mod tests { #[test] fn child_storage_uuid() { - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); - const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2"); + + let child_info_1 = ChildInfo::new_default(b"sub_test1"); + let child_info_2 = ChildInfo::new_default(b"sub_test2"); + use crate::trie_backend::tests::test_trie; let mut overlay = OverlayedChanges::default(); + let mut offchain_overlay = OffchainOverlayedChanges::default(); - let subtrie1 = ChildStorageKey::from_slice(b":child_storage:default:sub_test1").unwrap(); - let subtrie2 = ChildStorageKey::from_slice(b":child_storage:default:sub_test2").unwrap(); let mut transaction = { let backend = test_trie(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, + &mut offchain_overlay, &mut cache, &backend, changes_trie::disabled_state::<_, u64>(), None, ); - ext.set_child_storage(subtrie1, CHILD_INFO_1, b"abc".to_vec(), b"def".to_vec()); - ext.set_child_storage(subtrie2, CHILD_INFO_2, b"abc".to_vec(), b"def".to_vec()); + ext.set_child_storage(&child_info_1, b"abc".to_vec(), b"def".to_vec()); + ext.set_child_storage(&child_info_2, b"abc".to_vec(), b"def".to_vec()); ext.storage_root(); cache.transaction.unwrap() }; diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index ab50c61391bdb9f696bba1db1ca4d7874e9807a4..f5b3c88ac8196cc9003f4b5f6771ae25063f9b1d 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -29,7 +29,8 @@ use crate::{ use std::iter::FromIterator; use std::collections::{HashMap, BTreeMap, BTreeSet}; use codec::{Decode, Encode}; -use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, OwnedChildInfo, ChildInfo}; +use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, ChildInfo}; +use sp_core::offchain::storage::OffchainOverlayedChanges; use std::{mem, ops}; use hash_db::Hasher; @@ -79,8 +80,8 @@ pub struct OverlayedValue { pub struct OverlayedChangeSet { /// Top level storage changes. pub top: BTreeMap, - /// Child storage changes. - pub children: HashMap, OwnedChildInfo)>, + /// Child storage changes. The map key is the child storage key without the common prefix. + pub children_default: HashMap, ChildInfo)>, } /// A storage changes structure that can be generated by the data collected in [`OverlayedChanges`]. @@ -94,9 +95,12 @@ pub struct StorageChanges { pub main_storage_changes: StorageCollection, /// All changes to the child storages. pub child_storage_changes: ChildStorageCollection, + /// Offchain state changes to write to the offchain database. + pub offchain_storage_changes: OffchainOverlayedChanges, /// A transaction for the backend that contains all changes from /// [`main_storage_changes`](Self::main_storage_changes) and from /// [`child_storage_changes`](Self::child_storage_changes). + /// [`offchain_storage_changes`](Self::offchain_storage_changes). pub transaction: Transaction, /// The storage root after applying the transaction. pub transaction_storage_root: H::Out, @@ -111,6 +115,7 @@ impl StorageChanges { pub fn into_inner(self) -> ( StorageCollection, ChildStorageCollection, + OffchainOverlayedChanges, Transaction, H::Out, Option>, @@ -118,6 +123,7 @@ impl StorageChanges { ( self.main_storage_changes, self.child_storage_changes, + self.offchain_storage_changes, self.transaction, self.transaction_storage_root, self.changes_trie_transaction, @@ -162,6 +168,7 @@ impl Default for StorageChanges Self { main_storage_changes: Default::default(), child_storage_changes: Default::default(), + offchain_storage_changes: Default::default(), transaction: Default::default(), transaction_storage_root: Default::default(), changes_trie_transaction: None, @@ -174,7 +181,7 @@ impl FromIterator<(StorageKey, OverlayedValue)> for OverlayedChangeSet { fn from_iter>(iter: T) -> Self { Self { top: iter.into_iter().collect(), - children: Default::default(), + children_default: Default::default(), } } } @@ -182,13 +189,13 @@ impl FromIterator<(StorageKey, OverlayedValue)> for OverlayedChangeSet { impl OverlayedChangeSet { /// Whether the change set is empty. pub fn is_empty(&self) -> bool { - self.top.is_empty() && self.children.is_empty() + self.top.is_empty() && self.children_default.is_empty() } /// Clear the change set. pub fn clear(&mut self) { self.top.clear(); - self.children.clear(); + self.children_default.clear(); } } @@ -216,11 +223,18 @@ impl OverlayedChanges { }) } + /// Returns mutable reference to either commited or propsective value. + pub fn value_mut(&mut self, key: &[u8]) -> Option<&mut Option> { + // not using map because of double borrow inside closure + if let Some(entry) = self.prospective.top.get_mut(key) { return Some(&mut entry.value) } + return self.committed.top.get_mut(key).map(|entry| &mut entry.value); + } + /// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose /// value has been set. - pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option> { - if let Some(map) = self.prospective.children.get(storage_key) { + pub fn child_storage(&self, child_info: &ChildInfo, key: &[u8]) -> Option> { + if let Some(map) = self.prospective.children_default.get(child_info.storage_key()) { if let Some(val) = map.0.get(key) { let size_read = val.value.as_ref().map(|x| x.len() as u64).unwrap_or(0); self.stats.tally_read_modified(size_read); @@ -228,7 +242,7 @@ impl OverlayedChanges { } } - if let Some(map) = self.committed.children.get(storage_key) { + if let Some(map) = self.committed.children_default.get(child_info.storage_key()) { if let Some(val) = map.0.get(key) { let size_read = val.value.as_ref().map(|x| x.len() as u64).unwrap_or(0); self.stats.tally_read_modified(size_read); @@ -260,15 +274,15 @@ impl OverlayedChanges { /// `None` can be used to delete a value specified by the given key. pub(crate) fn set_child_storage( &mut self, - storage_key: StorageKey, - child_info: ChildInfo, + child_info: &ChildInfo, key: StorageKey, val: Option, ) { let size_write = val.as_ref().map(|x| x.len() as u64).unwrap_or(0); self.stats.tally_write_overlay(size_write); let extrinsic_index = self.extrinsic_index(); - let map_entry = self.prospective.children.entry(storage_key) + let storage_key = child_info.storage_key().to_vec(); + let map_entry = self.prospective.children_default.entry(storage_key) .or_insert_with(|| (Default::default(), child_info.to_owned())); let updatable = map_entry.1.try_update(child_info); debug_assert!(updatable); @@ -290,11 +304,11 @@ impl OverlayedChanges { /// [`discard_prospective`]: #method.discard_prospective pub(crate) fn clear_child_storage( &mut self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, ) { let extrinsic_index = self.extrinsic_index(); - let map_entry = self.prospective.children.entry(storage_key.to_vec()) + let storage_key = child_info.storage_key(); + let map_entry = self.prospective.children_default.entry(storage_key.to_vec()) .or_insert_with(|| (Default::default(), child_info.to_owned())); let updatable = map_entry.1.try_update(child_info); debug_assert!(updatable); @@ -308,7 +322,7 @@ impl OverlayedChanges { e.value = None; }); - if let Some((committed_map, _child_info)) = self.committed.children.get(storage_key) { + if let Some((committed_map, _child_info)) = self.committed.children_default.get(storage_key) { for (key, value) in committed_map.iter() { if !map_entry.0.contains_key(key) { map_entry.0.insert(key.clone(), OverlayedValue { @@ -364,12 +378,12 @@ impl OverlayedChanges { pub(crate) fn clear_child_prefix( &mut self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], ) { let extrinsic_index = self.extrinsic_index(); - let map_entry = self.prospective.children.entry(storage_key.to_vec()) + let storage_key = child_info.storage_key(); + let map_entry = self.prospective.children_default.entry(storage_key.to_vec()) .or_insert_with(|| (Default::default(), child_info.to_owned())); let updatable = map_entry.1.try_update(child_info); debug_assert!(updatable); @@ -385,7 +399,7 @@ impl OverlayedChanges { } } - if let Some((child_committed, _child_info)) = self.committed.children.get(storage_key) { + if let Some((child_committed, _child_info)) = self.committed.children_default.get(storage_key) { // Then do the same with keys from committed changes. // NOTE that we are making changes in the prospective change set. for key in child_committed.keys() { @@ -422,8 +436,8 @@ impl OverlayedChanges { .extend(prospective_extrinsics); } } - for (storage_key, (map, child_info)) in self.prospective.children.drain() { - let child_content = self.committed.children.entry(storage_key) + for (storage_key, (map, child_info)) in self.prospective.children_default.drain() { + let child_content = self.committed.children_default.entry(storage_key) .or_insert_with(|| (Default::default(), child_info)); // No update to child info at this point (will be needed for deletion). for (key, val) in map.into_iter() { @@ -445,14 +459,14 @@ impl OverlayedChanges { /// Will panic if there are any uncommitted prospective changes. fn drain_committed(&mut self) -> ( impl Iterator)>, - impl Iterator)>, OwnedChildInfo))>, + impl Iterator)>, ChildInfo))>, ) { assert!(self.prospective.is_empty()); ( std::mem::replace(&mut self.committed.top, Default::default()) .into_iter() .map(|(k, v)| (k, v.value)), - std::mem::replace(&mut self.committed.children, Default::default()) + std::mem::replace(&mut self.committed.children_default, Default::default()) .into_iter() .map(|(sk, (v, ci))| (sk, (v.into_iter().map(|(k, v)| (k, v.value)), ci))), ) @@ -503,11 +517,13 @@ impl OverlayedChanges { .take() .expect("Changes trie transaction was generated by `changes_trie_root`; qed"); + let offchain_storage_changes = Default::default(); let (main_storage_changes, child_storage_changes) = self.drain_committed(); Ok(StorageChanges { main_storage_changes: main_storage_changes.collect(), child_storage_changes: child_storage_changes.map(|(sk, it)| (sk, it.0.collect())).collect(), + offchain_storage_changes, transaction, transaction_storage_root, changes_trie_transaction, @@ -549,21 +565,20 @@ impl OverlayedChanges { ) -> H::Out where H::Out: Ord + Encode, { - let child_storage_keys = self.prospective.children.keys() - .chain(self.committed.children.keys()); + let child_storage_keys = self.prospective.children_default.keys() + .chain(self.committed.children_default.keys()); let child_delta_iter = child_storage_keys.map(|storage_key| ( - storage_key.clone(), - self.committed.children.get(storage_key) + self.default_child_info(storage_key).cloned() + .expect("child info initialized in either committed or prospective"), + self.committed.children_default.get(storage_key) .into_iter() .flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone()))) .chain( - self.prospective.children.get(storage_key) + self.prospective.children_default.get(storage_key) .into_iter() .flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone()))) ), - self.child_info(storage_key).cloned() - .expect("child info initialized in either committed or prospective"), ) ); @@ -610,11 +625,11 @@ impl OverlayedChanges { /// Get child info for a storage key. /// Take the latest value so prospective first. - pub fn child_info(&self, storage_key: &[u8]) -> Option<&OwnedChildInfo> { - if let Some((_, ci)) = self.prospective.children.get(storage_key) { + pub fn default_child_info(&self, storage_key: &[u8]) -> Option<&ChildInfo> { + if let Some((_, ci)) = self.prospective.children_default.get(storage_key) { return Some(&ci); } - if let Some((_, ci)) = self.committed.children.get(storage_key) { + if let Some((_, ci)) = self.committed.children_default.get(storage_key) { return Some(&ci); } None @@ -654,10 +669,10 @@ impl OverlayedChanges { ) -> Option<(&[u8], &OverlayedValue)> { let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded); - let next_prospective_key = self.prospective.children.get(storage_key) + let next_prospective_key = self.prospective.children_default.get(storage_key) .and_then(|(map, _)| map.range::<[u8], _>(range).next().map(|(k, v)| (&k[..], v))); - let next_committed_key = self.committed.children.get(storage_key) + let next_committed_key = self.committed.children_default.get(storage_key) .and_then(|(map, _)| map.range::<[u8], _>(range).next().map(|(k, v)| (&k[..], v))); match (next_committed_key, next_prospective_key) { @@ -746,9 +761,11 @@ mod tests { ..Default::default() }; + let mut offchain_overlay = Default::default(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, + &mut offchain_overlay, &mut cache, &backend, crate::changes_trie::disabled_state::<_, u64>(), @@ -866,39 +883,40 @@ mod tests { #[test] fn next_child_storage_key_change_works() { - let child = b"Child1".to_vec(); - let child_info = ChildInfo::new_default(b"uniqueid"); + let child_info = ChildInfo::new_default(b"Child1"); + let child_info = &child_info; + let child = child_info.storage_key(); let mut overlay = OverlayedChanges::default(); - overlay.set_child_storage(child.clone(), child_info, vec![20], Some(vec![20])); - overlay.set_child_storage(child.clone(), child_info, vec![30], Some(vec![30])); - overlay.set_child_storage(child.clone(), child_info, vec![40], Some(vec![40])); + overlay.set_child_storage(child_info, vec![20], Some(vec![20])); + overlay.set_child_storage(child_info, vec![30], Some(vec![30])); + overlay.set_child_storage(child_info, vec![40], Some(vec![40])); overlay.commit_prospective(); - overlay.set_child_storage(child.clone(), child_info, vec![10], Some(vec![10])); - overlay.set_child_storage(child.clone(), child_info, vec![30], None); + overlay.set_child_storage(child_info, vec![10], Some(vec![10])); + overlay.set_child_storage(child_info, vec![30], None); // next_prospective < next_committed - let next_to_5 = overlay.next_child_storage_key_change(&child, &[5]).unwrap(); + let next_to_5 = overlay.next_child_storage_key_change(child, &[5]).unwrap(); assert_eq!(next_to_5.0.to_vec(), vec![10]); assert_eq!(next_to_5.1.value, Some(vec![10])); // next_committed < next_prospective - let next_to_10 = overlay.next_child_storage_key_change(&child, &[10]).unwrap(); + let next_to_10 = overlay.next_child_storage_key_change(child, &[10]).unwrap(); assert_eq!(next_to_10.0.to_vec(), vec![20]); assert_eq!(next_to_10.1.value, Some(vec![20])); // next_committed == next_prospective - let next_to_20 = overlay.next_child_storage_key_change(&child, &[20]).unwrap(); + let next_to_20 = overlay.next_child_storage_key_change(child, &[20]).unwrap(); assert_eq!(next_to_20.0.to_vec(), vec![30]); assert_eq!(next_to_20.1.value, None); // next_committed, no next_prospective - let next_to_30 = overlay.next_child_storage_key_change(&child, &[30]).unwrap(); + let next_to_30 = overlay.next_child_storage_key_change(child, &[30]).unwrap(); assert_eq!(next_to_30.0.to_vec(), vec![40]); assert_eq!(next_to_30.1.value, Some(vec![40])); - overlay.set_child_storage(child.clone(), child_info, vec![50], Some(vec![50])); + overlay.set_child_storage(child_info, vec![50], Some(vec![50])); // next_prospective, no next_committed - let next_to_40 = overlay.next_child_storage_key_change(&child, &[40]).unwrap(); + let next_to_40 = overlay.next_child_storage_key_change(child, &[40]).unwrap(); assert_eq!(next_to_40.0.to_vec(), vec![50]); assert_eq!(next_to_40.1.value, Some(vec![50])); } diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 747872af831ac44fbebaccf0c6c571b735b4f73a..1cb281b070b4c104bae235426a579e8316c284e8 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -22,7 +22,7 @@ use codec::{Decode, Codec}; use log::debug; use hash_db::{Hasher, HashDB, EMPTY_PREFIX, Prefix}; use sp_trie::{ - MemoryDB, default_child_trie_root, read_trie_value_with, read_child_trie_value_with, + MemoryDB, empty_child_trie_root, read_trie_value_with, read_child_trie_value_with, record_all_keys, StorageProof, }; pub use sp_trie::Recorder; @@ -67,13 +67,13 @@ impl<'a, S, H> ProvingBackendRecorder<'a, S, H> /// Produce proof for a child key query. pub fn child_storage( &mut self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8] ) -> Result>, String> { + let storage_key = child_info.storage_key(); let root = self.storage(storage_key)? .and_then(|r| Decode::decode(&mut &r[..]).ok()) - .unwrap_or(default_child_trie_root::>(storage_key)); + .unwrap_or(empty_child_trie_root::>()); let mut read_overlay = S::Overlay::default(); let eph = Ephemeral::new( @@ -84,7 +84,6 @@ impl<'a, S, H> ProvingBackendRecorder<'a, S, H> let map_e = |e| format!("Trie lookup error: {}", e); read_child_trie_value_with::, _, _>( - storage_key, child_info.keyspace(), &eph, &root.as_ref(), @@ -201,20 +200,18 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> fn child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result>, Self::Error> { - self.0.child_storage(storage_key, child_info, key) + self.0.child_storage(child_info, key) } fn for_keys_in_child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, f: F, ) { - self.0.for_keys_in_child_storage(storage_key, child_info, f) + self.0.for_keys_in_child_storage(child_info, f) } fn next_storage_key(&self, key: &[u8]) -> Result>, Self::Error> { @@ -223,11 +220,10 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> fn next_child_storage_key( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result>, Self::Error> { - self.0.next_child_storage_key(storage_key, child_info, key) + self.0.next_child_storage_key(child_info, key) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { @@ -240,12 +236,11 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> fn for_child_keys_with_prefix( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], f: F, ) { - self.0.for_child_keys_with_prefix(storage_key, child_info, prefix, f) + self.0.for_child_keys_with_prefix( child_info, prefix, f) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -258,11 +253,10 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> fn child_keys( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], ) -> Vec> { - self.0.child_keys(storage_key, child_info, prefix) + self.0.child_keys(child_info, prefix) } fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) @@ -273,15 +267,14 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> fn child_storage_root( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, delta: I, ) -> (H::Out, bool, Self::Transaction) where I: IntoIterator, Option>)>, H::Out: Ord { - self.0.child_storage_root(storage_key, child_info, delta) + self.0.child_storage_root(child_info, delta) } fn register_overlay_stats(&mut self, _stats: &crate::stats::StateMachineStats) { } @@ -314,14 +307,10 @@ mod tests { use crate::InMemoryBackend; use crate::trie_backend::tests::test_trie; use super::*; - use sp_core::storage::ChildStorageKey; use crate::proving_backend::create_proof_check_backend; use sp_trie::PrefixedMemoryDB; use sp_runtime::traits::BlakeTwo256; - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); - const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2"); - fn test_proving<'a>( trie_backend: &'a TrieBackend,BlakeTwo256>, ) -> ProvingBackend<'a, PrefixedMemoryDB, BlakeTwo256> { @@ -389,33 +378,34 @@ mod tests { #[test] fn proof_recorded_and_checked_with_child() { - let subtrie1 = ChildStorageKey::from_slice(b":child_storage:default:sub1").unwrap(); - let subtrie2 = ChildStorageKey::from_slice(b":child_storage:default:sub2").unwrap(); - let own1 = subtrie1.into_owned(); - let own2 = subtrie2.into_owned(); + let child_info_1 = ChildInfo::new_default(b"sub1"); + let child_info_2 = ChildInfo::new_default(b"sub2"); + let child_info_1 = &child_info_1; + let child_info_2 = &child_info_2; let contents = vec![ (None, (0..64).map(|i| (vec![i], Some(vec![i]))).collect()), - (Some((own1.clone(), CHILD_INFO_1.to_owned())), + (Some(child_info_1.clone()), (28..65).map(|i| (vec![i], Some(vec![i]))).collect()), - (Some((own2.clone(), CHILD_INFO_2.to_owned())), + (Some(child_info_2.clone()), (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), ]; let in_memory = InMemoryBackend::::default(); let mut in_memory = in_memory.update(contents); + let child_storage_keys = vec![child_info_1.to_owned(), child_info_2.to_owned()]; let in_memory_root = in_memory.full_storage_root::<_, Vec<_>, _>( ::std::iter::empty(), - in_memory.child_storage_keys().map(|k|(k.0.to_vec(), Vec::new(), k.1.to_owned())) + child_storage_keys.into_iter().map(|k|(k.to_owned(), Vec::new())) ).0; (0..64).for_each(|i| assert_eq!( in_memory.storage(&[i]).unwrap().unwrap(), vec![i] )); (28..65).for_each(|i| assert_eq!( - in_memory.child_storage(&own1[..], CHILD_INFO_1, &[i]).unwrap().unwrap(), + in_memory.child_storage(child_info_1, &[i]).unwrap().unwrap(), vec![i] )); (10..15).for_each(|i| assert_eq!( - in_memory.child_storage(&own2[..], CHILD_INFO_2, &[i]).unwrap().unwrap(), + in_memory.child_storage(child_info_2, &[i]).unwrap().unwrap(), vec![i] )); @@ -443,7 +433,7 @@ mod tests { assert_eq!(proof_check.storage(&[64]).unwrap(), None); let proving = ProvingBackend::new(trie); - assert_eq!(proving.child_storage(&own1[..], CHILD_INFO_1, &[64]), Ok(Some(vec![64]))); + assert_eq!(proving.child_storage(child_info_1, &[64]), Ok(Some(vec![64]))); let proof = proving.extract_proof(); let proof_check = create_proof_check_backend::( @@ -451,7 +441,7 @@ mod tests { proof ).unwrap(); assert_eq!( - proof_check.child_storage(&own1[..], CHILD_INFO_1, &[64]).unwrap().unwrap(), + proof_check.child_storage(child_info_1, &[64]).unwrap().unwrap(), vec![64] ); } diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index aec42c76787074597a6e766a34042a1af574a714..70ee1f77d2e65387589f4af442f17d4d1fd8549d 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -30,6 +30,7 @@ use crate::{ }, }; use sp_core::{ + offchain::storage::OffchainOverlayedChanges, storage::{ well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key}, Storage, @@ -41,9 +42,10 @@ use sp_externalities::{Extensions, Extension}; /// Simple HashMap-based Externalities impl. pub struct TestExternalities where - H::Out: codec::Codec, + H::Out: codec::Codec + Ord, { overlay: OverlayedChanges, + offchain_overlay: OffchainOverlayedChanges, storage_transaction_cache: StorageTransactionCache< as Backend>::Transaction, H, N >, @@ -61,6 +63,7 @@ impl TestExternalities pub fn ext(&mut self) -> Ext> { Ext::new( &mut self.overlay, + &mut self.offchain_overlay, &mut self.storage_transaction_cache, &self.backend, match self.changes_trie_config.clone() { @@ -80,6 +83,11 @@ impl TestExternalities Self::new_with_code(&[], storage) } + /// New empty test externalities. + pub fn new_empty() -> Self { + Self::new_with_code(&[], Storage::default()) + } + /// Create a new instance of `TestExternalities` with code and storage. pub fn new_with_code(code: &[u8], mut storage: Storage) -> Self { let mut overlay = OverlayedChanges::default(); @@ -88,24 +96,31 @@ impl TestExternalities overlay.set_collect_extrinsics(changes_trie_config.is_some()); assert!(storage.top.keys().all(|key| !is_child_storage_key(key))); - assert!(storage.children.keys().all(|key| is_child_storage_key(key))); + assert!(storage.children_default.keys().all(|key| is_child_storage_key(key))); storage.top.insert(HEAP_PAGES.to_vec(), 8u64.encode()); storage.top.insert(CODE.to_vec(), code.to_vec()); + let offchain_overlay = OffchainOverlayedChanges::enabled(); + + let mut extensions = Extensions::default(); + extensions.register(sp_core::traits::TaskExecutorExt(sp_core::tasks::executor())); + + TestExternalities { overlay, + offchain_overlay, changes_trie_config, + extensions, changes_trie_storage: ChangesTrieInMemoryStorage::new(), backend: storage.into(), - extensions: Default::default(), storage_transaction_cache: Default::default(), } } /// Insert key/value into backend pub fn insert(&mut self, k: StorageKey, v: StorageValue) { - self.backend = self.backend.update(vec![(None, vec![(k, Some(v))])]); + self.backend.insert(vec![(None, vec![(k, Some(v))])]); } /// Registers the given extension for this instance. @@ -125,11 +140,11 @@ impl TestExternalities .map(|(k, v)| (k, v.value)).collect(); let mut transaction = vec![(None, top)]; - self.overlay.committed.children.clone().into_iter() - .chain(self.overlay.prospective.children.clone().into_iter()) - .for_each(|(keyspace, (map, child_info))| { + self.overlay.committed.children_default.clone().into_iter() + .chain(self.overlay.prospective.children_default.clone().into_iter()) + .for_each(|(_storage_key, (map, child_info))| { transaction.push(( - Some((keyspace, child_info)), + Some(child_info), map.into_iter() .map(|(k, v)| (k, v.value)) .collect::>(), @@ -149,7 +164,7 @@ impl TestExternalities } impl std::fmt::Debug for TestExternalities - where H::Out: codec::Codec, + where H::Out: Ord + codec::Codec, { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "overlay: {:?}\nbackend: {:?}", self.overlay, self.backend.pairs()) @@ -185,12 +200,27 @@ impl From for TestExternalities sp_externalities::ExtensionStore for TestExternalities where H: Hasher, - H::Out: codec::Codec, + H::Out: Ord + codec::Codec, N: ChangesTrieBlockNumber, { fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> { self.extensions.get_mut(type_id) } + + fn register_extension_with_type_id( + &mut self, + type_id: TypeId, + extension: Box, + ) -> Result<(), sp_externalities::Error> { + self.extensions.register_with_type_id(type_id, extension) + } + + fn deregister_extension_by_type_id(&mut self, type_id: TypeId) -> Result<(), sp_externalities::Error> { + self.extensions + .deregister(type_id) + .expect("There should be an extension we try to remove in TestExternalities"); + Ok(()) + } } #[cfg(test)] @@ -207,7 +237,7 @@ mod tests { ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); ext.set_storage(b"dog".to_vec(), b"puppy".to_vec()); ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec()); - const ROOT: [u8; 32] = hex!("2a340d3dfd52f5992c6b117e9e45f479e6da5afffafeb26ab619cf137a95aeb8"); + const ROOT: [u8; 32] = hex!("555d4777b52e9196e3f6373c556cc661e79cd463f881ab9e921e70fc30144bf4"); assert_eq!(&ext.storage_root()[..], &ROOT); } diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index f88e306a2fc3dad8ee1b7cfa6e45ead0f7e1a8b9..c757f05f5df42a617450c60c26050fa3aa16ad79 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -18,9 +18,9 @@ use log::{warn, debug}; use hash_db::Hasher; -use sp_trie::{Trie, delta_trie_root, default_child_trie_root, child_delta_trie_root}; +use sp_trie::{Trie, delta_trie_root, empty_child_trie_root, child_delta_trie_root}; use sp_trie::trie_types::{TrieDB, TrieError, Layout}; -use sp_core::storage::ChildInfo; +use sp_core::storage::{ChildInfo, ChildType}; use codec::{Codec, Decode}; use crate::{ StorageKey, StorageValue, Backend, @@ -29,7 +29,7 @@ use crate::{ /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. pub struct TrieBackend, H: Hasher> { - essence: TrieBackendEssence, + pub (crate) essence: TrieBackendEssence, } impl, H: Hasher> TrieBackend where H::Out: Codec { @@ -50,6 +50,11 @@ impl, H: Hasher> TrieBackend where H::Out: Codec self.essence.backend_storage() } + /// Get backend storage reference. + pub fn backend_storage_mut(&mut self) -> &mut S { + self.essence.backend_storage_mut() + } + /// Get trie root. pub fn root(&self) -> &H::Out { self.essence.root() @@ -80,11 +85,10 @@ impl, H: Hasher> Backend for TrieBackend where fn child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result, Self::Error> { - self.essence.child_storage(storage_key, child_info, key) + self.essence.child_storage(child_info, key) } fn next_storage_key(&self, key: &[u8]) -> Result, Self::Error> { @@ -93,11 +97,10 @@ impl, H: Hasher> Backend for TrieBackend where fn next_child_storage_key( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result, Self::Error> { - self.essence.next_child_storage_key(storage_key, child_info, key) + self.essence.next_child_storage_key(child_info, key) } fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { @@ -110,29 +113,24 @@ impl, H: Hasher> Backend for TrieBackend where fn for_keys_in_child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, f: F, ) { - self.essence.for_keys_in_child_storage(storage_key, child_info, f) + self.essence.for_keys_in_child_storage(child_info, f) } fn for_child_keys_with_prefix( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], f: F, ) { - self.essence.for_child_keys_with_prefix(storage_key, child_info, prefix, f) + self.essence.for_child_keys_with_prefix(child_info, prefix, f) } fn pairs(&self) -> Vec<(StorageKey, StorageValue)> { - let mut read_overlay = S::Overlay::default(); - let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay); - let collect_all = || -> Result<_, Box>> { - let trie = TrieDB::::new(&eph, self.essence.root())?; + let trie = TrieDB::::new(self.essence(), self.essence.root())?; let mut v = Vec::new(); for x in trie.iter()? { let (key, value) = x?; @@ -152,11 +150,8 @@ impl, H: Hasher> Backend for TrieBackend where } fn keys(&self, prefix: &[u8]) -> Vec { - let mut read_overlay = S::Overlay::default(); - let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay); - let collect_all = || -> Result<_, Box>> { - let trie = TrieDB::::new(&eph, self.essence.root())?; + let trie = TrieDB::::new(self.essence(), self.essence.root())?; let mut v = Vec::new(); for x in trie.iter()? { let (key, _) = x?; @@ -183,6 +178,7 @@ impl, H: Hasher> Backend for TrieBackend where &mut write_overlay, ); + let delta: Vec<_> = delta.into_iter().collect(); match delta_trie_root::, _, _, _, _>(&mut eph, root, delta) { Ok(ret) => root = ret, Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e), @@ -194,18 +190,20 @@ impl, H: Hasher> Backend for TrieBackend where fn child_storage_root( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, delta: I, ) -> (H::Out, bool, Self::Transaction) where I: IntoIterator)>, H::Out: Ord, { - let default_root = default_child_trie_root::>(storage_key); + let default_root = match child_info.child_type() { + ChildType::ParentKeyId => empty_child_trie_root::>() + }; let mut write_overlay = S::Overlay::default(); - let mut root = match self.storage(storage_key) { + let prefixed_storage_key = child_info.prefixed_storage_key(); + let mut root = match self.storage(prefixed_storage_key.as_slice()) { Ok(value) => value.and_then(|r| Decode::decode(&mut &r[..]).ok()).unwrap_or(default_root.clone()), Err(e) => { @@ -221,7 +219,6 @@ impl, H: Hasher> Backend for TrieBackend where ); match child_delta_trie_root::, _, _, _, _, _>( - storage_key, child_info.keyspace(), &mut eph, root, @@ -246,6 +243,10 @@ impl, H: Hasher> Backend for TrieBackend where fn usage_info(&self) -> crate::UsageInfo { crate::UsageInfo::empty() } + + fn wipe(&self) -> Result<(), Self::Error> { + Ok(()) + } } #[cfg(test)] @@ -257,16 +258,14 @@ pub mod tests { use sp_runtime::traits::BlakeTwo256; use super::*; - const CHILD_KEY_1: &[u8] = b":child_storage:default:sub1"; - - const CHILD_UUID_1: &[u8] = b"unique_id_1"; - const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(CHILD_UUID_1); + const CHILD_KEY_1: &[u8] = b"sub1"; fn test_db() -> (PrefixedMemoryDB, H256) { + let child_info = ChildInfo::new_default(CHILD_KEY_1); let mut root = H256::default(); let mut mdb = PrefixedMemoryDB::::default(); { - let mut mdb = KeySpacedDBMut::new(&mut mdb, CHILD_UUID_1); + let mut mdb = KeySpacedDBMut::new(&mut mdb, child_info.keyspace()); let mut trie = TrieDBMut::new(&mut mdb, &mut root); trie.insert(b"value3", &[142]).expect("insert failed"); trie.insert(b"value4", &[124]).expect("insert failed"); @@ -276,7 +275,8 @@ pub mod tests { let mut sub_root = Vec::new(); root.encode_to(&mut sub_root); let mut trie = TrieDBMut::new(&mut mdb, &mut root); - trie.insert(CHILD_KEY_1, &sub_root[..]).expect("insert failed"); + trie.insert(child_info.prefixed_storage_key().as_slice(), &sub_root[..]) + .expect("insert failed"); trie.insert(b"key", b"value").expect("insert failed"); trie.insert(b"value1", &[42]).expect("insert failed"); trie.insert(b"value2", &[24]).expect("insert failed"); @@ -302,7 +302,7 @@ pub mod tests { fn read_from_child_storage_returns_some() { let test_trie = test_trie(); assert_eq!( - test_trie.child_storage(CHILD_KEY_1, CHILD_INFO_1, b"value3").unwrap(), + test_trie.child_storage(&ChildInfo::new_default(CHILD_KEY_1), b"value3").unwrap(), Some(vec![142u8]), ); } diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 125a823f57fc1c505118b4ccde480ec017fd1129..8f1b8c7a58017ce145cb4eb5daff518feb2cc016 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -20,9 +20,9 @@ use std::ops::Deref; use std::sync::Arc; use log::{debug, warn}; -use hash_db::{self, Hasher, EMPTY_PREFIX, Prefix}; +use hash_db::{self, Hasher, Prefix}; use sp_trie::{Trie, MemoryDB, PrefixedMemoryDB, DBValue, - default_child_trie_root, read_trie_value, read_child_trie_value, + empty_child_trie_root, read_trie_value, read_child_trie_value, for_keys_in_child_trie, KeySpacedDB, TrieDBIterator}; use sp_trie::trie_types::{TrieDB, TrieError, Layout}; use crate::{backend::Consolidate, StorageKey, StorageValue}; @@ -39,6 +39,7 @@ pub trait Storage: Send + Sync { pub struct TrieBackendEssence, H: Hasher> { storage: S, root: H::Out, + empty: H::Out, } impl, H: Hasher> TrieBackendEssence where H::Out: Encode { @@ -47,6 +48,7 @@ impl, H: Hasher> TrieBackendEssence where H::Out: TrieBackendEssence { storage, root, + empty: H::hash(&[0u8]), } } @@ -55,11 +57,21 @@ impl, H: Hasher> TrieBackendEssence where H::Out: &self.storage } + /// Get backend storage reference. + pub fn backend_storage_mut(&mut self) -> &mut S { + &mut self.storage + } + /// Get trie root. pub fn root(&self) -> &H::Out { &self.root } + /// Set trie root. This is useful for testing. + pub fn set_root(&mut self, root: H::Out) { + self.root = root; + } + /// Consumes self and returns underlying storage. pub fn into_storage(self) -> S { self.storage @@ -71,15 +83,19 @@ impl, H: Hasher> TrieBackendEssence where H::Out: self.next_storage_key_from_root(&self.root, None, key) } + /// Access the root of the child storage in its parent trie + fn child_root(&self, child_info: &ChildInfo) -> Result, String> { + self.storage(child_info.prefixed_storage_key().as_slice()) + } + /// Return the next key in the child trie i.e. the minimum key that is strictly superior to /// `key` in lexicographic order. pub fn next_child_storage_key( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result, String> { - let child_root = match self.storage(storage_key)? { + let child_root = match self.child_root(child_info)? { Some(child_root) => child_root, None => return Ok(None), }; @@ -87,7 +103,7 @@ impl, H: Hasher> TrieBackendEssence where H::Out: let mut hash = H::Out::default(); if child_root.len() != hash.as_ref().len() { - return Err(format!("Invalid child storage hash at {:?}", storage_key)); + return Err(format!("Invalid child storage hash at {:?}", child_info.storage_key())); } // note: child_root and hash must be same size, panics otherwise. hash.as_mut().copy_from_slice(&child_root[..]); @@ -99,21 +115,16 @@ impl, H: Hasher> TrieBackendEssence where H::Out: fn next_storage_key_from_root( &self, root: &H::Out, - child_info: Option, + child_info: Option<&ChildInfo>, key: &[u8], ) -> Result, String> { - let mut read_overlay = S::Overlay::default(); - let eph = Ephemeral { - storage: &self.storage, - overlay: &mut read_overlay, - }; let dyn_eph: &dyn hash_db::HashDBRef<_, _>; let keyspace_eph; if let Some(child_info) = child_info.as_ref() { - keyspace_eph = KeySpacedDB::new(&eph, child_info.keyspace()); + keyspace_eph = KeySpacedDB::new(self, child_info.keyspace()); dyn_eph = &keyspace_eph; } else { - dyn_eph = &eph; + dyn_eph = self; } let trie = TrieDB::::new(dyn_eph, root) @@ -147,64 +158,43 @@ impl, H: Hasher> TrieBackendEssence where H::Out: /// Get the value of storage at given key. pub fn storage(&self, key: &[u8]) -> Result, String> { - let mut read_overlay = S::Overlay::default(); - let eph = Ephemeral { - storage: &self.storage, - overlay: &mut read_overlay, - }; - let map_e = |e| format!("Trie lookup error: {}", e); - read_trie_value::, _>(&eph, &self.root, key).map_err(map_e) + read_trie_value::, _>(self, &self.root, key).map_err(map_e) } /// Get the value of child storage at given key. pub fn child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, key: &[u8], ) -> Result, String> { - let root = self.storage(storage_key)? - .unwrap_or(default_child_trie_root::>(storage_key).encode()); - - let mut read_overlay = S::Overlay::default(); - let eph = Ephemeral { - storage: &self.storage, - overlay: &mut read_overlay, - }; + let root = self.child_root(child_info)? + .unwrap_or(empty_child_trie_root::>().encode()); let map_e = |e| format!("Trie lookup error: {}", e); - read_child_trie_value::, _>(storage_key, child_info.keyspace(), &eph, &root, key) + read_child_trie_value::, _>(child_info.keyspace(), self, &root, key) .map_err(map_e) } /// Retrieve all entries keys of child storage and call `f` for each of those keys. pub fn for_keys_in_child_storage( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, f: F, ) { - let root = match self.storage(storage_key) { - Ok(v) => v.unwrap_or(default_child_trie_root::>(storage_key).encode()), + let root = match self.child_root(child_info) { + Ok(v) => v.unwrap_or(empty_child_trie_root::>().encode()), Err(e) => { debug!(target: "trie", "Error while iterating child storage: {}", e); return; } }; - let mut read_overlay = S::Overlay::default(); - let eph = Ephemeral { - storage: &self.storage, - overlay: &mut read_overlay, - }; - - if let Err(e) = for_keys_in_child_trie::, _, Ephemeral>( - storage_key, + if let Err(e) = for_keys_in_child_trie::, _, _>( child_info.keyspace(), - &eph, + self, &root, f, ) { @@ -215,13 +205,12 @@ impl, H: Hasher> TrieBackendEssence where H::Out: /// Execute given closure for all keys starting with prefix. pub fn for_child_keys_with_prefix( &self, - storage_key: &[u8], - child_info: ChildInfo, + child_info: &ChildInfo, prefix: &[u8], mut f: F, ) { - let root_vec = match self.storage(storage_key) { - Ok(v) => v.unwrap_or(default_child_trie_root::>(storage_key).encode()), + let root_vec = match self.child_root(child_info) { + Ok(v) => v.unwrap_or(empty_child_trie_root::>().encode()), Err(e) => { debug!(target: "trie", "Error while iterating child storage: {}", e); return; @@ -242,14 +231,8 @@ impl, H: Hasher> TrieBackendEssence where H::Out: root: &H::Out, prefix: &[u8], mut f: F, - child_info: Option, + child_info: Option<&ChildInfo>, ) { - let mut read_overlay = S::Overlay::default(); - let eph = Ephemeral { - storage: &self.storage, - overlay: &mut read_overlay, - }; - let mut iter = move |db| -> Result<(), Box>> { let trie = TrieDB::::new(db, root)?; @@ -265,10 +248,10 @@ impl, H: Hasher> TrieBackendEssence where H::Out: }; let result = if let Some(child_info) = child_info { - let db = KeySpacedDB::new(&eph, child_info.keyspace()); + let db = KeySpacedDB::new(self, child_info.keyspace()); iter(&db) } else { - iter(&eph) + iter(self) }; if let Err(e) = result { debug!(target: "trie", "Error while iterating by prefix: {}", e); @@ -286,15 +269,6 @@ pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { overlay: &'a mut S::Overlay, } -impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> hash_db::AsPlainDB - for Ephemeral<'a, S, H> -{ - fn as_plain_db<'b>(&'b self) -> &'b (dyn hash_db::PlainDB + 'b) { self } - fn as_plain_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::PlainDB + 'b) { - self - } -} - impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> hash_db::AsHashDB for Ephemeral<'a, S, H> { @@ -311,43 +285,6 @@ impl<'a, S: TrieBackendStorage, H: Hasher> Ephemeral<'a, S, H> { } } -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::PlainDB - for Ephemeral<'a, S, H> -{ - fn get(&self, key: &H::Out) -> Option { - if let Some(val) = hash_db::HashDB::get(self.overlay, key, EMPTY_PREFIX) { - Some(val) - } else { - match self.storage.get(&key, EMPTY_PREFIX) { - Ok(x) => x, - Err(e) => { - warn!(target: "trie", "Failed to read from DB: {}", e); - None - }, - } - } - } - - fn contains(&self, key: &H::Out) -> bool { - hash_db::HashDB::get(self, key, EMPTY_PREFIX).is_some() - } - - fn emplace(&mut self, key: H::Out, value: DBValue) { - hash_db::HashDB::emplace(self.overlay, key, EMPTY_PREFIX, value) - } - - fn remove(&mut self, key: &H::Out) { - hash_db::HashDB::remove(self.overlay, key, EMPTY_PREFIX) - } -} - -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::PlainDBRef - for Ephemeral<'a, S, H> -{ - fn get(&self, key: &H::Out) -> Option { hash_db::PlainDB::get(self, key) } - fn contains(&self, key: &H::Out) -> bool { hash_db::PlainDB::contains(self, key) } -} - impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB for Ephemeral<'a, S, H> { @@ -428,6 +365,59 @@ impl TrieBackendStorage for MemoryDB { } } +impl, H: Hasher> hash_db::AsHashDB + for TrieBackendEssence +{ + fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB + 'b) { self } + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { self } +} + +impl, H: Hasher> hash_db::HashDB + for TrieBackendEssence +{ + fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + if *key == self.empty { + return Some([0u8].to_vec()) + } + match self.storage.get(&key, prefix) { + Ok(x) => x, + Err(e) => { + warn!(target: "trie", "Failed to read from DB: {}", e); + None + }, + } + } + + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + hash_db::HashDB::get(self, key, prefix).is_some() + } + + fn insert(&mut self, _prefix: Prefix, _value: &[u8]) -> H::Out { + unimplemented!(); + } + + fn emplace(&mut self, _key: H::Out, _prefix: Prefix, _value: DBValue) { + unimplemented!(); + } + + fn remove(&mut self, _key: &H::Out, _prefix: Prefix) { + unimplemented!(); + } +} + +impl, H: Hasher> hash_db::HashDBRef + for TrieBackendEssence +{ + fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + hash_db::HashDB::get(self, key, prefix) + } + + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + hash_db::HashDB::contains(self, key, prefix) + } +} + + #[cfg(test)] mod test { use sp_core::{Blake2Hasher, H256}; @@ -436,7 +426,8 @@ mod test { #[test] fn next_storage_key_and_next_child_storage_key_work() { - let child_info = ChildInfo::new_default(b"uniqueid"); + let child_info = ChildInfo::new_default(b"MyChild"); + let child_info = &child_info; // Contains values let mut root_1 = H256::default(); // Contains child trie @@ -460,7 +451,8 @@ mod test { } { let mut trie = TrieDBMut::new(&mut mdb, &mut root_2); - trie.insert(b"MyChild", root_1.as_ref()).expect("insert failed"); + trie.insert(child_info.prefixed_storage_key().as_slice(), root_1.as_ref()) + .expect("insert failed"); }; let essence_1 = TrieBackendEssence::new(mdb, root_1); @@ -475,19 +467,19 @@ mod test { let essence_2 = TrieBackendEssence::new(mdb, root_2); assert_eq!( - essence_2.next_child_storage_key(b"MyChild", child_info, b"2"), Ok(Some(b"3".to_vec())) + essence_2.next_child_storage_key(child_info, b"2"), Ok(Some(b"3".to_vec())) ); assert_eq!( - essence_2.next_child_storage_key(b"MyChild", child_info, b"3"), Ok(Some(b"4".to_vec())) + essence_2.next_child_storage_key(child_info, b"3"), Ok(Some(b"4".to_vec())) ); assert_eq!( - essence_2.next_child_storage_key(b"MyChild", child_info, b"4"), Ok(Some(b"6".to_vec())) + essence_2.next_child_storage_key(child_info, b"4"), Ok(Some(b"6".to_vec())) ); assert_eq!( - essence_2.next_child_storage_key(b"MyChild", child_info, b"5"), Ok(Some(b"6".to_vec())) + essence_2.next_child_storage_key(child_info, b"5"), Ok(Some(b"6".to_vec())) ); assert_eq!( - essence_2.next_child_storage_key(b"MyChild", child_info, b"6"), Ok(None) + essence_2.next_child_storage_key(child_info, b"6"), Ok(None) ); } } diff --git a/primitives/std/Cargo.toml b/primitives/std/Cargo.toml index 58ff78f2bbb510adc98a7de88415503cf71b0061..2b58167f17e92880d488caaa0f125290dbdf7fcf 100644 --- a/primitives/std/Cargo.toml +++ b/primitives/std/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-std" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -10,9 +10,9 @@ repository = "https://github.com/paritytech/substrate/" description = "Lowest-abstraction level for the Substrate runtime: just exports useful primitives from std or client/alloc to be used with any code that depends on the runtime." documentation = "https://docs.rs/sp-std" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [features] default = ["std"] std = [] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/storage/Cargo.toml b/primitives/storage/Cargo.toml index e4e842848dd0fe002e613099352b9f731bfeeeba..76174d13b03c720a824b5972e67493f13ffe478a 100644 --- a/primitives/storage/Cargo.toml +++ b/primitives/storage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-storage" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" description = "Storage related primitives" @@ -9,15 +9,16 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sp-storage/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } serde = { version = "1.0.101", optional = true, features = ["derive"] } impl-serde = { version = "0.2.3", optional = true } -sp-debug-derive = { version = "2.0.0-alpha.5", path = "../debug-derive" } +ref-cast = "1.0.0" +sp-debug-derive = { version = "2.0.0-dev", path = "../debug-derive" } [features] default = [ "std" ] std = [ "sp-std/std", "serde", "impl-serde" ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index 76fd4baac9cf323c51c9cd94f597b79f68b46e77..d2c4a73e23d59a138ff7b0602037004656f8942e 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -22,7 +22,9 @@ use serde::{Serialize, Deserialize}; use sp_debug_derive::RuntimeDebug; -use sp_std::{vec::Vec, borrow::Cow}; +use sp_std::vec::Vec; +use sp_std::ops::{Deref, DerefMut}; +use ref_cast::RefCast; /// Storage key. #[derive(PartialEq, Eq, RuntimeDebug)] @@ -32,6 +34,51 @@ pub struct StorageKey( pub Vec, ); +/// Storage key of a child trie, it contains the prefix to the key. +#[derive(PartialEq, Eq, RuntimeDebug)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone))] +#[repr(transparent)] +#[derive(RefCast)] +pub struct PrefixedStorageKey( + #[cfg_attr(feature = "std", serde(with="impl_serde::serialize"))] + Vec, +); + +impl Deref for PrefixedStorageKey { + type Target = Vec; + + fn deref(&self) -> &Vec { + &self.0 + } +} + +impl DerefMut for PrefixedStorageKey { + fn deref_mut(&mut self) -> &mut Vec { + &mut self.0 + } +} + +impl PrefixedStorageKey { + /// Create a prefixed storage key from its byte array + /// representation. + pub fn new(inner: Vec) -> Self { + PrefixedStorageKey(inner) + } + + /// Create a prefixed storage key reference. + pub fn new_ref(inner: &Vec) -> &Self { + PrefixedStorageKey::ref_cast(inner) + } + + /// Get inner key, this should + /// only be needed when writing + /// into parent trie to avoid an + /// allocation. + pub fn into_inner(self) -> Vec { + self.0 + } +} + /// Storage data associated to a [`StorageKey`]. #[derive(PartialEq, Eq, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone))] @@ -53,7 +100,7 @@ pub struct StorageChild { pub data: StorageMap, /// Associated child info for a child /// trie. - pub child_info: OwnedChildInfo, + pub child_info: ChildInfo, } #[cfg(feature = "std")] @@ -62,8 +109,11 @@ pub struct StorageChild { pub struct Storage { /// Top trie storage data. pub top: StorageMap, - /// Children trie storage data by storage key. - pub children: std::collections::HashMap, StorageChild>, + /// Children trie storage data. + /// The key does not including prefix, for the `default` + /// trie kind, so this is exclusively for the `ChildType::ParentKeyId` + /// tries. + pub children_default: std::collections::HashMap, StorageChild>, } /// Storage change set @@ -106,133 +156,87 @@ pub mod well_known_keys { // Other code might depend on this, so be careful changing this. key.starts_with(CHILD_STORAGE_KEY_PREFIX) } - - /// Determine whether a child trie key is valid. - /// - /// For now, the only valid child trie keys are those starting with `:child_storage:default:`. - /// - /// `child_trie_root` and `child_delta_trie_root` can panic if invalid value is provided to them. - pub fn is_child_trie_key_valid(storage_key: &[u8]) -> bool { - let has_right_prefix = storage_key.starts_with(b":child_storage:default:"); - if has_right_prefix { - // This is an attempt to catch a change of `is_child_storage_key`, which - // just checks if the key has prefix `:child_storage:` at the moment of writing. - debug_assert!( - is_child_storage_key(&storage_key), - "`is_child_trie_key_valid` is a subset of `is_child_storage_key`", - ); - } - has_right_prefix - } -} - -/// A wrapper around a child storage key. -/// -/// This wrapper ensures that the child storage key is correct and properly used. It is -/// impossible to create an instance of this struct without providing a correct `storage_key`. -pub struct ChildStorageKey<'a> { - storage_key: Cow<'a, [u8]>, } -impl<'a> ChildStorageKey<'a> { - /// Create new instance of `Self`. - fn new(storage_key: Cow<'a, [u8]>) -> Option { - if well_known_keys::is_child_trie_key_valid(&storage_key) { - Some(ChildStorageKey { storage_key }) - } else { - None - } - } - - /// Create a new `ChildStorageKey` from a vector. - /// - /// `storage_key` need to start with `:child_storage:default:` - /// See `is_child_trie_key_valid` for more details. - pub fn from_vec(key: Vec) -> Option { - Self::new(Cow::Owned(key)) - } - - /// Create a new `ChildStorageKey` from a slice. - /// - /// `storage_key` need to start with `:child_storage:default:` - /// See `is_child_trie_key_valid` for more details. - pub fn from_slice(key: &'a [u8]) -> Option { - Self::new(Cow::Borrowed(key)) - } - - /// Get access to the byte representation of the storage key. - /// - /// This key is guaranteed to be correct. - pub fn as_ref(&self) -> &[u8] { - &*self.storage_key - } - - /// Destruct this instance into an owned vector that represents the storage key. - /// - /// This key is guaranteed to be correct. - pub fn into_owned(self) -> Vec { - self.storage_key.into_owned() - } -} - -#[derive(Clone, Copy)] /// Information related to a child state. -pub enum ChildInfo<'a> { - Default(ChildTrie<'a>), -} - -/// Owned version of `ChildInfo`. -/// To be use in persistence layers. #[derive(Debug, Clone)] #[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord))] -pub enum OwnedChildInfo { - Default(OwnedChildTrie), +pub enum ChildInfo { + /// This is the one used by default. + ParentKeyId(ChildTrieParentKeyId), } -impl<'a> ChildInfo<'a> { - /// Instantiates information for a default child trie. - pub const fn new_default(unique_id: &'a[u8]) -> Self { - ChildInfo::Default(ChildTrie { - data: unique_id, +impl ChildInfo { + /// Instantiates child information for a default child trie + /// of kind `ChildType::ParentKeyId`, using an unprefixed parent + /// storage key. + pub fn new_default(storage_key: &[u8]) -> Self { + let data = storage_key.to_vec(); + ChildInfo::ParentKeyId(ChildTrieParentKeyId { data }) + } + + /// Same as `new_default` but with `Vec` as input. + pub fn new_default_from_vec(storage_key: Vec) -> Self { + ChildInfo::ParentKeyId(ChildTrieParentKeyId { + data: storage_key, }) } - /// Instantiates a owned version of this child info. - pub fn to_owned(&self) -> OwnedChildInfo { + /// Try to update with another instance, return false if both instance + /// are not compatible. + pub fn try_update(&mut self, other: &ChildInfo) -> bool { match self { - ChildInfo::Default(ChildTrie { data }) - => OwnedChildInfo::Default(OwnedChildTrie { - data: data.to_vec(), - }), + ChildInfo::ParentKeyId(child_trie) => child_trie.try_update(other), } } - /// Create child info from a linear byte packed value and a given type. - pub fn resolve_child_info(child_type: u32, data: &'a[u8]) -> Option { - match child_type { - x if x == ChildType::CryptoUniqueId as u32 => Some(ChildInfo::new_default(data)), - _ => None, + /// Returns byte sequence (keyspace) that can be use by underlying db to isolate keys. + /// This is a unique id of the child trie. The collision resistance of this value + /// depends on the type of child info use. For `ChildInfo::Default` it is and need to be. + pub fn keyspace(&self) -> &[u8] { + match self { + ChildInfo::ParentKeyId(..) => self.storage_key(), } } - /// Return a single byte vector containing packed child info content and its child info type. - /// This can be use as input for `resolve_child_info`. - pub fn info(&self) -> (&[u8], u32) { + /// Returns a reference to the location in the direct parent of + /// this trie but without the common prefix for this kind of + /// child trie. + pub fn storage_key(&self) -> &[u8] { match self { - ChildInfo::Default(ChildTrie { + ChildInfo::ParentKeyId(ChildTrieParentKeyId { data, - }) => (data, ChildType::CryptoUniqueId as u32), + }) => &data[..], } } - /// Return byte sequence (keyspace) that can be use by underlying db to isolate keys. - /// This is a unique id of the child trie. The collision resistance of this value - /// depends on the type of child info use. For `ChildInfo::Default` it is and need to be. - pub fn keyspace(&self) -> &[u8] { + /// Return a the full location in the direct parent of + /// this trie. + pub fn prefixed_storage_key(&self) -> PrefixedStorageKey { match self { - ChildInfo::Default(ChildTrie { + ChildInfo::ParentKeyId(ChildTrieParentKeyId { data, - }) => &data[..], + }) => ChildType::ParentKeyId.new_prefixed_key(data.as_slice()), + } + } + + /// Returns a the full location in the direct parent of + /// this trie. + pub fn into_prefixed_storage_key(self) -> PrefixedStorageKey { + match self { + ChildInfo::ParentKeyId(ChildTrieParentKeyId { + mut data, + }) => { + ChildType::ParentKeyId.do_prefix_key(&mut data); + PrefixedStorageKey(data) + }, + } + } + + /// Returns the type for this child info. + pub fn child_type(&self) -> ChildType { + match self { + ChildInfo::ParentKeyId(..) => ChildType::ParentKeyId, } } } @@ -241,65 +245,98 @@ impl<'a> ChildInfo<'a> { /// It does not strictly define different child type, it can also /// be related to technical consideration or api variant. #[repr(u32)] +#[derive(Clone, Copy, PartialEq)] +#[cfg_attr(feature = "std", derive(Debug))] pub enum ChildType { - /// Default, it uses a cryptographic strong unique id as input. - CryptoUniqueId = 1, + /// If runtime module ensures that the child key is a unique id that will + /// only be used once, its parent key is used as a child trie unique id. + ParentKeyId = 1, } -impl OwnedChildInfo { - /// Instantiates info for a default child trie. - pub fn new_default(unique_id: Vec) -> Self { - OwnedChildInfo::Default(OwnedChildTrie { - data: unique_id, +impl ChildType { + /// Try to get a child type from its `u32` representation. + pub fn new(repr: u32) -> Option { + Some(match repr { + r if r == ChildType::ParentKeyId as u32 => ChildType::ParentKeyId, + _ => return None, }) } - /// Try to update with another instance, return false if both instance - /// are not compatible. - pub fn try_update(&mut self, other: ChildInfo) -> bool { - match self { - OwnedChildInfo::Default(owned_child_trie) => owned_child_trie.try_update(other), + /// Transform a prefixed key into a tuple of the child type + /// and the unprefixed representation of the key. + pub fn from_prefixed_key<'a>(storage_key: &'a PrefixedStorageKey) -> Option<(Self, &'a [u8])> { + let match_type = |storage_key: &'a [u8], child_type: ChildType| { + let prefix = child_type.parent_prefix(); + if storage_key.starts_with(prefix) { + Some((child_type, &storage_key[prefix.len()..])) + } else { + None + } + }; + match_type(storage_key, ChildType::ParentKeyId) + } + + /// Produce a prefixed key for a given child type. + fn new_prefixed_key(&self, key: &[u8]) -> PrefixedStorageKey { + let parent_prefix = self.parent_prefix(); + let mut result = Vec::with_capacity(parent_prefix.len() + key.len()); + result.extend_from_slice(parent_prefix); + result.extend_from_slice(key); + PrefixedStorageKey(result) + } + + /// Prefixes a vec with the prefix for this child type. + fn do_prefix_key(&self, key: &mut Vec) { + let parent_prefix = self.parent_prefix(); + let key_len = key.len(); + if parent_prefix.len() > 0 { + key.resize(key_len + parent_prefix.len(), 0); + key.copy_within(..key_len, parent_prefix.len()); + key[..parent_prefix.len()].copy_from_slice(parent_prefix); } } - /// Get `ChildInfo` reference to this owned child info. - pub fn as_ref(&self) -> ChildInfo { + /// Returns the location reserved for this child trie in their parent trie if there + /// is one. + pub fn parent_prefix(&self) -> &'static [u8] { match self { - OwnedChildInfo::Default(OwnedChildTrie { data }) - => ChildInfo::Default(ChildTrie { - data: data.as_slice(), - }), + &ChildType::ParentKeyId => DEFAULT_CHILD_TYPE_PARENT_PREFIX, } } } /// A child trie of default type. -/// Default is the same implementation as the top trie. -/// It share its trie node storage with any kind of key, -/// and its unique id needs to be collision free (eg strong -/// crypto hash). -#[derive(Clone, Copy)] -pub struct ChildTrie<'a> { - /// Data containing unique id. - /// Unique id must but unique and free of any possible key collision - /// (depending on its storage behavior). - data: &'a[u8], -} - -/// Owned version of default child trie `ChildTrie`. +/// It uses the same default implementation as the top trie, +/// top trie being a child trie with no keyspace and no storage key. +/// Its keyspace is the variable (unprefixed) part of its storage key. +/// It shares its trie nodes backend storage with every other +/// child trie, so its storage key needs to be a unique id +/// that will be use only once. +/// Those unique id also required to be long enough to avoid any +/// unique id to be prefixed by an other unique id. #[derive(Debug, Clone)] #[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord))] -pub struct OwnedChildTrie { - /// See `ChildTrie` reference field documentation. +pub struct ChildTrieParentKeyId { + /// Data is the storage key without prefix. data: Vec, } -impl OwnedChildTrie { +impl ChildTrieParentKeyId { /// Try to update with another instance, return false if both instance /// are not compatible. - fn try_update(&mut self, other: ChildInfo) -> bool { + fn try_update(&mut self, other: &ChildInfo) -> bool { match other { - ChildInfo::Default(other) => self.data[..] == other.data[..], + ChildInfo::ParentKeyId(other) => self.data[..] == other.data[..], } } } + +const DEFAULT_CHILD_TYPE_PARENT_PREFIX: &'static [u8] = b":child_storage:default:"; + +#[test] +fn test_prefix_default_child_info() { + let child_info = ChildInfo::new_default(b"any key"); + let prefix = child_info.child_type().parent_prefix(); + assert!(prefix.starts_with(well_known_keys::CHILD_STORAGE_KEY_PREFIX)); + assert!(prefix.starts_with(DEFAULT_CHILD_TYPE_PARENT_PREFIX)); +} diff --git a/primitives/test-primitives/Cargo.toml b/primitives/test-primitives/Cargo.toml index b8cb583835d89b4c25c0fa1583c36c8f66489729..386f7be17c92e035cf744c6e42f43277e2341ceb 100644 --- a/primitives/test-primitives/Cargo.toml +++ b/primitives/test-primitives/Cargo.toml @@ -8,13 +8,16 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" publish = false +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../application-crypto" } +sp-application-crypto = { version = "2.0.0-dev", default-features = false, path = "../application-crypto" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../core" } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } -parity-util-mem = { version = "0.6.0", default-features = false, features = ["primitive-types"] } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../runtime" } +parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } [features] default = [ @@ -24,6 +27,3 @@ std = [ "sp-application-crypto/std", "serde", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/timestamp/Cargo.toml b/primitives/timestamp/Cargo.toml index 4a0851ccb19a843da2537b15d31843b0100e8cb3..9de079d1a79d51242f0d6798e2de6580536d3155 100644 --- a/primitives/timestamp/Cargo.toml +++ b/primitives/timestamp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-timestamp" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,14 +8,17 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Substrate core types and inherents for timestamps." +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../api" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../api" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../runtime" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../inherents" } +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../inherents" } impl-trait-for-tuples = "0.1.3" -wasm-timer = "0.2" +wasm-timer = { version = "0.2", optional = true } [features] default = [ "std" ] @@ -25,7 +28,5 @@ std = [ "sp-runtime/std", "codec/std", "sp-inherents/std", + "wasm-timer", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/tracing/Cargo.toml b/primitives/tracing/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..8eb3bc2beaad5c4d53159db7a591b2997ab2b452 --- /dev/null +++ b/primitives/tracing/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "sp-tracing" +version = "2.0.0-dev" +license = "GPL-3.0" +authors = ["Parity Technologies "] +edition = "2018" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Instrumentation primitives and macros for Substrate." + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +tracing = { version = "0.1.13", optional = true } + +[features] +default = [ "std" ] +std = [ "tracing" ] diff --git a/primitives/tracing/src/lib.rs b/primitives/tracing/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..e5bc93cf9ee52c2e7f69c78f6637fad71b641b34 --- /dev/null +++ b/primitives/tracing/src/lib.rs @@ -0,0 +1,84 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Substrate tracing primitives and macros. +//! +//! To trace functions or invidual code in Substrate, this crate provides [`tracing_span`] +//! and [`enter_span`]. See the individual docs for how to use these macros. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(feature = "std")] +#[doc(hidden)] +pub use tracing; + +/// Runs given code within a tracing span, measuring it's execution time. +/// +/// If tracing is not enabled, the code is still executed. +/// +/// # Example +/// +/// ``` +/// sp_tracing::tracing_span! { +/// "test-span"; +/// 1 + 1; +/// // some other complex code +/// } +/// ``` +#[macro_export] +macro_rules! tracing_span { + ( + $name:expr; + $( $code:tt )* + ) => { + { + $crate::enter_span!($name); + $( $code )* + } + } +} + +/// Enter a span. +/// +/// The span will be valid, until the scope is left. +/// +/// # Example +/// +/// ``` +/// sp_tracing::enter_span!("test-span"); +/// ``` +#[macro_export] +macro_rules! enter_span { + ( $name:expr ) => { + let __tracing_span__ = $crate::if_tracing!( + $crate::tracing::span!($crate::tracing::Level::TRACE, $name) + ); + let __tracing_guard__ = $crate::if_tracing!(__tracing_span__.enter()); + } +} + +/// Generates the given code if the tracing dependency is enabled. +#[macro_export] +#[cfg(feature = "std")] +macro_rules! if_tracing { + ( $if:expr ) => {{ $if }} +} + +#[macro_export] +#[cfg(not(feature = "std"))] +macro_rules! if_tracing { + ( $if:expr ) => {{}} +} diff --git a/primitives/transaction-pool/Cargo.toml b/primitives/transaction-pool/Cargo.toml index 6e30fb4ddc16eb167fa786d27f110e53ab5857ff..b33687246f768838d41cdfe7349645af6d4489b9 100644 --- a/primitives/transaction-pool/Cargo.toml +++ b/primitives/transaction-pool/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-transaction-pool" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,6 +9,9 @@ repository = "https://github.com/paritytech/substrate/" description = "Transaction pool primitives types & Runtime API." documentation = "https://docs.rs/sp-transaction-pool" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", optional = true } @@ -16,9 +19,9 @@ derive_more = { version = "0.99.2", optional = true } futures = { version = "0.3.1", optional = true } log = { version = "0.4.8", optional = true } serde = { version = "1.0.101", features = ["derive"], optional = true} -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../api" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } -sp-utils = { version = "2.0.0-alpha.5", default-features = false, path = "../utils" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../api" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../runtime" } +sp-utils = { version = "2.0.0-dev", default-features = false, path = "../utils" } [features] default = [ "std" ] @@ -31,6 +34,3 @@ std = [ "sp-api/std", "sp-runtime/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/trie/Cargo.toml b/primitives/trie/Cargo.toml index 3876f4652624be83193661e83ea74f85c108920a..c010b3262d7e22f0cff38c10324e0aef2ddc3dd9 100644 --- a/primitives/trie/Cargo.toml +++ b/primitives/trie/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-trie" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] description = "Patricia trie stuff using a parity-scale-codec node format" repository = "https://github.com/paritytech/substrate/" @@ -9,25 +9,28 @@ edition = "2018" homepage = "https://substrate.dev" documentation = "https://docs.rs/sp-trie" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [[bench]] name = "bench" harness = false [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } hash-db = { version = "0.15.2", default-features = false } -trie-db = { version = "0.20.0", default-features = false } +trie-db = { version = "0.20.1", default-features = false } trie-root = { version = "0.16.0", default-features = false } memory-db = { version = "0.20.0", default-features = false } -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../core" } +sp-core = { version = "2.0.0-dev", default-features = false, path = "../core" } [dev-dependencies] trie-bench = "0.21.0" trie-standardmap = "0.15.2" criterion = "0.2.11" hex-literal = "0.2.1" -sp-runtime = { version = "2.0.0-alpha.5", path = "../runtime" } +sp-runtime = { version = "2.0.0-dev", path = "../runtime" } [features] default = ["std"] @@ -40,6 +43,3 @@ std = [ "trie-root/std", "sp-core/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 80570a9792b107bfdfa14cd4fe6988c5088fbebd..f328b71750ba42a2252b19f71375f2db10d34076 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -86,8 +86,6 @@ pub trait AsHashDB: hash_db::AsHashDB {} impl> AsHashDB for T {} /// Reexport from `hash_db`, with genericity set for `Hasher` trait. pub type HashDB<'a, H> = dyn hash_db::HashDB + 'a; -/// Reexport from `hash_db`, with genericity set for key only. -pub type PlainDB<'a, K> = dyn hash_db::PlainDB + 'a; /// Reexport from `hash_db`, with genericity set for `Hasher` trait. /// This uses a `KeyFunction` for prefixing keys internally (avoiding /// key conflict for non random keys). @@ -211,9 +209,8 @@ pub fn read_trie_value_with< Ok(TrieDB::::new(&*db, root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?) } -/// Determine the default child trie root. -pub fn default_child_trie_root( - _storage_key: &[u8], +/// Determine the empty child trie root. +pub fn empty_child_trie_root( ) -> ::Out { L::trie_root::<_, Vec, Vec>(core::iter::empty()) } @@ -221,7 +218,6 @@ pub fn default_child_trie_root( /// Determine a child trie root given its ordered contents, closed form. H is the default hasher, /// but a generic implementation may ignore this type parameter and use other hashers. pub fn child_trie_root( - _storage_key: &[u8], input: I, ) -> ::Out where @@ -235,7 +231,6 @@ pub fn child_trie_root( /// Determine a child trie root given a hash DB and delta values. H is the default hasher, /// but a generic implementation may ignore this type parameter and use other hashers. pub fn child_delta_trie_root( - _storage_key: &[u8], keyspace: &[u8], db: &mut DB, root_data: RD, @@ -247,7 +242,6 @@ pub fn child_delta_trie_root( B: AsRef<[u8]>, RD: AsRef<[u8]>, DB: hash_db::HashDB - + hash_db::PlainDB, trie_db::DBValue>, { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -270,7 +264,6 @@ pub fn child_delta_trie_root( /// Call `f` for all keys in a child trie. pub fn for_keys_in_child_trie( - _storage_key: &[u8], keyspace: &[u8], db: &DB, root_slice: &[u8], @@ -278,7 +271,6 @@ pub fn for_keys_in_child_trie( ) -> Result<(), Box>> where DB: hash_db::HashDBRef - + hash_db::PlainDBRef, trie_db::DBValue>, { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -321,7 +313,6 @@ pub fn record_all_keys( /// Read a value from the child trie. pub fn read_child_trie_value( - _storage_key: &[u8], keyspace: &[u8], db: &DB, root_slice: &[u8], @@ -329,7 +320,6 @@ pub fn read_child_trie_value( ) -> Result>, Box>> where DB: hash_db::HashDBRef - + hash_db::PlainDBRef, trie_db::DBValue>, { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -341,7 +331,6 @@ pub fn read_child_trie_value( /// Read a value from the child trie with given query. pub fn read_child_trie_value_with, DB>( - _storage_key: &[u8], keyspace: &[u8], db: &DB, root_slice: &[u8], @@ -350,7 +339,6 @@ pub fn read_child_trie_value_with Result>, Box>> where DB: hash_db::HashDBRef - + hash_db::PlainDBRef, trie_db::DBValue>, { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. diff --git a/primitives/utils/Cargo.toml b/primitives/utils/Cargo.toml index 97e5ce1d9b618c2aa250ffd7054f4dbbc6165f14..79a3a3315474748bf61a7504c72689650ef9d8a2 100644 --- a/primitives/utils/Cargo.toml +++ b/primitives/utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-utils" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" diff --git a/primitives/version/Cargo.toml b/primitives/version/Cargo.toml index 726d064642fb355a9d037ac840c44a5708db2850..1d492ee41c4061083aebbe35a70a26f23d437f12 100644 --- a/primitives/version/Cargo.toml +++ b/primitives/version/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-version" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,13 +9,16 @@ repository = "https://github.com/paritytech/substrate/" description = "Version module for the Substrate runtime; Provides a function that returns the runtime version." documentation = "https://docs.rs/sp-version" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] impl-serde = { version = "0.2.3", optional = true } serde = { version = "1.0.101", optional = true, features = ["derive"] } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../std" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../runtime" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../runtime" } [features] default = ["std"] @@ -26,6 +29,3 @@ std = [ "sp-std/std", "sp-runtime/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/primitives/version/src/lib.rs b/primitives/version/src/lib.rs index 0534f87490868469294dbab078afeb2790155a1f..613b23156a59bca110fd532385d69d304dbe7a96 100644 --- a/primitives/version/src/lib.rs +++ b/primitives/version/src/lib.rs @@ -93,17 +93,29 @@ pub struct RuntimeVersion { ) )] pub apis: ApisVec, + + /// All existing dispatches are fully compatible when this number doesn't change. If this + /// number changes, then `spec_version` must change, also. + /// + /// This number must change when an existing dispatchable (module ID, dispatch ID) is changed, + /// either through an alteration in its user-level semantics, a parameter added/removed/changed, + /// a dispatchable being removed, a module being removed, or a dispatchable/module changing its + /// index. + /// + /// It need *not* change when a new module is added or when a dispatchable is added. + pub transaction_version: u32, } #[cfg(feature = "std")] impl fmt::Display for RuntimeVersion { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}-{}:{}({}-{})", + write!(f, "{}-{} ({}-{}.tx{}.au{})", self.spec_name, self.spec_version, - self.authoring_version, self.impl_name, - self.impl_version + self.impl_version, + self.transaction_version, + self.authoring_version, ) } } diff --git a/primitives/wasm-interface/Cargo.toml b/primitives/wasm-interface/Cargo.toml index 4a35d5b5180ff07529762e003370e7d2f21eef2b..8bea87d4902d4b5186965ad93d4286ed8eda26d6 100644 --- a/primitives/wasm-interface/Cargo.toml +++ b/primitives/wasm-interface/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sp-wasm-interface" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,15 +9,15 @@ repository = "https://github.com/paritytech/substrate/" description = "Types and traits for interfacing between the host and the wasm runtime." documentation = "https://docs.rs/sp-wasm-interface" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] wasmi = { version = "0.6.2", optional = true } impl-trait-for-tuples = "0.1.2" -sp-std = { version = "2.0.0-alpha.5", path = "../std", default-features = false } +sp-std = { version = "2.0.0-dev", path = "../std", default-features = false } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } [features] default = [ "std" ] std = [ "wasmi", "sp-std/std", "codec/std" ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index b8c9f9bd606059b73adb080f68c78dda3e8026a5..43979be55329a70aad5386b86a9d5e0443bdcfbc 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-test-utils" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" diff --git a/test-utils/client/Cargo.toml b/test-utils/client/Cargo.toml index ec87e7cd168f05585f9f70a25793fc4840af5017..37c99292ec910ec90ad8d5d255dee0840654985f 100644 --- a/test-utils/client/Cargo.toml +++ b/test-utils/client/Cargo.toml @@ -8,20 +8,21 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" publish = false +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sc-client-api = { version = "2.0.0-alpha.5", path = "../../client/api" } -sc-client = { version = "0.8.0-alpha.5", path = "../../client/" } -sc-client-db = { version = "0.8.0-alpha.5", features = ["test-helpers"], path = "../../client/db" } -sp-consensus = { version = "0.8.0-alpha.5", path = "../../primitives/consensus/common" } -sc-executor = { version = "0.8.0-alpha.5", path = "../../client/executor" } +sc-client-api = { version = "2.0.0-dev", path = "../../client/api" } +sc-client-db = { version = "0.8.0-dev", features = ["test-helpers"], path = "../../client/db" } +sp-consensus = { version = "0.8.0-dev", path = "../../primitives/consensus/common" } +sc-executor = { version = "0.8.0-dev", path = "../../client/executor" } +sc-consensus = { version = "0.8.0-dev", path = "../../client/consensus/common" } +sc-service = { version = "0.8.0-dev", default-features = false, features = ["test-helpers"], path = "../../client/service" } futures = "0.3.4" hash-db = "0.15.2" -sp-keyring = { version = "2.0.0-alpha.5", path = "../../primitives/keyring" } +sp-keyring = { version = "2.0.0-dev", path = "../../primitives/keyring" } codec = { package = "parity-scale-codec", version = "1.3.0" } -sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../primitives/runtime" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../primitives/blockchain" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../primitives/state-machine" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sp-core = { version = "2.0.0-dev", path = "../../primitives/core" } +sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" } +sp-blockchain = { version = "2.0.0-dev", path = "../../primitives/blockchain" } +sp-state-machine = { version = "0.8.0-dev", path = "../../primitives/state-machine" } diff --git a/test-utils/client/src/client_ext.rs b/test-utils/client/src/client_ext.rs index 6d6b539483e3999ae79a9f78f3b7fdc6381230b1..d663dda7a9323981772e3f88778b8c607ba911c8 100644 --- a/test-utils/client/src/client_ext.rs +++ b/test-utils/client/src/client_ext.rs @@ -16,7 +16,7 @@ //! Client extension for tests. -use sc_client::{self, Client}; +use sc_service::client::Client; use sc_client_api::backend::Finalizer; use sp_consensus::{ BlockImportParams, BlockImport, BlockOrigin, Error as ConsensusError, @@ -64,7 +64,7 @@ pub trait ClientBlockImportExt: Sized { impl ClientExt for Client where B: sc_client_api::backend::Backend, - E: sc_client::CallExecutor + 'static, + E: sc_client_api::CallExecutor + 'static, Self: BlockImport, Block: BlockT, { diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index d04e85fd10c2330161dc3636deb66cc4ba26b710..22173ca04edb0d6325a1b3217dc03bd7bb7c4725 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -20,7 +20,6 @@ pub mod client_ext; -pub use sc_client::{blockchain, self}; pub use sc_client_api::{ execution_extensions::{ExecutionStrategies, ExecutionExtensions}, ForkBlocks, BadBlocks, CloneableSpawn, @@ -36,16 +35,17 @@ pub use sp_keyring::{ pub use sp_core::{traits::BareCryptoStorePtr, tasks::executor as tasks_executor}; pub use sp_runtime::{Storage, StorageChild}; pub use sp_state_machine::ExecutionStrategy; +pub use sc_service::client; pub use self::client_ext::{ClientExt, ClientBlockImportExt}; use std::sync::Arc; use std::collections::HashMap; -use sp_core::storage::{well_known_keys, ChildInfo}; +use sp_core::storage::ChildInfo; use sp_runtime::traits::{Block as BlockT, BlakeTwo256}; -use sc_client::LocalCallExecutor; +use sc_service::client::{LocalCallExecutor, ClientConfig}; /// Test client light database backend. -pub type LightBackend = sc_client::light::backend::Backend< +pub type LightBackend = client::light::backend::Backend< sc_client_db::light::LightStorage, BlakeTwo256, >; @@ -66,6 +66,8 @@ impl GenesisInit for () { pub struct TestClientBuilder { execution_strategies: ExecutionStrategies, genesis_init: G, + /// The key is an unprefixed storage key, this only contains + /// default child trie content. child_storage_extension: HashMap, StorageChild>, backend: Arc, _executor: std::marker::PhantomData, @@ -129,17 +131,17 @@ impl TestClientBuilder, - child_key: impl AsRef<[u8]>, - child_info: ChildInfo, value: impl AsRef<[u8]>, ) -> Self { - let entry = self.child_storage_extension.entry(key.as_ref().to_vec()) + let storage_key = child_info.storage_key(); + let entry = self.child_storage_extension.entry(storage_key.to_vec()) .or_insert_with(|| StorageChild { data: Default::default(), - child_info: child_info.to_owned(), + child_info: child_info.clone(), }); - entry.data.insert(child_key.as_ref().to_vec(), value.as_ref().to_vec()); + entry.data.insert(key.as_ref().to_vec(), value.as_ref().to_vec()); self } @@ -173,15 +175,15 @@ impl TestClientBuilder ( - sc_client::Client< + client::Client< Backend, Executor, Block, RuntimeApi, >, - sc_client::LongestChain, + sc_consensus::LongestChain, ) where - Executor: sc_client::CallExecutor + 'static, + Executor: sc_client_api::CallExecutor + 'static, Backend: sc_client_api::backend::Backend, { let storage = { @@ -189,8 +191,8 @@ impl TestClientBuilder TestClientBuilder TestClientBuilder TestClientBuilder TestClientBuilder< Block, - sc_client::LocalCallExecutor>, + client::LocalCallExecutor>, Backend, G, > { @@ -231,13 +234,13 @@ impl TestClientBuilder< self, executor: I, ) -> ( - sc_client::Client< + client::Client< Backend, - sc_client::LocalCallExecutor>, + client::LocalCallExecutor>, Block, RuntimeApi >, - sc_client::LongestChain, + sc_consensus::LongestChain, ) where I: Into>>, E: sc_executor::NativeExecutionDispatch + 'static, @@ -246,7 +249,7 @@ impl TestClientBuilder< let executor = executor.into().unwrap_or_else(|| NativeExecutor::new(WasmExecutionMethod::Interpreted, None, 8) ); - let executor = LocalCallExecutor::new(self.backend.clone(), executor, tasks_executor()); + let executor = LocalCallExecutor::new(self.backend.clone(), executor, tasks_executor(), Default::default()); self.build_with_executor(executor) } diff --git a/test-utils/runtime/Cargo.toml b/test-utils/runtime/Cargo.toml index be22747ea69166d91f83102e79171c18c8e5441a..f4582d0b7091852e1cac80dd012e2cd4fdabc1de 100644 --- a/test-utils/runtime/Cargo.toml +++ b/test-utils/runtime/Cargo.toml @@ -9,44 +9,49 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" publish = false +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-application-crypto = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/application-crypto" } -sp-consensus-aura = { version = "0.8.0-alpha.5", default-features = false, path = "../../primitives/consensus/aura" } -sp-consensus-babe = { version = "0.8.0-alpha.5", default-features = false, path = "../../primitives/consensus/babe" } -sp-block-builder = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/block-builder" } -cfg-if = "0.1.10" +sp-application-crypto = { version = "2.0.0-dev", default-features = false, path = "../../primitives/application-crypto" } +sp-consensus-aura = { version = "0.8.0-dev", default-features = false, path = "../../primitives/consensus/aura" } +sp-consensus-babe = { version = "0.8.0-dev", default-features = false, path = "../../primitives/consensus/babe" } +sp-block-builder = { version = "2.0.0-dev", default-features = false, path = "../../primitives/block-builder" } codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } -frame-executive = { version = "2.0.0-alpha.5", default-features = false, path = "../../frame/executive" } -sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" } -sp-keyring = { version = "2.0.0-alpha.5", optional = true, path = "../../primitives/keyring" } -log = { version = "0.4.8", optional = true } +frame-executive = { version = "2.0.0-dev", default-features = false, path = "../../frame/executive" } +sp-inherents = { version = "2.0.0-dev", default-features = false, path = "../../primitives/inherents" } +sp-keyring = { version = "2.0.0-dev", optional = true, path = "../../primitives/keyring" } memory-db = { version = "0.20.0", default-features = false } -sp-offchain = { path = "../../primitives/offchain", default-features = false, version = "2.0.0-alpha.5"} -sp-core = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/core" } -sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" } -sp-runtime-interface = { path = "../../primitives/runtime-interface", default-features = false, version = "2.0.0-alpha.5"} -sp-io = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/io" } -frame-support = { version = "2.0.0-alpha.5", default-features = false, path = "../../frame/support" } -sp-version = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/version" } +sp-offchain = { path = "../../primitives/offchain", default-features = false, version = "2.0.0-dev"} +sp-core = { version = "2.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-std = { version = "2.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-runtime-interface = { path = "../../primitives/runtime-interface", default-features = false, version = "2.0.0-dev"} +sp-io = { version = "2.0.0-dev", default-features = false, path = "../../primitives/io" } +frame-support = { version = "2.0.0-dev", default-features = false, path = "../../frame/support" } +sp-version = { version = "2.0.0-dev", default-features = false, path = "../../primitives/version" } +sp-session = { version = "2.0.0-dev", default-features = false, path = "../../primitives/session" } +sp-api = { version = "2.0.0-dev", default-features = false, path = "../../primitives/api" } +sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../primitives/runtime" } +pallet-babe = { version = "2.0.0-dev", default-features = false, path = "../../frame/babe" } +frame-system = { version = "2.0.0-dev", default-features = false, path = "../../frame/system" } +frame-system-rpc-runtime-api = { version = "2.0.0-dev", default-features = false, path = "../../frame/system/rpc/runtime-api" } +pallet-timestamp = { version = "2.0.0-dev", default-features = false, path = "../../frame/timestamp" } +sp-trie = { version = "2.0.0-dev", default-features = false, path = "../../primitives/trie" } +sp-transaction-pool = { version = "2.0.0-dev", default-features = false, path = "../../primitives/transaction-pool" } +trie-db = { version = "0.20.1", default-features = false } +parity-util-mem = { version = "0.6.1", default-features = false, features = ["primitive-types"] } +sc-service = { version = "0.8.0-dev", default-features = false, optional = true, features = ["test-helpers"], path = "../../client/service" } + +# 3rd party +cfg-if = "0.1.10" +log = { version = "0.4.8", optional = true } serde = { version = "1.0.101", optional = true, features = ["derive"] } -sp-session = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/session" } -sp-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/api" } -sp-runtime = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/runtime" } -pallet-babe = { version = "2.0.0-alpha.5", default-features = false, path = "../../frame/babe" } -frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../../frame/system" } -frame-system-rpc-runtime-api = { version = "2.0.0-alpha.5", default-features = false, path = "../../frame/system/rpc/runtime-api" } -pallet-timestamp = { version = "2.0.0-alpha.5", default-features = false, path = "../../frame/timestamp" } -sc-client = { version = "0.8.0-alpha.5", optional = true, path = "../../client" } -sp-trie = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/trie" } -sp-transaction-pool = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/transaction-pool" } -trie-db = { version = "0.20.0", default-features = false } -parity-util-mem = { version = "0.6.0", default-features = false, features = ["primitive-types"] } [dev-dependencies] -sc-block-builder = { version = "0.8.0-alpha.5", path = "../../client/block-builder" } -sc-executor = { version = "0.8.0-alpha.5", path = "../../client/executor" } +sc-block-builder = { version = "0.8.0-dev", path = "../../client/block-builder" } +sc-executor = { version = "0.8.0-dev", path = "../../client/executor" } substrate-test-runtime-client = { version = "2.0.0-dev", path = "./client" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../primitives/state-machine" } +sp-state-machine = { version = "0.8.0-dev", path = "../../primitives/state-machine" } [build-dependencies] wasm-builder-runner = { version = "1.0.5", package = "substrate-wasm-builder-runner", path = "../../utils/wasm-builder-runner" } @@ -82,11 +87,8 @@ std = [ "frame-system-rpc-runtime-api/std", "frame-system/std", "pallet-timestamp/std", - "sc-client", + "sc-service", "sp-trie/std", "sp-transaction-pool/std", "trie-db/std", ] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/test-utils/runtime/client/Cargo.toml b/test-utils/runtime/client/Cargo.toml index 4be45fe46659a2583df6c02b2c67a32f8bf64a18..f622878404f44ae2c4fd9941615bf7ed6127da98 100644 --- a/test-utils/runtime/client/Cargo.toml +++ b/test-utils/runtime/client/Cargo.toml @@ -8,18 +8,20 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" publish = false +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sc-block-builder = { version = "0.8.0-alpha.5", path = "../../../client/block-builder" } +sp-consensus = { version = "0.8.0-dev", path = "../../../primitives/consensus/common" } +sc-block-builder = { version = "0.8.0-dev", path = "../../../client/block-builder" } substrate-test-client = { version = "2.0.0-dev", path = "../../client" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } substrate-test-runtime = { version = "2.0.0-dev", path = "../../runtime" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.5", path = "../../../primitives/api" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-api = { version = "2.0.0-dev", path = "../../../primitives/api" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } codec = { package = "parity-scale-codec", version = "1.3.0" } -sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api" } -sc-client = { version = "0.8.0-alpha.5", path = "../../../client/" } +sc-client-api = { version = "2.0.0-dev", path = "../../../client/api" } +sc-consensus = { version = "0.8.0-dev", path = "../../../client/consensus/common" } +sc-service = { version = "0.8.0-dev", default-features = false, path = "../../../client/service" } futures = "0.3.4" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index f0a405e67e469012f031bb0e3ab676a39e7c5c5b..7b51d88e069b0ad7a056c3bac17901213d524e69 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -26,7 +26,7 @@ use std::sync::Arc; use std::collections::HashMap; pub use substrate_test_client::*; pub use substrate_test_runtime as runtime; -pub use sc_client::LongestChain; +pub use sc_consensus::LongestChain; pub use self::block_builder_ext::BlockBuilderExt; @@ -34,7 +34,7 @@ use sp_core::{sr25519, ChangesTrieConfiguration}; use sp_core::storage::{ChildInfo, Storage, StorageChild}; use substrate_test_runtime::genesismap::{GenesisConfig, additional_storage_with_genesis}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Hash as HashT, NumberFor, HashFor}; -use sc_client::{ +use sc_service::client::{ light::fetcher::{ Fetcher, RemoteHeaderRequest, RemoteReadRequest, RemoteReadChildRequest, @@ -68,7 +68,7 @@ sc_executor::native_executor_instance! { pub type Backend = substrate_test_client::Backend; /// Test client executor. -pub type Executor = sc_client::LocalCallExecutor< +pub type Executor = client::LocalCallExecutor< Backend, NativeExecutor, >; @@ -77,10 +77,10 @@ pub type Executor = sc_client::LocalCallExecutor< pub type LightBackend = substrate_test_client::LightBackend; /// Test client light executor. -pub type LightExecutor = sc_client::light::call_executor::GenesisCallExecutor< +pub type LightExecutor = client::light::call_executor::GenesisCallExecutor< LightBackend, - sc_client::LocalCallExecutor< - sc_client::light::backend::Backend< + client::LocalCallExecutor< + client::light::backend::Backend< sc_client_db::light::LightStorage, HashFor >, @@ -123,16 +123,17 @@ impl substrate_test_client::GenesisInit for GenesisParameters { let mut storage = self.genesis_config().genesis_map(); - let child_roots = storage.children.iter().map(|(sk, child_content)| { + let child_roots = storage.children_default.iter().map(|(_sk, child_content)| { let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( child_content.data.clone().into_iter().collect() ); - (sk.clone(), state_root.encode()) + let prefixed_storage_key = child_content.child_info.prefixed_storage_key(); + (prefixed_storage_key.into_inner(), state_root.encode()) }); let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( storage.top.clone().into_iter().chain(child_roots).collect() ); - let block: runtime::Block = sc_client::genesis::construct_genesis_block(state_root); + let block: runtime::Block = client::genesis::construct_genesis_block(state_root); storage.top.extend(additional_storage_with_genesis(&block)); storage @@ -148,9 +149,9 @@ pub type TestClientBuilder = substrate_test_client::TestClientBuilder< >; /// Test client type with `LocalExecutor` and generic Backend. -pub type Client = sc_client::Client< +pub type Client = client::Client< B, - sc_client::LocalCallExecutor>, + client::LocalCallExecutor>, substrate_test_runtime::Block, substrate_test_runtime::RuntimeApi, >; @@ -192,22 +193,21 @@ pub trait TestClientBuilderExt: Sized { /// # Panics /// /// Panics if the key is empty. - fn add_extra_child_storage>, K: Into>, V: Into>>( + fn add_extra_child_storage>, V: Into>>( mut self, - storage_key: SK, - child_info: ChildInfo, + child_info: &ChildInfo, key: K, value: V, ) -> Self { - let storage_key = storage_key.into(); + let storage_key = child_info.storage_key().to_vec(); let key = key.into(); assert!(!storage_key.is_empty()); assert!(!key.is_empty()); - self.genesis_init_mut().extra_storage.children + self.genesis_init_mut().extra_storage.children_default .entry(storage_key) .or_insert_with(|| StorageChild { data: Default::default(), - child_info: child_info.to_owned(), + child_info: child_info.clone(), }).data.insert(key, value.into()); self } @@ -230,14 +230,14 @@ pub trait TestClientBuilderExt: Sized { } /// Build the test client and longest chain selector. - fn build_with_longest_chain(self) -> (Client, sc_client::LongestChain); + fn build_with_longest_chain(self) -> (Client, sc_consensus::LongestChain); /// Build the test client and the backend. fn build_with_backend(self) -> (Client, Arc); } impl TestClientBuilderExt for TestClientBuilder< - sc_client::LocalCallExecutor>, + client::LocalCallExecutor>, B > where B: sc_client_api::backend::Backend + 'static, @@ -249,7 +249,7 @@ impl TestClientBuilderExt for TestClientBuilder< Self::genesis_init_mut(self) } - fn build_with_longest_chain(self) -> (Client, sc_client::LongestChain) { + fn build_with_longest_chain(self) -> (Client, sc_consensus::LongestChain) { self.build_with_native_executor(None) } @@ -311,7 +311,10 @@ impl Fetcher for LightFetcher { unimplemented!() } - fn remote_read_child(&self, _: RemoteReadChildRequest) -> Self::RemoteReadResult { + fn remote_read_child( + &self, + _: RemoteReadChildRequest, + ) -> Self::RemoteReadResult { unimplemented!() } @@ -341,15 +344,15 @@ pub fn new() -> Client { /// Creates new light client instance used for tests. pub fn new_light() -> ( - sc_client::Client, + client::Client, Arc, ) { let storage = sc_client_db::light::LightStorage::new_test(); - let blockchain = Arc::new(sc_client::light::blockchain::Blockchain::new(storage)); + let blockchain = Arc::new(client::light::blockchain::Blockchain::new(storage)); let backend = Arc::new(LightBackend::new(blockchain.clone())); let executor = new_native_executor(); - let local_call_executor = sc_client::LocalCallExecutor::new(backend.clone(), executor, sp_core::tasks::executor()); + let local_call_executor = client::LocalCallExecutor::new(backend.clone(), executor, sp_core::tasks::executor(), Default::default()); let call_executor = LightExecutor::new( backend.clone(), local_call_executor, diff --git a/test-utils/runtime/client/src/trait_tests.rs b/test-utils/runtime/client/src/trait_tests.rs index 4af8aa37b640a27cd41f7859837a3808525b0327..2a377fabba1294a1343eb0c6475d70848bde4acd 100644 --- a/test-utils/runtime/client/src/trait_tests.rs +++ b/test-utils/runtime/client/src/trait_tests.rs @@ -26,7 +26,7 @@ use crate::{ }; use sc_client_api::backend; use sc_client_api::blockchain::{Backend as BlockChainBackendT, HeaderBackend}; -use substrate_test_client::sp_consensus::BlockOrigin; +use sp_consensus::BlockOrigin; use substrate_test_runtime::{self, Transfer}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{Block as BlockT, HashFor}; diff --git a/test-utils/runtime/src/genesismap.rs b/test-utils/runtime/src/genesismap.rs index 25d9a807ccee15982c02f040001cdfd6c059983f..9426cd6433cec47ecd5462cb808355c4979c837a 100644 --- a/test-utils/runtime/src/genesismap.rs +++ b/test-utils/runtime/src/genesismap.rs @@ -23,6 +23,7 @@ use codec::{Encode, KeyedVec, Joiner}; use sp_core::{ChangesTrieConfiguration, map}; use sp_core::storage::{well_known_keys, Storage}; use sp_runtime::traits::{Block as BlockT, Hash as HashT, Header as HeaderT}; +use sc_service::client::genesis; /// Configuration of a general Substrate test genesis block. pub struct GenesisConfig { @@ -73,7 +74,7 @@ impl GenesisConfig { map.extend(self.extra_storage.top.clone().into_iter()); // Assimilate the system genesis config. - let mut storage = Storage { top: map, children: self.extra_storage.children.clone()}; + let mut storage = Storage { top: map, children_default: self.extra_storage.children_default.clone()}; let mut config = system::GenesisConfig::default(); config.authorities = self.authorities.clone(); config.assimilate_storage(&mut storage).expect("Adding `system::GensisConfig` to the genesis"); @@ -85,7 +86,7 @@ impl GenesisConfig { pub fn insert_genesis_block( storage: &mut Storage, ) -> sp_core::hash::H256 { - let child_roots = storage.children.iter().map(|(sk, child_content)| { + let child_roots = storage.children_default.iter().map(|(sk, child_content)| { let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( child_content.data.clone().into_iter().collect(), ); @@ -96,7 +97,7 @@ pub fn insert_genesis_block( let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( storage.top.clone().into_iter().collect() ); - let block: crate::Block = sc_client::genesis::construct_genesis_block(state_root); + let block: crate::Block = genesis::construct_genesis_block(state_root); let genesis_hash = block.header.hash(); storage.top.extend(additional_storage_with_genesis(&block)); genesis_hash diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index f3efb4bea7c763f89aa9e8e79a1770d2c73c3c97..745eb8f74582195c7f95894a40c1014df345ed1f 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -47,13 +47,12 @@ use sp_version::RuntimeVersion; pub use sp_core::hash::H256; #[cfg(any(feature = "std", test))] use sp_version::NativeVersion; -use frame_support::{impl_outer_origin, parameter_types, weights::Weight}; +use frame_support::{impl_outer_origin, parameter_types, weights::{Weight, RuntimeDbWeight}}; use sp_inherents::{CheckInherentsResult, InherentData}; use cfg_if::cfg_if; -use sp_core::storage::ChildType; // Ensure Babe and Aura use the same crypto to simplify things a bit. -pub use sp_consensus_babe::{AuthorityId, SlotNumber}; +pub use sp_consensus_babe::{AuthorityId, SlotNumber, AllowedSlots}; pub type AuraId = sp_consensus_aura::sr25519::AuthorityId; // Include the WASM binary @@ -68,6 +67,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_version: 2, impl_version: 2, apis: RUNTIME_API_VERSIONS, + transaction_version: 1, }; fn version() -> RuntimeVersion { @@ -181,6 +181,16 @@ impl ExtrinsicT for Extrinsic { } } +impl sp_runtime::traits::Dispatchable for Extrinsic { + type Origin = (); + type Trait = (); + type Info = (); + type PostInfo = (); + fn dispatch(self, _origin: Self::Origin) -> sp_runtime::DispatchResultWithInfo { + panic!("This implemention should not be used for actual dispatch."); + } +} + impl Extrinsic { pub fn transfer(&self) -> &Transfer { match self { @@ -370,6 +380,10 @@ parameter_types! { pub const BlockHashCount: BlockNumber = 250; pub const MinimumPeriod: u64 = 5; pub const MaximumBlockWeight: Weight = 4 * 1024 * 1024; + pub const DbWeight: RuntimeDbWeight = RuntimeDbWeight { + read: 100, + write: 1000, + }; pub const MaximumBlockLength: u32 = 4 * 1024 * 1024; pub const AvailableBlockRatio: Perbill = Perbill::from_percent(75); } @@ -387,6 +401,9 @@ impl frame_system::Trait for Runtime { type Event = Event; type BlockHashCount = BlockHashCount; type MaximumBlockWeight = MaximumBlockWeight; + type DbWeight = (); + type BlockExecutionWeight = (); + type ExtrinsicBaseWeight = (); type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; type Version = (); @@ -618,15 +635,15 @@ cfg_if! { } impl sp_consensus_babe::BabeApi for Runtime { - fn configuration() -> sp_consensus_babe::BabeConfiguration { - sp_consensus_babe::BabeConfiguration { + fn configuration() -> sp_consensus_babe::BabeGenesisConfiguration { + sp_consensus_babe::BabeGenesisConfiguration { slot_duration: 1000, epoch_length: EpochDuration::get(), c: (3, 10), genesis_authorities: system::authorities() .into_iter().map(|x|(x, 1)).collect(), randomness: >::randomness(), - secondary_slots: true, + allowed_slots: AllowedSlots::PrimaryAndSecondaryPlainSlots, } } @@ -812,15 +829,15 @@ cfg_if! { } impl sp_consensus_babe::BabeApi for Runtime { - fn configuration() -> sp_consensus_babe::BabeConfiguration { - sp_consensus_babe::BabeConfiguration { + fn configuration() -> sp_consensus_babe::BabeGenesisConfiguration { + sp_consensus_babe::BabeGenesisConfiguration { slot_duration: 1000, epoch_length: EpochDuration::get(), c: (3, 10), genesis_authorities: system::authorities() .into_iter().map(|x|(x, 1)).collect(), randomness: >::randomness(), - secondary_slots: true, + allowed_slots: AllowedSlots::PrimaryAndSecondaryPlainSlots, } } @@ -907,22 +924,17 @@ fn test_read_storage() { } fn test_read_child_storage() { - const CHILD_KEY: &[u8] = b":child_storage:default:read_child_storage"; - const UNIQUE_ID: &[u8] = b":unique_id"; + const STORAGE_KEY: &[u8] = b"unique_id_1"; const KEY: &[u8] = b":read_child_storage"; - sp_io::storage::child_set( - CHILD_KEY, - UNIQUE_ID, - ChildType::CryptoUniqueId as u32, + sp_io::default_child_storage::set( + STORAGE_KEY, KEY, b"test", ); let mut v = [0u8; 4]; - let r = sp_io::storage::child_read( - CHILD_KEY, - UNIQUE_ID, - ChildType::CryptoUniqueId as u32, + let r = sp_io::default_child_storage::read( + STORAGE_KEY, KEY, &mut v, 0, @@ -931,10 +943,8 @@ fn test_read_child_storage() { assert_eq!(&v, b"test"); let mut v = [0u8; 4]; - let r = sp_io::storage::child_read( - CHILD_KEY, - UNIQUE_ID, - ChildType::CryptoUniqueId as u32, + let r = sp_io::default_child_storage::read( + STORAGE_KEY, KEY, &mut v, 8, diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index c35850ae950e928ae46bb882457f1bf3ef3baa83..9cfec187ddb797a619f87e52ffd71952c1b792bf 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -373,7 +373,7 @@ mod tests { vec![111u8, 0, 0, 0, 0, 0, 0, 0] } ], - children: map![], + children_default: map![], }, ) } diff --git a/test-utils/runtime/transaction-pool/Cargo.toml b/test-utils/runtime/transaction-pool/Cargo.toml index 52e2020dc8dd18dfc82de735e66b98b469bfefb9..9ccf9d2e448701cad8bd25b3dae3a55ad6930856 100644 --- a/test-utils/runtime/transaction-pool/Cargo.toml +++ b/test-utils/runtime/transaction-pool/Cargo.toml @@ -8,16 +8,16 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" publish = false +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] substrate-test-runtime-client = { version = "2.0.0-dev", path = "../client" } parking_lot = "0.10.0" codec = { package = "parity-scale-codec", version = "1.3.0" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../primitives/blockchain" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../primitives/transaction-pool" } -sc-transaction-graph = { version = "2.0.0-alpha.5", path = "../../../client/transaction-pool/graph" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../primitives/blockchain" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-transaction-pool = { version = "2.0.0-dev", path = "../../../primitives/transaction-pool" } +sc-transaction-graph = { version = "2.0.0-dev", path = "../../../client/transaction-pool/graph" } futures = { version = "0.3.1", features = ["compat"] } derive_more = "0.99.2" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/utils/browser/Cargo.toml b/utils/browser/Cargo.toml index 188f46bf197b831aaad001528befd4cc4b60f47c..29211298b731e1c3a78020919e421817fd058bc1 100644 --- a/utils/browser/Cargo.toml +++ b/utils/browser/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-browser-utils" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" authors = ["Parity Technologies "] description = "Utilities for creating a browser light-client." edition = "2018" @@ -8,21 +8,25 @@ license = "GPL-3.0" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] futures = "0.3" futures01 = { package = "futures", version = "0.1.29" } log = "0.4.8" -libp2p-wasm-ext = { version = "0.16.2", features = ["websocket"] } +libp2p-wasm-ext = { version = "0.18.0", features = ["websocket"] } console_error_panic_hook = "0.1.6" console_log = "0.1.2" js-sys = "0.3.34" wasm-bindgen = "0.2.57" wasm-bindgen-futures = "0.4.7" kvdb-web = "0.5" -sc-informant = { version = "0.8.0-alpha.5", path = "../../client/informant" } -sc-service = { version = "0.8.0-alpha.5", path = "../../client/service", default-features = false } -sc-network = { path = "../../client/network", version = "0.8.0-alpha.5"} -sc-chain-spec = { path = "../../client/chain-spec", version = "2.0.0-alpha.5"} +sp-database = { version = "2.0.0-dev", path = "../../primitives/database" } +sc-informant = { version = "0.8.0-dev", path = "../../client/informant" } +sc-service = { version = "0.8.0-dev", path = "../../client/service", default-features = false } +sc-network = { path = "../../client/network", version = "0.8.0-dev"} +sc-chain-spec = { path = "../../client/chain-spec", version = "2.0.0-dev"} # Imported just for the `no_cc` feature clear_on_drop = { version = "0.2.3", features = ["no_cc"] } @@ -31,6 +35,3 @@ rand6 = { package = "rand", version = "0.6", features = ["wasm-bindgen"] } rand = { version = "0.7", features = ["wasm-bindgen"] } futures-timer = { version = "3.0.1", features = ["wasm-bindgen"]} chrono = { version = "0.4", features = ["wasmbind"] } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/utils/browser/src/lib.rs b/utils/browser/src/lib.rs index 80dfa5e2d2470828d5d883a0f0e2532a4478995d..9be248883cb91411f52a24aa17c3ec2884742552 100644 --- a/utils/browser/src/lib.rs +++ b/utils/browser/src/lib.rs @@ -17,8 +17,10 @@ use futures01::sync::mpsc as mpsc01; use log::{debug, info}; use std::sync::Arc; +use sc_network::config::TransportConfig; use sc_service::{ - AbstractService, RpcSession, Role, Configuration, config::{DatabaseConfig, KeystoreConfig}, + AbstractService, RpcSession, Role, Configuration, + config::{DatabaseConfig, KeystoreConfig, NetworkConfiguration}, GenericChainSpec, RuntimeGenesis }; use wasm_bindgen::prelude::*; @@ -43,29 +45,58 @@ where let name = chain_spec.name().to_string(); let transport = ExtTransport::new(ffi::websocket_transport()); - let mut config = Configuration::default(); - config.network.boot_nodes = chain_spec.boot_nodes().to_vec(); - config.telemetry_endpoints = chain_spec.telemetry_endpoints().clone(); - config.chain_spec = Some(Box::new(chain_spec)); - config.network.transport = sc_network::config::TransportConfig::Normal { + let mut network = NetworkConfiguration::new( + format!("{} (Browser)", name), + "unknown", + Default::default(), + None, + ); + network.boot_nodes = chain_spec.boot_nodes().to_vec(); + network.transport = TransportConfig::Normal { wasm_external_transport: Some(transport.clone()), allow_private_ipv4: true, enable_mdns: false, use_yamux_flow_control: true, }; - config.task_executor = Some(Arc::new(move |fut| { - wasm_bindgen_futures::spawn_local(fut) - })); - config.telemetry_external_transport = Some(transport); - config.role = Role::Light; - config.name = format!("{} (Browser)", name); - config.database = Some({ - info!("Opening Indexed DB database '{}'...", name); - let db = kvdb_web::Database::open(name, 10) - .await?; - DatabaseConfig::Custom(Arc::new(db)) - }); - config.keystore = KeystoreConfig::InMemory; + + let config = Configuration { + network, + telemetry_endpoints: chain_spec.telemetry_endpoints().clone(), + chain_spec: Box::new(chain_spec), + task_executor: Arc::new(move |fut| wasm_bindgen_futures::spawn_local(fut)), + telemetry_external_transport: Some(transport), + role: Role::Light, + database: { + info!("Opening Indexed DB database '{}'...", name); + let db = kvdb_web::Database::open(name, 10).await?; + + DatabaseConfig::Custom(sp_database::as_database(db)) + }, + keystore: KeystoreConfig::InMemory, + default_heap_pages: Default::default(), + dev_key_seed: Default::default(), + disable_grandpa: Default::default(), + execution_strategies: Default::default(), + force_authoring: Default::default(), + impl_name: "parity-substrate", + impl_version: "0.0.0", + offchain_worker: Default::default(), + prometheus_config: Default::default(), + pruning: Default::default(), + rpc_cors: Default::default(), + rpc_http: Default::default(), + rpc_ws: Default::default(), + unsafe_rpc_expose: false, + rpc_ws_max_connections: Default::default(), + state_cache_child_ratio: Default::default(), + state_cache_size: Default::default(), + tracing_receiver: Default::default(), + tracing_targets: Default::default(), + transaction_pool: Default::default(), + wasm_method: Default::default(), + max_runtime_instances: 8, + announce_block: true, + }; Ok(config) } diff --git a/utils/build-script-utils/Cargo.toml b/utils/build-script-utils/Cargo.toml index 3fe10f6a825a82585396eccdd3d3c447edf9a9d4..89c584808ab50a11d0359c60780f0dd63ca55979 100644 --- a/utils/build-script-utils/Cargo.toml +++ b/utils/build-script-utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-build-script-utils" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,7 +8,8 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Crate with utility functions for `build.rs` scripts." -[dependencies] - [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +platforms = "0.2.1" diff --git a/utils/build-script-utils/src/git.rs b/utils/build-script-utils/src/git.rs new file mode 100644 index 0000000000000000000000000000000000000000..10f5446cb44f709e9208b89105d8246bf14ea23b --- /dev/null +++ b/utils/build-script-utils/src/git.rs @@ -0,0 +1,124 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::{env, fs, fs::File, io, io::Read, path::PathBuf}; + +/// Make sure the calling `build.rs` script is rerun when `.git/HEAD` or the ref of `.git/HEAD` +/// changed. +/// +/// The file is searched from the `CARGO_MANIFEST_DIR` upwards. If the file can not be found, +/// a warning is generated. +pub fn rerun_if_git_head_changed() { + let mut manifest_dir = PathBuf::from( + env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is always set by cargo."), + ); + let manifest_dir_copy = manifest_dir.clone(); + + while manifest_dir.parent().is_some() { + match get_git_paths(&manifest_dir) { + Err(err) => { + eprintln!("cargo:warning=Unable to read the Git repository: {}", err); + + return; + } + Ok(None) => {} + Ok(Some(paths)) => { + for p in paths { + println!("cargo:rerun-if-changed={}", p.display()); + } + + return; + } + } + + manifest_dir.pop(); + } + + println!( + "cargo:warning=Could not find `.git/HEAD` searching from `{}` upwards!", + manifest_dir_copy.display(), + ); +} + +// Code taken from https://github.com/rustyhorde/vergen/blob/8d522db8c8e16e26c0fc9ea8e6b0247cbf5cca84/src/output/envvar.rs +fn get_git_paths(path: &PathBuf) -> Result>, io::Error> { + let git_dir_or_file = path.join(".git"); + + if let Ok(metadata) = fs::metadata(&git_dir_or_file) { + if metadata.is_dir() { + // Echo the HEAD path + let git_head_path = git_dir_or_file.join("HEAD"); + + // Determine where HEAD points and echo that path also. + let mut f = File::open(&git_head_path)?; + let mut git_head_contents = String::new(); + let _ = f.read_to_string(&mut git_head_contents)?; + let ref_vec: Vec<&str> = git_head_contents.split(": ").collect(); + + if ref_vec.len() == 2 { + let current_head_file = ref_vec[1]; + let git_refs_path = git_dir_or_file.join(current_head_file); + + Ok(Some(vec![git_head_path, git_refs_path])) + } else { + Err(io::Error::new( + io::ErrorKind::Other, + "You are most likely in a detached HEAD state", + )) + } + } else if metadata.is_file() { + // We are in a worktree, so find out where the actual worktrees//HEAD file is. + let mut git_file = File::open(&git_dir_or_file)?; + let mut git_contents = String::new(); + let _ = git_file.read_to_string(&mut git_contents)?; + let dir_vec: Vec<&str> = git_contents.split(": ").collect(); + let git_path = dir_vec[1].trim(); + + // Echo the HEAD psth + let git_head_path = PathBuf::from(git_path).join("HEAD"); + + // Find out what the full path to the .git dir is. + let mut actual_git_dir = PathBuf::from(git_path); + actual_git_dir.pop(); + actual_git_dir.pop(); + + // Determine where HEAD points and echo that path also. + let mut f = File::open(&git_head_path)?; + let mut git_head_contents = String::new(); + let _ = f.read_to_string(&mut git_head_contents)?; + let ref_vec: Vec<&str> = git_head_contents.split(": ").collect(); + + if ref_vec.len() == 2 { + let current_head_file = ref_vec[1]; + let git_refs_path = actual_git_dir.join(current_head_file); + + Ok(Some(vec![git_head_path, git_refs_path])) + } else { + Err(io::Error::new( + io::ErrorKind::Other, + "You are most likely in a detached HEAD state", + )) + } + } else { + Err(io::Error::new( + io::ErrorKind::Other, + "Invalid .git format (Not a directory or a file)", + )) + } + } else { + Ok(None) + } +} diff --git a/utils/build-script-utils/src/lib.rs b/utils/build-script-utils/src/lib.rs index 1b915bdcafacb1e5ecc0b81c60d062d0537ed496..57a1e7c5cdb3e8cae406b4e6d49b1dbc8cc27fb6 100644 --- a/utils/build-script-utils/src/lib.rs +++ b/utils/build-script-utils/src/lib.rs @@ -16,29 +16,8 @@ //! Crate with utility functions for `build.rs` scripts. -use std::{env, path::PathBuf}; +mod version; +mod git; -/// Make sure the calling `build.rs` script is rerun when `.git/HEAD` changed. -/// -/// The file is searched from the `CARGO_MANIFEST_DIR` upwards. If the file can not be found, -/// a warning is generated. -pub fn rerun_if_git_head_changed() { - let mut manifest_dir = PathBuf::from( - env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is always set by cargo.") - ); - let manifest_dir_copy = manifest_dir.clone(); - - while manifest_dir.parent().is_some() { - if manifest_dir.join(".git/HEAD").exists() { - println!("cargo:rerun-if-changed={}", manifest_dir.join(".git/HEAD").display()); - return - } - - manifest_dir.pop(); - } - - println!( - "cargo:warning=Could not find `.git/HEAD` searching from `{}` upwards!", - manifest_dir_copy.display(), - ); -} +pub use git::*; +pub use version::*; diff --git a/utils/build-script-utils/src/version.rs b/utils/build-script-utils/src/version.rs new file mode 100644 index 0000000000000000000000000000000000000000..01a97c6f383dde71f383cb0a9d5e617a380d0ac7 --- /dev/null +++ b/utils/build-script-utils/src/version.rs @@ -0,0 +1,66 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use platforms::*; +use std::{borrow::Cow, process::Command}; + +/// Generate the `cargo:` key output +pub fn generate_cargo_keys() { + let output = Command::new("git") + .args(&["rev-parse", "--short", "HEAD"]) + .output(); + + let commit = match output { + Ok(o) if o.status.success() => { + let sha = String::from_utf8_lossy(&o.stdout).trim().to_owned(); + Cow::from(sha) + } + Ok(o) => { + println!("cargo:warning=Git command failed with status: {}", o.status); + Cow::from("unknown-commit") + }, + Err(err) => { + println!("cargo:warning=Failed to execute git command: {}", err); + Cow::from("unknown-commit") + }, + }; + + println!("cargo:rustc-env=SUBSTRATE_CLI_IMPL_VERSION={}", get_version(&commit)) +} + +fn get_platform() -> String { + let env_dash = if TARGET_ENV.is_some() { "-" } else { "" }; + + format!( + "{}-{}{}{}", + TARGET_ARCH.as_str(), + TARGET_OS.as_str(), + env_dash, + TARGET_ENV.map(|x| x.as_str()).unwrap_or(""), + ) +} + +fn get_version(impl_commit: &str) -> String { + let commit_dash = if impl_commit.is_empty() { "" } else { "-" }; + + format!( + "{}{}{}-{}", + std::env::var("CARGO_PKG_VERSION").unwrap_or_default(), + commit_dash, + impl_commit, + get_platform(), + ) +} diff --git a/utils/fork-tree/Cargo.toml b/utils/fork-tree/Cargo.toml index e46618feb8e5a9d8dc4aed8f100c5f6bfcea1323..c18826d16fe506ea506953189590b6389971279e 100644 --- a/utils/fork-tree/Cargo.toml +++ b/utils/fork-tree/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fork-tree" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -9,8 +9,8 @@ repository = "https://github.com/paritytech/substrate/" description = "Utility library for managing tree-like ordered data with logic for pruning the tree while finalizing nodes." documentation = "https://docs.rs/fork-tree" -[dependencies] -codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } - [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "1.3.0", features = ["derive"] } diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index f4b72187478d47c63575c01ad27d86500c160816..b02f42d7593fae6f38feab02be31022a0c56e499 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "frame-benchmarking-cli" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,23 +8,22 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "CLI for benchmarking FRAME" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -frame-benchmarking = { version = "2.0.0-alpha.5", path = "../../../frame/benchmarking" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../primitives/core" } -sc-service = { version = "0.8.0-alpha.5", default-features = false, path = "../../../client/service" } -sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" } -sc-client = { version = "0.8.0-alpha.5", path = "../../../client" } -sc-client-db = { version = "0.8.0-alpha.5", path = "../../../client/db" } -sc-executor = { version = "0.8.0-alpha.5", path = "../../../client/executor" } -sp-externalities = { version = "0.8.0-alpha.5", path = "../../../primitives/externalities" } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } -sp-state-machine = { version = "0.8.0-alpha.5", path = "../../../primitives/state-machine" } +frame-benchmarking = { version = "2.0.0-dev", path = "../../../frame/benchmarking" } +sp-core = { version = "2.0.0-dev", path = "../../../primitives/core" } +sc-service = { version = "0.8.0-dev", default-features = false, path = "../../../client/service" } +sc-cli = { version = "0.8.0-dev", path = "../../../client/cli" } +sc-client-db = { version = "0.8.0-dev", path = "../../../client/db" } +sc-executor = { version = "0.8.0-dev", path = "../../../client/executor" } +sp-externalities = { version = "0.8.0-dev", path = "../../../primitives/externalities" } +sp-runtime = { version = "2.0.0-dev", path = "../../../primitives/runtime" } +sp-state-machine = { version = "0.8.0-dev", path = "../../../primitives/state-machine" } structopt = "0.3.8" codec = { version = "1.3.0", package = "parity-scale-codec" } [features] -default = ["rocksdb"] -rocksdb = ["sc-client-db/kvdb-rocksdb"] - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +default = ["db"] +db = ["sc-client-db/kvdb-rocksdb", "sc-client-db/parity-db"] diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs new file mode 100644 index 0000000000000000000000000000000000000000..ebca380baff2db82c2f9598449f0bc6f78395e86 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -0,0 +1,146 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::BenchmarkCmd; +use codec::{Decode, Encode}; +use frame_benchmarking::{Analysis, BenchmarkBatch}; +use sc_cli::{SharedParams, CliConfiguration, ExecutionStrategy, Result}; +use sc_client_db::BenchmarkingState; +use sc_executor::NativeExecutor; +use sp_state_machine::StateMachine; +use sp_externalities::Extensions; +use sc_service::{Configuration, NativeExecutionDispatch}; +use sp_runtime::{ + traits::{Block as BlockT, Header as HeaderT, NumberFor}, +}; +use sp_core::{tasks, testing::KeyStore, traits::KeystoreExt}; +use std::fmt::Debug; + +impl BenchmarkCmd { + /// Runs the command and benchmarks the chain. + pub fn run(&self, config: Configuration) -> Result<()> + where + BB: BlockT + Debug, + <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, + ::Hash: std::str::FromStr, + ExecDispatch: NativeExecutionDispatch + 'static, + { + let spec = config.chain_spec; + let wasm_method = self.wasm_method.into(); + let strategy = self.execution.unwrap_or(ExecutionStrategy::Native); + + let genesis_storage = spec.build_storage()?; + let mut changes = Default::default(); + let mut offchain_changes = Default::default(); + let cache_size = Some(self.database_cache_size as usize); + let state = BenchmarkingState::::new(genesis_storage, cache_size)?; + let executor = NativeExecutor::::new( + wasm_method, + None, // heap pages + 2, // The runtime instances cache size. + ); + + let mut extensions = Extensions::default(); + extensions.register(KeystoreExt(KeyStore::new())); + + let result = StateMachine::<_, _, NumberFor, _>::new( + &state, + None, + &mut changes, + &mut offchain_changes, + &executor, + "Benchmark_dispatch_benchmark", + &( + &self.pallet, + &self.extrinsic, + self.lowest_range_values.clone(), + self.highest_range_values.clone(), + self.steps.clone(), + self.repeat, + ).encode(), + extensions, + &sp_state_machine::backend::BackendRuntimeCode::new(&state).runtime_code()?, + tasks::executor(), + ) + .execute(strategy.into()) + .map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?; + + let results = , String> as Decode>::decode(&mut &result[..]) + .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?; + + match results { + Ok(batches) => for batch in batches.into_iter() { + // Print benchmark metadata + println!( + "Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}", + String::from_utf8(batch.pallet).expect("Encoded from String; qed"), + String::from_utf8(batch.benchmark).expect("Encoded from String; qed"), + self.lowest_range_values, + self.highest_range_values, + self.steps, + self.repeat, + ); + + // Skip raw data + analysis if there are no results + if batch.results.len() == 0 { continue } + + if self.raw_data { + // Print the table header + batch.results[0].0.iter().for_each(|param| print!("{:?},", param.0)); + + print!("extrinsic_time,storage_root_time\n"); + // Print the values + batch.results.iter().for_each(|result| { + let parameters = &result.0; + parameters.iter().for_each(|param| print!("{:?},", param.1)); + // Print extrinsic time and storage root time + print!("{:?},{:?}\n", result.1, result.2); + }); + + println!(); + } + + // Conduct analysis. + if !self.no_median_slopes { + if let Some(analysis) = Analysis::median_slopes(&batch.results) { + println!("Median Slopes Analysis\n========\n{}", analysis); + } + } + if !self.no_min_squares { + if let Some(analysis) = Analysis::min_squares_iqr(&batch.results) { + println!("Min Squares Analysis\n========\n{}", analysis); + } + } + }, + Err(error) => eprintln!("Error: {:?}", error), + } + + Ok(()) + } +} + +impl CliConfiguration for BenchmarkCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } + + fn chain_id(&self, _is_dev: bool) -> Result { + Ok(match self.shared_params.chain { + Some(ref chain) => chain.clone(), + None => "dev".into(), + }) + } +} diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 926d140e024098cc0a30787d4a458e27c05ae8db..96204d1ae576382feb3ac68ab9b5c26ed5a90d61 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -14,21 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +mod command; + +use sc_cli::{ExecutionStrategy, WasmExecutionMethod}; use std::fmt::Debug; -use sp_runtime::{traits::{Block as BlockT, Header as HeaderT, NumberFor}}; -use sc_client::StateMachine; -use sc_cli::{ExecutionStrategy, WasmExecutionMethod, VersionInfo}; -use sc_client_db::BenchmarkingState; -use sc_service::{Configuration, ChainSpec}; -use sc_executor::{NativeExecutor, NativeExecutionDispatch}; -use codec::{Encode, Decode}; -use frame_benchmarking::{BenchmarkBatch, Analysis}; -use sp_core::{ - tasks, - traits::KeystoreExt, - testing::KeyStore, -}; -use sp_externalities::Extensions; /// The `benchmark` command used to benchmark FRAME Pallets. #[derive(Debug, structopt::StructOpt, Clone)] @@ -96,128 +85,3 @@ pub struct BenchmarkCmd { #[structopt(long = "db-cache", value_name = "MiB", default_value = "128")] pub database_cache_size: u32, } - -impl BenchmarkCmd { - /// Initialize - pub fn init(&self, version: &sc_cli::VersionInfo) -> sc_cli::Result<()> { - self.shared_params.init(version) - } - - /// Runs the command and benchmarks the chain. - pub fn run( - self, - config: Configuration, - ) -> sc_cli::Result<()> - where - BB: BlockT + Debug, - <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, - ::Hash: std::str::FromStr, - ExecDispatch: NativeExecutionDispatch + 'static, - { - let spec = config.chain_spec.expect("chain_spec is always Some"); - let wasm_method = self.wasm_method.into(); - let strategy = self.execution.unwrap_or(ExecutionStrategy::Native); - - let genesis_storage = spec.build_storage()?; - let mut changes = Default::default(); - let cache_size = Some(self.database_cache_size as usize); - let state = BenchmarkingState::::new(genesis_storage, cache_size)?; - let executor = NativeExecutor::::new( - wasm_method, - None, // heap pages - 2, // The runtime instances cache size. - ); - - let mut extensions = Extensions::default(); - extensions.register(KeystoreExt(KeyStore::new())); - - let result = StateMachine::<_, _, NumberFor, _>::new( - &state, - None, - &mut changes, - &executor, - "Benchmark_dispatch_benchmark", - &( - &self.pallet, - &self.extrinsic, - self.lowest_range_values.clone(), - self.highest_range_values.clone(), - self.steps.clone(), - self.repeat, - ).encode(), - extensions, - &sp_state_machine::backend::BackendRuntimeCode::new(&state).runtime_code()?, - tasks::executor(), - ) - .execute(strategy.into()) - .map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?; - - let results = , String> as Decode>::decode(&mut &result[..]) - .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?; - - match results { - Ok(batches) => for batch in batches.into_iter() { - // Print benchmark metadata - println!( - "Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}", - String::from_utf8(batch.pallet).expect("Encoded from String; qed"), - String::from_utf8(batch.benchmark).expect("Encoded from String; qed"), - self.lowest_range_values, - self.highest_range_values, - self.steps, - self.repeat, - ); - - if self.raw_data { - // Print the table header - batch.results[0].0.iter().for_each(|param| print!("{:?},", param.0)); - - print!("extrinsic_time,storage_root_time\n"); - // Print the values - batch.results.iter().for_each(|result| { - let parameters = &result.0; - parameters.iter().for_each(|param| print!("{:?},", param.1)); - // Print extrinsic time and storage root time - print!("{:?},{:?}\n", result.1, result.2); - }); - - print!("\n"); - } - - // Conduct analysis. - if !self.no_median_slopes { - if let Some(analysis) = Analysis::median_slopes(&batch.results) { - println!("Median Slopes Analysis\n========\n{}", analysis); - } - } - if !self.no_min_squares { - if let Some(analysis) = Analysis::min_squares_iqr(&batch.results) { - println!("Min Squares Analysis\n========\n{}", analysis); - } - } - }, - Err(error) => eprintln!("Error: {:?}", error), - } - - Ok(()) - } - - /// Update and prepare a `Configuration` with command line parameters - pub fn update_config( - &self, - mut config: &mut Configuration, - spec_factory: impl FnOnce(&str) -> Result, String>, - _version: &VersionInfo, - ) -> sc_cli::Result<()> - { - // Configure chain spec. - let chain_key = self.shared_params.chain.clone().unwrap_or("dev".into()); - let spec = spec_factory(&chain_key)?; - config.chain_spec = Some(spec); - - // Make sure to configure keystore. - config.use_in_memory_keystore()?; - - Ok(()) - } -} diff --git a/utils/frame/rpc/support/Cargo.toml b/utils/frame/rpc/support/Cargo.toml index 72884330d2e8224cc9959beb5c23bcd67166dea1..c635471bb930476ef84e0f84dafa31f6e43b2489 100644 --- a/utils/frame/rpc/support/Cargo.toml +++ b/utils/frame/rpc/support/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-frame-rpc-support" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies ", "Andrew Dirksen "] edition = "2018" license = "GPL-3.0" @@ -8,19 +8,19 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "Substrate RPC for FRAME's support" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] futures = { version = "0.3.0", features = ["compat"] } jsonrpc-client-transports = { version = "14.0.5", default-features = false, features = ["http"] } jsonrpc-core = "14" codec = { package = "parity-scale-codec", version = "1" } serde = "1" -frame-support = { version = "2.0.0-alpha.5", path = "../../../../frame/support" } -sp-storage = { version = "2.0.0-alpha.5", path = "../../../../primitives/storage" } -sc-rpc-api = { version = "0.8.0-alpha.5", path = "../../../../client/rpc-api" } +frame-support = { version = "2.0.0-dev", path = "../../../../frame/support" } +sp-storage = { version = "2.0.0-dev", path = "../../../../primitives/storage" } +sc-rpc-api = { version = "0.8.0-dev", path = "../../../../client/rpc-api" } [dev-dependencies] -frame-system = { version = "2.0.0-alpha.5", path = "../../../../frame/system" } +frame-system = { version = "2.0.0-dev", path = "../../../../frame/system" } tokio = "0.2" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/utils/frame/rpc/system/Cargo.toml b/utils/frame/rpc/system/Cargo.toml index a9e775393dfb17702933d17399a80824872ba5ee..a33a9dfd73163afb1d7b5a76024506e548ec5e1b 100644 --- a/utils/frame/rpc/system/Cargo.toml +++ b/utils/frame/rpc/system/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-frame-rpc-system" -version = "2.0.0-alpha.5" +version = "2.0.0-dev" authors = ["Parity Technologies "] edition = "2018" license = "GPL-3.0" @@ -8,8 +8,11 @@ homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" description = "FRAME's system exposed over Substrate RPC" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sc-client = { version = "0.8.0-alpha.5", path = "../../../../client/" } +sc-client-api = { version = "2.0.0-dev", path = "../../../../client/api" } codec = { package = "parity-scale-codec", version = "1.3.0" } futures = "0.3.4" jsonrpc-core = "14.0.3" @@ -17,17 +20,14 @@ jsonrpc-core-client = "14.0.5" jsonrpc-derive = "14.0.3" log = "0.4.8" serde = { version = "1.0.101", features = ["derive"] } -sp-runtime = { version = "2.0.0-alpha.5", path = "../../../../primitives/runtime" } -sp-api = { version = "2.0.0-alpha.5", path = "../../../../primitives/api" } -frame-system-rpc-runtime-api = { version = "2.0.0-alpha.5", path = "../../../../frame/system/rpc/runtime-api" } -sp-core = { version = "2.0.0-alpha.5", path = "../../../../primitives/core" } -sp-blockchain = { version = "2.0.0-alpha.5", path = "../../../../primitives/blockchain" } -sp-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../../primitives/transaction-pool" } +sp-runtime = { version = "2.0.0-dev", path = "../../../../primitives/runtime" } +sp-api = { version = "2.0.0-dev", path = "../../../../primitives/api" } +frame-system-rpc-runtime-api = { version = "2.0.0-dev", path = "../../../../frame/system/rpc/runtime-api" } +sp-core = { version = "2.0.0-dev", path = "../../../../primitives/core" } +sp-blockchain = { version = "2.0.0-dev", path = "../../../../primitives/blockchain" } +sp-transaction-pool = { version = "2.0.0-dev", path = "../../../../primitives/transaction-pool" } [dev-dependencies] substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../../test-utils/runtime/client" } env_logger = "0.7.0" -sc-transaction-pool = { version = "2.0.0-alpha.5", path = "../../../../client/transaction-pool" } - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] +sc-transaction-pool = { version = "2.0.0-dev", path = "../../../../client/transaction-pool" } diff --git a/utils/frame/rpc/system/src/lib.rs b/utils/frame/rpc/system/src/lib.rs index c73ddfe93efa067d8188f9abb66c554c9865baad..8dff3a641d1edf435ef50853c0a32ca9ad45ff8d 100644 --- a/utils/frame/rpc/system/src/lib.rs +++ b/utils/frame/rpc/system/src/lib.rs @@ -19,10 +19,7 @@ use std::sync::Arc; use codec::{self, Codec, Decode, Encode}; -use sc_client::{ - light::blockchain::{future_header, RemoteBlockchain}, - light::fetcher::{Fetcher, RemoteCallRequest}, -}; +use sc_client_api::light::{future_header, RemoteBlockchain, Fetcher, RemoteCallRequest}; use jsonrpc_core::{ Error, ErrorCode, futures::future::{result, Future}, @@ -236,7 +233,11 @@ mod tests { let _ = env_logger::try_init(); let client = Arc::new(substrate_test_runtime_client::new()); let pool = Arc::new( - BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone()))).0 + BasicPool::new( + Default::default(), + Arc::new(FullChainApi::new(client.clone())), + None, + ).0 ); let source = sp_runtime::transaction_validity::TransactionSource::External; diff --git a/utils/prometheus/Cargo.toml b/utils/prometheus/Cargo.toml index add4b0da5ff9565b76e0efb25c5c7ceb888e709a..0b46540903159f86d4ece7774d76973750b505b6 100644 --- a/utils/prometheus/Cargo.toml +++ b/utils/prometheus/Cargo.toml @@ -1,13 +1,16 @@ [package] description = "Endpoint to expose Prometheus metrics" name = "substrate-prometheus-endpoint" -version = "0.8.0-alpha.5" +version = "0.8.0-dev" license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" homepage = "https://substrate.dev" repository = "https://github.com/paritytech/substrate/" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] log = "0.4.8" prometheus = "0.8" @@ -18,6 +21,3 @@ derive_more = "0.99" async-std = { version = "1.0.1", features = ["unstable"] } hyper = { version = "0.13.1", default-features = false, features = ["stream"] } tokio = "0.2" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] diff --git a/utils/prometheus/src/lib.rs b/utils/prometheus/src/lib.rs index 00f0fb4f972e476122ac70007c1fe52c443f0625..9030704cb746ffbb4fa9ebfe5f468751a67a28d2 100644 --- a/utils/prometheus/src/lib.rs +++ b/utils/prometheus/src/lib.rs @@ -19,6 +19,7 @@ pub use prometheus::{ self, Registry, Error as PrometheusError, Opts, Histogram, HistogramOpts, HistogramVec, + exponential_buckets, core::{ GenericGauge as Gauge, GenericCounter as Counter, GenericGaugeVec as GaugeVec, GenericCounterVec as CounterVec, diff --git a/utils/wasm-builder-runner/Cargo.toml b/utils/wasm-builder-runner/Cargo.toml index 77796ea8d9a8b15c21d7531fb772afc43c47ee6a..c8cbeb169ea7471a0833d000fa5038f7e45c56ab 100644 --- a/utils/wasm-builder-runner/Cargo.toml +++ b/utils/wasm-builder-runner/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/paritytech/substrate/" license = "GPL-3.0" homepage = "https://substrate.dev" -[dependencies] - [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] diff --git a/utils/wasm-builder/Cargo.toml b/utils/wasm-builder/Cargo.toml index ed953cca577ddf1deaec0f8600b7d473293ab24c..1aa38a790c25644f459025ec58f2acd56869eed3 100644 --- a/utils/wasm-builder/Cargo.toml +++ b/utils/wasm-builder/Cargo.toml @@ -9,6 +9,9 @@ repository = "https://github.com/paritytech/substrate/" license = "GPL-3.0" homepage = "https://substrate.dev" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] build-helper = "0.1.1" cargo_metadata = "0.9.0" @@ -19,6 +22,3 @@ fs2 = "0.4.3" wasm-gc-api = "0.1.11" atty = "0.2.13" itertools = "0.8.2" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"]